B #5186: Exit HA node in case of DB failure (#712)

(cherry picked from commit b36cf9f33940652c59bf1cf8a0195f7d062445c3)

include/SqlDB.h

@@ -166,6 +166,15 @@ public:
         }
     }
 
+    /**
+     * Set maximal number of consecutive errors to terminate oned in HA environment
+     * -1 to disable this feature
+     */
+    void set_errors_limit(int limit)
+    {
+        errors_limit = limit;
+    }
+
 protected:
     /**
      *  Performs a DB transaction
@@ -174,17 +183,7 @@ protected:
      *    @param quiet True to log errors with DDEBUG level instead of ERROR
      *    @return 0 on success -1 on failure
      */
-    int exec(std::ostringstream& cmd, Callbackable* obj, bool quiet)
-    {
-        int rc = exec_ext(cmd, obj, quiet);
-
-        if (rc != 0)
-        {
-            rc = -1;
-        };
-
-        return rc;
-    }
+    int exec(std::ostringstream& cmd, Callbackable* obj, bool quiet);
 
     /**
      *  This function performs a DB transaction and returns and extended error code
@@ -201,6 +200,18 @@ protected:
         {SqlFeature::FTS, false},
         {SqlFeature::COMPARE_BINARY, false}
     };
+
+private:
+    /**
+     * Actual number of consecutive errors
+     */
+    int consecutive_errors = 0;
+
+    /**
+     * Maximal number of consecutive errors to terminate oned in HA environment
+     * -1 to disable this feature
+     */
+    int errors_limit = -1;
 };
 
 #endif /*SQL_DB_H_*/

src/nebula/Nebula.cc

@@ -391,6 +391,7 @@ void Nebula::start(bool bootstrap_only)
         string compare_binary;
         int    timeout;
         int    connections;
+        int    errors_limit;
 
         const VectorAttribute * _db = nebula_configuration->get("DB");
 
@@ -407,6 +408,7 @@ void Nebula::start(bool bootstrap_only)
             _db->vector_value<string>("COMPARE_BINARY", compare_binary, "NO");
             _db->vector_value("TIMEOUT", timeout, 2500);
             _db->vector_value("CONNECTIONS", connections, 25);
+            _db->vector_value("ERRORS_LIMIT", errors_limit, 25);
         }
 
         if ( db_backend_type == "sqlite" )
@@ -457,6 +459,11 @@ void Nebula::start(bool bootstrap_only)
         // ---------------------------------------------------------------------
         solo = server_id == -1;
 
+        if  (!solo)
+        {
+            db_backend->set_errors_limit(errors_limit);
+        }
+
         if ( (solo && local_bootstrap) || bootstrap_only)
         {
             if (cache)

src/scheduler/src/sched/SConstruct

@@ -42,6 +42,8 @@ sched_env.Prepend(LIBS=[
     'nebula_vm',
     'nebula_host',
     'nebula_parsers',
+    'nebula_sql',
+    'nebula_sql_const',
     'crypto',
     'xml2'
 ])

src/sql/SConstruct

@@ -20,7 +20,7 @@ Import('env')
 
 lib_name='nebula_sql'
 
-source_files=['LogDB.cc']
+source_files=['LogDB.cc', 'SqlDB.cc']
 
 # Sources to generate the library
 if env['sqlite']=='yes':

src/sql/SqlDB.cc

@@ -0,0 +1,48 @@
+/* -------------------------------------------------------------------------- */
+/* Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                */
+/*                                                                            */
+/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
+/* not use this file except in compliance with the License. You may obtain    */
+/* a copy of the License at                                                   */
+/*                                                                            */
+/* http://www.apache.org/licenses/LICENSE-2.0                                 */
+/*                                                                            */
+/* Unless required by applicable law or agreed to in writing, software        */
+/* distributed under the License is distributed on an "AS IS" BASIS,          */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+/* See the License for the specific language governing permissions and        */
+/* limitations under the License.                                             */
+/* -------------------------------------------------------------------------- */
+
+#include "SqlDB.h"
+#include "NebulaLog.h"
+
+#include <unistd.h>
+#include <csignal>
+#include <thread>
+
+int SqlDB::exec(std::ostringstream& cmd, Callbackable* obj, bool quiet)
+{
+    int rc = exec_ext(cmd, obj, quiet);
+
+    if (rc != 0)
+    {
+        consecutive_errors++;
+        rc = -1;
+
+        if (errors_limit > 0 && consecutive_errors > errors_limit)
+        {
+            NebulaLog::error("SQL", "Lost connection to DB server, exiting...");
+
+            // Kill the master process. The call is in the thread to avoid deadlock
+            std::thread thr([]{ kill(getpid(), SIGTERM); });
+            thr.detach();
+        }
+    }
+    else
+    {
+        consecutive_errors = 0;
+    }
+
+    return rc;
+}