diff --git a/include/SqlDB.h b/include/SqlDB.h index 836a3210ea..a9d7db8605 100644 --- a/include/SqlDB.h +++ b/include/SqlDB.h @@ -166,6 +166,15 @@ public: } } + /** + * Set maximal number of consecutive errors to terminate oned in HA environment + * -1 to disable this feature + */ + void set_errors_limit(int limit) + { + errors_limit = limit; + } + protected: /** * Performs a DB transaction @@ -174,17 +183,7 @@ protected: * @param quiet True to log errors with DDEBUG level instead of ERROR * @return 0 on success -1 on failure */ - int exec(std::ostringstream& cmd, Callbackable* obj, bool quiet) - { - int rc = exec_ext(cmd, obj, quiet); - - if (rc != 0) - { - rc = -1; - }; - - return rc; - } + int exec(std::ostringstream& cmd, Callbackable* obj, bool quiet); /** * This function performs a DB transaction and returns and extended error code @@ -201,6 +200,18 @@ protected: {SqlFeature::FTS, false}, {SqlFeature::COMPARE_BINARY, false} }; + +private: + /** + * Actual number of consecutive errors + */ + int consecutive_errors = 0; + + /** + * Maximal number of consecutive errors to terminate oned in HA environment + * -1 to disable this feature + */ + int errors_limit = -1; }; #endif /*SQL_DB_H_*/ diff --git a/src/nebula/Nebula.cc b/src/nebula/Nebula.cc index 22d6333a0e..2d8f892548 100644 --- a/src/nebula/Nebula.cc +++ b/src/nebula/Nebula.cc @@ -391,6 +391,7 @@ void Nebula::start(bool bootstrap_only) string compare_binary; int timeout; int connections; + int errors_limit; const VectorAttribute * _db = nebula_configuration->get("DB"); @@ -407,6 +408,7 @@ void Nebula::start(bool bootstrap_only) _db->vector_value("COMPARE_BINARY", compare_binary, "NO"); _db->vector_value("TIMEOUT", timeout, 2500); _db->vector_value("CONNECTIONS", connections, 25); + _db->vector_value("ERRORS_LIMIT", errors_limit, 25); } if ( db_backend_type == "sqlite" ) @@ -457,6 +459,11 @@ void Nebula::start(bool bootstrap_only) // --------------------------------------------------------------------- solo = server_id == -1; + if (!solo) + { + db_backend->set_errors_limit(errors_limit); + } + if ( (solo && local_bootstrap) || bootstrap_only) { if (cache) diff --git a/src/scheduler/src/sched/SConstruct b/src/scheduler/src/sched/SConstruct index fb6c5c1357..6c900b2d7e 100644 --- a/src/scheduler/src/sched/SConstruct +++ b/src/scheduler/src/sched/SConstruct @@ -42,6 +42,8 @@ sched_env.Prepend(LIBS=[ 'nebula_vm', 'nebula_host', 'nebula_parsers', + 'nebula_sql', + 'nebula_sql_const', 'crypto', 'xml2' ]) diff --git a/src/sql/SConstruct b/src/sql/SConstruct index d25a7809cf..c7094bc3f1 100644 --- a/src/sql/SConstruct +++ b/src/sql/SConstruct @@ -20,7 +20,7 @@ Import('env') lib_name='nebula_sql' -source_files=['LogDB.cc'] +source_files=['LogDB.cc', 'SqlDB.cc'] # Sources to generate the library if env['sqlite']=='yes': diff --git a/src/sql/SqlDB.cc b/src/sql/SqlDB.cc new file mode 100644 index 0000000000..5566f617b3 --- /dev/null +++ b/src/sql/SqlDB.cc @@ -0,0 +1,48 @@ +/* -------------------------------------------------------------------------- */ +/* Copyright 2002-2020, OpenNebula Project, OpenNebula Systems */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); you may */ +/* not use this file except in compliance with the License. You may obtain */ +/* a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ +/* See the License for the specific language governing permissions and */ +/* limitations under the License. */ +/* -------------------------------------------------------------------------- */ + +#include "SqlDB.h" +#include "NebulaLog.h" + +#include +#include +#include + +int SqlDB::exec(std::ostringstream& cmd, Callbackable* obj, bool quiet) +{ + int rc = exec_ext(cmd, obj, quiet); + + if (rc != 0) + { + consecutive_errors++; + rc = -1; + + if (errors_limit > 0 && consecutive_errors > errors_limit) + { + NebulaLog::error("SQL", "Lost connection to DB server, exiting..."); + + // Kill the master process. The call is in the thread to avoid deadlock + std::thread thr([]{ kill(getpid(), SIGTERM); }); + thr.detach(); + } + } + else + { + consecutive_errors = 0; + } + + return rc; +}