From 9aa10411039938c0a89dac473f66bada483f0cf7 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Mon, 13 Apr 2020 17:32:21 +0200 Subject: [PATCH] F #3600: Initial PostgreSQL Support co-authored-by: Igor Sivy co-authored-by: Pavel Czerny co-authored-by: Vlastimil Holer (cherry picked from commit c52f62018c32281c6e418211f33f1bba46388e98) --- SConstruct | 10 + include/ClusterPool.h | 7 +- include/DatastorePool.h | 11 +- include/DocumentPool.h | 9 +- include/GroupPool.h | 6 +- include/HookPool.h | 8 +- include/HostPool.h | 7 +- include/ImagePool.h | 11 +- include/LogDB.h | 27 +- include/MarketPlaceAppPool.h | 11 +- include/MarketPlacePool.h | 11 +- include/MySqlDB.h | 88 ++--- include/PoolSQL.h | 32 +- include/PostgreSqlDB.h | 196 +++++++++++ include/SecurityGroupPool.h | 9 +- include/SqlDB.h | 73 +++- include/SqliteDB.h | 37 +- include/UserPool.h | 6 +- include/VMGroupPool.h | 11 +- include/VMTemplatePool.h | 11 +- include/VNTemplatePool.h | 11 +- include/VdcPool.h | 10 +- include/VirtualMachinePool.h | 13 +- include/VirtualNetworkPool.h | 11 +- include/VirtualRouterPool.h | 11 +- include/ZonePool.h | 10 +- share/etc/oned.conf | 2 +- share/install_gems/CentOS7/Gemfile.lock | 2 + share/install_gems/CentOS8/Gemfile.lock | 2 + share/install_gems/Debian10/Gemfile.lock | 2 + share/install_gems/Debian9/Gemfile.lock | 2 + share/install_gems/Gemfile | 2 + share/install_gems/Ubuntu1604/Gemfile.lock | 2 + share/install_gems/Ubuntu1804/Gemfile.lock | 2 + share/install_gems/Ubuntu1904/Gemfile.lock | 2 + share/install_gems/Ubuntu1910/Gemfile.lock | 2 + share/install_gems/install_gems | 6 +- src/acl/AclManager.cc | 6 +- src/group/GroupPool.cc | 7 +- src/host/HostPool.cc | 2 +- src/im/InformationManager.cc | 3 +- src/monitor/etc/monitord.conf | 9 + src/monitor/src/monitor/Monitor.cc | 24 +- .../src/monitor/MonitorConfigTemplate.cc | 4 + src/nebula/Nebula.cc | 14 +- src/nebula/SystemDB.cc | 2 +- src/onedb/database_schema.rb | 2 + src/onedb/onedb | 47 +-- src/onedb/onedb.rb | 29 +- src/onedb/onedb_backend.rb | 308 ++++++++++++++-- src/onedb/shared/5.10.0_to_5.12.0.rb | 21 ++ src/pool/PoolSQL.cc | 8 +- src/rm/RequestManagerPoolInfoFilter.cc | 21 +- src/sql/LogDB.cc | 26 +- src/sql/MySqlDB.cc | 52 +-- src/sql/PostgreSqlDB.cc | 333 ++++++++++++++++++ src/sql/SConstruct | 10 + src/sql/SqliteDB.cc | 30 +- src/um/UserPool.cc | 7 +- src/vm/VirtualMachine.cc | 2 +- src/vm/VirtualMachinePool.cc | 10 +- 61 files changed, 1263 insertions(+), 387 deletions(-) create mode 100644 include/PostgreSqlDB.h create mode 100644 src/sql/PostgreSqlDB.cc diff --git a/SConstruct b/SConstruct index 94f5fd8e2c..b03bff7d25 100755 --- a/SConstruct +++ b/SConstruct @@ -153,6 +153,16 @@ if mysql == 'yes': else: main_env.Append(mysql='no') +# PostgreSql +postgresql = ARGUMENTS.get('postgresql', 'no') +if postgresql == 'yes': + main_env.Append(postgresql='yes') + main_env.Append(CPPPATH=['/usr/include/postgresql']) + main_env.Append(CPPFLAGS=["-DPOSTGRESQL_DB"]) + main_env.Append(LIBS=['libpq']) +else: + main_env.Append(postgresql='no') + # Flag to compile with xmlrpc-c versions prior to 1.31 (September 2012) new_xmlrpc = ARGUMENTS.get('new_xmlrpc', 'no') if new_xmlrpc == 'yes': diff --git a/include/ClusterPool.h b/include/ClusterPool.h index 9a375abf2f..8a94f33288 100644 --- a/include/ClusterPool.h +++ b/include/ClusterPool.h @@ -200,16 +200,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, + int dump(std::string& oss, const std::string& where, int sid, int eid, bool desc) { return PoolSQL::dump(oss, "CLUSTER_POOL", "body", Cluster::table, where, - limit, desc); + sid, eid, desc); }; /** diff --git a/include/DatastorePool.h b/include/DatastorePool.h index 97e47dca0e..37d6bf8e4d 100644 --- a/include/DatastorePool.h +++ b/include/DatastorePool.h @@ -143,16 +143,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table, where, - limit, desc); + return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table, + where, sid, eid, desc); }; /** diff --git a/include/DocumentPool.h b/include/DocumentPool.h index 3421e897a7..f4979b0b93 100644 --- a/include/DocumentPool.h +++ b/include/DocumentPool.h @@ -94,16 +94,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { return PoolSQL::dump(oss, "DOCUMENT_POOL", "body", Document::table, where, - limit, desc); + sid, eid, desc); }; /** diff --git a/include/GroupPool.h b/include/GroupPool.h index a24302aff1..457542a21e 100644 --- a/include/GroupPool.h +++ b/include/GroupPool.h @@ -160,13 +160,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc); + int dump(string& oss, const string& where, int sid, int eid, bool desc); private: diff --git a/include/HookPool.h b/include/HookPool.h index 403af7741a..722473bab6 100644 --- a/include/HookPool.h +++ b/include/HookPool.h @@ -74,15 +74,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, + int dump(std::string& oss, const std::string& where, int sid, int eid, bool desc) { - return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where, limit, desc); + return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where, + sid, eid, desc); }; /** diff --git a/include/HostPool.h b/include/HostPool.h index 0dcd99065e..4bf487072e 100644 --- a/include/HostPool.h +++ b/include/HostPool.h @@ -212,16 +212,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, + int dump(std::string& oss, const std::string& where, int sid, int eid, bool desc) { return PoolSQL::dump(oss, "HOST_POOL", "body", one_db::host_table, - where, limit, desc); + where, sid, eid, desc); }; /** diff --git a/include/ImagePool.h b/include/ImagePool.h index 30bd3ad59b..fc6bf8bdb2 100644 --- a/include/ImagePool.h +++ b/include/ImagePool.h @@ -155,16 +155,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, limit, - desc); + return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, sid, + eid, desc); } /** diff --git a/include/LogDB.h b/include/LogDB.h index 36e4adcae9..833efad0d9 100644 --- a/include/LogDB.h +++ b/include/LogDB.h @@ -208,19 +208,14 @@ public: db->free_str(str); } - bool multiple_values_support() + bool supports(SqlDB::SqlFeature ft) { - return db->multiple_values_support(); + return db->supports(ft); } - bool limit_support() + std::string limit_string(int start_id, int end_id) { - return db->limit_support(); - } - - bool fts_available() - { - return db->fts_available(); + return db->limit_string(start_id, end_id); } // ------------------------------------------------------------------------- // Database methods @@ -422,19 +417,9 @@ public: _logdb->free_str(str); } - bool multiple_values_support() + bool supports(SqlDB::SqlFeature ft) { - return _logdb->multiple_values_support(); - } - - bool limit_support() - { - return _logdb->limit_support(); - } - - bool fts_available() - { - return _logdb->fts_available(); + return _logdb->supports(ft); } /** diff --git a/include/MarketPlaceAppPool.h b/include/MarketPlaceAppPool.h index b8266b767f..36b58f881b 100644 --- a/include/MarketPlaceAppPool.h +++ b/include/MarketPlaceAppPool.h @@ -144,16 +144,17 @@ public: * the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(std::string& oss, const std::string& where, - const std::string& limit, bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body", MarketPlaceApp::table, - where, limit, desc); + return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body", + MarketPlaceApp::table, where, sid, eid, desc); }; /** Update a particular MarketPlaceApp diff --git a/include/MarketPlacePool.h b/include/MarketPlacePool.h index b499aed634..4e12c7d8e3 100644 --- a/include/MarketPlacePool.h +++ b/include/MarketPlacePool.h @@ -111,16 +111,17 @@ public: * the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(std::string& oss, const std::string& where, - const std::string& limit, bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table, where, - limit, desc); + return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table, + where, sid, eid, desc); }; /** diff --git a/include/MySqlDB.h b/include/MySqlDB.h index 4661237dfa..cbf0b1ed86 100644 --- a/include/MySqlDB.h +++ b/include/MySqlDB.h @@ -30,9 +30,6 @@ #include "SqlDB.h" #include "ObjectSQL.h" - -using namespace std; - #ifdef MYSQL_DB #include @@ -44,13 +41,13 @@ class MySqlDB : public SqlDB { public: - MySqlDB(const string& _server, - int _port, - const string& _user, - const string& _password, - const string& _database, - const string& _encoding, - int _connections); + MySqlDB(const std::string& _server, + int _port, + const std::string& _user, + const std::string& _password, + const std::string& _database, + const std::string& _encoding, + int _connections); ~MySqlDB(); @@ -61,7 +58,7 @@ public: * @param str the string to be escaped * @return a valid SQL string or NULL in case of failure */ - char * escape_str(const string& str); + char * escape_str(const std::string& str); /** * Frees a previously scaped string @@ -72,36 +69,6 @@ public: delete[] str; } - /** - * Returns true if the syntax INSERT VALUES (data), (data), (data) - * is supported - * - * @return true if supported - */ - bool multiple_values_support() - { - return true; - } - - /** - * Returns true if this Database can use LIMIT in queries with DELETE - * and UPDATE - * - * @return true if supported - */ - bool limit_support() - { - return true; - } - - /** - * Return true if the backend allows FTS index - */ - bool fts_available() - { - return _fts_available; - } - protected: /** * Wraps the mysql_query function call @@ -109,7 +76,7 @@ protected: * @param obj Callbackable obj to call if the query succeeds * @return 0 on success */ - int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet); + int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override; private: @@ -137,19 +104,17 @@ private: /** * MySQL Connection parameters */ - string server; + std::string server; int port; - string user; + std::string user; - string password; + std::string password; - string database; + std::string database; - string encoding; - - bool _fts_available; + std::string encoding; /** * Fine-grain mutex for DB access (pool of DB connections) @@ -177,12 +142,12 @@ class MySqlDB : public SqlDB { public: - MySqlDB(const string& _server, + MySqlDB(const std::string& _server, int _port, - const string& _user, - const string& _password, - const string& _database, - const string& _encoding, + const std::string& _user, + const std::string& _password, + const std::string& _database, + const std::string& _encoding, int _connections) { throw runtime_error("Aborting oned, MySQL support not compiled!"); @@ -190,19 +155,14 @@ public: ~MySqlDB(){}; + char * escape_str(const std::string& str) override {return nullptr;}; - char * escape_str(const string& str){return 0;}; - - void free_str(char * str){}; - - bool multiple_values_support(){return true;}; - - bool limit_support(){return true;}; - - bool fts_available(){return false;}; + void free_str(char * str) override {}; protected: - int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet){return -1;}; + int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override { + return -1; + }; }; #endif diff --git a/include/PoolSQL.h b/include/PoolSQL.h index 5033f99a90..696b1b8568 100644 --- a/include/PoolSQL.h +++ b/include/PoolSQL.h @@ -169,7 +169,7 @@ public: */ int dump(string& oss, const string& where, bool desc) { - return dump(oss, where, "", desc); + return dump(oss, where, 0, -1, desc); } /** @@ -177,30 +177,33 @@ public: * to the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - virtual int dump(string& oss, const string& where, - const string& limit, bool desc) = 0; + virtual int dump(string& oss, const string& where, int sid, int eid, + bool desc) = 0; /** * Dumps the pool in extended XML format * A filter and limit can be also added to the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ virtual int dump_extended(string& oss, const string& where, - const string& limit, + int sid, + int eid, bool desc) { - return dump(oss, where, limit, desc); + return dump(oss, where, sid, eid, desc); } // ------------------------------------------------------------------------- @@ -276,12 +279,13 @@ public: } /** - * Return true if FTS is available. + * Return true if feature is supported */ - bool is_fts_available() + bool supports(SqlDB::SqlFeature ft) { - return db->fts_available(); + return db->supports(ft); } + protected: /** @@ -316,7 +320,8 @@ protected: * @param elem_name Name of the root xml pool name * @param table Pool table name * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param start_id first element used for pagination + * @param end_id last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success @@ -326,7 +331,8 @@ protected: const string& column, const char * table, const string& where, - const string& limit, + int start_id, + int end_id, bool desc); /** @@ -346,7 +352,7 @@ protected: const string& where, bool desc) { - return dump(oss, elem_name, "body", table, where, "", desc); + return dump(oss, elem_name, "body", table, where, 0, -1, desc); } /** diff --git a/include/PostgreSqlDB.h b/include/PostgreSqlDB.h new file mode 100644 index 0000000000..2cd1572dd3 --- /dev/null +++ b/include/PostgreSqlDB.h @@ -0,0 +1,196 @@ +/* -------------------------------------------------------------------------- */ +/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); you may */ +/* not use this file except in compliance with the License. You may obtain */ +/* a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ +/* See the License for the specific language governing permissions and */ +/* limitations under the License. */ +/* -------------------------------------------------------------------------- */ + +#ifndef POSTGRESQL_DB_H_ +#define POSTGRESQL_DB_H_ + +#include +#include +#include + +#include "NebulaLog.h" +#include "SqlDB.h" +#include "ObjectSQL.h" + +#ifdef POSTGRESQL_DB + +#include + +/** + * PostgreSqlDB class. Provides a wrapper to the PostgreSQL database interface. + */ +class PostgreSqlDB : public SqlDB +{ +public: + PostgreSqlDB( + const string& _server, + int _port, + const string& _user, + const string& _password, + const string& _database, + int _connections); + + ~PostgreSqlDB(); + + /** + * This function returns a legal SQL string that can be used in an SQL + * statement. + * @param str the string to be escaped + * @return a valid SQL string or NULL in case of failure + */ + char * escape_str(const string& str) override; + + /** + * Frees a previously scaped string + * @param str pointer to the str + */ + void free_str(char * str) override; + + /** + * @param sid the offset + * @param eid the rowcount + * @return string with compatible LIMIT clause syntax + * LIMIT row_count OFFSET offset + * + * +---+---+---+---+---+---+---+---+-- + * | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |... + * +---+---+---+---+---+---+---+---+-- + * | | + * /-------------------/ + * LIMIT 5 OFFSET 3 + */ + + std::string limit_string(int sid, int eid) override + { + std::ostringstream oss; + oss << "LIMIT " << eid << " OFFSET " << sid; + + return oss.str(); + } + + std::string limit_string(int sid) override + { + std::ostringstream oss; + oss << "LIMIT " << sid << " OFFSET 0"; + + return oss.str(); + } + +protected: + int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override; + +private: + + /** + * Number of concurrent DB connections. + */ + int max_connections; + + /** + * Connection pool + */ + queue db_connect; + + /** + * DB connection to escape strings + */ + PGconn * db_escape_connect; + + /** + * Connection parameters + */ + string server; + int port; + string user; + string password; + string database; + + /** + * Fine-grain mutex for DB access (pool of DB connections) + */ + pthread_mutex_t mutex; + + /** + * Conditional variable to wake-up waiting threads. + */ + pthread_cond_t cond; + + /** + * Gets a free DB connection from the pool. + */ + PGconn * get_db_connection(); + + /** + * Returns the connection to the pool. + */ + void free_db_connection(PGconn * db); + + /** + * Preprocesses the query to be compatible with PostgreSQL syntax + * + * Any change to this method should be reflected in BackEndPostgreSQL class + * in src/onedb/onedb_backend.rb + * + * This method alters to queries: + * - CREATE TABLE to adjust type names + * . MEDIUMTEXT -> TEXT + * . LONGTEXT -> TEXT + * . BIGINT UNSIGNED -> NUMERIC + * + * - REPLACE INTO into PostgreSQL INSERT INTO query with ON CONFLICT + * clause. For example: + * REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0) + * changes to: + * INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0) + * ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid + *************************************************************************** + * Any change to this method should be reflected in BackEndPostgreSQL class + * in src/onedb/onedb_backend.rb + *************************************************************************** + */ + static std::string preprocess_query(std::ostringstream& cmd); +}; +#else +// Class stub +class PostgreSqlDB : public SqlDB +{ +public: + PostgreSqlDB( + const string& _server, + int _port, + const string& _user, + const string& _password, + const string& _database, + int _connections) + { + throw runtime_error("Aborting oned, PostgreSQL support not compiled!"); + } + ~PostgreSqlDB(){} + + char * escape_str(const string& str) override {return 0;}; + + void free_str(char * str) override {}; + + std::string limit_string(int sid, int eid) override {return "";} + +protected: + int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override { + return -1; + }; +}; +#endif + +#endif /*POSTGRESQL_DB_H*/ diff --git a/include/SecurityGroupPool.h b/include/SecurityGroupPool.h index b297fd435c..82078208d3 100644 --- a/include/SecurityGroupPool.h +++ b/include/SecurityGroupPool.h @@ -106,16 +106,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { return PoolSQL::dump(oss, "SECURITY_GROUP_POOL", "body", SecurityGroup::table, - where, limit, desc); + where, sid, eid, desc); }; /** diff --git a/include/SqlDB.h b/include/SqlDB.h index fe4f6c2d24..52099a9534 100644 --- a/include/SqlDB.h +++ b/include/SqlDB.h @@ -18,6 +18,7 @@ #define SQL_DB_H_ #include +#include #include "Callbackable.h" /** @@ -40,6 +41,13 @@ public: SQL_DUP_KEY = -201 }; + enum class SqlFeature + { + MULTIPLE_VALUE, // syntax INSERT VALUES (data), (data), (data) + LIMIT, // LIMIT in queries with DELETE and UPDATE + FTS // Full Text Search + }; + /* ---------------------------------------------------------------------- */ /* Database Operations */ /* ---------------------------------------------------------------------- */ @@ -99,27 +107,18 @@ public: */ virtual void free_str(char * str) = 0; - /** - * Returns true if the syntax INSERT VALUES (data), (data), (data) - * is supported - * - * @return true if supported - */ - virtual bool multiple_values_support() = 0; - /** - * Returns true if this Database can use LIMIT in queries with DELETE - * and UPDATE - * - * @return true if supported - */ - virtual bool limit_support() = 0; + bool supports(SqlFeature ft) + { + auto it = features.find(ft); - /** - * Return true if the backend allows FTS index - */ - virtual bool fts_available() = 0; + if ( it == features.end() ) + { + return false; + } + return it->second; + } /** * @return pointer to a non-federated version of this database @@ -129,6 +128,35 @@ public: return this; } + /** + * @return string with compatible LIMIT clause syntax + * LIMIT [offset], row_count + * + * +---+---+---+---+---+---+---+---+-- + * | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |... + * +---+---+---+---+---+---+---+---+-- + * | | + * /-------------------/ + * LIMIT 3, 5 + */ + virtual std::string limit_string(int sid, int eid) + { + std::ostringstream oss; + + oss << "LIMIT " << sid << "," << eid; + + return oss.str(); + } + + virtual std::string limit_string(int sid) + { + std::ostringstream oss; + + oss << "LIMIT " << sid; + + return oss.str(); + } + protected: /** * Performs a DB transaction @@ -154,6 +182,15 @@ protected: * @return SqlError enum */ virtual int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) = 0; + + /** + * Feature set + */ + std::map features = { + {SqlFeature::MULTIPLE_VALUE, false}, + {SqlFeature::LIMIT, false}, + {SqlFeature::FTS, false} + }; }; #endif /*SQL_DB_H_*/ diff --git a/include/SqliteDB.h b/include/SqliteDB.h index 98eced00d6..594a6fd03d 100644 --- a/include/SqliteDB.h +++ b/include/SqliteDB.h @@ -63,26 +63,6 @@ public: */ void free_str(char * str) override; - /** - * Returns true if the syntax INSERT VALUES (data), (data), (data) - * is supported - * - * @return true if supported - */ - bool multiple_values_support() override; - - /** - * Returns true if this Database can use LIMIT in queries with DELETE - * and UPDATE - * - * @return true if supported - */ - bool limit_support() override; - - bool fts_available() override - { - return false; - } protected: /** * Wraps the sqlite3_exec function call, and locks the DB mutex. @@ -98,17 +78,12 @@ private: /** * Fine-grain mutex for DB access */ - pthread_mutex_t mutex; + pthread_mutex_t mutex; /** * Pointer to the database. */ - sqlite3 * db; - - /** - * LIMIT for DELETE and UPDATE queries is enabled - */ - int enable_limit; + sqlite3 * db; /** * Function to lock the DB @@ -141,13 +116,9 @@ public: char * escape_str(const string& str) override { return 0; } - void free_str(char * str) override {} + void free_str(char * str) override {}; - bool multiple_values_support() override { return true; } - - bool limit_support() override { return true; } - - bool fts_available() override { return false; } + std::string get_limit_string(const std::string& str) override { return str; } protected: int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override diff --git a/include/UserPool.h b/include/UserPool.h index 5a7e2e4db4..f26b0200fc 100644 --- a/include/UserPool.h +++ b/include/UserPool.h @@ -212,13 +212,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc); + int dump(string& oss, const string& where, int sid, int eid, bool desc); /** * Name for the OpenNebula core authentication process diff --git a/include/VMGroupPool.h b/include/VMGroupPool.h index 55facc9f86..772ad96c11 100644 --- a/include/VMGroupPool.h +++ b/include/VMGroupPool.h @@ -99,16 +99,17 @@ public: * Dumps the VMGroup pool in XML format. A filter can be added to the query * @param os the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(std::string& os, const std::string& where, - const std::string& limit, bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(os, "VM_GROUP_POOL", "body", VMGroup::table, where, limit, - desc); + return PoolSQL::dump(oss, "VM_GROUP_POOL", "body", VMGroup::table, where, + sid, eid, desc); }; /** diff --git a/include/VMTemplatePool.h b/include/VMTemplatePool.h index 61b82fb084..b1d133610d 100644 --- a/include/VMTemplatePool.h +++ b/include/VMTemplatePool.h @@ -84,16 +84,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table, where, - limit, desc); + return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table, + where, sid, eid, desc); }; /** diff --git a/include/VNTemplatePool.h b/include/VNTemplatePool.h index 8650ccc62e..254beda0bd 100644 --- a/include/VNTemplatePool.h +++ b/include/VNTemplatePool.h @@ -84,16 +84,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table, where, - limit, desc); + return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table, + where, sid, eid, desc); }; /** diff --git a/include/VdcPool.h b/include/VdcPool.h index 8d61b208e1..f2a355c179 100644 --- a/include/VdcPool.h +++ b/include/VdcPool.h @@ -90,15 +90,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, limit, desc); + return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, sid, + eid, desc); }; /** diff --git a/include/VirtualMachinePool.h b/include/VirtualMachinePool.h index 198e1c1526..1b271c0b67 100644 --- a/include/VirtualMachinePool.h +++ b/include/VirtualMachinePool.h @@ -250,16 +250,17 @@ public: * pool * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { return PoolSQL::dump(oss, "VM_POOL", "short_body", one_db::vm_table, where, - limit, desc); + sid, eid, desc); }; /** @@ -274,11 +275,11 @@ public: * * @return 0 on success */ - int dump_extended(string& oss, const string& where, const string& limit, + int dump_extended(string& oss, const string& where, int sid, int eid, bool desc) { return PoolSQL::dump(oss, "VM_POOL", "body", one_db::vm_table, where, - limit, desc); + sid, eid, desc); }; /** diff --git a/include/VirtualNetworkPool.h b/include/VirtualNetworkPool.h index b00dec080d..16d152d2fe 100644 --- a/include/VirtualNetworkPool.h +++ b/include/VirtualNetworkPool.h @@ -157,16 +157,17 @@ public: * to the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table, where, - limit, desc); + return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table, + where, sid, eid, desc); } /** diff --git a/include/VirtualRouterPool.h b/include/VirtualRouterPool.h index 8b073deb51..5d54b46d22 100644 --- a/include/VirtualRouterPool.h +++ b/include/VirtualRouterPool.h @@ -92,16 +92,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table, where, - limit, desc); + return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table, + where, sid, eid, desc); }; /** diff --git a/include/ZonePool.h b/include/ZonePool.h index 74a4f768d1..8ef81d9b70 100644 --- a/include/ZonePool.h +++ b/include/ZonePool.h @@ -91,15 +91,17 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all - * @param limit parameters used for pagination + * @param sid first element used for pagination + * @param eid last element used for pagination, -1 to disable * @param desc descending order of pool elements * * @return 0 on success */ - int dump(string& oss, const string& where, const string& limit, - bool desc) + int dump(std::string& oss, const std::string& where, int sid, int eid, + bool desc) { - return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where, limit, desc); + return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where, + sid, eid, desc); }; /** diff --git a/share/etc/oned.conf b/share/etc/oned.conf index 97609c7ce0..f4795dd358 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -84,7 +84,7 @@ DB = [ BACKEND = "sqlite" ] # USER = "oneadmin", # PASSWD = "oneadmin", # DB_NAME = "opennebula", -# CONNECTIONS = 50 ] +# CONNECTIONS = 25 ] VNC_PORTS = [ START = 5900, diff --git a/share/install_gems/CentOS7/Gemfile.lock b/share/install_gems/CentOS7/Gemfile.lock index 4846ba09b0..1bb4bc0076 100644 --- a/share/install_gems/CentOS7/Gemfile.lock +++ b/share/install_gems/CentOS7/Gemfile.lock @@ -89,6 +89,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.1.4) polyglot (0.3.5) public_suffix (2.0.5) rack (1.6.13) @@ -163,6 +164,7 @@ DEPENDENCIES nokogiri (< 1.7) ox parse-cron + pg (< 1.2.0) public_suffix (< 3.0.0) rack (< 2.0.0) rbvmomi (~> 2.2.0) diff --git a/share/install_gems/CentOS8/Gemfile.lock b/share/install_gems/CentOS8/Gemfile.lock index bbe6b74060..551d618911 100644 --- a/share/install_gems/CentOS8/Gemfile.lock +++ b/share/install_gems/CentOS8/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -168,6 +169,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Debian10/Gemfile.lock b/share/install_gems/Debian10/Gemfile.lock index bbe6b74060..551d618911 100644 --- a/share/install_gems/Debian10/Gemfile.lock +++ b/share/install_gems/Debian10/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -168,6 +169,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Debian9/Gemfile.lock b/share/install_gems/Debian9/Gemfile.lock index 4143482d15..d796b50ca6 100644 --- a/share/install_gems/Debian9/Gemfile.lock +++ b/share/install_gems/Debian9/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -167,6 +168,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Gemfile b/share/install_gems/Gemfile index 202561bed3..c3721572d7 100644 --- a/share/install_gems/Gemfile +++ b/share/install_gems/Gemfile @@ -28,8 +28,10 @@ end if RUBY_VERSION < '2.2.0' gem 'rack', '< 2.0.0' # sunstone, cloud, oneflow gem 'minitest', '< 5.12.0' # packethost + gem 'pg', '< 1.2.0' # onedb else gem 'rack' # sunstone, cloud, oneflow + gem 'pg' # onedb end if RUBY_VERSION >= '2.4.0' diff --git a/share/install_gems/Ubuntu1604/Gemfile.lock b/share/install_gems/Ubuntu1604/Gemfile.lock index 4143482d15..d796b50ca6 100644 --- a/share/install_gems/Ubuntu1604/Gemfile.lock +++ b/share/install_gems/Ubuntu1604/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -167,6 +168,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Ubuntu1804/Gemfile.lock b/share/install_gems/Ubuntu1804/Gemfile.lock index bbe6b74060..551d618911 100644 --- a/share/install_gems/Ubuntu1804/Gemfile.lock +++ b/share/install_gems/Ubuntu1804/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -168,6 +169,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Ubuntu1904/Gemfile.lock b/share/install_gems/Ubuntu1904/Gemfile.lock index bbe6b74060..551d618911 100644 --- a/share/install_gems/Ubuntu1904/Gemfile.lock +++ b/share/install_gems/Ubuntu1904/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -168,6 +169,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/Ubuntu1910/Gemfile.lock b/share/install_gems/Ubuntu1910/Gemfile.lock index bbe6b74060..551d618911 100644 --- a/share/install_gems/Ubuntu1910/Gemfile.lock +++ b/share/install_gems/Ubuntu1910/Gemfile.lock @@ -91,6 +91,7 @@ GEM optimist (3.0.0) ox (2.13.2) parse-cron (0.1.4) + pg (1.2.2) polyglot (0.3.5) public_suffix (4.0.3) rack (2.2.2) @@ -168,6 +169,7 @@ DEPENDENCIES nokogiri ox parse-cron + pg public_suffix rack rbvmomi (~> 2.2.0) diff --git a/share/install_gems/install_gems b/share/install_gems/install_gems index 7a628b4358..2868dc2b95 100755 --- a/share/install_gems/install_gems +++ b/share/install_gems/install_gems @@ -27,8 +27,10 @@ end if !defined?(RUBY_VERSION) || RUBY_VERSION < '2.2.0' RACK = 'rack --version "< 2.0.0"' + PG = 'pg --version "< 1.2.0"' else RACK = 'rack' + PG = 'pg' end TREETOP = 'treetop --version ">= 1.6.3"' @@ -47,7 +49,7 @@ GROUPS={ :ec2_hybrid => 'aws-sdk --version "~> 2.5"', :oca => 'ox', :market => 'aws-sdk', - :onedb => "mysql2", + :onedb => ['mysql2', PG], :hooks => %w[zeromq ffi-rzmq], :serversync => "augeas" } @@ -66,6 +68,7 @@ DISTRIBUTIONS={ :dependencies => { SQLITE => ['gcc', 'libsqlite3-dev'], 'mysql2' => ['gcc', 'libssl-dev', ['default-libmysqlclient-dev', 'libmysqlclient-dev']], + PG => ['gcc', 'postgresql-server-dev-all'], 'curb' => ['gcc', 'libcurl4-openssl-dev'], $nokogiri => %w{gcc rake libxml2-dev libxslt1-dev patch}, 'xmlparser' => ['gcc', 'libexpat1-dev'], @@ -87,6 +90,7 @@ DISTRIBUTIONS={ :dependencies => { SQLITE => ['gcc', 'sqlite-devel'], 'mysql2' => ['gcc', 'mysql-devel', 'openssl-devel'], + PG => ['gcc', 'postgresql-devel'], 'curb' => ['gcc', 'curl-devel'], $nokogiri => %w{gcc rubygem-rake libxml2-devel libxslt-devel patch}, 'xmlparser' => ['gcc', 'expat-devel'], diff --git a/src/acl/AclManager.cc b/src/acl/AclManager.cc index 91468c5962..fa39c74593 100644 --- a/src/acl/AclManager.cc +++ b/src/acl/AclManager.cc @@ -25,11 +25,11 @@ const char * AclManager::table = "acl"; -const char * AclManager::db_names = "oid, user, resource, rights, zone"; +const char * AclManager::db_names = "oid, userset, resource, rights, zone"; const char * AclManager::db_bootstrap = "CREATE TABLE IF NOT EXISTS " - "acl (oid INT PRIMARY KEY, user BIGINT, resource BIGINT, " - "rights BIGINT, zone BIGINT, UNIQUE(user, resource, rights, zone))"; + "acl (oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, " + "rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone))"; /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ diff --git a/src/group/GroupPool.cc b/src/group/GroupPool.cc index 935c43297a..e40407132b 100644 --- a/src/group/GroupPool.cc +++ b/src/group/GroupPool.cc @@ -218,8 +218,7 @@ int GroupPool::drop(PoolObjectSQL * objsql, string& error_msg) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -int GroupPool::dump(string& oss, const string& where, - const string& limit, bool desc) +int GroupPool::dump(string& oss, const string& where, int sid, int eid, bool desc) { int rc; string def_quota_xml; @@ -243,9 +242,9 @@ int GroupPool::dump(string& oss, const string& where, cmd << " DESC"; } - if ( !limit.empty() ) + if ( eid != -1 ) { - cmd << " LIMIT " << limit; + cmd << " " << db->limit_string(sid, eid); } oss.append(""); diff --git a/src/host/HostPool.cc b/src/host/HostPool.cc index 56e60bebf0..111ef62dcc 100644 --- a/src/host/HostPool.cc +++ b/src/host/HostPool.cc @@ -171,7 +171,7 @@ int HostPool::dump_monitoring( cmd << "SELECT " << one_db::host_monitor_table << ".body FROM " << one_db::host_monitor_table << " INNER JOIN " << one_db::host_table - << " WHERE hid = oid"; + << " ON hid = oid"; if ( !where.empty() ) { diff --git a/src/im/InformationManager.cc b/src/im/InformationManager.cc index db28c9a1ab..46d32f9265 100644 --- a/src/im/InformationManager.cc +++ b/src/im/InformationManager.cc @@ -72,7 +72,8 @@ int InformationManager::start() } string xml_hosts; - hpool->dump(xml_hosts, "", "", false); + + hpool->dump(xml_hosts, "", 0, -1, false); Message msg; diff --git a/src/monitor/etc/monitord.conf b/src/monitor/etc/monitord.conf index 711a644908..625c01fe92 100644 --- a/src/monitor/etc/monitord.conf +++ b/src/monitor/etc/monitord.conf @@ -18,6 +18,11 @@ # VM_MONITORING_EXPIRATION_TIME: Time, in seconds, to expire monitoring # information. Use 0 to disable VM monitoring recording. # +# DB: Database configuration attributes. Monitord will use the DB configuration +# in oned.conf. The following attributes can be tuned: +# - CONNECTIONS: Number of DB connections. The DB needs to be configure to +# support oned + monitord connections. +# # LOG: Configuration for the logging system # - system: defines the logging system: # file to log in the sched.log file @@ -39,6 +44,10 @@ #HOST_MONITORING_EXPIRATION_TIME = 43200 #VM_MONITORING_EXPIRATION_TIME = 43200 +DB = [ + CONNECTIONS = 15 +] + LOG = [ system = "FILE", debug_level = 3 diff --git a/src/monitor/src/monitor/Monitor.cc b/src/monitor/src/monitor/Monitor.cc index cfe75aa809..8d7a299099 100644 --- a/src/monitor/src/monitor/Monitor.cc +++ b/src/monitor/src/monitor/Monitor.cc @@ -22,6 +22,7 @@ #include "StreamManager.h" #include "SqliteDB.h" #include "MySqlDB.h" +#include "PostgreSqlDB.h" #include #include @@ -35,6 +36,7 @@ void Monitor::start() // Configuration File // ------------------------------------------------------------------------- OpenNebulaTemplate oned_config(get_defaults_location(), oned_filename); + if (oned_config.load_configuration() != 0) { throw runtime_error("Error reading oned configuration file " + @@ -91,7 +93,8 @@ void Monitor::start() // ------------------------------------------------------------------------- // Database // ------------------------------------------------------------------------- - const VectorAttribute * _db = oned_config.get("DB"); + const VectorAttribute * _db = oned_config.get("DB"); + const VectorAttribute * _db_m = config->get("DB"); std::string db_backend = _db->vector_value("BACKEND"); @@ -115,10 +118,23 @@ void Monitor::start() _db->vector_value("PASSWD", passwd, "oneadmin"); _db->vector_value("DB_NAME", db_name, "opennebula"); _db->vector_value("ENCODING", encoding, ""); - _db->vector_value("CONNECTIONS", connections, 50); - sqlDB.reset(new MySqlDB(server, port, user, passwd, db_name, - encoding, connections)); + _db_m->vector_value("CONNECTIONS", connections, 15); + + if ( db_backend == "postgresql" ) + { + sqlDB.reset(new PostgreSqlDB(server, port, user, passwd, db_name, + connections)); + } + else if ( db_backend == "mysql" ) + { + sqlDB.reset(new MySqlDB(server, port, user, passwd, db_name, + encoding, connections)); + } + else + { + throw runtime_error("Unknown DB backend " + db_backend); + } } // ------------------------------------------------------------------------- diff --git a/src/monitor/src/monitor/MonitorConfigTemplate.cc b/src/monitor/src/monitor/MonitorConfigTemplate.cc index 849ec52a11..cd2fc3fa03 100644 --- a/src/monitor/src/monitor/MonitorConfigTemplate.cc +++ b/src/monitor/src/monitor/MonitorConfigTemplate.cc @@ -25,6 +25,7 @@ void MonitorConfigTemplate::set_conf_default() /* HOST_MONITORING_EXPIRATION_TIME VM_MONITORING_EXPIRATION_TIME + DB LOG UDP_LISTENER PROBES_PERIOD @@ -36,6 +37,9 @@ void MonitorConfigTemplate::set_conf_default() set_conf_single("HOST_MONITORING_EXPIRATION_TIME", "43200"); set_conf_single("VM_MONITORING_EXPIRATION_TIME", "43200"); + va = new VectorAttribute("DB", {{"CONNECTIONS", "15"}}); + conf_default.insert(make_pair(va->name(), va)); + va = new VectorAttribute("LOG", {{"SYSTEM", "FILE"}, {"DEBUG_LEVEL", "3"}}); conf_default.insert(make_pair(va->name(), va)); diff --git a/src/nebula/Nebula.cc b/src/nebula/Nebula.cc index e909519a59..d5087d5cbe 100644 --- a/src/nebula/Nebula.cc +++ b/src/nebula/Nebula.cc @@ -19,6 +19,7 @@ #include "VirtualMachine.h" #include "SqliteDB.h" #include "MySqlDB.h" +#include "PostgreSqlDB.h" #include "Client.h" #include "LogDB.h" #include "SystemDB.h" @@ -388,7 +389,7 @@ void Nebula::start(bool bootstrap_only) if (_db->vector_value("CONNECTIONS", connections) == -1) { - connections = 50; + connections = 25; } if (_db->vector_value("ENCODING", encoding) == -1) @@ -401,11 +402,20 @@ void Nebula::start(bool bootstrap_only) { db_backend = new SqliteDB(var_location + "one.db"); } - else + else if ( db_backend_type == "mysql" ) { db_backend = new MySqlDB(server, port, user, passwd, db_name, encoding, connections); } + else if ( db_backend_type == "postgresql" ) + { + db_backend = new PostgreSqlDB(server, port, user, passwd, db_name, + connections); + } + else + { + throw runtime_error("DB BACKEND must be one of sqlite, mysql or postgresql."); + } // --------------------------------------------------------------------- // Check Database Versions diff --git a/src/nebula/SystemDB.cc b/src/nebula/SystemDB.cc index f1c8b16355..b86bf6c628 100644 --- a/src/nebula/SystemDB.cc +++ b/src/nebula/SystemDB.cc @@ -102,7 +102,7 @@ int SystemDB::local_bootstrap() oss << "INSERT INTO " << local_ver_table << " (" << local_ver_names << ") " << "VALUES (0, '" << Nebula::local_db_version() << "', " << time(0) << ", '" << Nebula::version() << " daemon bootstrap', " - << Nebula::instance().is_federation_slave() << ")"; + << "'" << Nebula::instance().is_federation_slave() << "')"; rc += db->exec_local_wr(oss); diff --git a/src/onedb/database_schema.rb b/src/onedb/database_schema.rb index 9e2439f3ab..f288666dc8 100644 --- a/src/onedb/database_schema.rb +++ b/src/onedb/database_schema.rb @@ -127,6 +127,8 @@ class OneDBBacKEnd "state INTEGER, lcm_state INTEGER, " << "owner_u INTEGER, group_u INTEGER, other_u INTEGER, short_body MEDIUMTEXT, " << "search_token MEDIUMTEXT", + acl: "oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, " << + "rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone)" } } diff --git a/src/onedb/onedb b/src/onedb/onedb index deb3412570..7ec6e93583 100755 --- a/src/onedb/onedb +++ b/src/onedb/onedb @@ -101,16 +101,26 @@ SQLITE={ } ############################################################################### -# MySQL options +# MySQL and PostgreSQL options ############################################################################### +TYPE={ + :name => "type", + :short => "-t type", + :large => "--type type", + :format => String, + :description => "Backend type.", + :proc => lambda { |o, options| + options[:backend] = o.to_sym + } +} + SERVER={ :name => 'server', :short => '-S host', :large => '--server host', :format => String, - :description => 'MySQL server hostname or IP. Defaults to localhost', - :proc => lambda {|o, options| - options[:backend] = :mysql + :description => "MySQL or PostgreSQL server hostname or IP. Defaults to localhost", + :proc => lambda { |o, options| options[:server] = o } } @@ -120,10 +130,9 @@ PORT={ :short => '-P port', :large => '--port port', :format => String, - :description => 'MySQL server port. Defaults to 3306', - :proc => lambda {|o, options| - options[:backend] = :mysql - options[:port] = o + :description => "MySQL or PostgreSQL server port. Defaults to 3306 for MySQL and 5432 for PostgreSQL", + :proc => lambda { |o, options| + options[:port] = o } } @@ -132,9 +141,8 @@ USERNAME={ :short => '-u user', :large => '--username user', :format => String, - :description => 'MySQL username', - :proc => lambda {|o, options| - options[:backend] = :mysql + :description => "MySQL or PostgreSQL username", + :proc => lambda { |o, options| options[:user] = o } } @@ -144,9 +152,8 @@ PASSWORD={ :short => '-p pass', :large => '--password pass', :format => String, - :description => 'MySQL password. Leave unset to be prompted for it', - :proc => lambda {|o, options| - options[:backend] = :mysql + :description => "MySQL or PostgreSQL password. Leave unset to be prompted for it", + :proc => lambda { |o, options| options[:passwd] = o } } @@ -156,9 +163,8 @@ DBNAME={ :short => '-d dbname', :large => '--dbname dbname', :format => String, - :description => 'MySQL DB name for OpenNebula', - :proc => lambda {|o, options| - options[:backend] = :mysql + :description => "MySQL or PostgreSQL DB name for OpenNebula", + :proc => lambda { |o, options| options[:db_name] = o } } @@ -167,9 +173,8 @@ ENCODING={ :name => 'encoding', :large => '--encoding charset', :format => String, - :description => 'MySQL encoding to use for the connection', - :proc => lambda {|o, options| - options[:backend] = :mysql + :description => "MySQL or PostgreSQL encoding to use for the connection", + :proc => lambda { |o, options| options[:encoding] = o } } @@ -332,7 +337,7 @@ CommandParser::CmdParser.new(ARGV) do # Global options ########################################################################### set :option, CommandParser::OPTIONS - set :option, [SQLITE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING] + set :option, [SQLITE, TYPE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING] ########################################################################### # Backup diff --git a/src/onedb/onedb.rb b/src/onedb/onedb.rb index 9deed5e231..7145bfb1ab 100644 --- a/src/onedb/onedb.rb +++ b/src/onedb/onedb.rb @@ -23,6 +23,11 @@ class OneDB attr_accessor :backend def initialize(ops) + # Set MySQL backend as default if any connection option is provided and --type is not + if ops[:backend].nil? and (!ops[:server].nil? || !ops[:port].nil? || !ops[:user].nil? || !ops[:password].nil? || !ops[:db_name].nil? || !ops[:encoding].nil?) + ops[:backend] = :mysql + end + if ops[:backend] == :sqlite begin require 'sqlite3' @@ -55,8 +60,30 @@ class OneDB :db_name => ops[:db_name], :encoding=> ops[:encoding] ) + elsif ops[:backend] == :postgresql + begin + require 'pg' + rescue + STDERR.puts "Ruby gem pg is needed for this operation:" + STDERR.puts " $ sudo gem install pg" + exit -1 + end + + passwd = ops[:passwd] + if !passwd + passwd = get_password("PostgreSQL Password: ") + end + + @backend = BackEndPostgreSQL.new( + :server => ops[:server], + :port => ops[:port], + :user => ops[:user], + :passwd => passwd, + :db_name => ops[:db_name], + :encoding=> ops[:encoding] + ) else - raise "You need to specify the SQLite or MySQL connection options." + raise "You need to specify the SQLite, MySQL or PostgreSQL connection options." end end diff --git a/src/onedb/onedb_backend.rb b/src/onedb/onedb_backend.rb index e00a961ed9..0de74c5da0 100644 --- a/src/onedb/onedb_backend.rb +++ b/src/onedb/onedb_backend.rb @@ -120,22 +120,13 @@ class OneDBBacKEnd comment = "Database migrated from #{version} to #{db_version}"+ " (#{one_version}) by onedb command." - max_oid = nil - @db.fetch("SELECT MAX(oid) FROM db_versioning") do |row| - max_oid = row[:"MAX(oid)"].to_i - end + max_oid = @db[:db_versioning].max(:oid) - max_oid = 0 if max_oid.nil? - - query = - @db.run( - "INSERT INTO db_versioning (oid, version, timestamp, comment) "<< - "VALUES (" << - "#{max_oid+1}, " << - "'#{db_version}', " << - "#{Time.new.to_i}, " << - "'#{comment}')" - ) + @db[:db_versioning].insert( + oid: max_oid + 1, + version: db_version, + timestamp: Time.new.to_i, + comment: comment) puts comment end @@ -144,29 +135,16 @@ class OneDBBacKEnd comment = "Database migrated from #{version} to #{db_version}"+ " (#{one_version}) by onedb command." - max_oid = nil - @db.fetch("SELECT MAX(oid) FROM local_db_versioning") do |row| - max_oid = row[:"MAX(oid)"].to_i - end + max_oid = @db[:local_db_versioning].max(:oid) - max_oid = 0 if max_oid.nil? + is_slave = @db[:local_db_versioning].select(:is_slave).where_single_value(oid: max_oid) - is_slave = 0 - - @db.fetch("SELECT is_slave FROM local_db_versioning "<< - "WHERE oid=#{max_oid}") do |row| - is_slave = row[:is_slave] ? 1 : 0 - end - - @db.run( - "INSERT INTO local_db_versioning (oid, version, timestamp, comment, is_slave) "<< - "VALUES (" << - "#{max_oid+1}, " << - "'#{db_version}', " << - "#{Time.new.to_i}, " << - "'#{comment}'," << - "#{is_slave})" - ) + @db[:local_db_versioning].insert( + oid: max_oid + 1, + version: db_version, + timestamp: Time.new.to_i, + comment: comment, + is_slave: is_slave) puts comment end @@ -638,3 +616,261 @@ class BackEndSQLite < OneDBBacKEnd end end end + +class BackEndPostgreSQL < OneDBBacKEnd + def initialize(opts={}) + @server = opts[:server] + @port = opts[:port] + @user = opts[:user] + @passwd = opts[:passwd] + @db_name = opts[:db_name] + @encoding= opts[:encoding] + + # Check for errors: + error = false + + (error = true; missing = "USER" ) if @user == nil + (error = true; missing = "DBNAME") if @db_name == nil + + if error + raise "PostgreSQL option #{missing} is needed" + end + + # Check for defaults: + @server = "localhost" if @server.nil? + @port = 5432 if @port.nil? + + encoding if @encoding.nil? + + # Clean leading and trailing quotes, if any + @server = @server [1..-2] if @server [0] == ?" + @port = @port [1..-2] if @port [0] == ?" + @user = @user [1..-2] if @user [0] == ?" + @passwd = @passwd [1..-2] if @passwd [0] == ?" + @db_name = @db_name[1..-2] if @db_name[0] == ?" + end + + def bck_file(federated = false) + t = Time.now + + bck_name = "#{VAR_LOCATION}/postgresql_#{@server}_#{@db_name}_" + + bck_name << "federated_" if federated + + bck_name << "#{t.year}-#{t.month}-#{t.day}_" + bck_name << "#{t.hour}:#{t.min}:#{t.sec}.sql" + + bck_name + end + + + def backup(bck_file, federated = false) + cmd = "PGPASSWORD=\"#{@passwd}\" pg_dump -U #{@user} -h #{@server} -p #{@port} -b #{@db_name} -Fp -f #{bck_file} " + + if federated + connect_db + + @db.create_table!(:logdb_tmp, as: @db[:logdb].where{fed_index != -1}) + + FEDERATED_TABLES.each do |table| + cmd << " -t " << table + end + + cmd << " -t logdb_tmp" + + rc = system(cmd) + if !rc + raise "Unknown error running '#{cmd}'" + end + + @db.drop_table(:logdb_tmp) + + File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("logdb_tmp","logdb")) + else + rc = system(cmd) + + if !rc + raise "Unknown error running '#{cmd}'" + end + end + + File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("COMMENT ON","-- COMMENT ON")) + + puts "PostgreSQL dump stored in #{bck_file}" + puts "Use 'onedb restore' to restore the DB" + puts + end + + def get_db_encoding + db_enc = '' + + @db.fetch("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = \'#{@db_name}\'") do |row| + db_enc = row[:pg_encoding_to_char] + db_enc ||= row[:PG_ENCODING_TO_CHAR] + end + + db_enc + end + + def encoding + @encoding = '' + + connect_db + + @encoding = get_db_encoding + end + + def get_table_enconding(table = nil) + encoding = get_db_encoding + + table_to_nk(encoding) + end + + def table_to_nk(encoding) + case encoding + when 'UTF8' + 'UTF-8' + when 'ISO_8859_5' + 'ISO-8859-5' + when 'ISO_8859_6' + 'ISO-8859-6' + when 'ISO_8859_7' + 'ISO-8859-7' + when 'ISO_8859_8' + 'ISO-8859-8' + when 'LATIN1' + 'ISO-8859-1' + when 'LATIN2' + 'ISO-8859-2' + when 'LATIN3' + 'ISO-8859-3' + when 'LATIN4' + 'ISO-8859-4' + when 'LATIN5' + 'ISO-8859-9' + when 'SJIS' + 'SHIFT-JIS' + when 'EUC_JP' + 'EUC-JP' + when 'SQL_ASCII' + 'ASCII' + else + NOKOGIRI_ENCODING + end + end + + def restore(bck_file, force=nil, federated=false) + if !federated && !force && db_exists? + raise "PostgreSQL database #{@db_name} at #{@server} exists," << + " use -f to overwrite." + end + + connect_db + if federated + FEDERATED_TABLES.each do |table| + @db.drop_table?(table) + end + + @db.drop_table?(:logdb) + else + @db.tables.each do |table| + @db.drop_table(table) + end + end + + rc = system("PGPASSWORD=\"#{@passwd}\" psql -U #{@user} -h #{@server} -p #{@port} -d #{@db_name} -f #{bck_file} --set ON_ERROR_STOP=on --quiet -o /dev/null") + if !rc + raise "Error while restoring PostgreSQL DB #{@db_name} at #{@server}." + end + + puts "PostgreSQL DB #{@db_name} at #{@server} restored." + end + + def fts_index(recreate = false) + raise "FTS index not supported for PostgreSQL." + end + + private + + def connect_db + passwd = CGI.escape(@passwd) + + endpoint = "postgres://#{@user}:#{passwd}@#{@server}:#{@port}/#{@db_name}" + + begin + options = {} + options[:encoding] = @encoding unless @encoding.empty? + + @db = Sequel.connect(endpoint, options) + rescue Exception => e + raise "Error connecting to DB: " + e.message + end + + redefine_db_methods + end + + def redefine_db_methods + def @db.fetch(query) + preprocessed = BackEndPostgreSQL.preprocess(query) + super(preprocessed) + end + + def @db.run(query) + preprocessed = BackEndPostgreSQL.preprocess(query) + super(preprocessed) + end + end + + # Any change to this method should be reflected in PostgreSqlDB class + # in src/sql/PostgreSqlDB.cc + def self.preprocess(query) + pp_query = query.dup + + if pp_query.upcase.start_with?('CREATE TABLE') + pp_query = replace_type(pp_query, 'MEDIUMTEXT', 'TEXT') + pp_query = replace_type(pp_query, 'LONGTEXT', 'TEXT') + pp_query = replace_type(pp_query, 'BIGINT UNSIGNED', 'NUMERIC(20)') + end + + if pp_query.upcase.start_with?('REPLACE') + pp_query = replace_replace_into(query) + end + + pp_query + end + + def self.replace_type(query, type, replacement) + query = query.gsub(type.upcase, replacement.upcase) + query = query.gsub(type.downcase, replacement.downcase) + return query + end + + # This method changes MySQL/SQLite REPLACE INTO into PostgreSQL + # INSERT INTO query with ON CONFLICT clause. + # For example: + # REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0) + # changes to: + # INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0) + # ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid" + # + # Any change to this method should be reflected in PostgreSqlDB class + # in src/sql/PostgreSqlDB.cc + def self.replace_replace_into(query) + query[0, 7] = "INSERT" + db_names_start = query.index('(') + 1 + db_names_end = query.index(')') + db_names = query[db_names_start, db_names_end - db_names_start] + + splits = db_names.split(',') + + query += " ON CONFLICT (" + splits[0] + ") DO UPDATE SET " + sep = "" + splits[1..-1].each do |split| + split = split.strip + query += sep + split + " = EXCLUDED." + split + sep = ", " + end + + query + end +end diff --git a/src/onedb/shared/5.10.0_to_5.12.0.rb b/src/onedb/shared/5.10.0_to_5.12.0.rb index cd6b157e3f..dbfccf4db6 100644 --- a/src/onedb/shared/5.10.0_to_5.12.0.rb +++ b/src/onedb/shared/5.10.0_to_5.12.0.rb @@ -27,7 +27,28 @@ module Migrator end def up + feature_3600 true end + private + + #Rename acl column name from user to userset to support postgresql + def feature_3600 + @db.run 'DROP TABLE IF EXISTS old_acl;' + @db.run 'ALTER TABLE acl RENAME TO old_acl;' + + create_table(:acl) + + @db.transaction do + @db.fetch('SELECT * FROM old_acl') do |row| + row[:userset] = row.delete[:user] + + @db[:acl].insert(row) + end + end + + @db.run "DROP TABLE old_acl;" + end + end diff --git a/src/pool/PoolSQL.cc b/src/pool/PoolSQL.cc index 21c0241f65..889633418b 100644 --- a/src/pool/PoolSQL.cc +++ b/src/pool/PoolSQL.cc @@ -274,8 +274,8 @@ PoolObjectSQL * PoolSQL::get_ro(const string& name, int uid) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -int PoolSQL::dump(string& oss, const string& elem_name, const string& column, const char* table, - const string& where, const string& limit, bool desc) +int PoolSQL::dump(string& oss, const string& elem_name, const string& column, + const char* table, const string& where, int sid, int eid, bool desc) { ostringstream cmd; @@ -293,9 +293,9 @@ int PoolSQL::dump(string& oss, const string& elem_name, const string& column, co cmd << " DESC"; } - if ( !limit.empty() ) + if ( eid != -1 ) { - cmd << " LIMIT " << limit; + cmd << " " << db->limit_string(sid, eid); } return dump(oss, elem_name, cmd); diff --git a/src/rm/RequestManagerPoolInfoFilter.cc b/src/rm/RequestManagerPoolInfoFilter.cc index 44fb61bf63..395d9f111e 100644 --- a/src/rm/RequestManagerPoolInfoFilter.cc +++ b/src/rm/RequestManagerPoolInfoFilter.cc @@ -200,6 +200,8 @@ void RequestManagerPoolInfoFilter::dump( std::string where_string, limit_clause; std::string desc; + int limit_end_id = -1; + int rc; if ( filter_flag < GROUP ) @@ -222,8 +224,7 @@ void RequestManagerPoolInfoFilter::dump( if ( end_id < -1 ) { - oss << start_id << "," << -end_id; - limit_clause = oss.str(); + limit_end_id = -end_id; } Nebula::instance().get_configuration_attribute(att.uid, att.gid, @@ -233,14 +234,16 @@ void RequestManagerPoolInfoFilter::dump( { rc = pool->dump_extended(str, where_string, - limit_clause, + start_id, + limit_end_id, one_util::toupper(desc) == "DESC"); } else { rc = pool->dump(str, where_string, - limit_clause, + start_id, + limit_end_id, one_util::toupper(desc) == "DESC"); } @@ -300,7 +303,7 @@ void VirtualMachinePoolInfo::request_execute( { fts_query = xmlrpc_c::value_string(paramList.getString(5)); - if (!fts_query.empty() && !pool->is_fts_available()) + if (!fts_query.empty() && !pool->supports(SqlDB::SqlFeature::FTS)) { att.resp_msg = "Full text search is not supported by the SQL backend"; @@ -556,6 +559,8 @@ void VirtualNetworkPoolInfo::request_execute( int start_id = xmlrpc_c::value_int(paramList.getInt(2)); int end_id = xmlrpc_c::value_int(paramList.getInt(3)); + int limit_end_id = -1; + if ( filter_flag < GROUP ) { att.resp_msg = "Incorrect filter_flag"; @@ -588,7 +593,7 @@ void VirtualNetworkPoolInfo::request_execute( if ( end_id < -1 ) { - limit_clause << start_id << "," << -end_id; + limit_end_id = -end_id; } /* ---------------------------------------------------------------------- */ @@ -600,7 +605,7 @@ void VirtualNetworkPoolInfo::request_execute( Nebula::instance().get_configuration_attribute(att.uid, att.gid, "API_LIST_ORDER", desc); - int rc = pool->dump(pool_oss, where_string.str(), limit_clause.str(), + int rc = pool->dump(pool_oss, where_string.str(), start_id, limit_end_id, one_util::toupper(desc) == "DESC"); if ( rc != 0 ) @@ -1075,4 +1080,4 @@ void HookLogInfo::request_execute(xmlrpc_c::paramList const& _paramList, success_response(dump_xml, att); return; -} \ No newline at end of file +} diff --git a/src/sql/LogDB.cc b/src/sql/LogDB.cc index 0478a98156..c3a49de4fd 100644 --- a/src/sql/LogDB.cc +++ b/src/sql/LogDB.cc @@ -184,7 +184,7 @@ int LogDB::setup_index(uint64_t& _last_applied, uint64_t& _last_index) cb.set_callback(&_last_applied); - oss << "SELECT MAX(log_index) FROM logdb WHERE applied = 1"; + oss << "SELECT MAX(log_index) FROM logdb WHERE applied = '1'"; rc += db->exec_rd(oss, &cb); @@ -345,7 +345,7 @@ int LogDB::insert(uint64_t index, unsigned int term, const std::string& sql, << "'" << sql_db << "'," << tstamp << "," << fed_index << "," - << applied << ")"; + << "'" << applied << "')"; int rc = db->exec_wr(oss); @@ -383,7 +383,7 @@ int LogDB::apply_log_record(LogDBRecord * lr) { std::ostringstream oss; - oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = 1" + oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = '1'" << " WHERE log_index = " << lr->index << " AND timestamp = 0"; if ( db->exec_wr(oss) != 0 ) @@ -642,8 +642,8 @@ int LogDB::purge_log() oss.str(""); oss << " SELECT MIN(i.log_index) FROM (" << " SELECT log_index FROM logdb WHERE fed_index = " << UINT64_MAX - << " AND applied = 1 AND log_index >= 0 " - << " ORDER BY log_index DESC LIMIT " << log_retention + << " AND applied = '1' AND log_index >= 0 " + << " ORDER BY log_index DESC " << db->limit_string(log_retention) << " ) AS i"; cb_min_idx.set_callback(&min_idx); @@ -655,12 +655,12 @@ int LogDB::purge_log() cb.set_affected_rows(0); oss.str(""); - oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 " + oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 " << "AND fed_index = " << UINT64_MAX << " AND log_index < " << min_idx; - if ( db->limit_support() ) + if ( db->supports(SqlDB::SqlFeature::LIMIT) ) { - oss << " LIMIT " << limit_purge; + oss << " " << db->limit_string(limit_purge); } if ( db->exec_wr(oss, &cb) != -1 ) @@ -705,8 +705,8 @@ int LogDB::purge_log() oss.str(""); oss << " SELECT MIN(i.log_index) FROM (" << " SELECT log_index FROM logdb WHERE fed_index != " << UINT64_MAX - << " AND applied = 1 AND log_index >= 0 " - << " ORDER BY log_index DESC LIMIT " << log_retention + << " AND applied = '1' AND log_index >= 0 " + << " ORDER BY log_index DESC " << db->limit_string(log_retention) << " ) AS i"; cb_min_idx.set_callback(&min_idx); @@ -718,12 +718,12 @@ int LogDB::purge_log() cb.set_affected_rows(0); oss.str(""); - oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 " + oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 " << "AND fed_index != " << UINT64_MAX << " AND log_index < " << min_idx; - if ( db->limit_support() ) + if ( db->supports(SqlDB::SqlFeature::LIMIT) ) { - oss << " LIMIT " << limit_purge; + oss << " " << db->limit_string(limit_purge); } if ( db->exec_wr(oss, &cb) != -1 ) diff --git a/src/sql/MySqlDB.cc b/src/sql/MySqlDB.cc index 27692bb191..cd3906bc84 100644 --- a/src/sql/MySqlDB.cc +++ b/src/sql/MySqlDB.cc @@ -21,16 +21,17 @@ /********* * Doc: https://dev.mysql.com/doc/refman/8.0/en/c-api.html ********/ +using namespace std; /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ static unsigned long get_server_version(MYSQL * db) { - std::string version_str; - std::vector ids; + string version_str; + vector ids; - std::string version_query = "SELECT @@GLOBAL.version"; + string version_query = "SELECT @@GLOBAL.version"; if ( mysql_query(db, version_query.c_str()) != 0 ) { @@ -56,8 +57,8 @@ static unsigned long get_server_version(MYSQL * db) mysql_free_result(result); - std::string version_number; - std::stringstream version_iss(version_str); + string version_number; + stringstream version_iss(version_str); if (!getline(version_iss, version_number, '-') || version_number.empty()) { @@ -72,10 +73,9 @@ static unsigned long get_server_version(MYSQL * db) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -static std::string get_encoding(MYSQL * c, const std::string& sql, - std::string& error) +static string get_encoding(MYSQL * c, const string& sql, string& error) { - std::string encoding; + string encoding; if ( mysql_query(c, sql.c_str()) != 0 ) { @@ -111,7 +111,7 @@ static std::string get_encoding(MYSQL * c, const std::string& sql, /* -------------------------------------------------------------------------- */ -int MySqlDB::db_encoding(std::string& error) +int MySqlDB::db_encoding(string& error) { MYSQL * connection = mysql_init(nullptr); @@ -124,7 +124,7 @@ int MySqlDB::db_encoding(std::string& error) return -1; } - std::string create_sql = "CREATE DATABASE IF NOT EXISTS " + database; + string create_sql = "CREATE DATABASE IF NOT EXISTS " + database; if ( mysql_query(connection, create_sql.c_str()) != 0 ) { @@ -140,22 +140,22 @@ int MySqlDB::db_encoding(std::string& error) } //Get encodings for database and tables - std::string db_sql = "SELECT default_character_set_name FROM " + string db_sql = "SELECT default_character_set_name FROM " "information_schema.SCHEMATA WHERE schema_name = \"" + database + "\""; - std::string db_enc = get_encoding(connection, db_sql, error); + string db_enc = get_encoding(connection, db_sql, error); if ( db_enc.empty() ) { return -1; } - std::string table_sql = "SELECT CCSA.character_set_name FROM " - "information_schema.`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`" + string table_sql = "SELECT CCSA.character_set_name FROM information_schema."\ + "`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`" " CCSA WHERE CCSA.collation_name = T.table_collation AND T.table_schema = " "\"" + database + "\" AND T.table_name = \"system_attributes\""; - std::string table_enc = get_encoding(connection, table_sql, error); + string table_enc = get_encoding(connection, table_sql, error); if ( !table_enc.empty() && table_enc != db_enc) { @@ -176,13 +176,13 @@ int MySqlDB::db_encoding(std::string& error) MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, const string& d, const string& e, int m):max_connections(m), server(s), - port(p), user(u), password(_p), database(d), encoding(e), _fts_available(false) + port(p), user(u), password(_p), database(d), encoding(e) { vector connections(max_connections); MYSQL * rc; ostringstream oss; - std::string error; + string error; // ------------------------------------------------------------------------- // Initialize the MySQL library @@ -240,13 +240,13 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, // ------------------------------------------------------------------------- // Connect to OpenNebula Database // ------------------------------------------------------------------------- - std::string use_sql = "USE " + database; + string use_sql = "USE " + database; for (int i=0 ; i < max_connections ; i++) { if ( mysql_query(connections[i], use_sql.c_str()) != 0 ) { - std::string error = "Could not connect to database: "; + string error = "Could not connect to database: "; error.append(mysql_error(connections[i])); throw runtime_error(error); @@ -254,7 +254,7 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, if ( mysql_set_character_set(connections[i], encoding.c_str()) != 0 ) { - std::string error = "Could not set encoding : "; + string error = "Could not set encoding : "; error.append(mysql_error(connections[i])); throw runtime_error(error); @@ -271,12 +271,12 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, oss.clear(); //-------------------------------------------------------------------------- - // Get server information and FTS support + // Get server information and FTS support & features //-------------------------------------------------------------------------- unsigned long version; unsigned long min_fts_version; - std::string server_info = mysql_get_server_info(db_escape_connect); + string server_info = mysql_get_server_info(db_escape_connect); version = get_server_version(db_escape_connect); @@ -299,15 +299,19 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, if (version >= min_fts_version) { - _fts_available = true; NebulaLog::log("ONE", Log::INFO, "\tFTS enabled"); } else { - _fts_available = false; NebulaLog::log("ONE", Log::INFO, "\tFTS disabled"); } + features = { + {SqlFeature::MULTIPLE_VALUE, true}, + {SqlFeature::LIMIT, true}, + {SqlFeature::FTS, version >= min_fts_version} + }; + pthread_mutex_init(&mutex,0); pthread_cond_init(&cond,0); diff --git a/src/sql/PostgreSqlDB.cc b/src/sql/PostgreSqlDB.cc new file mode 100644 index 0000000000..0af1b7fc36 --- /dev/null +++ b/src/sql/PostgreSqlDB.cc @@ -0,0 +1,333 @@ +/* -------------------------------------------------------------------------- */ +/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); you may */ +/* not use this file except in compliance with the License. You may obtain */ +/* a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ +/* See the License for the specific language governing permissions and */ +/* limitations under the License. */ +/* -------------------------------------------------------------------------- */ + +#include "NebulaUtil.h" +#include "PostgreSqlDB.h" + +#include + +#include + +/********* + * Doc: https://www.postgresql.org/docs/current/libpq.html + ********/ + +#define PG_DEFAULT_PORT 5432 + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +PostgreSqlDB::PostgreSqlDB( + const string& _server, + int _port, + const string& _user, + const string& _password, + const string& _database, + int _connections) +{ + PGconn* conn; + + server = _server; + port = _port; + user = _user; + + password = _password; + database = _database; + + if ( port == 0 ) + { + port = PG_DEFAULT_PORT; + } + + max_connections = _connections; + + // Set up connection parameters + string params = "host=" + _server + + " port=" + to_string(port) + + " user=" + user + + " password=" + password + + " dbname=" + database; + + // Create connection pool + for (int i = 0; i < max_connections; i++) + { + conn = PQconnectdb(params.c_str()); + + if ( PQstatus(conn) == CONNECTION_BAD ) + { + ostringstream oss; + oss << "Could not open connect to database server: " + << PQerrorMessage(conn); + + throw runtime_error(oss.str()); + } + + db_connect.push(conn); + } + + db_escape_connect = PQconnectdb(params.c_str()); + + if ( PQserverVersion(db_escape_connect) < 90500 ) + { + std::string error = "PostgreSQL version error: must be 9.5 or higher."; + + NebulaLog::log("ONE", Log::ERROR, error); + + throw runtime_error(error); + } + + features = { + {SqlFeature::MULTIPLE_VALUE, PQlibVersion() < 80200}, + {SqlFeature::LIMIT, false}, + {SqlFeature::FTS, false} + }; + + pthread_mutex_init(&mutex, 0); + + pthread_cond_init(&cond, 0); +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +PostgreSqlDB::~PostgreSqlDB() +{ + while (!db_connect.empty()) + { + PGconn* conn = db_connect.front(); + db_connect.pop(); + + PQfinish(conn); + } + + PQfinish(db_escape_connect); + + pthread_mutex_destroy(&mutex); + pthread_cond_destroy(&cond); +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +char * PostgreSqlDB::escape_str(const string& str) +{ + char* buf = new char[str.size() * 2 + 1]; + int err; + + PQescapeStringConn(db_escape_connect, buf, str.c_str(), str.length(), &err); + + if ( err != 0 ) + { + delete[] buf; + + return nullptr; + } + + return buf; +} + +/* -------------------------------------------------------------------------- */ + +void PostgreSqlDB::free_str(char * str) +{ + delete[] str; +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +int PostgreSqlDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) +{ + string str = preprocess_query(cmd); + + const char* c_str = str.c_str(); + + Log::MessageType error_type = quiet ? Log::DDEBUG : Log::ERROR; + + + PGconn* conn = get_db_connection(); + + PGresult* res = PQexec(conn, c_str); + + if ( PQresultStatus(res) != PGRES_COMMAND_OK && + PQresultStatus(res) != PGRES_TUPLES_OK ) + { + const char* err_msg = PQerrorMessage(conn); + + ostringstream oss; + oss << "SQL command was: " << c_str; + oss << ", error " << err_msg; + + NebulaLog::log("ONE", error_type, oss); + + PQclear(res); + free_db_connection(conn); + + return SqlDB::SQL; + } + + if ( obj == 0 ) + { + // Free the result and db connection + PQclear(res); + free_db_connection(conn); + + return SqlDB::SUCCESS; + } + + int ec = SqlDB::SUCCESS; + + // Get number of fields and rows of the result + int n_fields = PQnfields(res); + int n_rows = PQntuples(res); + + if ( obj->isCallBackSet() && PQresultStatus(res) == PGRES_TUPLES_OK ) + { + char** names = new char*[n_fields]; + char** values = new char*[n_fields]; + + // Get column names + for (int i = 0; i < n_fields; i++) + { + names[i] = PQfname(res, i); + } + + // For each row + for (int row = 0; row < n_rows; row++) + { + // get values in that row + for (int field = 0; field < n_fields; field++) + { + values[field] = PQgetvalue(res, row, field); + } + + // and do a callback on them + if ( obj->do_callback(n_fields, values, names) != 0 ) + { + ec = SqlDB::SQL; + break; + } + } + + delete[] names; + delete[] values; + } + + if ( obj->get_affected_rows() == 0 && n_rows ) + { + obj->set_affected_rows(n_rows); + } + + // Free the result and db connection + PQclear(res); + free_db_connection(conn); + + return ec; +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +PGconn * PostgreSqlDB::get_db_connection() +{ + PGconn* conn; + + pthread_mutex_lock(&mutex); + + while (db_connect.empty() == true) + { + pthread_cond_wait(&cond, &mutex); + } + + conn = db_connect.front(); + db_connect.pop(); + + pthread_mutex_unlock(&mutex); + + return conn; +} + +/* -------------------------------------------------------------------------- */ + +void PostgreSqlDB::free_db_connection(PGconn * db) +{ + pthread_mutex_lock(&mutex); + + db_connect.push(db); + + pthread_cond_signal(&cond); + + pthread_mutex_unlock(&mutex); +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +static void replace_substring(std::string& cmd, const std::string& s1, + const std::string& s2) +{ + size_t pos = cmd.find(s1); + + while (pos != std::string::npos) + { + cmd.replace(pos, s1.length(), s2); + + pos = cmd.find(s1, pos + s2.length()); + } +} + +std::string PostgreSqlDB::preprocess_query(std::ostringstream& cmd) +{ + std::string query = cmd.str(); + size_t pos; + + // Both CREATE TABLE and REPLACE should be at the start + // so we don't change user data + if ((pos = query.find("CREATE TABLE")) == 0) + { + replace_substring(query, "MEDIUMTEXT", "TEXT"); + replace_substring(query, "LONGTEXT", "TEXT"); + replace_substring(query, "BIGINT UNSIGNED", "NUMERIC(20)"); + } + else if ((pos = query.find("REPLACE")) == 0) + { + query.replace(0, 7, "INSERT"); + + size_t table_start = query.find("INTO ", 7) + 5; + + size_t names_start = query.find("(", table_start) + 1; + size_t names_end = query.find(")", names_start); + + std::string db_names = query.substr(names_start, names_end - names_start); + std::string table = query.substr(table_start, names_start - 2 - table_start); + + std::vector splits; + one_util::split(db_names, ',', splits); + + query += " ON CONFLICT ON CONSTRAINT " + table + "_pkey DO UPDATE SET "; + + const char* sep = ""; + + for (size_t i = 1; i < splits.size(); i++) + { + query += sep + splits[i] + " = EXCLUDED." + splits[i]; + sep = ", "; + } + } + + return query; +} + diff --git a/src/sql/SConstruct b/src/sql/SConstruct index cab8c12680..4981538046 100644 --- a/src/sql/SConstruct +++ b/src/sql/SConstruct @@ -29,6 +29,16 @@ if env['sqlite']=='yes': if env['mysql']=='yes': source_files.append('MySqlDB.cc') +if env['postgresql']=='yes': + source_files.append('PostgreSqlDB.cc') + +# Build library +env.StaticLibrary(lib_name, source_files) + +lib_name = 'nebula_sql_const' + +source_files = ['OneDB.cc'] + # Build library env.StaticLibrary(lib_name, source_files) diff --git a/src/sql/SqliteDB.cc b/src/sql/SqliteDB.cc index 3681f34e48..9e38ccfe79 100644 --- a/src/sql/SqliteDB.cc +++ b/src/sql/SqliteDB.cc @@ -13,8 +13,6 @@ /* See the License for the specific language governing permissions and */ /* limitations under the License. */ /* -------------------------------------------------------------------------- */ - - #include "SqliteDB.h" using namespace std; @@ -52,14 +50,21 @@ SqliteDB::SqliteDB(const string& db_name) throw runtime_error("Could not open database."); } - enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT"); + int enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT"); - if (enable_limit) + if (enable_limit == 1) { - NebulaLog::log("ONE",Log::INFO , "sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT"); + NebulaLog::log("ONE",Log::INFO , + "sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT"); } sqlite3_extended_result_codes(db, 1); + + features = { + {SqlFeature::MULTIPLE_VALUE, false}, + {SqlFeature::LIMIT, enable_limit == 1}, + {SqlFeature::FTS, false} + }; } /* -------------------------------------------------------------------------- */ @@ -72,21 +77,6 @@ SqliteDB::~SqliteDB() } /* -------------------------------------------------------------------------- */ - -bool SqliteDB::multiple_values_support() -{ - // Versions > 3.7.11 support multiple value inserts, but tests - // have ended in segfault. A transaction seems to perform better - //return SQLITE_VERSION_NUMBER >= 3007011; - return false; -} - -/* -------------------------------------------------------------------------- */ -bool SqliteDB::limit_support() -{ - return enable_limit == 1; -} - /* -------------------------------------------------------------------------- */ int SqliteDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) diff --git a/src/um/UserPool.cc b/src/um/UserPool.cc index 13cb82ebd5..710637910d 100644 --- a/src/um/UserPool.cc +++ b/src/um/UserPool.cc @@ -1428,8 +1428,7 @@ int UserPool::authorize(AuthRequest& ar) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -int UserPool::dump(string& oss, const string& where, const string& limit, - bool desc) +int UserPool::dump(string& oss, const string& where, int sid, int eid, bool desc) { int rc; string def_quota_xml; @@ -1453,9 +1452,9 @@ int UserPool::dump(string& oss, const string& where, const string& limit, cmd << " DESC"; } - if ( !limit.empty() ) + if ( eid != -1 ) { - cmd << " LIMIT " << limit; + cmd << " " << db->limit_string(sid, eid); } oss.append(""); diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 7d28a439f7..c2045f1571 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -455,7 +455,7 @@ int VirtualMachine::bootstrap(SqlDB * db) oss_vm << one_db::vm_db_bootstrap; - if (db->fts_available()) + if (db->supports(SqlDB::SqlFeature::FTS)) { oss_vm << ", FULLTEXT ftidx(search_token))"; } diff --git a/src/vm/VirtualMachinePool.cc b/src/vm/VirtualMachinePool.cc index 03e5f7d264..0bc95cb89d 100644 --- a/src/vm/VirtualMachinePool.cc +++ b/src/vm/VirtualMachinePool.cc @@ -230,7 +230,7 @@ int VirtualMachinePool::get_running( << " state = " << VirtualMachine::ACTIVE << " and ( lcm_state = " << VirtualMachine::RUNNING << " or lcm_state = " << VirtualMachine::UNKNOWN << " )" - << " ORDER BY last_poll ASC LIMIT " << vm_limit; + << " ORDER BY last_poll ASC " << db->limit_string(vm_limit); where = os.str(); @@ -262,8 +262,7 @@ int VirtualMachinePool::dump_acct(string& oss, const string& where, ostringstream cmd; cmd << "SELECT " << History::table << ".body FROM " << History::table - << " INNER JOIN " << one_db::vm_table - << " WHERE vid=oid"; + << " INNER JOIN " << one_db::vm_table << " ON vid=oid"; if ( !where.empty() ) { @@ -302,8 +301,7 @@ int VirtualMachinePool::dump_showback(string& oss, cmd << "SELECT " << one_db::vm_showback_table << ".body FROM " << one_db::vm_showback_table - << " INNER JOIN " << one_db::vm_table - << " WHERE vmid=oid"; + << " INNER JOIN " << one_db::vm_table << " ON vmid=oid"; if ( !where.empty() ) { @@ -719,7 +717,7 @@ int VirtualMachinePool::calculate_showback( // Write to DB - if (db->multiple_values_support()) + if (db->supports(SqlDB::SqlFeature::MULTIPLE_VALUE)) { oss.str("");