F #3600: Initial PostgreSQL Support

co-authored-by: Igor Sivy <igorsivy@gmail.com>
co-authored-by: Pavel Czerny <pczerny@opennebula.io>
co-authored-by: Vlastimil Holer <vholer@opennebula.io>
(cherry picked from commit c52f62018c32281c6e418211f33f1bba46388e98)

SConstruct

@@ -153,6 +153,16 @@ if mysql == 'yes':
 else:
     main_env.Append(mysql='no')
 
+# PostgreSql
+postgresql = ARGUMENTS.get('postgresql', 'no')
+if postgresql == 'yes':
+    main_env.Append(postgresql='yes')
+    main_env.Append(CPPPATH=['/usr/include/postgresql'])
+    main_env.Append(CPPFLAGS=["-DPOSTGRESQL_DB"])
+    main_env.Append(LIBS=['libpq'])
+else:
+    main_env.Append(postgresql='no')
+
 # Flag to compile with xmlrpc-c versions prior to 1.31 (September 2012)
 new_xmlrpc = ARGUMENTS.get('new_xmlrpc', 'no')
 if new_xmlrpc == 'yes':

include/ClusterPool.h

@@ -200,16 +200,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
         bool desc)
     {
         return PoolSQL::dump(oss, "CLUSTER_POOL", "body", Cluster::table, where,
-                             limit, desc);
+                             sid, eid, desc);
     };
 
     /**

include/DatastorePool.h

@@ -143,16 +143,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table, where,
+        return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table,
-                             limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/DocumentPool.h

@@ -94,16 +94,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
         return PoolSQL::dump(oss, "DOCUMENT_POOL", "body", Document::table, where,
-                             limit, desc);
+                             sid, eid, desc);
     };
 
     /**

include/GroupPool.h

@@ -160,13 +160,13 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(string& oss, const string& where, int sid, int eid, bool desc);
-            bool desc);
 
 private:
 

include/HookPool.h

@@ -74,15 +74,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
         bool desc)
     {
-        return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where, limit, desc);
+        return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where,
+                sid, eid, desc);
     };
 
     /**

include/HostPool.h

@@ -212,16 +212,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
         bool desc)
     {
         return PoolSQL::dump(oss, "HOST_POOL", "body", one_db::host_table,
-                where, limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/ImagePool.h

@@ -155,16 +155,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, limit, 
+        return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, sid,
-            desc);
+                eid, desc);
     }
 
     /**

include/LogDB.h

@@ -208,19 +208,14 @@ public:
         db->free_str(str);
     }
 
-    bool multiple_values_support()
+    bool supports(SqlDB::SqlFeature ft)
     {
-        return db->multiple_values_support();
+        return db->supports(ft);
     }
 
-    bool limit_support()
+    std::string limit_string(int start_id, int end_id)
     {
-        return db->limit_support();
+        return db->limit_string(start_id, end_id);
-    }
-
-    bool fts_available()
-    {
-        return db->fts_available();
     }
     // -------------------------------------------------------------------------
     // Database methods
@@ -422,19 +417,9 @@ public:
         _logdb->free_str(str);
     }
 
-    bool multiple_values_support()
+    bool supports(SqlDB::SqlFeature ft)
     {
-        return _logdb->multiple_values_support();
+        return _logdb->supports(ft);
-    }
-
-    bool limit_support()
-    {
-        return _logdb->limit_support();
-    }
-
-    bool fts_available()
-    {
-        return _logdb->fts_available();
     }
 
     /**

include/MarketPlaceAppPool.h

@@ -144,16 +144,17 @@ public:
      *  the query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(std::string& oss, const std::string& where,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-		const std::string& limit, bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body", MarketPlaceApp::table,
+        return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body",
-                where, limit, desc);
+                MarketPlaceApp::table, where, sid, eid, desc);
     };
 
     /** Update a particular MarketPlaceApp

include/MarketPlacePool.h

@@ -111,16 +111,17 @@ public:
      *  the query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(std::string& oss, const std::string& where,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-		const std::string& limit, bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table, where,
+        return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table,
-                             limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/MySqlDB.h

@@ -30,9 +30,6 @@
 #include "SqlDB.h"
 #include "ObjectSQL.h"
 
-
-using namespace std;
-
 #ifdef MYSQL_DB
 
 #include <mysql.h>
@@ -44,13 +41,13 @@ class MySqlDB : public SqlDB
 {
 public:
 
-    MySqlDB(const string& _server,
+    MySqlDB(const std::string& _server,
-            int           _port,
+            int                _port,
-            const string& _user,
+            const std::string& _user,
-            const string& _password,
+            const std::string& _password,
-            const string& _database,
+            const std::string& _database,
-            const string& _encoding,
+            const std::string& _encoding,
-            int           _connections);
+            int                _connections);
 
     ~MySqlDB();
 
@@ -61,7 +58,7 @@ public:
      *    @param str the string to be escaped
      *    @return a valid SQL string or NULL in case of failure
      */
-    char * escape_str(const string& str);
+    char * escape_str(const std::string& str);
 
     /**
      *  Frees a previously scaped string
@@ -72,36 +69,6 @@ public:
         delete[] str;
     }
 
-    /**
-     * Returns true if the syntax INSERT VALUES (data), (data), (data)
-     * is supported
-     *
-     * @return true if supported
-     */
-    bool multiple_values_support()
-    {
-        return true;
-    }
-
-    /**
-     * Returns true if this Database can use LIMIT in queries with DELETE
-     *  and UPDATE
-     *
-     * @return true if supported
-     */
-    bool limit_support()
-    {
-        return true;
-    }
-
-    /**
-     *  Return true if the backend allows FTS index
-     */
-     bool fts_available()
-     {
-         return _fts_available;
-     }
-
 protected:
     /**
      *  Wraps the mysql_query function call
@@ -109,7 +76,7 @@ protected:
      *    @param obj Callbackable obj to call if the query succeeds
      *    @return 0 on success
      */
-    int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet);
+    int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override;
 
 private:
 
@@ -137,19 +104,17 @@ private:
     /**
      *  MySQL Connection parameters
      */
-    string server;
+    std::string server;
 
     int    port;
 
-    string user;
+    std::string user;
 
-    string password;
+    std::string password;
 
-    string database;
+    std::string database;
 
-    string encoding;
+    std::string encoding;
-
-    bool _fts_available;
 
     /**
      *  Fine-grain mutex for DB access (pool of DB connections)
@@ -177,12 +142,12 @@ class MySqlDB : public SqlDB
 {
 public:
 
-    MySqlDB(const string& _server,
+    MySqlDB(const std::string& _server,
             int           _port,
-            const string& _user,
+            const std::string& _user,
-            const string& _password,
+            const std::string& _password,
-            const string& _database,
+            const std::string& _database,
-            const string& _encoding,
+            const std::string& _encoding,
             int           _connections)
     {
         throw runtime_error("Aborting oned, MySQL support not compiled!");
@@ -190,19 +155,14 @@ public:
 
     ~MySqlDB(){};
 
+    char * escape_str(const std::string& str) override {return nullptr;};
 
-    char * escape_str(const string& str){return 0;};
+    void free_str(char * str) override {};
-
-    void free_str(char * str){};
-
-    bool multiple_values_support(){return true;};
-
-    bool limit_support(){return true;};
-
-    bool fts_available(){return false;};
 
 protected:
-    int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet){return -1;};
+    int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override {
+        return -1;
+    };
 };
 #endif
 

include/PoolSQL.h

@@ -169,7 +169,7 @@ public:
      */
     int dump(string& oss, const string& where, bool desc)
     {
-        return dump(oss, where, "", desc);
+        return dump(oss, where, 0, -1, desc);
     }
 
     /**
@@ -177,30 +177,33 @@ public:
      *  to the query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    virtual int dump(string& oss, const string& where,
+    virtual int dump(string& oss, const string& where, int sid, int eid,
-                     const string& limit, bool desc) = 0;
+            bool desc) = 0;
 
     /**
      *  Dumps the pool in extended XML format
      *  A filter and limit can be also added to the query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
     virtual int dump_extended(string& oss,
                       const string& where,
-                      const string& limit,
+                      int sid,
+                      int eid,
                       bool desc)
     {
-        return dump(oss, where, limit, desc);
+        return dump(oss, where, sid, eid, desc);
     }
 
     // -------------------------------------------------------------------------
@@ -276,12 +279,13 @@ public:
     }
 
     /**
-     * Return true if FTS is available.
+     * Return true if feature is supported
      */
-     bool is_fts_available()
+     bool supports(SqlDB::SqlFeature ft)
      {
-         return db->fts_available();
+         return db->supports(ft);
      }
+
 protected:
 
     /**
@@ -316,7 +320,8 @@ protected:
      *  @param elem_name Name of the root xml pool name
      *  @param table Pool table name
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param start_id first element used for pagination
+     *  @param end_id last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
@@ -326,7 +331,8 @@ protected:
              const string&  column,
              const char *   table,
              const string&  where,
-             const string&  limit,
+             int            start_id,
+             int            end_id,
              bool           desc);
 
     /**
@@ -346,7 +352,7 @@ protected:
              const string&  where,
              bool           desc)
     {
-        return dump(oss, elem_name, "body", table, where, "", desc);
+        return dump(oss, elem_name, "body", table, where, 0, -1, desc);
     }
 
     /**

include/PostgreSqlDB.h

@@ -0,0 +1,196 @@
+/* -------------------------------------------------------------------------- */
+/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems                */
+/*                                                                            */
+/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
+/* not use this file except in compliance with the License. You may obtain    */
+/* a copy of the License at                                                   */
+/*                                                                            */
+/* http://www.apache.org/licenses/LICENSE-2.0                                 */
+/*                                                                            */
+/* Unless required by applicable law or agreed to in writing, software        */
+/* distributed under the License is distributed on an "AS IS" BASIS,          */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+/* See the License for the specific language governing permissions and        */
+/* limitations under the License.                                             */
+/* -------------------------------------------------------------------------- */
+
+#ifndef POSTGRESQL_DB_H_
+#define POSTGRESQL_DB_H_
+
+#include <string>
+#include <stdexcept>
+#include <queue>
+
+#include "NebulaLog.h"
+#include "SqlDB.h"
+#include "ObjectSQL.h"
+
+#ifdef POSTGRESQL_DB
+
+#include <libpq-fe.h>
+
+/**
+ *  PostgreSqlDB class. Provides a wrapper to the PostgreSQL database interface.
+ */
+class PostgreSqlDB : public SqlDB
+{
+public:
+    PostgreSqlDB(
+        const string& _server,
+        int           _port,
+        const string& _user,
+        const string& _password,
+        const string& _database,
+        int           _connections);
+
+    ~PostgreSqlDB();
+
+    /**
+     *  This function returns a legal SQL string that can be used in an SQL
+     *  statement.
+     *    @param str the string to be escaped
+     *    @return a valid SQL string or NULL in case of failure
+     */
+    char * escape_str(const string& str) override;
+
+    /**
+     *  Frees a previously scaped string
+     *    @param str pointer to the str
+     */
+    void free_str(char * str) override;
+
+    /**
+     *  @param sid the offset
+     *  @param eid the rowcount
+     *  @return string with compatible LIMIT clause syntax
+     *  LIMIT row_count OFFSET offset
+     *  
+     *  +---+---+---+---+---+---+---+---+--
+     *  | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |...
+     *  +---+---+---+---+---+---+---+---+--
+     *          |                   |
+     *          /-------------------/
+     *            LIMIT 5 OFFSET 3
+     */
+
+    std::string limit_string(int sid, int eid) override
+    {
+        std::ostringstream oss;
+        oss << "LIMIT " << eid << " OFFSET " << sid;
+
+        return oss.str();
+    }
+
+    std::string limit_string(int sid) override
+    {
+        std::ostringstream oss;
+        oss << "LIMIT " << sid << " OFFSET 0";
+
+        return oss.str();
+    }
+
+protected:
+    int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override;
+
+private:
+
+    /**
+     *  Number of concurrent DB connections.
+     */
+    int max_connections;
+
+    /**
+     *  Connection pool
+     */
+    queue<PGconn *> db_connect;
+
+    /**
+     *  DB connection to escape strings
+     */
+    PGconn * db_escape_connect;
+
+    /**
+     *  Connection parameters
+     */
+    string  server;
+    int     port;
+    string  user;
+    string  password;
+    string  database;
+
+    /**
+     *  Fine-grain mutex for DB access (pool of DB connections)
+     */
+    pthread_mutex_t mutex;
+
+    /**
+     *  Conditional variable to wake-up waiting threads.
+     */
+    pthread_cond_t  cond;
+
+    /**
+     *  Gets a free DB connection from the pool.
+     */
+    PGconn * get_db_connection();
+
+    /**
+     *  Returns the connection to the pool.
+     */
+    void free_db_connection(PGconn * db);
+
+    /**
+     *  Preprocesses the query to be compatible with PostgreSQL syntax
+     *
+     *  Any change to this method should be reflected in BackEndPostgreSQL class
+     *  in src/onedb/onedb_backend.rb
+     *
+     *  This method alters to queries:
+     *    - CREATE TABLE to adjust type names
+     *      . MEDIUMTEXT -> TEXT
+     *      . LONGTEXT -> TEXT
+     *      . BIGINT UNSIGNED -> NUMERIC
+     *
+     *    - REPLACE INTO into PostgreSQL INSERT INTO query with ON CONFLICT
+     *    clause. For example:
+     *       REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0)
+     *    changes to:
+     *       INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0)
+     *          ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid
+     ***************************************************************************
+     *  Any change to this method should be reflected in BackEndPostgreSQL class
+     *  in src/onedb/onedb_backend.rb
+     ***************************************************************************
+     */
+    static std::string preprocess_query(std::ostringstream& cmd);
+};
+#else
+// Class stub
+class PostgreSqlDB : public SqlDB
+{
+public:
+    PostgreSqlDB(
+        const string& _server,
+        int           _port,
+        const string& _user,
+        const string& _password,
+        const string& _database,
+        int           _connections)
+    {
+        throw runtime_error("Aborting oned, PostgreSQL support not compiled!");
+    }
+    ~PostgreSqlDB(){}
+
+    char * escape_str(const string& str) override {return 0;};
+
+    void free_str(char * str) override {};
+
+    std::string limit_string(int sid, int eid) override {return "";}
+
+protected:
+    int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override {
+        return -1;
+    };
+};
+#endif
+
+#endif /*POSTGRESQL_DB_H*/

include/SecurityGroupPool.h

@@ -106,16 +106,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
         return PoolSQL::dump(oss, "SECURITY_GROUP_POOL", "body", SecurityGroup::table,
-                where, limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/SqlDB.h

@@ -18,6 +18,7 @@
 #define SQL_DB_H_
 
 #include <sstream>
+#include <map>
 #include "Callbackable.h"
 
 /**
@@ -40,6 +41,13 @@ public:
         SQL_DUP_KEY = -201
     };
 
+    enum class SqlFeature
+    {
+        MULTIPLE_VALUE, // syntax INSERT VALUES (data), (data), (data)
+        LIMIT,          // LIMIT in queries with DELETE and UPDATE
+        FTS             // Full Text Search
+    };
+
     /* ---------------------------------------------------------------------- */
     /* Database Operations                                                    */
     /* ---------------------------------------------------------------------- */
@@ -99,27 +107,18 @@ public:
      */
     virtual void free_str(char * str) = 0;
 
-    /**
-     * Returns true if the syntax INSERT VALUES (data), (data), (data)
-     * is supported
-     *
-     * @return true if supported
-     */
-    virtual bool multiple_values_support() = 0;
 
-    /**
+    bool supports(SqlFeature ft)
-     * Returns true if this Database can use LIMIT in queries with DELETE
+    {
-     *  and UPDATE
+        auto it = features.find(ft);
-     *
-     * @return true if supported
-     */
-    virtual bool limit_support() = 0;
 
-    /**
+        if ( it == features.end() )
-     *  Return true if the backend allows FTS index
+        {
-     */
+            return false;
-    virtual bool fts_available() = 0;
+        }
 
+        return it->second;
+    }
 
     /**
      *  @return pointer to a non-federated version of this database
@@ -129,6 +128,35 @@ public:
         return this;
     }
 
+    /**
+     *  @return string with compatible LIMIT clause syntax
+     *  LIMIT [offset], row_count
+     *  
+     *  +---+---+---+---+---+---+---+---+--
+     *  | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |...
+     *  +---+---+---+---+---+---+---+---+--
+     *          |                   |
+     *          /-------------------/
+     *              LIMIT 3, 5
+     */
+    virtual std::string limit_string(int sid, int eid)
+    {
+        std::ostringstream oss;
+
+        oss << "LIMIT " << sid << "," << eid;
+
+        return oss.str();
+    }
+
+    virtual std::string limit_string(int sid)
+    {
+        std::ostringstream oss;
+
+        oss << "LIMIT " << sid;
+
+        return oss.str();
+    }
+
 protected:
     /**
      *  Performs a DB transaction
@@ -154,6 +182,15 @@ protected:
      *    @return SqlError enum
      */
     virtual int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) = 0;
+
+    /**
+     *  Feature set
+     */
+    std::map<SqlFeature, bool> features = {
+        {SqlFeature::MULTIPLE_VALUE, false},
+        {SqlFeature::LIMIT, false},
+        {SqlFeature::FTS, false}
+    };
 };
 
 #endif /*SQL_DB_H_*/

include/SqliteDB.h

@@ -63,26 +63,6 @@ public:
      */
     void free_str(char * str) override;
 
-    /**
-     * Returns true if the syntax INSERT VALUES (data), (data), (data)
-     * is supported
-     *
-     * @return true if supported
-     */
-    bool multiple_values_support() override;
-
-    /**
-     * Returns true if this Database can use LIMIT in queries with DELETE
-     *  and UPDATE
-     *
-     * @return true if supported
-     */
-    bool limit_support() override;
-
-    bool fts_available() override
-    {
-        return false;
-    }
 protected:
     /**
      *  Wraps the sqlite3_exec function call, and locks the DB mutex.
@@ -98,17 +78,12 @@ private:
     /**
      *  Fine-grain mutex for DB access
      */
-    pthread_mutex_t     mutex;
+    pthread_mutex_t mutex;
 
     /**
      *  Pointer to the database.
      */
-    sqlite3 *           db;
+    sqlite3 * db;
-
-    /**
-     *  LIMIT for DELETE and UPDATE queries is enabled
-     */
-    int                 enable_limit;
 
     /**
      *  Function to lock the DB
@@ -141,13 +116,9 @@ public:
 
     char * escape_str(const string& str) override { return 0; }
 
-    void free_str(char * str) override {}
+    void free_str(char * str) override {};
 
-    bool multiple_values_support() override { return true; }
+    std::string get_limit_string(const std::string& str) override { return str; }
-
-    bool limit_support() override { return true; }
-
-    bool fts_available() override { return false; }
 
 protected:
     int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override

include/UserPool.h

@@ -212,13 +212,13 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(string& oss, const string& where, int sid, int eid, bool desc);
-            bool desc);
 
     /**
      *  Name for the OpenNebula core authentication process

include/VMGroupPool.h

@@ -99,16 +99,17 @@ public:
      *  Dumps the VMGroup pool in XML format. A filter can be added to the query
      *  @param os the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(std::string& os, const std::string& where,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            const std::string& limit, bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(os, "VM_GROUP_POOL", "body", VMGroup::table, where, limit,
+        return PoolSQL::dump(oss, "VM_GROUP_POOL", "body", VMGroup::table, where,
-                desc);
+                sid, eid, desc);
     };
 
     /**

include/VMTemplatePool.h

@@ -84,16 +84,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table, where,
+        return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table,
-                             limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/VNTemplatePool.h

@@ -84,16 +84,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table, where,
+        return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table,
-                             limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/VdcPool.h

@@ -90,15 +90,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, limit, desc);
+        return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, sid,
+                eid, desc);
     };
 
     /**

include/VirtualMachinePool.h

@@ -250,16 +250,17 @@ public:
      *  pool
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
         return PoolSQL::dump(oss, "VM_POOL", "short_body", one_db::vm_table, where,
-                             limit, desc);
+                             sid, eid, desc);
     };
 
     /**
@@ -274,11 +275,11 @@ public:
      *
      *  @return 0 on success
      */
-    int dump_extended(string& oss, const string& where, const string& limit,
+    int dump_extended(string& oss, const string& where, int sid, int eid,
             bool desc)
     {
         return PoolSQL::dump(oss, "VM_POOL", "body", one_db::vm_table, where,
-                             limit, desc);
+                             sid, eid, desc);
     };
 
     /**

include/VirtualNetworkPool.h

@@ -157,16 +157,17 @@ public:
      *  to the query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit, 
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table, where,
+        return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table,
-                             limit, desc);
+                where, sid, eid, desc);
     }
 
     /**

include/VirtualRouterPool.h

@@ -92,16 +92,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit,
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table, where,
+        return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table,
-                             limit, desc);
+                where, sid, eid, desc);
     };
 
     /**

include/ZonePool.h

@@ -91,15 +91,17 @@ public:
      *  query
      *  @param oss the output stream to dump the pool contents
      *  @param where filter for the objects, defaults to all
-     *  @param limit parameters used for pagination
+     *  @param sid first element used for pagination
+     *  @param eid last element used for pagination, -1 to disable
      *  @param desc descending order of pool elements
      *
      *  @return 0 on success
      */
-    int dump(string& oss, const string& where, const string& limit, 
+    int dump(std::string& oss, const std::string& where, int sid, int eid,
-            bool desc)
+        bool desc)
     {
-        return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where, limit, desc);
+        return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where,
+                sid, eid, desc);
     };
 
     /**

share/etc/oned.conf

@@ -84,7 +84,7 @@ DB = [ BACKEND = "sqlite" ]
 #        USER    = "oneadmin",
 #        PASSWD  = "oneadmin",
 #        DB_NAME = "opennebula",
-#        CONNECTIONS = 50 ]
+#        CONNECTIONS = 25 ]
 
 VNC_PORTS = [
     START    = 5900,

share/install_gems/CentOS7/Gemfile.lock

@@ -89,6 +89,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.1.4)
     polyglot (0.3.5)
     public_suffix (2.0.5)
     rack (1.6.13)
@@ -163,6 +164,7 @@ DEPENDENCIES
   nokogiri (< 1.7)
   ox
   parse-cron
+  pg (< 1.2.0)
   public_suffix (< 3.0.0)
   rack (< 2.0.0)
   rbvmomi (~> 2.2.0)

share/install_gems/CentOS8/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -168,6 +169,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Debian10/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -168,6 +169,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Debian9/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -167,6 +168,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Gemfile

@@ -28,8 +28,10 @@ end
 if RUBY_VERSION < '2.2.0'
     gem 'rack', '< 2.0.0'                 # sunstone, cloud, oneflow
     gem 'minitest', '< 5.12.0'            # packethost
+    gem 'pg', '< 1.2.0'                   # onedb
 else
     gem 'rack'                            # sunstone, cloud, oneflow
+    gem 'pg'                              # onedb
 end
 
 if RUBY_VERSION >= '2.4.0'

share/install_gems/Ubuntu1604/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -167,6 +168,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Ubuntu1804/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -168,6 +169,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Ubuntu1904/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -168,6 +169,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/Ubuntu1910/Gemfile.lock

@@ -91,6 +91,7 @@ GEM
     optimist (3.0.0)
     ox (2.13.2)
     parse-cron (0.1.4)
+    pg (1.2.2)
     polyglot (0.3.5)
     public_suffix (4.0.3)
     rack (2.2.2)
@@ -168,6 +169,7 @@ DEPENDENCIES
   nokogiri
   ox
   parse-cron
+  pg
   public_suffix
   rack
   rbvmomi (~> 2.2.0)

share/install_gems/install_gems

@@ -27,8 +27,10 @@ end
 
 if !defined?(RUBY_VERSION) || RUBY_VERSION < '2.2.0'
     RACK = 'rack --version "< 2.0.0"'
+    PG = 'pg --version "< 1.2.0"'
 else
     RACK = 'rack'
+    PG = 'pg'
 end
 
 TREETOP = 'treetop --version ">= 1.6.3"'
@@ -47,7 +49,7 @@ GROUPS={
     :ec2_hybrid => 'aws-sdk --version "~> 2.5"',
     :oca        => 'ox',
     :market     => 'aws-sdk',
-    :onedb      => "mysql2",
+    :onedb      => ['mysql2', PG],
     :hooks      => %w[zeromq ffi-rzmq],
     :serversync => "augeas"
 }
@@ -66,6 +68,7 @@ DISTRIBUTIONS={
         :dependencies => {
             SQLITE      => ['gcc', 'libsqlite3-dev'],
             'mysql2'     => ['gcc', 'libssl-dev', ['default-libmysqlclient-dev', 'libmysqlclient-dev']],
+            PG          => ['gcc', 'postgresql-server-dev-all'],
             'curb'      => ['gcc', 'libcurl4-openssl-dev'],
             $nokogiri   => %w{gcc rake libxml2-dev libxslt1-dev patch},
             'xmlparser' => ['gcc', 'libexpat1-dev'],
@@ -87,6 +90,7 @@ DISTRIBUTIONS={
         :dependencies => {
             SQLITE      => ['gcc', 'sqlite-devel'],
             'mysql2'     => ['gcc', 'mysql-devel', 'openssl-devel'],
+            PG          => ['gcc', 'postgresql-devel'],
             'curb'      => ['gcc', 'curl-devel'],
             $nokogiri   => %w{gcc rubygem-rake libxml2-devel libxslt-devel patch},
             'xmlparser' => ['gcc', 'expat-devel'],

src/acl/AclManager.cc

@@ -25,11 +25,11 @@
 
 const char * AclManager::table = "acl";
 
-const char * AclManager::db_names = "oid, user, resource, rights, zone";
+const char * AclManager::db_names = "oid, userset, resource, rights, zone";
 
 const char * AclManager::db_bootstrap = "CREATE TABLE IF NOT EXISTS "
-    "acl (oid INT PRIMARY KEY, user BIGINT, resource BIGINT, "
+    "acl (oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, "
-    "rights BIGINT, zone BIGINT, UNIQUE(user, resource, rights, zone))";
+    "rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone))";
 
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */

src/group/GroupPool.cc

@@ -218,8 +218,7 @@ int GroupPool::drop(PoolObjectSQL * objsql, string& error_msg)
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
-int GroupPool::dump(string& oss, const string& where,
+int GroupPool::dump(string& oss, const string& where, int sid, int eid, bool desc)
-    const string& limit, bool desc)
 {
     int     rc;
     string  def_quota_xml;
@@ -243,9 +242,9 @@ int GroupPool::dump(string& oss, const string& where,
         cmd << " DESC";
     }
 
-    if ( !limit.empty() )
+    if ( eid != -1 )
     {
-        cmd << " LIMIT " << limit;
+        cmd << " " << db->limit_string(sid, eid);
     }
 
     oss.append("<GROUP_POOL>");

src/host/HostPool.cc

@@ -171,7 +171,7 @@ int HostPool::dump_monitoring(
 
     cmd << "SELECT " << one_db::host_monitor_table << ".body FROM "
         << one_db::host_monitor_table << " INNER JOIN " << one_db::host_table
-        << " WHERE hid = oid";
+        << " ON hid = oid";
 
     if ( !where.empty() )
     {

src/im/InformationManager.cc

@@ -72,7 +72,8 @@ int InformationManager::start()
     }
 
     string xml_hosts;
-    hpool->dump(xml_hosts, "", "", false);
+
+    hpool->dump(xml_hosts, "", 0, -1, false);
 
     Message<OpenNebulaMessages> msg;
 

src/monitor/etc/monitord.conf

@@ -18,6 +18,11 @@
 #  VM_MONITORING_EXPIRATION_TIME: Time, in seconds, to expire monitoring
 #       information. Use 0 to disable VM monitoring recording.
 #
+#  DB: Database configuration attributes. Monitord will use the DB configuration
+#      in oned.conf. The following attributes can be tuned:
+#      - CONNECTIONS: Number of DB connections. The DB needs to be configure to
+#      support oned + monitord connections.
+#
 #  LOG: Configuration for the logging system
 #    - system: defines the logging system:
 #          file      to log in the sched.log file
@@ -39,6 +44,10 @@
 #HOST_MONITORING_EXPIRATION_TIME = 43200
 #VM_MONITORING_EXPIRATION_TIME = 43200
 
+DB = [
+  CONNECTIONS = 15
+]
+
 LOG = [
   system      = "FILE",
   debug_level = 3

src/monitor/src/monitor/Monitor.cc

@@ -22,6 +22,7 @@
 #include "StreamManager.h"
 #include "SqliteDB.h"
 #include "MySqlDB.h"
+#include "PostgreSqlDB.h"
 
 #include <fcntl.h>
 #include <unistd.h>
@@ -35,6 +36,7 @@ void Monitor::start()
     // Configuration File
     // -------------------------------------------------------------------------
     OpenNebulaTemplate oned_config(get_defaults_location(), oned_filename);
+
     if (oned_config.load_configuration() != 0)
     {
         throw runtime_error("Error reading oned configuration file " +
@@ -91,7 +93,8 @@ void Monitor::start()
     // -------------------------------------------------------------------------
     // Database
     // -------------------------------------------------------------------------
-    const VectorAttribute * _db = oned_config.get("DB");
+    const VectorAttribute * _db   = oned_config.get("DB");
+    const VectorAttribute * _db_m = config->get("DB");
 
     std::string db_backend = _db->vector_value("BACKEND");
 
@@ -115,10 +118,23 @@ void Monitor::start()
         _db->vector_value<string>("PASSWD", passwd, "oneadmin");
         _db->vector_value<string>("DB_NAME", db_name, "opennebula");
         _db->vector_value<string>("ENCODING", encoding, "");
-        _db->vector_value("CONNECTIONS", connections, 50);
 
-        sqlDB.reset(new MySqlDB(server, port, user, passwd, db_name,
+        _db_m->vector_value("CONNECTIONS", connections, 15);
-                    encoding, connections));
+
+        if ( db_backend == "postgresql" )
+        {
+            sqlDB.reset(new PostgreSqlDB(server, port, user, passwd, db_name,
+                    connections));
+        }
+        else if ( db_backend == "mysql" )
+        {
+            sqlDB.reset(new MySqlDB(server, port, user, passwd, db_name,
+                        encoding, connections));
+        }
+        else
+        {
+            throw runtime_error("Unknown DB backend " + db_backend);
+        }
     }
 
     // -------------------------------------------------------------------------

src/monitor/src/monitor/MonitorConfigTemplate.cc

@@ -25,6 +25,7 @@ void MonitorConfigTemplate::set_conf_default()
 /*
  HOST_MONITORING_EXPIRATION_TIME
  VM_MONITORING_EXPIRATION_TIME
+ DB
  LOG
  UDP_LISTENER
  PROBES_PERIOD
@@ -36,6 +37,9 @@ void MonitorConfigTemplate::set_conf_default()
     set_conf_single("HOST_MONITORING_EXPIRATION_TIME", "43200");
     set_conf_single("VM_MONITORING_EXPIRATION_TIME", "43200");
 
+    va = new VectorAttribute("DB", {{"CONNECTIONS", "15"}});
+    conf_default.insert(make_pair(va->name(), va));
+
     va = new VectorAttribute("LOG", {{"SYSTEM", "FILE"}, {"DEBUG_LEVEL", "3"}});
     conf_default.insert(make_pair(va->name(), va));
 

src/nebula/Nebula.cc

@@ -19,6 +19,7 @@
 #include "VirtualMachine.h"
 #include "SqliteDB.h"
 #include "MySqlDB.h"
+#include "PostgreSqlDB.h"
 #include "Client.h"
 #include "LogDB.h"
 #include "SystemDB.h"
@@ -388,7 +389,7 @@ void Nebula::start(bool bootstrap_only)
 
             if (_db->vector_value("CONNECTIONS", connections) == -1)
             {
-                connections = 50;
+                connections = 25;
             }
 
             if (_db->vector_value("ENCODING", encoding) == -1)
@@ -401,11 +402,20 @@ void Nebula::start(bool bootstrap_only)
         {
             db_backend = new SqliteDB(var_location + "one.db");
         }
-        else
+        else if ( db_backend_type == "mysql" )
         {
             db_backend = new MySqlDB(server, port, user, passwd, db_name,
                     encoding, connections);
         }
+        else if ( db_backend_type == "postgresql" )
+        {
+            db_backend = new PostgreSqlDB(server, port, user, passwd, db_name,
+                    connections);
+        }
+        else
+        {
+            throw runtime_error("DB BACKEND must be one of sqlite, mysql or postgresql.");
+        }
 
         // ---------------------------------------------------------------------
         // Check Database Versions

src/nebula/SystemDB.cc

@@ -102,7 +102,7 @@ int SystemDB::local_bootstrap()
     oss << "INSERT INTO " << local_ver_table << " (" << local_ver_names << ") "
         << "VALUES (0, '" << Nebula::local_db_version() << "', " << time(0)
         << ", '" << Nebula::version() << " daemon bootstrap', "
-        << Nebula::instance().is_federation_slave() << ")";
+        << "'" << Nebula::instance().is_federation_slave() << "')";
 
     rc += db->exec_local_wr(oss);
 

src/onedb/database_schema.rb

@@ -127,6 +127,8 @@ class OneDBBacKEnd
                 "state INTEGER, lcm_state INTEGER, " <<
                 "owner_u INTEGER, group_u INTEGER, other_u INTEGER, short_body MEDIUMTEXT, " <<
                 "search_token MEDIUMTEXT",
+            acl: "oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, " <<
+                 "rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone)"
         }
     }
 

src/onedb/onedb

@@ -101,16 +101,26 @@ SQLITE={
 }
 
 ###############################################################################
-# MySQL options
+# MySQL and PostgreSQL options
 ###############################################################################
+TYPE={
+    :name => "type",
+    :short => "-t type",
+    :large => "--type type",
+    :format => String,
+    :description => "Backend type.",
+    :proc => lambda { |o, options|
+        options[:backend] = o.to_sym
+    }
+}
+
 SERVER={
     :name => 'server',
     :short => '-S host',
     :large => '--server host',
     :format => String,
-    :description => 'MySQL server hostname or IP. Defaults to localhost',
+    :description => "MySQL or PostgreSQL server hostname or IP. Defaults to localhost",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend] = :mysql
         options[:server]  = o
     }
 }
@@ -120,10 +130,9 @@ PORT={
     :short => '-P port',
     :large => '--port port',
     :format => String,
-    :description => 'MySQL server port. Defaults to 3306',
+    :description => "MySQL or PostgreSQL server port. Defaults to 3306 for MySQL and 5432 for PostgreSQL",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend] = :mysql
+        options[:port]  = o
-        options[:port] = o
     }
 }
 
@@ -132,9 +141,8 @@ USERNAME={
     :short => '-u user',
     :large => '--username user',
     :format => String,
-    :description => 'MySQL username',
+    :description => "MySQL or PostgreSQL username",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend] = :mysql
         options[:user]    = o
     }
 }
@@ -144,9 +152,8 @@ PASSWORD={
     :short => '-p pass',
     :large => '--password pass',
     :format => String,
-    :description => 'MySQL password. Leave unset to be prompted for it',
+    :description => "MySQL or PostgreSQL password. Leave unset to be prompted for it",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend] = :mysql
         options[:passwd]  = o
     }
 }
@@ -156,9 +163,8 @@ DBNAME={
     :short => '-d dbname',
     :large => '--dbname dbname',
     :format => String,
-    :description => 'MySQL DB name for OpenNebula',
+    :description => "MySQL or PostgreSQL DB name for OpenNebula",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend] = :mysql
         options[:db_name] = o
     }
 }
@@ -167,9 +173,8 @@ ENCODING={
     :name => 'encoding',
     :large => '--encoding charset',
     :format => String,
-    :description => 'MySQL encoding to use for the connection',
+    :description => "MySQL or PostgreSQL encoding to use for the connection",
-    :proc => lambda {|o, options|
+    :proc => lambda { |o, options|
-        options[:backend]  = :mysql
         options[:encoding] = o
     }
 }
@@ -332,7 +337,7 @@ CommandParser::CmdParser.new(ARGV) do
     # Global options
     ###########################################################################
     set :option, CommandParser::OPTIONS
-    set :option, [SQLITE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING]
+    set :option, [SQLITE, TYPE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING]
 
     ###########################################################################
     # Backup

src/onedb/onedb.rb

@@ -23,6 +23,11 @@ class OneDB
     attr_accessor :backend
 
     def initialize(ops)
+        # Set MySQL backend as default if any connection option is provided and --type is not
+        if ops[:backend].nil? and (!ops[:server].nil? || !ops[:port].nil? || !ops[:user].nil? || !ops[:password].nil? || !ops[:db_name].nil? || !ops[:encoding].nil?)
+            ops[:backend] = :mysql
+        end
+
         if ops[:backend] == :sqlite
             begin
                 require 'sqlite3'
@@ -55,8 +60,30 @@ class OneDB
                 :db_name => ops[:db_name],
                 :encoding=> ops[:encoding]
             )
+        elsif ops[:backend] == :postgresql
+            begin
+                require 'pg'
+            rescue
+                STDERR.puts "Ruby gem pg is needed for this operation:"
+                STDERR.puts "   $ sudo gem install pg"
+                exit -1
+            end
+
+            passwd = ops[:passwd]
+            if !passwd
+                passwd = get_password("PostgreSQL Password: ")
+            end
+
+            @backend = BackEndPostgreSQL.new(
+                :server  => ops[:server],
+                :port    => ops[:port],
+                :user    => ops[:user],
+                :passwd  => passwd,
+                :db_name => ops[:db_name],
+                :encoding=> ops[:encoding]
+            )
         else
-            raise "You need to specify the SQLite or MySQL connection options."
+            raise "You need to specify the SQLite, MySQL or PostgreSQL connection options."
         end
     end
 

src/onedb/onedb_backend.rb

@@ -120,22 +120,13 @@ class OneDBBacKEnd
         comment = "Database migrated from #{version} to #{db_version}"+
                   " (#{one_version}) by onedb command."
 
-        max_oid = nil
+        max_oid = @db[:db_versioning].max(:oid)
-        @db.fetch("SELECT MAX(oid) FROM db_versioning") do |row|
-            max_oid = row[:"MAX(oid)"].to_i
-        end
 
-        max_oid = 0 if max_oid.nil?
+        @db[:db_versioning].insert(
-
+            oid: max_oid + 1,
-        query =
+            version: db_version,
-        @db.run(
+            timestamp: Time.new.to_i,
-            "INSERT INTO db_versioning (oid, version, timestamp, comment) "<<
+            comment: comment)
-            "VALUES ("                                                     <<
-                "#{max_oid+1}, "                                           <<
-                "'#{db_version}', "                                        <<
-                "#{Time.new.to_i}, "                                       <<
-                "'#{comment}')"
-        )
 
         puts comment
     end
@@ -144,29 +135,16 @@ class OneDBBacKEnd
         comment = "Database migrated from #{version} to #{db_version}"+
                   " (#{one_version}) by onedb command."
 
-        max_oid = nil
+        max_oid = @db[:local_db_versioning].max(:oid)
-        @db.fetch("SELECT MAX(oid) FROM local_db_versioning") do |row|
-            max_oid = row[:"MAX(oid)"].to_i
-        end
 
-        max_oid = 0 if max_oid.nil?
+        is_slave = @db[:local_db_versioning].select(:is_slave).where_single_value(oid: max_oid)
 
-        is_slave = 0
+        @db[:local_db_versioning].insert(
-
+            oid: max_oid + 1, 
-        @db.fetch("SELECT is_slave FROM local_db_versioning "<<
+            version: db_version, 
-                  "WHERE oid=#{max_oid}") do |row|
+            timestamp: Time.new.to_i, 
-            is_slave = row[:is_slave] ? 1 : 0
+            comment: comment, 
-        end
+            is_slave: is_slave)
-
-        @db.run(
-            "INSERT INTO local_db_versioning (oid, version, timestamp, comment, is_slave) "<<
-            "VALUES ("                                                     <<
-                "#{max_oid+1}, "                                           <<
-                "'#{db_version}', "                                        <<
-                "#{Time.new.to_i}, "                                       <<
-                "'#{comment}',"                                            <<
-                "#{is_slave})"
-        )
 
         puts comment
     end
@@ -638,3 +616,261 @@ class BackEndSQLite < OneDBBacKEnd
         end
     end
 end
+
+class BackEndPostgreSQL < OneDBBacKEnd
+    def initialize(opts={})
+        @server  = opts[:server]
+        @port    = opts[:port]
+        @user    = opts[:user]
+        @passwd  = opts[:passwd]
+        @db_name = opts[:db_name]
+        @encoding= opts[:encoding]
+
+        # Check for errors:
+        error   = false
+
+        (error = true; missing = "USER"  )  if @user    == nil
+        (error = true; missing = "DBNAME")  if @db_name == nil
+
+        if error
+            raise "PostgreSQL option #{missing} is needed"
+        end
+
+        # Check for defaults:
+        @server = "localhost" if @server.nil?
+        @port   = 5432        if @port.nil?
+
+        encoding if @encoding.nil?
+
+        # Clean leading and trailing quotes, if any
+        @server  = @server [1..-2] if @server [0] == ?"
+        @port    = @port   [1..-2] if @port   [0] == ?"
+        @user    = @user   [1..-2] if @user   [0] == ?"
+        @passwd  = @passwd [1..-2] if @passwd [0] == ?"
+        @db_name = @db_name[1..-2] if @db_name[0] == ?"
+    end
+
+    def bck_file(federated = false)
+        t = Time.now
+
+        bck_name = "#{VAR_LOCATION}/postgresql_#{@server}_#{@db_name}_"
+
+        bck_name << "federated_" if federated
+
+        bck_name << "#{t.year}-#{t.month}-#{t.day}_"
+        bck_name << "#{t.hour}:#{t.min}:#{t.sec}.sql"
+
+        bck_name
+    end
+
+
+    def backup(bck_file, federated = false)
+        cmd = "PGPASSWORD=\"#{@passwd}\" pg_dump -U #{@user} -h #{@server} -p #{@port} -b #{@db_name} -Fp -f #{bck_file} "
+        
+        if federated
+            connect_db
+
+            @db.create_table!(:logdb_tmp, as: @db[:logdb].where{fed_index != -1})
+
+            FEDERATED_TABLES.each do |table|
+                cmd << " -t " << table
+            end
+
+            cmd << " -t logdb_tmp"
+
+            rc = system(cmd)
+            if !rc
+                raise "Unknown error running '#{cmd}'"
+            end
+
+            @db.drop_table(:logdb_tmp)
+            
+            File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("logdb_tmp","logdb"))
+        else
+            rc = system(cmd)
+
+            if !rc
+                raise "Unknown error running '#{cmd}'"
+            end
+        end
+
+        File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("COMMENT ON","-- COMMENT ON"))
+
+        puts "PostgreSQL dump stored in #{bck_file}"
+        puts "Use 'onedb restore' to restore the DB"
+        puts
+    end
+
+    def get_db_encoding
+        db_enc = ''
+
+        @db.fetch("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = \'#{@db_name}\'") do |row|
+            db_enc = row[:pg_encoding_to_char]
+            db_enc ||= row[:PG_ENCODING_TO_CHAR]
+        end
+
+        db_enc
+    end
+
+    def encoding
+        @encoding = ''
+
+        connect_db
+
+        @encoding = get_db_encoding
+    end
+
+    def get_table_enconding(table = nil)
+        encoding = get_db_encoding
+
+        table_to_nk(encoding)
+    end
+
+    def table_to_nk(encoding)
+        case encoding
+        when 'UTF8'
+            'UTF-8'
+        when 'ISO_8859_5'
+            'ISO-8859-5'
+        when 'ISO_8859_6'
+            'ISO-8859-6'
+        when 'ISO_8859_7'
+            'ISO-8859-7'
+        when 'ISO_8859_8'
+            'ISO-8859-8'
+        when 'LATIN1'
+            'ISO-8859-1'
+        when 'LATIN2'
+            'ISO-8859-2'
+        when 'LATIN3'
+            'ISO-8859-3'
+        when 'LATIN4'
+            'ISO-8859-4'
+        when 'LATIN5'
+            'ISO-8859-9'
+        when 'SJIS'
+            'SHIFT-JIS'
+        when 'EUC_JP'
+            'EUC-JP'
+        when 'SQL_ASCII'
+            'ASCII'
+        else
+            NOKOGIRI_ENCODING
+        end
+    end
+
+    def restore(bck_file, force=nil, federated=false)
+        if !federated && !force && db_exists?
+            raise "PostgreSQL database #{@db_name} at #{@server} exists," <<
+                  " use -f to overwrite."
+        end
+
+        connect_db
+        if federated
+            FEDERATED_TABLES.each do |table|
+                @db.drop_table?(table)
+            end
+
+            @db.drop_table?(:logdb)
+        else
+            @db.tables.each do |table|
+                @db.drop_table(table)
+            end
+        end
+
+        rc = system("PGPASSWORD=\"#{@passwd}\" psql -U #{@user} -h #{@server} -p #{@port} -d #{@db_name} -f #{bck_file} --set ON_ERROR_STOP=on --quiet -o /dev/null")
+        if !rc
+            raise "Error while restoring PostgreSQL DB #{@db_name} at #{@server}."
+        end
+
+        puts "PostgreSQL DB #{@db_name} at #{@server} restored."
+    end
+
+    def fts_index(recreate = false)
+        raise "FTS index not supported for PostgreSQL." 
+    end
+
+    private
+
+    def connect_db
+        passwd = CGI.escape(@passwd)
+
+        endpoint = "postgres://#{@user}:#{passwd}@#{@server}:#{@port}/#{@db_name}"
+
+        begin
+            options = {}
+            options[:encoding] = @encoding unless @encoding.empty?
+
+            @db = Sequel.connect(endpoint, options)
+        rescue Exception => e
+            raise "Error connecting to DB: " + e.message
+        end
+
+        redefine_db_methods
+    end
+
+    def redefine_db_methods
+        def @db.fetch(query)
+            preprocessed = BackEndPostgreSQL.preprocess(query)
+            super(preprocessed)
+        end
+
+        def @db.run(query)
+            preprocessed = BackEndPostgreSQL.preprocess(query)
+            super(preprocessed)
+        end
+    end
+
+    # Any change to this method should be reflected in PostgreSqlDB class
+    # in src/sql/PostgreSqlDB.cc
+    def self.preprocess(query)
+        pp_query = query.dup
+
+        if pp_query.upcase.start_with?('CREATE TABLE')
+            pp_query = replace_type(pp_query, 'MEDIUMTEXT', 'TEXT')
+            pp_query = replace_type(pp_query, 'LONGTEXT', 'TEXT')
+            pp_query = replace_type(pp_query, 'BIGINT UNSIGNED', 'NUMERIC(20)')
+        end
+
+        if pp_query.upcase.start_with?('REPLACE')
+            pp_query = replace_replace_into(query)
+        end
+
+        pp_query
+    end
+
+    def self.replace_type(query, type, replacement)
+        query = query.gsub(type.upcase, replacement.upcase)
+        query = query.gsub(type.downcase, replacement.downcase)
+        return query
+    end
+
+    # This method changes MySQL/SQLite REPLACE INTO into PostgreSQL
+    # INSERT INTO query with ON CONFLICT clause. 
+    # For example:
+    #   REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0)
+    # changes to:
+    #   INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0) 
+    #       ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid"
+    #
+    # Any change to this method should be reflected in PostgreSqlDB class
+    # in src/sql/PostgreSqlDB.cc
+    def self.replace_replace_into(query)
+        query[0, 7] = "INSERT" 
+        db_names_start = query.index('(') + 1
+        db_names_end = query.index(')')
+        db_names = query[db_names_start, db_names_end - db_names_start]
+    
+        splits = db_names.split(',')
+    
+        query += " ON CONFLICT (" + splits[0] + ") DO UPDATE SET "
+        sep = ""
+        splits[1..-1].each do |split|
+            split = split.strip
+            query += sep + split + " = EXCLUDED." + split
+            sep = ", "
+        end
+    
+        query    
+    end
+end

src/onedb/shared/5.10.0_to_5.12.0.rb

@@ -27,7 +27,28 @@ module Migrator
     end
 
     def up
+        feature_3600
         true
     end
 
+    private
+
+    #Rename acl column name from user to userset to support postgresql
+    def feature_3600
+        @db.run 'DROP TABLE IF EXISTS old_acl;'
+        @db.run 'ALTER TABLE acl RENAME TO old_acl;'
+
+        create_table(:acl)
+
+        @db.transaction do
+            @db.fetch('SELECT * FROM old_acl') do |row|
+                row[:userset] = row.delete[:user]
+
+                @db[:acl].insert(row)
+            end
+        end
+
+        @db.run "DROP TABLE old_acl;"
+    end
+
 end

src/pool/PoolSQL.cc

@@ -274,8 +274,8 @@ PoolObjectSQL * PoolSQL::get_ro(const string& name, int uid)
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
-int PoolSQL::dump(string& oss, const string& elem_name, const string& column, const char* table,
+int PoolSQL::dump(string& oss, const string& elem_name, const string& column, 
-    const string& where, const string& limit, bool desc)
+        const char* table, const string& where, int sid, int eid, bool desc)
 {
     ostringstream   cmd;
 
@@ -293,9 +293,9 @@ int PoolSQL::dump(string& oss, const string& elem_name, const string& column, co
         cmd << " DESC";
     }
 
-    if ( !limit.empty() )
+    if ( eid != -1 )
     {
-        cmd << " LIMIT " << limit;
+        cmd << " " << db->limit_string(sid, eid);
     }
 
     return dump(oss, elem_name, cmd);

src/rm/RequestManagerPoolInfoFilter.cc

@@ -200,6 +200,8 @@ void RequestManagerPoolInfoFilter::dump(
     std::string where_string, limit_clause;
     std::string desc;
 
+    int limit_end_id = -1;
+
     int rc;
 
     if ( filter_flag < GROUP )
@@ -222,8 +224,7 @@ void RequestManagerPoolInfoFilter::dump(
 
     if ( end_id < -1 )
     {
-        oss << start_id << "," << -end_id;
+        limit_end_id = -end_id;
-        limit_clause = oss.str();
     }
 
     Nebula::instance().get_configuration_attribute(att.uid, att.gid,
@@ -233,14 +234,16 @@ void RequestManagerPoolInfoFilter::dump(
     {
         rc = pool->dump_extended(str,
                                  where_string,
-                                 limit_clause,
+                                 start_id,
+                                 limit_end_id,
                                  one_util::toupper(desc) == "DESC");
     }
     else
     {
         rc = pool->dump(str,
                         where_string,
-                        limit_clause,
+                        start_id,
+                        limit_end_id,
                         one_util::toupper(desc) == "DESC");
     }
 
@@ -300,7 +303,7 @@ void VirtualMachinePoolInfo::request_execute(
     {
         fts_query = xmlrpc_c::value_string(paramList.getString(5));
 
-        if (!fts_query.empty() && !pool->is_fts_available())
+        if (!fts_query.empty() && !pool->supports(SqlDB::SqlFeature::FTS))
         {
             att.resp_msg = "Full text search is not supported by the SQL backend";
 
@@ -556,6 +559,8 @@ void VirtualNetworkPoolInfo::request_execute(
     int start_id    = xmlrpc_c::value_int(paramList.getInt(2));
     int end_id      = xmlrpc_c::value_int(paramList.getInt(3));
 
+    int limit_end_id = -1;
+
     if ( filter_flag < GROUP )
     {
         att.resp_msg = "Incorrect filter_flag";
@@ -588,7 +593,7 @@ void VirtualNetworkPoolInfo::request_execute(
 
     if ( end_id < -1 )
     {
-        limit_clause << start_id << "," << -end_id;
+        limit_end_id = -end_id;
     }
 
     /* ---------------------------------------------------------------------- */
@@ -600,7 +605,7 @@ void VirtualNetworkPoolInfo::request_execute(
     Nebula::instance().get_configuration_attribute(att.uid, att.gid,
             "API_LIST_ORDER", desc);
 
-    int rc = pool->dump(pool_oss, where_string.str(), limit_clause.str(),
+    int rc = pool->dump(pool_oss, where_string.str(), start_id, limit_end_id,
             one_util::toupper(desc) == "DESC");
 
     if ( rc != 0 )
@@ -1075,4 +1080,4 @@ void HookLogInfo::request_execute(xmlrpc_c::paramList const& _paramList,
     success_response(dump_xml, att);
 
     return;
-}
+}

src/sql/LogDB.cc

@@ -184,7 +184,7 @@ int LogDB::setup_index(uint64_t& _last_applied, uint64_t& _last_index)
 
     cb.set_callback(&_last_applied);
 
-    oss << "SELECT MAX(log_index) FROM logdb WHERE applied = 1";
+    oss << "SELECT MAX(log_index) FROM logdb WHERE applied = '1'";
 
     rc += db->exec_rd(oss, &cb);
 
@@ -345,7 +345,7 @@ int LogDB::insert(uint64_t index, unsigned int term, const std::string& sql,
             << "'" << sql_db    << "',"
             <<        tstamp    << ","
             <<        fed_index << ","
-            <<        applied   << ")";
+            << "'" << applied   << "')";
 
     int rc = db->exec_wr(oss);
 
@@ -383,7 +383,7 @@ int LogDB::apply_log_record(LogDBRecord * lr)
     {
         std::ostringstream oss;
 
-        oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = 1"
+        oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = '1'"
             << " WHERE log_index = " << lr->index << " AND timestamp = 0";
 
         if ( db->exec_wr(oss) != 0 )
@@ -642,8 +642,8 @@ int LogDB::purge_log()
     oss.str("");
     oss << "  SELECT MIN(i.log_index) FROM ("
         << "    SELECT log_index FROM logdb WHERE fed_index = " << UINT64_MAX
-        << "      AND applied = 1 AND log_index >= 0 "
+        << "      AND applied = '1' AND log_index >= 0 "
-        << "      ORDER BY log_index DESC LIMIT " << log_retention
+        << "      ORDER BY log_index DESC " << db->limit_string(log_retention)
         << "  ) AS i";
 
     cb_min_idx.set_callback(&min_idx);
@@ -655,12 +655,12 @@ int LogDB::purge_log()
     cb.set_affected_rows(0);
 
     oss.str("");
-    oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 "
+    oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 "
         << "AND fed_index = " << UINT64_MAX << " AND log_index < " << min_idx;
 
-    if ( db->limit_support() )
+    if ( db->supports(SqlDB::SqlFeature::LIMIT) )
     {
-        oss << " LIMIT " << limit_purge;
+        oss << " " << db->limit_string(limit_purge);
     }
 
     if ( db->exec_wr(oss, &cb) != -1 )
@@ -705,8 +705,8 @@ int LogDB::purge_log()
     oss.str("");
     oss << "  SELECT MIN(i.log_index) FROM ("
         << "    SELECT log_index FROM logdb WHERE fed_index != " << UINT64_MAX
-        << "      AND applied = 1 AND log_index >= 0 "
+        << "      AND applied = '1' AND log_index >= 0 "
-        << "      ORDER BY log_index DESC LIMIT " << log_retention
+        << "      ORDER BY log_index DESC " << db->limit_string(log_retention)
         << "  ) AS i";
 
     cb_min_idx.set_callback(&min_idx);
@@ -718,12 +718,12 @@ int LogDB::purge_log()
     cb.set_affected_rows(0);
 
     oss.str("");
-    oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 "
+    oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 "
         << "AND fed_index != " << UINT64_MAX << " AND log_index < " << min_idx;
 
-    if ( db->limit_support() )
+    if ( db->supports(SqlDB::SqlFeature::LIMIT) )
     {
-        oss << " LIMIT " << limit_purge;
+        oss << " " << db->limit_string(limit_purge);
     }
 
     if ( db->exec_wr(oss, &cb) != -1 )

src/sql/MySqlDB.cc

@@ -21,16 +21,17 @@
 /*********
  * Doc: https://dev.mysql.com/doc/refman/8.0/en/c-api.html
  ********/
+using namespace std;
 
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
 static unsigned long get_server_version(MYSQL * db)
 {
-    std::string version_str;
+    string version_str;
-    std::vector<unsigned int> ids;
+    vector<unsigned int> ids;
 
-    std::string version_query = "SELECT @@GLOBAL.version";
+    string version_query = "SELECT @@GLOBAL.version";
 
     if ( mysql_query(db, version_query.c_str()) != 0 )
     {
@@ -56,8 +57,8 @@ static unsigned long get_server_version(MYSQL * db)
 
     mysql_free_result(result);
 
-    std::string version_number;
+    string version_number;
-    std::stringstream version_iss(version_str);
+    stringstream version_iss(version_str);
 
     if (!getline(version_iss, version_number, '-') || version_number.empty())
     {
@@ -72,10 +73,9 @@ static unsigned long get_server_version(MYSQL * db)
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
-static std::string get_encoding(MYSQL * c, const std::string& sql,
+static string get_encoding(MYSQL * c, const string& sql, string& error)
-        std::string& error)
 {
-    std::string encoding;
+    string encoding;
 
     if ( mysql_query(c, sql.c_str()) != 0 )
     {
@@ -111,7 +111,7 @@ static std::string get_encoding(MYSQL * c, const std::string& sql,
 
 /* -------------------------------------------------------------------------- */
 
-int MySqlDB::db_encoding(std::string& error)
+int MySqlDB::db_encoding(string& error)
 {
     MYSQL * connection = mysql_init(nullptr);
 
@@ -124,7 +124,7 @@ int MySqlDB::db_encoding(std::string& error)
         return -1;
     }
 
-    std::string create_sql = "CREATE DATABASE IF NOT EXISTS " + database;
+    string create_sql = "CREATE DATABASE IF NOT EXISTS " + database;
 
     if ( mysql_query(connection, create_sql.c_str()) != 0 )
     {
@@ -140,22 +140,22 @@ int MySqlDB::db_encoding(std::string& error)
     }
 
     //Get encodings for database and tables
-    std::string db_sql = "SELECT default_character_set_name FROM "
+    string db_sql = "SELECT default_character_set_name FROM "
      "information_schema.SCHEMATA WHERE schema_name = \"" + database + "\"";
 
-    std::string db_enc = get_encoding(connection, db_sql, error);
+    string db_enc = get_encoding(connection, db_sql, error);
 
     if ( db_enc.empty() )
     {
         return -1;
     }
 
-    std::string table_sql = "SELECT CCSA.character_set_name FROM "
+    string table_sql = "SELECT CCSA.character_set_name FROM information_schema."\
-     "information_schema.`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`"
+     "`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`"
      " CCSA WHERE CCSA.collation_name = T.table_collation AND T.table_schema = "
      "\"" + database + "\" AND T.table_name = \"system_attributes\"";
 
-    std::string table_enc = get_encoding(connection, table_sql, error);
+    string table_enc = get_encoding(connection, table_sql, error);
 
     if ( !table_enc.empty() && table_enc != db_enc)
     {
@@ -176,13 +176,13 @@ int MySqlDB::db_encoding(std::string& error)
 
 MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
     const string& d, const string& e, int m):max_connections(m), server(s),
-     port(p), user(u), password(_p), database(d), encoding(e), _fts_available(false)
+     port(p), user(u), password(_p), database(d), encoding(e)
 {
     vector<MYSQL *> connections(max_connections);
     MYSQL * rc;
 
     ostringstream oss;
-    std::string   error;
+    string   error;
 
     // -------------------------------------------------------------------------
     // Initialize the MySQL library
@@ -240,13 +240,13 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
     // -------------------------------------------------------------------------
     // Connect to OpenNebula Database
     // -------------------------------------------------------------------------
-    std::string use_sql = "USE " + database;
+    string use_sql = "USE " + database;
 
     for (int i=0 ; i < max_connections ; i++)
     {
         if ( mysql_query(connections[i], use_sql.c_str()) != 0 )
         {
-            std::string error = "Could not connect to database: ";
+            string error = "Could not connect to database: ";
             error.append(mysql_error(connections[i]));
 
             throw runtime_error(error);
@@ -254,7 +254,7 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
 
         if ( mysql_set_character_set(connections[i], encoding.c_str()) != 0 )
         {
-            std::string error = "Could not set encoding : ";
+            string error = "Could not set encoding : ";
             error.append(mysql_error(connections[i]));
 
             throw runtime_error(error);
@@ -271,12 +271,12 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
     oss.clear();
 
     //--------------------------------------------------------------------------
-    // Get server information and FTS support
+    // Get server information and FTS support & features
     //--------------------------------------------------------------------------
     unsigned long version;
     unsigned long min_fts_version;
 
-    std::string server_info = mysql_get_server_info(db_escape_connect);
+    string server_info = mysql_get_server_info(db_escape_connect);
 
     version = get_server_version(db_escape_connect);
 
@@ -299,15 +299,19 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
 
     if (version >= min_fts_version)
     {
-        _fts_available = true;
         NebulaLog::log("ONE", Log::INFO, "\tFTS enabled");
     }
     else
     {
-        _fts_available = false;
         NebulaLog::log("ONE", Log::INFO, "\tFTS disabled");
     }
 
+    features = {
+        {SqlFeature::MULTIPLE_VALUE, true},
+        {SqlFeature::LIMIT, true},
+        {SqlFeature::FTS, version >= min_fts_version}
+    };
+
     pthread_mutex_init(&mutex,0);
 
     pthread_cond_init(&cond,0);

src/sql/PostgreSqlDB.cc

@@ -0,0 +1,333 @@
+/* -------------------------------------------------------------------------- */
+/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems                */
+/*                                                                            */
+/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
+/* not use this file except in compliance with the License. You may obtain    */
+/* a copy of the License at                                                   */
+/*                                                                            */
+/* http://www.apache.org/licenses/LICENSE-2.0                                 */
+/*                                                                            */
+/* Unless required by applicable law or agreed to in writing, software        */
+/* distributed under the License is distributed on an "AS IS" BASIS,          */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+/* See the License for the specific language governing permissions and        */
+/* limitations under the License.                                             */
+/* -------------------------------------------------------------------------- */
+
+#include "NebulaUtil.h"
+#include "PostgreSqlDB.h"
+
+#include <libpq-fe.h>
+
+#include <iostream>
+
+/*********
+ * Doc: https://www.postgresql.org/docs/current/libpq.html
+ ********/
+
+#define PG_DEFAULT_PORT 5432
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+PostgreSqlDB::PostgreSqlDB(
+    const string& _server,
+    int           _port,
+    const string& _user,
+    const string& _password,
+    const string& _database,
+    int           _connections)
+{
+    PGconn* conn;
+
+    server = _server;
+    port   = _port;
+    user   = _user;
+
+    password = _password;
+    database = _database;
+
+    if ( port == 0 )
+    {
+        port = PG_DEFAULT_PORT;
+    }
+
+    max_connections = _connections;
+
+    // Set up connection parameters
+    string params = "host=" + _server
+                  + " port=" + to_string(port)
+                  + " user=" + user
+                  + " password=" + password
+                  + " dbname=" + database;
+
+    // Create connection pool
+    for (int i = 0; i < max_connections; i++)
+    {
+        conn = PQconnectdb(params.c_str());
+
+        if ( PQstatus(conn) == CONNECTION_BAD )
+        {
+            ostringstream oss;
+            oss << "Could not open connect to database server: "
+                << PQerrorMessage(conn);
+
+            throw runtime_error(oss.str());
+        }
+
+        db_connect.push(conn);
+    }
+
+    db_escape_connect = PQconnectdb(params.c_str());
+
+    if ( PQserverVersion(db_escape_connect) < 90500 )
+    {
+        std::string error = "PostgreSQL version error: must be 9.5 or higher.";
+
+        NebulaLog::log("ONE", Log::ERROR, error);
+
+        throw runtime_error(error);
+    }
+
+    features = {
+        {SqlFeature::MULTIPLE_VALUE, PQlibVersion() < 80200},
+        {SqlFeature::LIMIT, false},
+        {SqlFeature::FTS, false}
+    };
+
+    pthread_mutex_init(&mutex, 0);
+
+    pthread_cond_init(&cond, 0);
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+PostgreSqlDB::~PostgreSqlDB()
+{
+    while (!db_connect.empty())
+    {
+        PGconn* conn = db_connect.front();
+        db_connect.pop();
+
+        PQfinish(conn);
+    }
+
+    PQfinish(db_escape_connect);
+
+    pthread_mutex_destroy(&mutex);
+    pthread_cond_destroy(&cond);
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+char * PostgreSqlDB::escape_str(const string& str)
+{
+    char* buf = new char[str.size() * 2 + 1];
+    int err;
+
+    PQescapeStringConn(db_escape_connect, buf, str.c_str(), str.length(), &err);
+
+    if ( err != 0 )
+    {
+        delete[] buf;
+
+        return nullptr;
+    }
+
+    return buf;
+}
+
+/* -------------------------------------------------------------------------- */
+
+void PostgreSqlDB::free_str(char * str)
+{
+    delete[] str;
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+int PostgreSqlDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet)
+{
+    string str = preprocess_query(cmd);
+
+    const char* c_str = str.c_str();
+
+    Log::MessageType error_type = quiet ? Log::DDEBUG : Log::ERROR;
+
+
+    PGconn* conn  = get_db_connection();
+
+    PGresult* res = PQexec(conn, c_str);
+
+    if ( PQresultStatus(res) != PGRES_COMMAND_OK &&
+         PQresultStatus(res) != PGRES_TUPLES_OK )
+    {
+        const char* err_msg = PQerrorMessage(conn);
+
+        ostringstream oss;
+        oss << "SQL command was: " << c_str;
+        oss << ", error " << err_msg;
+
+        NebulaLog::log("ONE", error_type, oss);
+
+        PQclear(res);
+        free_db_connection(conn);
+
+        return SqlDB::SQL;
+    }
+
+    if ( obj == 0 )
+    {
+        // Free the result and db connection
+        PQclear(res);
+        free_db_connection(conn);
+
+        return SqlDB::SUCCESS;
+    }
+
+    int ec = SqlDB::SUCCESS;
+
+    // Get number of fields and rows of the result
+    int n_fields = PQnfields(res);
+    int n_rows   = PQntuples(res);
+
+    if ( obj->isCallBackSet() && PQresultStatus(res) == PGRES_TUPLES_OK )
+    {
+        char** names  = new char*[n_fields];
+        char** values = new char*[n_fields];
+
+        // Get column names
+        for (int i = 0; i < n_fields; i++)
+        {
+            names[i] = PQfname(res, i);
+        }
+
+        // For each row
+        for (int row = 0; row < n_rows; row++)
+        {
+            // get values in that row
+            for (int field = 0; field < n_fields; field++)
+            {
+                values[field] = PQgetvalue(res, row, field);
+            }
+
+            // and do a callback on them
+            if ( obj->do_callback(n_fields, values, names) != 0 )
+            {
+                ec = SqlDB::SQL;
+                break;
+            }
+        }
+
+        delete[] names;
+        delete[] values;
+    }
+
+    if ( obj->get_affected_rows() == 0 && n_rows )
+    {
+        obj->set_affected_rows(n_rows);
+    }
+
+    // Free the result and db connection
+    PQclear(res);
+    free_db_connection(conn);
+
+    return ec;
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+PGconn * PostgreSqlDB::get_db_connection()
+{
+    PGconn* conn;
+
+    pthread_mutex_lock(&mutex);
+
+    while (db_connect.empty() == true)
+    {
+        pthread_cond_wait(&cond, &mutex);
+    }
+
+    conn = db_connect.front();
+    db_connect.pop();
+
+    pthread_mutex_unlock(&mutex);
+
+    return conn;
+}
+
+/* -------------------------------------------------------------------------- */
+
+void PostgreSqlDB::free_db_connection(PGconn * db)
+{
+    pthread_mutex_lock(&mutex);
+
+    db_connect.push(db);
+
+    pthread_cond_signal(&cond);
+
+    pthread_mutex_unlock(&mutex);
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+static void replace_substring(std::string& cmd, const std::string& s1,
+        const std::string& s2)
+{
+    size_t pos = cmd.find(s1);
+
+    while (pos != std::string::npos)
+    {
+        cmd.replace(pos, s1.length(), s2);
+
+        pos = cmd.find(s1, pos + s2.length());
+    }
+}
+
+std::string PostgreSqlDB::preprocess_query(std::ostringstream& cmd)
+{
+    std::string query = cmd.str();
+    size_t pos;
+
+    // Both CREATE TABLE and REPLACE should be at the start
+    // so we don't change user data
+    if ((pos = query.find("CREATE TABLE")) == 0)
+    {
+        replace_substring(query, "MEDIUMTEXT", "TEXT");
+        replace_substring(query, "LONGTEXT", "TEXT");
+        replace_substring(query, "BIGINT UNSIGNED", "NUMERIC(20)");
+    }
+    else if ((pos = query.find("REPLACE")) == 0)
+    {
+        query.replace(0, 7, "INSERT");
+
+        size_t table_start = query.find("INTO ", 7) + 5;
+
+        size_t names_start = query.find("(", table_start) + 1;
+        size_t names_end   = query.find(")", names_start);
+
+        std::string db_names = query.substr(names_start, names_end - names_start);
+        std::string table    = query.substr(table_start, names_start - 2 - table_start);
+
+        std::vector<std::string> splits;
+        one_util::split(db_names, ',', splits);
+
+        query += " ON CONFLICT ON CONSTRAINT " + table + "_pkey DO UPDATE SET ";
+
+        const char* sep = "";
+
+        for (size_t i = 1; i < splits.size(); i++)
+        {
+            query += sep + splits[i] + " = EXCLUDED." + splits[i];
+            sep = ", ";
+        }
+    }
+
+    return query;
+}
+

src/sql/SConstruct

@@ -29,6 +29,16 @@ if env['sqlite']=='yes':
 if env['mysql']=='yes':
     source_files.append('MySqlDB.cc')
 
+if env['postgresql']=='yes':
+    source_files.append('PostgreSqlDB.cc')
+
+# Build library
+env.StaticLibrary(lib_name, source_files)
+
+lib_name = 'nebula_sql_const'
+
+source_files = ['OneDB.cc']
+
 # Build library
 env.StaticLibrary(lib_name, source_files)
 

src/sql/SqliteDB.cc

@@ -13,8 +13,6 @@
 /* See the License for the specific language governing permissions and        */
 /* limitations under the License.                                             */
 /* -------------------------------------------------------------------------- */
-
-
 #include "SqliteDB.h"
 
 using namespace std;
@@ -52,14 +50,21 @@ SqliteDB::SqliteDB(const string& db_name)
         throw runtime_error("Could not open database.");
     }
 
-    enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
+    int enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
 
-    if (enable_limit)
+    if (enable_limit == 1)
     {
-        NebulaLog::log("ONE",Log::INFO , "sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
+        NebulaLog::log("ONE",Log::INFO ,
+                "sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
     }
 
     sqlite3_extended_result_codes(db, 1);
+
+    features = {
+        {SqlFeature::MULTIPLE_VALUE, false},
+        {SqlFeature::LIMIT, enable_limit == 1},
+        {SqlFeature::FTS, false}
+    };
 }
 
 /* -------------------------------------------------------------------------- */
@@ -72,21 +77,6 @@ SqliteDB::~SqliteDB()
 }
 
 /* -------------------------------------------------------------------------- */
-
-bool SqliteDB::multiple_values_support()
-{
-    // Versions > 3.7.11 support multiple value inserts, but tests
-    // have ended in segfault. A transaction seems to perform better
-    //return SQLITE_VERSION_NUMBER >= 3007011;
-    return false;
-}
-
-/* -------------------------------------------------------------------------- */
-bool SqliteDB::limit_support()
-{
-    return enable_limit == 1;
-}
-
 /* -------------------------------------------------------------------------- */
 
 int SqliteDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet)

src/um/UserPool.cc

@@ -1428,8 +1428,7 @@ int UserPool::authorize(AuthRequest& ar)
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
-int UserPool::dump(string& oss, const string& where, const string& limit,
+int UserPool::dump(string& oss, const string& where, int sid, int eid, bool desc)
-        bool desc)
 {
     int     rc;
     string  def_quota_xml;
@@ -1453,9 +1452,9 @@ int UserPool::dump(string& oss, const string& where, const string& limit,
         cmd << " DESC";
     }
 
-    if ( !limit.empty() )
+    if ( eid != -1 )
     {
-        cmd << " LIMIT " << limit;
+        cmd << " " << db->limit_string(sid, eid);
     }
 
     oss.append("<USER_POOL>");

src/vm/VirtualMachine.cc

@@ -455,7 +455,7 @@ int VirtualMachine::bootstrap(SqlDB * db)
 
     oss_vm << one_db::vm_db_bootstrap;
 
-    if (db->fts_available())
+    if (db->supports(SqlDB::SqlFeature::FTS))
     {
         oss_vm << ", FULLTEXT ftidx(search_token))";
     }

src/vm/VirtualMachinePool.cc

@@ -230,7 +230,7 @@ int VirtualMachinePool::get_running(
        << " state = " << VirtualMachine::ACTIVE
        << " and ( lcm_state = " << VirtualMachine::RUNNING
        << " or lcm_state = " << VirtualMachine::UNKNOWN << " )"
-       << " ORDER BY last_poll ASC LIMIT " << vm_limit;
+       << " ORDER BY last_poll ASC " << db->limit_string(vm_limit);
 
     where = os.str();
 
@@ -262,8 +262,7 @@ int VirtualMachinePool::dump_acct(string& oss, const string&  where,
     ostringstream cmd;
 
     cmd << "SELECT " << History::table << ".body FROM " << History::table
-        << " INNER JOIN " << one_db::vm_table
+        << " INNER JOIN " << one_db::vm_table << " ON vid=oid";
-        << " WHERE vid=oid";
 
     if ( !where.empty() )
     {
@@ -302,8 +301,7 @@ int VirtualMachinePool::dump_showback(string& oss,
 
     cmd << "SELECT " << one_db::vm_showback_table << ".body FROM "
         << one_db::vm_showback_table
-        << " INNER JOIN " << one_db::vm_table
+        << " INNER JOIN " << one_db::vm_table << " ON vmid=oid";
-        << " WHERE vmid=oid";
 
     if ( !where.empty() )
     {
@@ -719,7 +717,7 @@ int VirtualMachinePool::calculate_showback(
 
     // Write to DB
 
-    if (db->multiple_values_support())
+    if (db->supports(SqlDB::SqlFeature::MULTIPLE_VALUE))
     {
         oss.str("");