shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-01-03 01:17:41 +03:00
Code Releases Wiki Activity

F #3600: Initial PostgreSQL Support

Browse Source 
co-authored-by: Igor Sivy <igorsivy@gmail.com>
co-authored-by: Pavel Czerny <pczerny@opennebula.io>
co-authored-by: Vlastimil Holer <vholer@opennebula.io>
(cherry picked from commit c52f62018c32281c6e418211f33f1bba46388e98)
This commit is contained in:
Ruben S. Montero 2020-04-13 17:32:21 +02:00
parent cbd3bda137
commit 9aa1041103
No known key found for this signature in database
GPG Key ID: A0CEA6FA880A1D87
61 changed files with 1263 additions and 387 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
SConstruct
View File

@ -153,6 +153,16 @@ if mysql == 'yes':
else:
main_env.Append(mysql='no')
# PostgreSql
postgresql = ARGUMENTS.get('postgresql', 'no')
if postgresql == 'yes':
main_env.Append(postgresql='yes')
main_env.Append(CPPPATH=['/usr/include/postgresql'])
main_env.Append(CPPFLAGS=["-DPOSTGRESQL_DB"])
main_env.Append(LIBS=['libpq'])
else:
main_env.Append(postgresql='no')
# Flag to compile with xmlrpc-c versions prior to 1.31 (September 2012)
new_xmlrpc = ARGUMENTS.get('new_xmlrpc', 'no')
if new_xmlrpc == 'yes':

7
include/ClusterPool.h
View File

@ -200,16 +200,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "CLUSTER_POOL", "body", Cluster::table, where,
limit, desc);
sid, eid, desc);
};
/**

9
include/DatastorePool.h
View File

@ -143,16 +143,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table, where,
limit, desc);
return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table,
where, sid, eid, desc);
};
/**

7
include/DocumentPool.h
View File

@ -94,16 +94,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "DOCUMENT_POOL", "body", Document::table, where,
limit, desc);
sid, eid, desc);
};
/**

6
include/GroupPool.h
View File

@ -160,13 +160,13 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
bool desc);
int dump(string& oss, const string& where, int sid, int eid, bool desc);
private:

8
include/HookPool.h
View File

@ -74,15 +74,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where, limit, desc);
return PoolSQL::dump(oss, "HOOK_POOL", "body", Hook::table, where,
sid, eid, desc);
};
/**

7
include/HostPool.h
View File

@ -212,16 +212,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "HOST_POOL", "body", one_db::host_table,
where, limit, desc);
where, sid, eid, desc);
};
/**

9
include/ImagePool.h
View File

@ -155,16 +155,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, limit,
desc);
return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, sid,
eid, desc);
}
/**

27
include/LogDB.h
View File

@ -208,19 +208,14 @@ public:
db->free_str(str);
}
bool multiple_values_support()
bool supports(SqlDB::SqlFeature ft)
{
return db->multiple_values_support();
return db->supports(ft);
}
bool limit_support()
std::string limit_string(int start_id, int end_id)
{
return db->limit_support();
}
bool fts_available()
{
return db->fts_available();
return db->limit_string(start_id, end_id);
}
// -------------------------------------------------------------------------
// Database methods
@ -422,19 +417,9 @@ public:
_logdb->free_str(str);
}
bool multiple_values_support()
bool supports(SqlDB::SqlFeature ft)
{
return _logdb->multiple_values_support();
}
bool limit_support()
{
return _logdb->limit_support();
}
bool fts_available()
{
return _logdb->fts_available();
return _logdb->supports(ft);
}
/**

11
include/MarketPlaceAppPool.h
View File

@ -144,16 +144,17 @@ public:
* the query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(std::string& oss, const std::string& where,
const std::string& limit, bool desc)
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body", MarketPlaceApp::table,
where, limit, desc);
return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body",
MarketPlaceApp::table, where, sid, eid, desc);
};
/** Update a particular MarketPlaceApp

11
include/MarketPlacePool.h
View File

@ -111,16 +111,17 @@ public:
* the query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(std::string& oss, const std::string& where,
const std::string& limit, bool desc)
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table, where,
limit, desc);
return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table,
where, sid, eid, desc);
};
/**

84
include/MySqlDB.h
View File

@ -30,9 +30,6 @@
#include "SqlDB.h"
#include "ObjectSQL.h"
using namespace std;
#ifdef MYSQL_DB
#include <mysql.h>
@ -44,12 +41,12 @@ class MySqlDB : public SqlDB
{
public:
MySqlDB(const string& _server,
MySqlDB(const std::string& _server,
int _port,
const string& _user,
const string& _password,
const string& _database,
const string& _encoding,
const std::string& _user,
const std::string& _password,
const std::string& _database,
const std::string& _encoding,
int _connections);
~MySqlDB();
@ -61,7 +58,7 @@ public:
* @param str the string to be escaped
* @return a valid SQL string or NULL in case of failure
*/
char * escape_str(const string& str);
char * escape_str(const std::string& str);
/**
* Frees a previously scaped string
@ -72,36 +69,6 @@ public:
delete[] str;
}
/**
* Returns true if the syntax INSERT VALUES (data), (data), (data)
* is supported
*
* @return true if supported
*/
bool multiple_values_support()
{
return true;
}
/**
* Returns true if this Database can use LIMIT in queries with DELETE
* and UPDATE
*
* @return true if supported
*/
bool limit_support()
{
return true;
}
/**
* Return true if the backend allows FTS index
*/
bool fts_available()
{
return _fts_available;
}
protected:
/**
* Wraps the mysql_query function call
@ -109,7 +76,7 @@ protected:
* @param obj Callbackable obj to call if the query succeeds
* @return 0 on success
*/
int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet);
int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override;
private:
@ -137,19 +104,17 @@ private:
/**
* MySQL Connection parameters
*/
string server;
std::string server;
int port;
string user;
std::string user;
string password;
std::string password;
string database;
std::string database;
string encoding;
bool _fts_available;
std::string encoding;
/**
* Fine-grain mutex for DB access (pool of DB connections)
@ -177,12 +142,12 @@ class MySqlDB : public SqlDB
{
public:
MySqlDB(const string& _server,
MySqlDB(const std::string& _server,
int _port,
const string& _user,
const string& _password,
const string& _database,
const string& _encoding,
const std::string& _user,
const std::string& _password,
const std::string& _database,
const std::string& _encoding,
int _connections)
{
throw runtime_error("Aborting oned, MySQL support not compiled!");
@ -190,19 +155,14 @@ public:
~MySqlDB(){};
char * escape_str(const std::string& str) override {return nullptr;};
char * escape_str(const string& str){return 0;};
void free_str(char * str){};
bool multiple_values_support(){return true;};
bool limit_support(){return true;};
bool fts_available(){return false;};
void free_str(char * str) override {};
protected:
int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet){return -1;};
int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override {
return -1;
};
};
#endif

32
include/PoolSQL.h
View File

@ -169,7 +169,7 @@ public:
*/
int dump(string& oss, const string& where, bool desc)
{
return dump(oss, where, "", desc);
return dump(oss, where, 0, -1, desc);
}
/**
@ -177,30 +177,33 @@ public:
* to the query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
virtual int dump(string& oss, const string& where,
const string& limit, bool desc) = 0;
virtual int dump(string& oss, const string& where, int sid, int eid,
bool desc) = 0;
/**
* Dumps the pool in extended XML format
* A filter and limit can be also added to the query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
virtual int dump_extended(string& oss,
const string& where,
const string& limit,
int sid,
int eid,
bool desc)
{
return dump(oss, where, limit, desc);
return dump(oss, where, sid, eid, desc);
}
// -------------------------------------------------------------------------
@ -276,12 +279,13 @@ public:
}
/**
* Return true if FTS is available.
* Return true if feature is supported
*/
bool is_fts_available()
bool supports(SqlDB::SqlFeature ft)
{
return db->fts_available();
return db->supports(ft);
}
protected:
/**
@ -316,7 +320,8 @@ protected:
* @param elem_name Name of the root xml pool name
* @param table Pool table name
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param start_id first element used for pagination
* @param end_id last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
@ -326,7 +331,8 @@ protected:
const string& column,
const char * table,
const string& where,
const string& limit,
int start_id,
int end_id,
bool desc);
/**
@ -346,7 +352,7 @@ protected:
const string& where,
bool desc)
{
return dump(oss, elem_name, "body", table, where, "", desc);
return dump(oss, elem_name, "body", table, where, 0, -1, desc);
}
/**

196
include/PostgreSqlDB.h Normal file
View File

@ -0,0 +1,196 @@
/* -------------------------------------------------------------------------- */
/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems */
/* */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may */
/* not use this file except in compliance with the License. You may obtain */
/* a copy of the License at */
/* */
/* http://www.apache.org/licenses/LICENSE-2.0 */
/* */
/* Unless required by applicable law or agreed to in writing, software */
/* distributed under the License is distributed on an "AS IS" BASIS, */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
/* See the License for the specific language governing permissions and */
/* limitations under the License. */
/* -------------------------------------------------------------------------- */
#ifndef POSTGRESQL_DB_H_
#define POSTGRESQL_DB_H_
#include <string>
#include <stdexcept>
#include <queue>
#include "NebulaLog.h"
#include "SqlDB.h"
#include "ObjectSQL.h"
#ifdef POSTGRESQL_DB
#include <libpq-fe.h>
/**
* PostgreSqlDB class. Provides a wrapper to the PostgreSQL database interface.
*/
class PostgreSqlDB : public SqlDB
{
public:
PostgreSqlDB(
const string& _server,
int _port,
const string& _user,
const string& _password,
const string& _database,
int _connections);
~PostgreSqlDB();
/**
* This function returns a legal SQL string that can be used in an SQL
* statement.
* @param str the string to be escaped
* @return a valid SQL string or NULL in case of failure
*/
char * escape_str(const string& str) override;
/**
* Frees a previously scaped string
* @param str pointer to the str
*/
void free_str(char * str) override;
/**
* @param sid the offset
* @param eid the rowcount
* @return string with compatible LIMIT clause syntax
* LIMIT row_count OFFSET offset
*
* +---+---+---+---+---+---+---+---+--
* | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |...
* +---+---+---+---+---+---+---+---+--
* | |
* /-------------------/
* LIMIT 5 OFFSET 3
*/
std::string limit_string(int sid, int eid) override
{
std::ostringstream oss;
oss << "LIMIT " << eid << " OFFSET " << sid;
return oss.str();
}
std::string limit_string(int sid) override
{
std::ostringstream oss;
oss << "LIMIT " << sid << " OFFSET 0";
return oss.str();
}
protected:
int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override;
private:
/**
* Number of concurrent DB connections.
*/
int max_connections;
/**
* Connection pool
*/
queue<PGconn *> db_connect;
/**
* DB connection to escape strings
*/
PGconn * db_escape_connect;
/**
* Connection parameters
*/
string server;
int port;
string user;
string password;
string database;
/**
* Fine-grain mutex for DB access (pool of DB connections)
*/
pthread_mutex_t mutex;
/**
* Conditional variable to wake-up waiting threads.
*/
pthread_cond_t cond;
/**
* Gets a free DB connection from the pool.
*/
PGconn * get_db_connection();
/**
* Returns the connection to the pool.
*/
void free_db_connection(PGconn * db);
/**
* Preprocesses the query to be compatible with PostgreSQL syntax
*
* Any change to this method should be reflected in BackEndPostgreSQL class
* in src/onedb/onedb_backend.rb
*
* This method alters to queries:
* - CREATE TABLE to adjust type names
* . MEDIUMTEXT -> TEXT
* . LONGTEXT -> TEXT
* . BIGINT UNSIGNED -> NUMERIC
*
* - REPLACE INTO into PostgreSQL INSERT INTO query with ON CONFLICT
* clause. For example:
* REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0)
* changes to:
* INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0)
* ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid
***************************************************************************
* Any change to this method should be reflected in BackEndPostgreSQL class
* in src/onedb/onedb_backend.rb
***************************************************************************
*/
static std::string preprocess_query(std::ostringstream& cmd);
};
#else
// Class stub
class PostgreSqlDB : public SqlDB
{
public:
PostgreSqlDB(
const string& _server,
int _port,
const string& _user,
const string& _password,
const string& _database,
int _connections)
{
throw runtime_error("Aborting oned, PostgreSQL support not compiled!");
}
~PostgreSqlDB(){}
char * escape_str(const string& str) override {return 0;};
void free_str(char * str) override {};
std::string limit_string(int sid, int eid) override {return "";}
protected:
int exec_ext(std::ostringstream& c, Callbackable *o, bool q) override {
return -1;
};
};
#endif
#endif /*POSTGRESQL_DB_H*/

7
include/SecurityGroupPool.h
View File

@ -106,16 +106,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "SECURITY_GROUP_POOL", "body", SecurityGroup::table,
where, limit, desc);
where, sid, eid, desc);
};
/**

73
include/SqlDB.h
View File

@ -18,6 +18,7 @@
#define SQL_DB_H_
#include <sstream>
#include <map>
#include "Callbackable.h"
/**
@ -40,6 +41,13 @@ public:
SQL_DUP_KEY = -201
};
enum class SqlFeature
{
MULTIPLE_VALUE, // syntax INSERT VALUES (data), (data), (data)
LIMIT, // LIMIT in queries with DELETE and UPDATE
FTS // Full Text Search
};
/* ---------------------------------------------------------------------- */
/* Database Operations */
/* ---------------------------------------------------------------------- */
@ -99,27 +107,18 @@ public:
*/
virtual void free_str(char * str) = 0;
/**
* Returns true if the syntax INSERT VALUES (data), (data), (data)
* is supported
*
* @return true if supported
*/
virtual bool multiple_values_support() = 0;
/**
* Returns true if this Database can use LIMIT in queries with DELETE
* and UPDATE
*
* @return true if supported
*/
virtual bool limit_support() = 0;
bool supports(SqlFeature ft)
{
auto it = features.find(ft);
/**
* Return true if the backend allows FTS index
*/
virtual bool fts_available() = 0;
if ( it == features.end() )
{
return false;
}
return it->second;
}
/**
* @return pointer to a non-federated version of this database
@ -129,6 +128,35 @@ public:
return this;
}
/**
* @return string with compatible LIMIT clause syntax
* LIMIT [offset], row_count
*
* +---+---+---+---+---+---+---+---+--
* | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 |...
* +---+---+---+---+---+---+---+---+--
* | |
* /-------------------/
* LIMIT 3, 5
*/
virtual std::string limit_string(int sid, int eid)
{
std::ostringstream oss;
oss << "LIMIT " << sid << "," << eid;
return oss.str();
}
virtual std::string limit_string(int sid)
{
std::ostringstream oss;
oss << "LIMIT " << sid;
return oss.str();
}
protected:
/**
* Performs a DB transaction
@ -154,6 +182,15 @@ protected:
* @return SqlError enum
*/
virtual int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) = 0;
/**
* Feature set
*/
std::map<SqlFeature, bool> features = {
{SqlFeature::MULTIPLE_VALUE, false},
{SqlFeature::LIMIT, false},
{SqlFeature::FTS, false}
};
};
#endif /*SQL_DB_H_*/

33
include/SqliteDB.h
View File

@ -63,26 +63,6 @@ public:
*/
void free_str(char * str) override;
/**
* Returns true if the syntax INSERT VALUES (data), (data), (data)
* is supported
*
* @return true if supported
*/
bool multiple_values_support() override;
/**
* Returns true if this Database can use LIMIT in queries with DELETE
* and UPDATE
*
* @return true if supported
*/
bool limit_support() override;
bool fts_available() override
{
return false;
}
protected:
/**
* Wraps the sqlite3_exec function call, and locks the DB mutex.
@ -105,11 +85,6 @@ private:
*/
sqlite3 * db;
/**
* LIMIT for DELETE and UPDATE queries is enabled
*/
int enable_limit;
/**
* Function to lock the DB
*/
@ -141,13 +116,9 @@ public:
char * escape_str(const string& str) override { return 0; }
void free_str(char * str) override {}
void free_str(char * str) override {};
bool multiple_values_support() override { return true; }
bool limit_support() override { return true; }
bool fts_available() override { return false; }
std::string get_limit_string(const std::string& str) override { return str; }
protected:
int exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet) override

6
include/UserPool.h
View File

@ -212,13 +212,13 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
bool desc);
int dump(string& oss, const string& where, int sid, int eid, bool desc);
/**
* Name for the OpenNebula core authentication process

11
include/VMGroupPool.h
View File

@ -99,16 +99,17 @@ public:
* Dumps the VMGroup pool in XML format. A filter can be added to the query
* @param os the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(std::string& os, const std::string& where,
const std::string& limit, bool desc)
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(os, "VM_GROUP_POOL", "body", VMGroup::table, where, limit,
desc);
return PoolSQL::dump(oss, "VM_GROUP_POOL", "body", VMGroup::table, where,
sid, eid, desc);
};
/**

9
include/VMTemplatePool.h
View File

@ -84,16 +84,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table, where,
limit, desc);
return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table,
where, sid, eid, desc);
};
/**

9
include/VNTemplatePool.h
View File

@ -84,16 +84,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table, where,
limit, desc);
return PoolSQL::dump(oss, "VNTEMPLATE_POOL", "body", VNTemplate::table,
where, sid, eid, desc);
};
/**

8
include/VdcPool.h
View File

@ -90,15 +90,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, limit, desc);
return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, sid,
eid, desc);
};
/**

11
include/VirtualMachinePool.h
View File

@ -250,16 +250,17 @@ public:
* pool
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VM_POOL", "short_body", one_db::vm_table, where,
limit, desc);
sid, eid, desc);
};
/**
@ -274,11 +275,11 @@ public:
*
* @return 0 on success
*/
int dump_extended(string& oss, const string& where, const string& limit,
int dump_extended(string& oss, const string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VM_POOL", "body", one_db::vm_table, where,
limit, desc);
sid, eid, desc);
};
/**

9
include/VirtualNetworkPool.h
View File

@ -157,16 +157,17 @@ public:
* to the query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table, where,
limit, desc);
return PoolSQL::dump(oss, "VNET_POOL", "body", VirtualNetwork::table,
where, sid, eid, desc);
}
/**

9
include/VirtualRouterPool.h
View File

@ -92,16 +92,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table, where,
limit, desc);
return PoolSQL::dump(oss, "VROUTER_POOL", "body", VirtualRouter::table,
where, sid, eid, desc);
};
/**

8
include/ZonePool.h
View File

@ -91,15 +91,17 @@ public:
* query
* @param oss the output stream to dump the pool contents
* @param where filter for the objects, defaults to all
* @param limit parameters used for pagination
* @param sid first element used for pagination
* @param eid last element used for pagination, -1 to disable
* @param desc descending order of pool elements
*
* @return 0 on success
*/
int dump(string& oss, const string& where, const string& limit,
int dump(std::string& oss, const std::string& where, int sid, int eid,
bool desc)
{
return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where, limit, desc);
return PoolSQL::dump(oss, "ZONE_POOL", "body", Zone::table, where,
sid, eid, desc);
};
/**

2
share/etc/oned.conf
View File

@ -84,7 +84,7 @@ DB = [ BACKEND = "sqlite" ]
# USER = "oneadmin",
# PASSWD = "oneadmin",
# DB_NAME = "opennebula",
# CONNECTIONS = 50 ]
# CONNECTIONS = 25 ]
VNC_PORTS = [
START = 5900,

2
share/install_gems/CentOS7/Gemfile.lock
View File

@ -89,6 +89,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.1.4)
polyglot (0.3.5)
public_suffix (2.0.5)
rack (1.6.13)
@ -163,6 +164,7 @@ DEPENDENCIES
nokogiri (< 1.7)
ox
parse-cron
pg (< 1.2.0)
public_suffix (< 3.0.0)
rack (< 2.0.0)
rbvmomi (~> 2.2.0)

2
share/install_gems/CentOS8/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -168,6 +169,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Debian10/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -168,6 +169,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Debian9/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -167,6 +168,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Gemfile
View File

@ -28,8 +28,10 @@ end
if RUBY_VERSION < '2.2.0'
gem 'rack', '< 2.0.0' # sunstone, cloud, oneflow
gem 'minitest', '< 5.12.0' # packethost
gem 'pg', '< 1.2.0' # onedb
else
gem 'rack' # sunstone, cloud, oneflow
gem 'pg' # onedb
end
if RUBY_VERSION >= '2.4.0'

2
share/install_gems/Ubuntu1604/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -167,6 +168,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Ubuntu1804/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -168,6 +169,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Ubuntu1904/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -168,6 +169,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

2
share/install_gems/Ubuntu1910/Gemfile.lock
View File

@ -91,6 +91,7 @@ GEM
optimist (3.0.0)
ox (2.13.2)
parse-cron (0.1.4)
pg (1.2.2)
polyglot (0.3.5)
public_suffix (4.0.3)
rack (2.2.2)
@ -168,6 +169,7 @@ DEPENDENCIES
nokogiri
ox
parse-cron
pg
public_suffix
rack
rbvmomi (~> 2.2.0)

6
share/install_gems/install_gems
View File

@ -27,8 +27,10 @@ end
if !defined?(RUBY_VERSION) || RUBY_VERSION < '2.2.0'
RACK = 'rack --version "< 2.0.0"'
PG = 'pg --version "< 1.2.0"'
else
RACK = 'rack'
PG = 'pg'
end
TREETOP = 'treetop --version ">= 1.6.3"'
@ -47,7 +49,7 @@ GROUPS={
:ec2_hybrid => 'aws-sdk --version "~> 2.5"',
:oca => 'ox',
:market => 'aws-sdk',
:onedb => "mysql2",
:onedb => ['mysql2', PG],
:hooks => %w[zeromq ffi-rzmq],
:serversync => "augeas"
}
@ -66,6 +68,7 @@ DISTRIBUTIONS={
:dependencies => {
SQLITE => ['gcc', 'libsqlite3-dev'],
'mysql2' => ['gcc', 'libssl-dev', ['default-libmysqlclient-dev', 'libmysqlclient-dev']],
PG => ['gcc', 'postgresql-server-dev-all'],
'curb' => ['gcc', 'libcurl4-openssl-dev'],
$nokogiri => %w{gcc rake libxml2-dev libxslt1-dev patch},
'xmlparser' => ['gcc', 'libexpat1-dev'],
@ -87,6 +90,7 @@ DISTRIBUTIONS={
:dependencies => {
SQLITE => ['gcc', 'sqlite-devel'],
'mysql2' => ['gcc', 'mysql-devel', 'openssl-devel'],
PG => ['gcc', 'postgresql-devel'],
'curb' => ['gcc', 'curl-devel'],
$nokogiri => %w{gcc rubygem-rake libxml2-devel libxslt-devel patch},
'xmlparser' => ['gcc', 'expat-devel'],

6
src/acl/AclManager.cc
View File

@ -25,11 +25,11 @@
const char * AclManager::table = "acl";
const char * AclManager::db_names = "oid, user, resource, rights, zone";
const char * AclManager::db_names = "oid, userset, resource, rights, zone";
const char * AclManager::db_bootstrap = "CREATE TABLE IF NOT EXISTS "
"acl (oid INT PRIMARY KEY, user BIGINT, resource BIGINT, "
"rights BIGINT, zone BIGINT, UNIQUE(user, resource, rights, zone))";
"acl (oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, "
"rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone))";
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

7
src/group/GroupPool.cc
View File

@ -218,8 +218,7 @@ int GroupPool::drop(PoolObjectSQL * objsql, string& error_msg)
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
int GroupPool::dump(string& oss, const string& where,
const string& limit, bool desc)
int GroupPool::dump(string& oss, const string& where, int sid, int eid, bool desc)
{
int rc;
string def_quota_xml;
@ -243,9 +242,9 @@ int GroupPool::dump(string& oss, const string& where,
cmd << " DESC";
}
if ( !limit.empty() )
if ( eid != -1 )
{
cmd << " LIMIT " << limit;
cmd << " " << db->limit_string(sid, eid);
}
oss.append("<GROUP_POOL>");

2
src/host/HostPool.cc
View File

@ -171,7 +171,7 @@ int HostPool::dump_monitoring(
cmd << "SELECT " << one_db::host_monitor_table << ".body FROM "
<< one_db::host_monitor_table << " INNER JOIN " << one_db::host_table
<< " WHERE hid = oid";
<< " ON hid = oid";
if ( !where.empty() )
{

3
src/im/InformationManager.cc
View File

@ -72,7 +72,8 @@ int InformationManager::start()
}
string xml_hosts;
hpool->dump(xml_hosts, "", "", false);
hpool->dump(xml_hosts, "", 0, -1, false);
Message<OpenNebulaMessages> msg;

9
src/monitor/etc/monitord.conf
View File

@ -18,6 +18,11 @@
# VM_MONITORING_EXPIRATION_TIME: Time, in seconds, to expire monitoring
# information. Use 0 to disable VM monitoring recording.
#
# DB: Database configuration attributes. Monitord will use the DB configuration
# in oned.conf. The following attributes can be tuned:
# - CONNECTIONS: Number of DB connections. The DB needs to be configure to
# support oned + monitord connections.
#
# LOG: Configuration for the logging system
# - system: defines the logging system:
# file to log in the sched.log file
@ -39,6 +44,10 @@
#HOST_MONITORING_EXPIRATION_TIME = 43200
#VM_MONITORING_EXPIRATION_TIME = 43200
DB = [
CONNECTIONS = 15
]
LOG = [
system = "FILE",
debug_level = 3

18
src/monitor/src/monitor/Monitor.cc
View File

@ -22,6 +22,7 @@
#include "StreamManager.h"
#include "SqliteDB.h"
#include "MySqlDB.h"
#include "PostgreSqlDB.h"
#include <fcntl.h>
#include <unistd.h>
@ -35,6 +36,7 @@ void Monitor::start()
// Configuration File
// -------------------------------------------------------------------------
OpenNebulaTemplate oned_config(get_defaults_location(), oned_filename);
if (oned_config.load_configuration() != 0)
{
throw runtime_error("Error reading oned configuration file " +
@ -92,6 +94,7 @@ void Monitor::start()
// Database
// -------------------------------------------------------------------------
const VectorAttribute * _db = oned_config.get("DB");
const VectorAttribute * _db_m = config->get("DB");
std::string db_backend = _db->vector_value("BACKEND");
@ -115,11 +118,24 @@ void Monitor::start()
_db->vector_value<string>("PASSWD", passwd, "oneadmin");
_db->vector_value<string>("DB_NAME", db_name, "opennebula");
_db->vector_value<string>("ENCODING", encoding, "");
_db->vector_value("CONNECTIONS", connections, 50);
_db_m->vector_value("CONNECTIONS", connections, 15);
if ( db_backend == "postgresql" )
{
sqlDB.reset(new PostgreSqlDB(server, port, user, passwd, db_name,
connections));
}
else if ( db_backend == "mysql" )
{
sqlDB.reset(new MySqlDB(server, port, user, passwd, db_name,
encoding, connections));
}
else
{
throw runtime_error("Unknown DB backend " + db_backend);
}
}
// -------------------------------------------------------------------------
// Pools

4
src/monitor/src/monitor/MonitorConfigTemplate.cc
View File

@ -25,6 +25,7 @@ void MonitorConfigTemplate::set_conf_default()
/*
HOST_MONITORING_EXPIRATION_TIME
VM_MONITORING_EXPIRATION_TIME
DB
LOG
UDP_LISTENER
PROBES_PERIOD
@ -36,6 +37,9 @@ void MonitorConfigTemplate::set_conf_default()
set_conf_single("HOST_MONITORING_EXPIRATION_TIME", "43200");
set_conf_single("VM_MONITORING_EXPIRATION_TIME", "43200");
va = new VectorAttribute("DB", {{"CONNECTIONS", "15"}});
conf_default.insert(make_pair(va->name(), va));
va = new VectorAttribute("LOG", {{"SYSTEM", "FILE"}, {"DEBUG_LEVEL", "3"}});
conf_default.insert(make_pair(va->name(), va));

14
src/nebula/Nebula.cc
View File

@ -19,6 +19,7 @@
#include "VirtualMachine.h"
#include "SqliteDB.h"
#include "MySqlDB.h"
#include "PostgreSqlDB.h"
#include "Client.h"
#include "LogDB.h"
#include "SystemDB.h"
@ -388,7 +389,7 @@ void Nebula::start(bool bootstrap_only)
if (_db->vector_value("CONNECTIONS", connections) == -1)
{
connections = 50;
connections = 25;
}
if (_db->vector_value("ENCODING", encoding) == -1)
@ -401,11 +402,20 @@ void Nebula::start(bool bootstrap_only)
{
db_backend = new SqliteDB(var_location + "one.db");
}
else
else if ( db_backend_type == "mysql" )
{
db_backend = new MySqlDB(server, port, user, passwd, db_name,
encoding, connections);
}
else if ( db_backend_type == "postgresql" )
{
db_backend = new PostgreSqlDB(server, port, user, passwd, db_name,
connections);
}
else
{
throw runtime_error("DB BACKEND must be one of sqlite, mysql or postgresql.");
}
// ---------------------------------------------------------------------
// Check Database Versions

2
src/nebula/SystemDB.cc
View File

@ -102,7 +102,7 @@ int SystemDB::local_bootstrap()
oss << "INSERT INTO " << local_ver_table << " (" << local_ver_names << ") "
<< "VALUES (0, '" << Nebula::local_db_version() << "', " << time(0)
<< ", '" << Nebula::version() << " daemon bootstrap', "
<< Nebula::instance().is_federation_slave() << ")";
<< "'" << Nebula::instance().is_federation_slave() << "')";
rc += db->exec_local_wr(oss);

2
src/onedb/database_schema.rb
View File

@ -127,6 +127,8 @@ class OneDBBacKEnd
"state INTEGER, lcm_state INTEGER, " <<
"owner_u INTEGER, group_u INTEGER, other_u INTEGER, short_body MEDIUMTEXT, " <<
"search_token MEDIUMTEXT",
acl: "oid INT PRIMARY KEY, userset BIGINT, resource BIGINT, " <<
"rights BIGINT, zone BIGINT, UNIQUE(userset, resource, rights, zone)"
}
}

45
src/onedb/onedb
View File

@ -101,16 +101,26 @@ SQLITE={
}
###############################################################################
# MySQL options
# MySQL and PostgreSQL options
###############################################################################
TYPE={
:name => "type",
:short => "-t type",
:large => "--type type",
:format => String,
:description => "Backend type.",
:proc => lambda { |o, options|
options[:backend] = o.to_sym
}
}
SERVER={
:name => 'server',
:short => '-S host',
:large => '--server host',
:format => String,
:description => 'MySQL server hostname or IP. Defaults to localhost',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL server hostname or IP. Defaults to localhost",
:proc => lambda { |o, options|
options[:server] = o
}
}
@ -120,9 +130,8 @@ PORT={
:short => '-P port',
:large => '--port port',
:format => String,
:description => 'MySQL server port. Defaults to 3306',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL server port. Defaults to 3306 for MySQL and 5432 for PostgreSQL",
:proc => lambda { |o, options|
options[:port] = o
}
}
@ -132,9 +141,8 @@ USERNAME={
:short => '-u user',
:large => '--username user',
:format => String,
:description => 'MySQL username',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL username",
:proc => lambda { |o, options|
options[:user] = o
}
}
@ -144,9 +152,8 @@ PASSWORD={
:short => '-p pass',
:large => '--password pass',
:format => String,
:description => 'MySQL password. Leave unset to be prompted for it',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL password. Leave unset to be prompted for it",
:proc => lambda { |o, options|
options[:passwd] = o
}
}
@ -156,9 +163,8 @@ DBNAME={
:short => '-d dbname',
:large => '--dbname dbname',
:format => String,
:description => 'MySQL DB name for OpenNebula',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL DB name for OpenNebula",
:proc => lambda { |o, options|
options[:db_name] = o
}
}
@ -167,9 +173,8 @@ ENCODING={
:name => 'encoding',
:large => '--encoding charset',
:format => String,
:description => 'MySQL encoding to use for the connection',
:proc => lambda {|o, options|
options[:backend] = :mysql
:description => "MySQL or PostgreSQL encoding to use for the connection",
:proc => lambda { |o, options|
options[:encoding] = o
}
}
@ -332,7 +337,7 @@ CommandParser::CmdParser.new(ARGV) do
# Global options
###########################################################################
set :option, CommandParser::OPTIONS
set :option, [SQLITE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING]
set :option, [SQLITE, TYPE, SERVER, PORT, USERNAME, PASSWORD, DBNAME, ENCODING]
###########################################################################
# Backup

29
src/onedb/onedb.rb
View File

@ -23,6 +23,11 @@ class OneDB
attr_accessor :backend
def initialize(ops)
# Set MySQL backend as default if any connection option is provided and --type is not
if ops[:backend].nil? and (!ops[:server].nil? || !ops[:port].nil? || !ops[:user].nil? || !ops[:password].nil? || !ops[:db_name].nil? || !ops[:encoding].nil?)
ops[:backend] = :mysql
end
if ops[:backend] == :sqlite
begin
require 'sqlite3'
@ -55,8 +60,30 @@ class OneDB
:db_name => ops[:db_name],
:encoding=> ops[:encoding]
)
elsif ops[:backend] == :postgresql
begin
require 'pg'
rescue
STDERR.puts "Ruby gem pg is needed for this operation:"
STDERR.puts " $ sudo gem install pg"
exit -1
end
passwd = ops[:passwd]
if !passwd
passwd = get_password("PostgreSQL Password: ")
end
@backend = BackEndPostgreSQL.new(
:server => ops[:server],
:port => ops[:port],
:user => ops[:user],
:passwd => passwd,
:db_name => ops[:db_name],
:encoding=> ops[:encoding]
)
else
raise "You need to specify the SQLite or MySQL connection options."
raise "You need to specify the SQLite, MySQL or PostgreSQL connection options."
end
end

308
src/onedb/onedb_backend.rb
View File

@ -120,22 +120,13 @@ class OneDBBacKEnd
comment = "Database migrated from #{version} to #{db_version}"+
" (#{one_version}) by onedb command."
max_oid = nil
@db.fetch("SELECT MAX(oid) FROM db_versioning") do |row|
max_oid = row[:"MAX(oid)"].to_i
end
max_oid = @db[:db_versioning].max(:oid)
max_oid = 0 if max_oid.nil?
query =
@db.run(
"INSERT INTO db_versioning (oid, version, timestamp, comment) "<<
"VALUES (" <<
"#{max_oid+1}, " <<
"'#{db_version}', " <<
"#{Time.new.to_i}, " <<
"'#{comment}')"
)
@db[:db_versioning].insert(
oid: max_oid + 1,
version: db_version,
timestamp: Time.new.to_i,
comment: comment)
puts comment
end
@ -144,29 +135,16 @@ class OneDBBacKEnd
comment = "Database migrated from #{version} to #{db_version}"+
" (#{one_version}) by onedb command."
max_oid = nil
@db.fetch("SELECT MAX(oid) FROM local_db_versioning") do |row|
max_oid = row[:"MAX(oid)"].to_i
end
max_oid = @db[:local_db_versioning].max(:oid)
max_oid = 0 if max_oid.nil?
is_slave = @db[:local_db_versioning].select(:is_slave).where_single_value(oid: max_oid)
is_slave = 0
@db.fetch("SELECT is_slave FROM local_db_versioning "<<
"WHERE oid=#{max_oid}") do |row|
is_slave = row[:is_slave] ? 1 : 0
end
@db.run(
"INSERT INTO local_db_versioning (oid, version, timestamp, comment, is_slave) "<<
"VALUES (" <<
"#{max_oid+1}, " <<
"'#{db_version}', " <<
"#{Time.new.to_i}, " <<
"'#{comment}'," <<
"#{is_slave})"
)
@db[:local_db_versioning].insert(
oid: max_oid + 1,
version: db_version,
timestamp: Time.new.to_i,
comment: comment,
is_slave: is_slave)
puts comment
end
@ -638,3 +616,261 @@ class BackEndSQLite < OneDBBacKEnd
end
end
end
class BackEndPostgreSQL < OneDBBacKEnd
def initialize(opts={})
@server = opts[:server]
@port = opts[:port]
@user = opts[:user]
@passwd = opts[:passwd]
@db_name = opts[:db_name]
@encoding= opts[:encoding]
# Check for errors:
error = false
(error = true; missing = "USER" ) if @user == nil
(error = true; missing = "DBNAME") if @db_name == nil
if error
raise "PostgreSQL option #{missing} is needed"
end
# Check for defaults:
@server = "localhost" if @server.nil?
@port = 5432 if @port.nil?
encoding if @encoding.nil?
# Clean leading and trailing quotes, if any
@server = @server [1..-2] if @server [0] == ?"
@port = @port [1..-2] if @port [0] == ?"
@user = @user [1..-2] if @user [0] == ?"
@passwd = @passwd [1..-2] if @passwd [0] == ?"
@db_name = @db_name[1..-2] if @db_name[0] == ?"
end
def bck_file(federated = false)
t = Time.now
bck_name = "#{VAR_LOCATION}/postgresql_#{@server}_#{@db_name}_"
bck_name << "federated_" if federated
bck_name << "#{t.year}-#{t.month}-#{t.day}_"
bck_name << "#{t.hour}:#{t.min}:#{t.sec}.sql"
bck_name
end
def backup(bck_file, federated = false)
cmd = "PGPASSWORD=\"#{@passwd}\" pg_dump -U #{@user} -h #{@server} -p #{@port} -b #{@db_name} -Fp -f #{bck_file} "
if federated
connect_db
@db.create_table!(:logdb_tmp, as: @db[:logdb].where{fed_index != -1})
FEDERATED_TABLES.each do |table|
cmd << " -t " << table
end
cmd << " -t logdb_tmp"
rc = system(cmd)
if !rc
raise "Unknown error running '#{cmd}'"
end
@db.drop_table(:logdb_tmp)
File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("logdb_tmp","logdb"))
else
rc = system(cmd)
if !rc
raise "Unknown error running '#{cmd}'"
end
end
File.write("#{bck_file}",File.open("#{bck_file}",&:read).gsub("COMMENT ON","-- COMMENT ON"))
puts "PostgreSQL dump stored in #{bck_file}"
puts "Use 'onedb restore' to restore the DB"
puts
end
def get_db_encoding
db_enc = ''
@db.fetch("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = \'#{@db_name}\'") do |row|
db_enc = row[:pg_encoding_to_char]
db_enc ||= row[:PG_ENCODING_TO_CHAR]
end
db_enc
end
def encoding
@encoding = ''
connect_db
@encoding = get_db_encoding
end
def get_table_enconding(table = nil)
encoding = get_db_encoding
table_to_nk(encoding)
end
def table_to_nk(encoding)
case encoding
when 'UTF8'
'UTF-8'
when 'ISO_8859_5'
'ISO-8859-5'
when 'ISO_8859_6'
'ISO-8859-6'
when 'ISO_8859_7'
'ISO-8859-7'
when 'ISO_8859_8'
'ISO-8859-8'
when 'LATIN1'
'ISO-8859-1'
when 'LATIN2'
'ISO-8859-2'
when 'LATIN3'
'ISO-8859-3'
when 'LATIN4'
'ISO-8859-4'
when 'LATIN5'
'ISO-8859-9'
when 'SJIS'
'SHIFT-JIS'
when 'EUC_JP'
'EUC-JP'
when 'SQL_ASCII'
'ASCII'
else
NOKOGIRI_ENCODING
end
end
def restore(bck_file, force=nil, federated=false)
if !federated && !force && db_exists?
raise "PostgreSQL database #{@db_name} at #{@server} exists," <<
" use -f to overwrite."
end
connect_db
if federated
FEDERATED_TABLES.each do |table|
@db.drop_table?(table)
end
@db.drop_table?(:logdb)
else
@db.tables.each do |table|
@db.drop_table(table)
end
end
rc = system("PGPASSWORD=\"#{@passwd}\" psql -U #{@user} -h #{@server} -p #{@port} -d #{@db_name} -f #{bck_file} --set ON_ERROR_STOP=on --quiet -o /dev/null")
if !rc
raise "Error while restoring PostgreSQL DB #{@db_name} at #{@server}."
end
puts "PostgreSQL DB #{@db_name} at #{@server} restored."
end
def fts_index(recreate = false)
raise "FTS index not supported for PostgreSQL."
end
private
def connect_db
passwd = CGI.escape(@passwd)
endpoint = "postgres://#{@user}:#{passwd}@#{@server}:#{@port}/#{@db_name}"
begin
options = {}
options[:encoding] = @encoding unless @encoding.empty?
@db = Sequel.connect(endpoint, options)
rescue Exception => e
raise "Error connecting to DB: " + e.message
end
redefine_db_methods
end
def redefine_db_methods
def @db.fetch(query)
preprocessed = BackEndPostgreSQL.preprocess(query)
super(preprocessed)
end
def @db.run(query)
preprocessed = BackEndPostgreSQL.preprocess(query)
super(preprocessed)
end
end
# Any change to this method should be reflected in PostgreSqlDB class
# in src/sql/PostgreSqlDB.cc
def self.preprocess(query)
pp_query = query.dup
if pp_query.upcase.start_with?('CREATE TABLE')
pp_query = replace_type(pp_query, 'MEDIUMTEXT', 'TEXT')
pp_query = replace_type(pp_query, 'LONGTEXT', 'TEXT')
pp_query = replace_type(pp_query, 'BIGINT UNSIGNED', 'NUMERIC(20)')
end
if pp_query.upcase.start_with?('REPLACE')
pp_query = replace_replace_into(query)
end
pp_query
end
def self.replace_type(query, type, replacement)
query = query.gsub(type.upcase, replacement.upcase)
query = query.gsub(type.downcase, replacement.downcase)
return query
end
# This method changes MySQL/SQLite REPLACE INTO into PostgreSQL
# INSERT INTO query with ON CONFLICT clause.
# For example:
# REPLACE INTO pool_control (tablename, last_oid) VALUES ('acl',0)
# changes to:
# INSERT INTO pool_control (tablename, last_oid) VALUES ('acl',0)
# ON CONFLICT (tablename) DO UPDATE SET last_oid = EXCLUDED.last_oid"
#
# Any change to this method should be reflected in PostgreSqlDB class
# in src/sql/PostgreSqlDB.cc
def self.replace_replace_into(query)
query[0, 7] = "INSERT"
db_names_start = query.index('(') + 1
db_names_end = query.index(')')
db_names = query[db_names_start, db_names_end - db_names_start]
splits = db_names.split(',')
query += " ON CONFLICT (" + splits[0] + ") DO UPDATE SET "
sep = ""
splits[1..-1].each do |split|
split = split.strip
query += sep + split + " = EXCLUDED." + split
sep = ", "
end
query
end
end

21
src/onedb/shared/5.10.0_to_5.12.0.rb
View File

@ -27,7 +27,28 @@ module Migrator
end
def up
feature_3600
true
end
private
#Rename acl column name from user to userset to support postgresql
def feature_3600
@db.run 'DROP TABLE IF EXISTS old_acl;'
@db.run 'ALTER TABLE acl RENAME TO old_acl;'
create_table(:acl)
@db.transaction do
@db.fetch('SELECT * FROM old_acl') do |row|
row[:userset] = row.delete[:user]
@db[:acl].insert(row)
end
end
@db.run "DROP TABLE old_acl;"
end
end

8
src/pool/PoolSQL.cc
View File

@ -274,8 +274,8 @@ PoolObjectSQL * PoolSQL::get_ro(const string& name, int uid)
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
int PoolSQL::dump(string& oss, const string& elem_name, const string& column, const char* table,
const string& where, const string& limit, bool desc)
int PoolSQL::dump(string& oss, const string& elem_name, const string& column,
const char* table, const string& where, int sid, int eid, bool desc)
{
ostringstream cmd;
@ -293,9 +293,9 @@ int PoolSQL::dump(string& oss, const string& elem_name, const string& column, co
cmd << " DESC";
}
if ( !limit.empty() )
if ( eid != -1 )
{
cmd << " LIMIT " << limit;
cmd << " " << db->limit_string(sid, eid);
}
return dump(oss, elem_name, cmd);

19
src/rm/RequestManagerPoolInfoFilter.cc
View File

@ -200,6 +200,8 @@ void RequestManagerPoolInfoFilter::dump(
std::string where_string, limit_clause;
std::string desc;
int limit_end_id = -1;
int rc;
if ( filter_flag < GROUP )
@ -222,8 +224,7 @@ void RequestManagerPoolInfoFilter::dump(
if ( end_id < -1 )
{
oss << start_id << "," << -end_id;
limit_clause = oss.str();
limit_end_id = -end_id;
}
Nebula::instance().get_configuration_attribute(att.uid, att.gid,
@ -233,14 +234,16 @@ void RequestManagerPoolInfoFilter::dump(
{
rc = pool->dump_extended(str,
where_string,
limit_clause,
start_id,
limit_end_id,
one_util::toupper(desc) == "DESC");
}
else
{
rc = pool->dump(str,
where_string,
limit_clause,
start_id,
limit_end_id,
one_util::toupper(desc) == "DESC");
}
@ -300,7 +303,7 @@ void VirtualMachinePoolInfo::request_execute(
{
fts_query = xmlrpc_c::value_string(paramList.getString(5));
if (!fts_query.empty() && !pool->is_fts_available())
if (!fts_query.empty() && !pool->supports(SqlDB::SqlFeature::FTS))
{
att.resp_msg = "Full text search is not supported by the SQL backend";
@ -556,6 +559,8 @@ void VirtualNetworkPoolInfo::request_execute(
int start_id = xmlrpc_c::value_int(paramList.getInt(2));
int end_id = xmlrpc_c::value_int(paramList.getInt(3));
int limit_end_id = -1;
if ( filter_flag < GROUP )
{
att.resp_msg = "Incorrect filter_flag";
@ -588,7 +593,7 @@ void VirtualNetworkPoolInfo::request_execute(
if ( end_id < -1 )
{
limit_clause << start_id << "," << -end_id;
limit_end_id = -end_id;
}
/* ---------------------------------------------------------------------- */
@ -600,7 +605,7 @@ void VirtualNetworkPoolInfo::request_execute(
Nebula::instance().get_configuration_attribute(att.uid, att.gid,
"API_LIST_ORDER", desc);
int rc = pool->dump(pool_oss, where_string.str(), limit_clause.str(),
int rc = pool->dump(pool_oss, where_string.str(), start_id, limit_end_id,
one_util::toupper(desc) == "DESC");
if ( rc != 0 )

26
src/sql/LogDB.cc
View File

@ -184,7 +184,7 @@ int LogDB::setup_index(uint64_t& _last_applied, uint64_t& _last_index)
cb.set_callback(&_last_applied);
oss << "SELECT MAX(log_index) FROM logdb WHERE applied = 1";
oss << "SELECT MAX(log_index) FROM logdb WHERE applied = '1'";
rc += db->exec_rd(oss, &cb);
@ -345,7 +345,7 @@ int LogDB::insert(uint64_t index, unsigned int term, const std::string& sql,
<< "'" << sql_db << "',"
<< tstamp << ","
<< fed_index << ","
<< applied << ")";
<< "'" << applied << "')";
int rc = db->exec_wr(oss);
@ -383,7 +383,7 @@ int LogDB::apply_log_record(LogDBRecord * lr)
{
std::ostringstream oss;
oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = 1"
oss << "UPDATE logdb SET timestamp = " << time(0) << ", applied = '1'"
<< " WHERE log_index = " << lr->index << " AND timestamp = 0";
if ( db->exec_wr(oss) != 0 )
@ -642,8 +642,8 @@ int LogDB::purge_log()
oss.str("");
oss << " SELECT MIN(i.log_index) FROM ("
<< " SELECT log_index FROM logdb WHERE fed_index = " << UINT64_MAX
<< " AND applied = 1 AND log_index >= 0 "
<< " ORDER BY log_index DESC LIMIT " << log_retention
<< " AND applied = '1' AND log_index >= 0 "
<< " ORDER BY log_index DESC " << db->limit_string(log_retention)
<< " ) AS i";
cb_min_idx.set_callback(&min_idx);
@ -655,12 +655,12 @@ int LogDB::purge_log()
cb.set_affected_rows(0);
oss.str("");
oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 "
oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 "
<< "AND fed_index = " << UINT64_MAX << " AND log_index < " << min_idx;
if ( db->limit_support() )
if ( db->supports(SqlDB::SqlFeature::LIMIT) )
{
oss << " LIMIT " << limit_purge;
oss << " " << db->limit_string(limit_purge);
}
if ( db->exec_wr(oss, &cb) != -1 )
@ -705,8 +705,8 @@ int LogDB::purge_log()
oss.str("");
oss << " SELECT MIN(i.log_index) FROM ("
<< " SELECT log_index FROM logdb WHERE fed_index != " << UINT64_MAX
<< " AND applied = 1 AND log_index >= 0 "
<< " ORDER BY log_index DESC LIMIT " << log_retention
<< " AND applied = '1' AND log_index >= 0 "
<< " ORDER BY log_index DESC " << db->limit_string(log_retention)
<< " ) AS i";
cb_min_idx.set_callback(&min_idx);
@ -718,12 +718,12 @@ int LogDB::purge_log()
cb.set_affected_rows(0);
oss.str("");
oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 "
oss << "DELETE FROM logdb WHERE applied = '1' AND log_index >= 0 "
<< "AND fed_index != " << UINT64_MAX << " AND log_index < " << min_idx;
if ( db->limit_support() )
if ( db->supports(SqlDB::SqlFeature::LIMIT) )
{
oss << " LIMIT " << limit_purge;
oss << " " << db->limit_string(limit_purge);
}
if ( db->exec_wr(oss, &cb) != -1 )

52
src/sql/MySqlDB.cc
View File

@ -21,16 +21,17 @@
/*********
* Doc: https://dev.mysql.com/doc/refman/8.0/en/c-api.html
********/
using namespace std;
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
static unsigned long get_server_version(MYSQL * db)
{
std::string version_str;
std::vector<unsigned int> ids;
string version_str;
vector<unsigned int> ids;
std::string version_query = "SELECT @@GLOBAL.version";
string version_query = "SELECT @@GLOBAL.version";
if ( mysql_query(db, version_query.c_str()) != 0 )
{
@ -56,8 +57,8 @@ static unsigned long get_server_version(MYSQL * db)
mysql_free_result(result);
std::string version_number;
std::stringstream version_iss(version_str);
string version_number;
stringstream version_iss(version_str);
if (!getline(version_iss, version_number, '-') || version_number.empty())
{
@ -72,10 +73,9 @@ static unsigned long get_server_version(MYSQL * db)
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
static std::string get_encoding(MYSQL * c, const std::string& sql,
std::string& error)
static string get_encoding(MYSQL * c, const string& sql, string& error)
{
std::string encoding;
string encoding;
if ( mysql_query(c, sql.c_str()) != 0 )
{
@ -111,7 +111,7 @@ static std::string get_encoding(MYSQL * c, const std::string& sql,
/* -------------------------------------------------------------------------- */
int MySqlDB::db_encoding(std::string& error)
int MySqlDB::db_encoding(string& error)
{
MYSQL * connection = mysql_init(nullptr);
@ -124,7 +124,7 @@ int MySqlDB::db_encoding(std::string& error)
return -1;
}
std::string create_sql = "CREATE DATABASE IF NOT EXISTS " + database;
string create_sql = "CREATE DATABASE IF NOT EXISTS " + database;
if ( mysql_query(connection, create_sql.c_str()) != 0 )
{
@ -140,22 +140,22 @@ int MySqlDB::db_encoding(std::string& error)
}
//Get encodings for database and tables
std::string db_sql = "SELECT default_character_set_name FROM "
string db_sql = "SELECT default_character_set_name FROM "
"information_schema.SCHEMATA WHERE schema_name = \"" + database + "\"";
std::string db_enc = get_encoding(connection, db_sql, error);
string db_enc = get_encoding(connection, db_sql, error);
if ( db_enc.empty() )
{
return -1;
}
std::string table_sql = "SELECT CCSA.character_set_name FROM "
"information_schema.`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`"
string table_sql = "SELECT CCSA.character_set_name FROM information_schema."\
"`TABLES` T, information_schema.`COLLATION_CHARACTER_SET_APPLICABILITY`"
" CCSA WHERE CCSA.collation_name = T.table_collation AND T.table_schema = "
"\"" + database + "\" AND T.table_name = \"system_attributes\"";
std::string table_enc = get_encoding(connection, table_sql, error);
string table_enc = get_encoding(connection, table_sql, error);
if ( !table_enc.empty() && table_enc != db_enc)
{
@ -176,13 +176,13 @@ int MySqlDB::db_encoding(std::string& error)
MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
const string& d, const string& e, int m):max_connections(m), server(s),
port(p), user(u), password(_p), database(d), encoding(e), _fts_available(false)
port(p), user(u), password(_p), database(d), encoding(e)
{
vector<MYSQL *> connections(max_connections);
MYSQL * rc;
ostringstream oss;
std::string error;
string error;
// -------------------------------------------------------------------------
// Initialize the MySQL library
@ -240,13 +240,13 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
// -------------------------------------------------------------------------
// Connect to OpenNebula Database
// -------------------------------------------------------------------------
std::string use_sql = "USE " + database;
string use_sql = "USE " + database;
for (int i=0 ; i < max_connections ; i++)
{
if ( mysql_query(connections[i], use_sql.c_str()) != 0 )
{
std::string error = "Could not connect to database: ";
string error = "Could not connect to database: ";
error.append(mysql_error(connections[i]));
throw runtime_error(error);
@ -254,7 +254,7 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
if ( mysql_set_character_set(connections[i], encoding.c_str()) != 0 )
{
std::string error = "Could not set encoding : ";
string error = "Could not set encoding : ";
error.append(mysql_error(connections[i]));
throw runtime_error(error);
@ -271,12 +271,12 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
oss.clear();
//--------------------------------------------------------------------------
// Get server information and FTS support
// Get server information and FTS support & features
//--------------------------------------------------------------------------
unsigned long version;
unsigned long min_fts_version;
std::string server_info = mysql_get_server_info(db_escape_connect);
string server_info = mysql_get_server_info(db_escape_connect);
version = get_server_version(db_escape_connect);
@ -299,15 +299,19 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
if (version >= min_fts_version)
{
_fts_available = true;
NebulaLog::log("ONE", Log::INFO, "\tFTS enabled");
}
else
{
_fts_available = false;
NebulaLog::log("ONE", Log::INFO, "\tFTS disabled");
}
features = {
{SqlFeature::MULTIPLE_VALUE, true},
{SqlFeature::LIMIT, true},
{SqlFeature::FTS, version >= min_fts_version}
};
pthread_mutex_init(&mutex,0);
pthread_cond_init(&cond,0);

333
src/sql/PostgreSqlDB.cc Normal file
View File

@ -0,0 +1,333 @@
/* -------------------------------------------------------------------------- */
/* Copyright 2002-2019, OpenNebula Project, OpenNebula Systems */
/* */
/* Licensed under the Apache License, Version 2.0 (the "License"); you may */
/* not use this file except in compliance with the License. You may obtain */
/* a copy of the License at */
/* */
/* http://www.apache.org/licenses/LICENSE-2.0 */
/* */
/* Unless required by applicable law or agreed to in writing, software */
/* distributed under the License is distributed on an "AS IS" BASIS, */
/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */
/* See the License for the specific language governing permissions and */
/* limitations under the License. */
/* -------------------------------------------------------------------------- */
#include "NebulaUtil.h"
#include "PostgreSqlDB.h"
#include <libpq-fe.h>
#include <iostream>
/*********
* Doc: https://www.postgresql.org/docs/current/libpq.html
********/
#define PG_DEFAULT_PORT 5432
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
PostgreSqlDB::PostgreSqlDB(
const string& _server,
int _port,
const string& _user,
const string& _password,
const string& _database,
int _connections)
{
PGconn* conn;
server = _server;
port = _port;
user = _user;
password = _password;
database = _database;
if ( port == 0 )
{
port = PG_DEFAULT_PORT;
}
max_connections = _connections;
// Set up connection parameters
string params = "host=" + _server
+ " port=" + to_string(port)
+ " user=" + user
+ " password=" + password
+ " dbname=" + database;
// Create connection pool
for (int i = 0; i < max_connections; i++)
{
conn = PQconnectdb(params.c_str());
if ( PQstatus(conn) == CONNECTION_BAD )
{
ostringstream oss;
oss << "Could not open connect to database server: "
<< PQerrorMessage(conn);
throw runtime_error(oss.str());
}
db_connect.push(conn);
}
db_escape_connect = PQconnectdb(params.c_str());
if ( PQserverVersion(db_escape_connect) < 90500 )
{
std::string error = "PostgreSQL version error: must be 9.5 or higher.";
NebulaLog::log("ONE", Log::ERROR, error);
throw runtime_error(error);
}
features = {
{SqlFeature::MULTIPLE_VALUE, PQlibVersion() < 80200},
{SqlFeature::LIMIT, false},
{SqlFeature::FTS, false}
};
pthread_mutex_init(&mutex, 0);
pthread_cond_init(&cond, 0);
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
PostgreSqlDB::~PostgreSqlDB()
{
while (!db_connect.empty())
{
PGconn* conn = db_connect.front();
db_connect.pop();
PQfinish(conn);
}
PQfinish(db_escape_connect);
pthread_mutex_destroy(&mutex);
pthread_cond_destroy(&cond);
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
char * PostgreSqlDB::escape_str(const string& str)
{
char* buf = new char[str.size() * 2 + 1];
int err;
PQescapeStringConn(db_escape_connect, buf, str.c_str(), str.length(), &err);
if ( err != 0 )
{
delete[] buf;
return nullptr;
}
return buf;
}
/* -------------------------------------------------------------------------- */
void PostgreSqlDB::free_str(char * str)
{
delete[] str;
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
int PostgreSqlDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet)
{
string str = preprocess_query(cmd);
const char* c_str = str.c_str();
Log::MessageType error_type = quiet ? Log::DDEBUG : Log::ERROR;
PGconn* conn = get_db_connection();
PGresult* res = PQexec(conn, c_str);
if ( PQresultStatus(res) != PGRES_COMMAND_OK &&
PQresultStatus(res) != PGRES_TUPLES_OK )
{
const char* err_msg = PQerrorMessage(conn);
ostringstream oss;
oss << "SQL command was: " << c_str;
oss << ", error " << err_msg;
NebulaLog::log("ONE", error_type, oss);
PQclear(res);
free_db_connection(conn);
return SqlDB::SQL;
}
if ( obj == 0 )
{
// Free the result and db connection
PQclear(res);
free_db_connection(conn);
return SqlDB::SUCCESS;
}
int ec = SqlDB::SUCCESS;
// Get number of fields and rows of the result
int n_fields = PQnfields(res);
int n_rows = PQntuples(res);
if ( obj->isCallBackSet() && PQresultStatus(res) == PGRES_TUPLES_OK )
{
char** names = new char*[n_fields];
char** values = new char*[n_fields];
// Get column names
for (int i = 0; i < n_fields; i++)
{
names[i] = PQfname(res, i);
}
// For each row
for (int row = 0; row < n_rows; row++)
{
// get values in that row
for (int field = 0; field < n_fields; field++)
{
values[field] = PQgetvalue(res, row, field);
}
// and do a callback on them
if ( obj->do_callback(n_fields, values, names) != 0 )
{
ec = SqlDB::SQL;
break;
}
}
delete[] names;
delete[] values;
}
if ( obj->get_affected_rows() == 0 && n_rows )
{
obj->set_affected_rows(n_rows);
}
// Free the result and db connection
PQclear(res);
free_db_connection(conn);
return ec;
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
PGconn * PostgreSqlDB::get_db_connection()
{
PGconn* conn;
pthread_mutex_lock(&mutex);
while (db_connect.empty() == true)
{
pthread_cond_wait(&cond, &mutex);
}
conn = db_connect.front();
db_connect.pop();
pthread_mutex_unlock(&mutex);
return conn;
}
/* -------------------------------------------------------------------------- */
void PostgreSqlDB::free_db_connection(PGconn * db)
{
pthread_mutex_lock(&mutex);
db_connect.push(db);
pthread_cond_signal(&cond);
pthread_mutex_unlock(&mutex);
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
static void replace_substring(std::string& cmd, const std::string& s1,
const std::string& s2)
{
size_t pos = cmd.find(s1);
while (pos != std::string::npos)
{
cmd.replace(pos, s1.length(), s2);
pos = cmd.find(s1, pos + s2.length());
}
}
std::string PostgreSqlDB::preprocess_query(std::ostringstream& cmd)
{
std::string query = cmd.str();
size_t pos;
// Both CREATE TABLE and REPLACE should be at the start
// so we don't change user data
if ((pos = query.find("CREATE TABLE")) == 0)
{
replace_substring(query, "MEDIUMTEXT", "TEXT");
replace_substring(query, "LONGTEXT", "TEXT");
replace_substring(query, "BIGINT UNSIGNED", "NUMERIC(20)");
}
else if ((pos = query.find("REPLACE")) == 0)
{
query.replace(0, 7, "INSERT");
size_t table_start = query.find("INTO ", 7) + 5;
size_t names_start = query.find("(", table_start) + 1;
size_t names_end = query.find(")", names_start);
std::string db_names = query.substr(names_start, names_end - names_start);
std::string table = query.substr(table_start, names_start - 2 - table_start);
std::vector<std::string> splits;
one_util::split(db_names, ',', splits);
query += " ON CONFLICT ON CONSTRAINT " + table + "_pkey DO UPDATE SET ";
const char* sep = "";
for (size_t i = 1; i < splits.size(); i++)
{
query += sep + splits[i] + " = EXCLUDED." + splits[i];
sep = ", ";
}
}
return query;
}

10
src/sql/SConstruct
View File

@ -29,6 +29,16 @@ if env['sqlite']=='yes':
if env['mysql']=='yes':
source_files.append('MySqlDB.cc')
if env['postgresql']=='yes':
source_files.append('PostgreSqlDB.cc')
# Build library
env.StaticLibrary(lib_name, source_files)
lib_name = 'nebula_sql_const'
source_files = ['OneDB.cc']
# Build library
env.StaticLibrary(lib_name, source_files)

30
src/sql/SqliteDB.cc
View File

@ -13,8 +13,6 @@
/* See the License for the specific language governing permissions and */
/* limitations under the License. */
/* -------------------------------------------------------------------------- */
#include "SqliteDB.h"
using namespace std;
@ -52,14 +50,21 @@ SqliteDB::SqliteDB(const string& db_name)
throw runtime_error("Could not open database.");
}
enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
int enable_limit = sqlite3_compileoption_used("SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
if (enable_limit)
if (enable_limit == 1)
{
NebulaLog::log("ONE",Log::INFO , "sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
NebulaLog::log("ONE",Log::INFO ,
"sqlite has enabled: SQLITE_ENABLE_UPDATE_DELETE_LIMIT");
}
sqlite3_extended_result_codes(db, 1);
features = {
{SqlFeature::MULTIPLE_VALUE, false},
{SqlFeature::LIMIT, enable_limit == 1},
{SqlFeature::FTS, false}
};
}
/* -------------------------------------------------------------------------- */
@ -72,21 +77,6 @@ SqliteDB::~SqliteDB()
}
/* -------------------------------------------------------------------------- */
bool SqliteDB::multiple_values_support()
{
// Versions > 3.7.11 support multiple value inserts, but tests
// have ended in segfault. A transaction seems to perform better
//return SQLITE_VERSION_NUMBER >= 3007011;
return false;
}
/* -------------------------------------------------------------------------- */
bool SqliteDB::limit_support()
{
return enable_limit == 1;
}
/* -------------------------------------------------------------------------- */
int SqliteDB::exec_ext(std::ostringstream& cmd, Callbackable *obj, bool quiet)

7
src/um/UserPool.cc
View File

@ -1428,8 +1428,7 @@ int UserPool::authorize(AuthRequest& ar)
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
int UserPool::dump(string& oss, const string& where, const string& limit,
bool desc)
int UserPool::dump(string& oss, const string& where, int sid, int eid, bool desc)
{
int rc;
string def_quota_xml;
@ -1453,9 +1452,9 @@ int UserPool::dump(string& oss, const string& where, const string& limit,
cmd << " DESC";
}
if ( !limit.empty() )
if ( eid != -1 )
{
cmd << " LIMIT " << limit;
cmd << " " << db->limit_string(sid, eid);
}
oss.append("<USER_POOL>");

2
src/vm/VirtualMachine.cc
View File

@ -455,7 +455,7 @@ int VirtualMachine::bootstrap(SqlDB * db)
oss_vm << one_db::vm_db_bootstrap;
if (db->fts_available())
if (db->supports(SqlDB::SqlFeature::FTS))
{
oss_vm << ", FULLTEXT ftidx(search_token))";
}

10
src/vm/VirtualMachinePool.cc
View File

@ -230,7 +230,7 @@ int VirtualMachinePool::get_running(
<< " state = " << VirtualMachine::ACTIVE
<< " and ( lcm_state = " << VirtualMachine::RUNNING
<< " or lcm_state = " << VirtualMachine::UNKNOWN << " )"
<< " ORDER BY last_poll ASC LIMIT " << vm_limit;
<< " ORDER BY last_poll ASC " << db->limit_string(vm_limit);
where = os.str();
@ -262,8 +262,7 @@ int VirtualMachinePool::dump_acct(string& oss, const string& where,
ostringstream cmd;
cmd << "SELECT " << History::table << ".body FROM " << History::table
<< " INNER JOIN " << one_db::vm_table
<< " WHERE vid=oid";
<< " INNER JOIN " << one_db::vm_table << " ON vid=oid";
if ( !where.empty() )
{
@ -302,8 +301,7 @@ int VirtualMachinePool::dump_showback(string& oss,
cmd << "SELECT " << one_db::vm_showback_table << ".body FROM "
<< one_db::vm_showback_table
<< " INNER JOIN " << one_db::vm_table
<< " WHERE vmid=oid";
<< " INNER JOIN " << one_db::vm_table << " ON vmid=oid";
if ( !where.empty() )
{
@ -719,7 +717,7 @@ int VirtualMachinePool::calculate_showback(
// Write to DB
if (db->multiple_values_support())
if (db->supports(SqlDB::SqlFeature::MULTIPLE_VALUE))
{
oss.str("");