shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2024-12-22 13:33:52 +03:00
Code Releases Wiki Activity

F #5861: Search VM using JSON querry (#3102)

Browse Source 
Signed-off-by: Pavel Czerny <pczerny@opennebula.io>
Co-authored-by: Alejandro Huertas <ahuertas@opennebula.io>
This commit is contained in:
Pavel Czerný 2024-06-13 12:35:15 +02:00 committed by GitHub
parent 27301e09ab
commit 95c51ff87f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
21 changed files with 62 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
include/Attribute.h
View File

@ -79,8 +79,6 @@ public:
virtual void to_json(std::ostringstream& s) const = 0;
virtual void to_token(std::ostringstream& s) const = 0;
virtual void to_xml(std::ostringstream& s,
const std::map<std::string, std::set<std::string>> &hidden) const = 0;
@ -203,19 +201,6 @@ public:
one_util::escape_json(attribute_value, s);
}
void to_token(std::ostringstream& s) const override
{
if (attribute_name.empty() || attribute_value.empty())
{
return;
}
one_util::escape_token(attribute_name, s);
s << "=";
one_util::escape_token(attribute_value, s);
s << std::endl;
}
/**
* Builds a new attribute from a string.
*/
@ -456,8 +441,6 @@ public:
void to_json(std::ostringstream& s) const override;
void to_token(std::ostringstream& s) const override;
/**
* Builds a new attribute from a string of the form:
* "VAL_NAME_1=VAL_VALUE_1,...,VAL_NAME_N=VAL_VALUE_N".

5
include/ExtendedAttribute.h
View File

@ -93,11 +93,6 @@ public:
return va->to_json(s);
};
void to_token(std::ostringstream& s) const override
{
return va->to_token(s);
};
void to_xml(std::ostringstream& s,
const std::map<std::string, std::set<std::string>> &hidden) const override
{

2
include/History.h
View File

@ -192,8 +192,6 @@ private:
std::string& to_json(std::string& json) const;
std::string& to_token(std::string& text) const;
/**
* Rebuilds the object from an xml node
* @param node The xml node pointer

4
include/SqlDB.h
View File

@ -45,7 +45,7 @@ public:
{
MULTIPLE_VALUE, // syntax INSERT VALUES (data), (data), (data)
LIMIT, // LIMIT in queries with DELETE and UPDATE
FTS, // Full Text Search
JSON_QUERY, // JSON queries Search
COMPARE_BINARY // Use BINARY for comparing name in DB
};
@ -198,7 +198,7 @@ protected:
{
{SqlFeature::MULTIPLE_VALUE, false},
{SqlFeature::LIMIT, false},
{SqlFeature::FTS, false},
{SqlFeature::JSON_QUERY, false},
{SqlFeature::COMPARE_BINARY, false}
};

2
include/Template.h
View File

@ -199,8 +199,6 @@ public:
std::string& to_json(std::string& xml) const;
std::string& to_token(std::string& xml) const;
/**
* Writes the template in a plain text string
* @param str string that hold the template representation

2
include/VirtualMachine.h
View File

@ -1965,8 +1965,6 @@ private:
std::string& to_json(std::string& json) const;
std::string& to_token(std::string& text) const;
// -------------------------------------------------------------------------
// Attribute Parser
// -------------------------------------------------------------------------

3
src/cli/one_helper/onevm_helper.rb
View File

@ -160,7 +160,8 @@ class OneVMHelper < OpenNebulaHelper::OneHelper
SEARCH = {
:name => 'search',
:large => '--search search',
:description => 'query in KEY=VALUE format',
:description => 'Query in PATH=VALUE format. For example: ' \
'onevm list --search "VM.NAME=abc&VM.TEMPLATE.DISK[*].IMAGE=db1"',
:format => String
}

16
src/common/Attribute.cc
View File

@ -147,22 +147,6 @@ void VectorAttribute::to_json(std::ostringstream& s) const
s << "}";
}
void VectorAttribute::to_token(std::ostringstream& s) const
{
for (auto it=attribute_value.begin(); it!=attribute_value.end(); it++)
{
if (it->first.empty() || it->second.empty())
{
continue;
}
one_util::escape_token(it->first, s);
s << "=";
one_util::escape_token(it->second, s);
s << std::endl;
}
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

16
src/host/HostMonitoringTemplate.cc
View File

@ -127,24 +127,18 @@ int NUMAMonitoring::from_template(const Template &tmpl)
unsigned long size;
unsigned long fr;
ostringstream oss;
if (page->vector_value("NODE_ID", node_id) != 0)
{
page->to_token(oss);
NebulaLog::warn("HMM", "Hugepage doesn't contain node ID: "
+ oss.str());
+ page->marshall(","));
continue;
}
if (page->vector_value("SIZE", size) != 0)
{
page->to_token(oss);
NebulaLog::warn("HMM", "Hugepage doesn't contain size: "
+ oss.str());
+ page->marshall(","));
continue;
}
@ -165,14 +159,10 @@ int NUMAMonitoring::from_template(const Template &tmpl)
unsigned long used;
unsigned long fr;
ostringstream oss;
if (mem->vector_value("NODE_ID", node_id) != 0)
{
mem->to_token(oss);
NebulaLog::warn("HMM", "Memory node doesn't contain node ID: "
+ oss.str());
+ mem->marshall(","));
continue;
}

1
src/onedb/database_schema.rb
View File

@ -106,7 +106,6 @@ class OneDBBacKEnd
"body MEDIUMTEXT, uid INTEGER, gid INTEGER," <<
"owner_u INTEGER, group_u INTEGER, other_u INTEGER",
index_sql: [{ :name =>'state_oid_idx', :table => 'vm_pool', :columns => '(state, oid)' },
{ :name =>'ftidx', :table => 'vm_pool', :columns => '(search_token)', :type => 'FULLTEXT' },
{ :name =>'applied_idx', :table => 'logdb', :columns => '(applied)' },
{ :name =>'fed_index_idx', :table => 'logdb', :columns => '(fed_index)' }],

25
src/onedb/onedb
View File

@ -104,12 +104,6 @@ FEDERATED = {
:description => 'Limit backup/restore to federated tables'
}
RECREATE = {
:name => 'recreate',
:large => '--recreate',
:description => 'Delete FTS index and create a new one'
}
###############################################################################
# SQLite options
###############################################################################
@ -891,23 +885,4 @@ CommandParser::CmdParser.new(ARGV) do
0 # exit code
end
###########################################################################
# Create and recreate FTS index in vmpool
###########################################################################
fts_desc = <<-EOT.unindent
Create or recreate FTS index in vm pool to be able to search VMs
EOT
command :'create-index', fts_desc, :options => [RECREATE] do
begin
helper = OneDB.new(options)
helper.fts_index(options.key?(:recreate))
rescue StandardError => e
STDERR.puts e.message
[-1, e.message]
end
0
end
end

11
src/onedb/onedb.rb
View File

@ -709,17 +709,6 @@ class OneDB
end
end
# Create or recreate FTS index on vm_pool search_token column
#
# @param recreate [Boolean] True to delete the index and create it again
def fts_index(recreate = false)
if backend.is_a? BackEndSQLite
raise 'This is operation is not supported for sqlite backend'
end
backend.fts_index(recreate)
end
private
def one_not_running()

25
src/onedb/onedb_backend.rb
View File

@ -386,23 +386,6 @@ class BackEndMySQL < OneDBBacKEnd
puts "MySQL DB #{@db_name} at #{@server} restored."
end
# Create or recreate FTS index on vm_pool search_token column
#
# @param recreate [Boolean] True to delete the index and create it again
def fts_index(recreate = false)
connect_db
if recreate
@db.alter_table(:vm_pool) do
drop_index :search_token, name: 'ftidx'
end
end
@db.alter_table(:vm_pool) do
add_full_text_index :search_token, name: 'ftidx'
end
end
def idx?(idx)
query = "SHOW INDEX FROM #{idx[:table]} WHERE KEY_NAME = '#{idx[:name]}'"
!@db.fetch(query).first.nil?
@ -800,16 +783,12 @@ class BackEndPostgreSQL < OneDBBacKEnd
puts "PostgreSQL DB #{@db_name} at #{@server} restored."
end
def fts_index(recreate = false)
raise "FTS index not supported for PostgreSQL."
end
def idx?(idx)
query = "SELECT * FROM pg_indexes WHERE indexname = '#{idx[:name]}'"
!@db.fetch(query).first.nil?
end
def create_idx(version = nil)
def create_idx(version = nil)
schema = get_schema(:index_sqlite, version)
schema.each do |idx|
@ -830,7 +809,7 @@ class BackEndPostgreSQL < OneDBBacKEnd
schema.each do |idx|
next unless idx? idx
query = "DROP INDEX #{idx[:name]};"
@db.run query

44
src/rm/RequestManagerPoolInfoFilter.cc
View File

@ -295,15 +295,15 @@ void VirtualMachinePoolInfo::request_execute(
int end_id = xmlrpc_c::value_int(paramList.getInt(3));
int state = xmlrpc_c::value_int(paramList.getInt(4));
std::string fts_query;
std::string json_query;
if (paramList.size() > 5)
{
fts_query = xmlrpc_c::value_string(paramList.getString(5));
json_query = xmlrpc_c::value_string(paramList.getString(5));
if (!fts_query.empty() && !pool->supports(SqlDB::SqlFeature::FTS))
if (!json_query.empty() && !pool->supports(SqlDB::SqlFeature::JSON_QUERY))
{
att.resp_msg = "Full text search is not supported by the SQL backend";
att.resp_msg = "JSON query search is not supported by the SQL backend";
failure_response(INTERNAL, att);
return;
@ -335,11 +335,11 @@ void VirtualMachinePoolInfo::request_execute(
break;
}
if (!fts_query.empty())
if (!json_query.empty())
{
char * _fts_query = pool->escape_str(fts_query);
char * _json_query = pool->escape_str(json_query);
if ( _fts_query == 0 )
if ( _json_query == 0 )
{
att.resp_msg = "Error building search query";
@ -352,11 +352,33 @@ void VirtualMachinePoolInfo::request_execute(
and_filter << " AND ";
}
and_filter << "MATCH(search_token) AGAINST ('+\"";
one_util::escape_token(_fts_query, and_filter);
and_filter << "\"' in boolean mode)";
vector<string> keys;
pool->free_str(_fts_query);
one_util::split(_json_query, '&', keys);
for(const auto& key: keys)
{
vector<string> kv;
one_util::split(key, '=', kv);
if (kv.size() == 1)
{
// No key specified, search whole json body
kv.push_back(kv[0]);
kv[0] = "*";
}
and_filter << "JSON_UNQUOTE(JSON_EXTRACT(body_json, '$." << kv[0] << "'))";
and_filter << " LIKE '%" << kv[1] << "%'";
if (key != keys.back())
{
and_filter << " AND ";
}
}
pool->free_str(_json_query);
}
dump(att, filter_flag, start_id, end_id, and_filter.str(), "");

16
src/sql/MySqlDB.cc
View File

@ -252,10 +252,10 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
oss.clear();
//--------------------------------------------------------------------------
// Get server information and FTS support & features
// Get server information and JSON query support & features
//--------------------------------------------------------------------------
unsigned long version;
unsigned long min_fts_version;
unsigned long min_json_version;
string server_info = mysql_get_server_info(db_escape_connect);
@ -271,27 +271,27 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p,
if (server_info.find("mariadb") == std::string::npos)
{
min_fts_version = 50600;
min_json_version = 50708;
}
else
{
min_fts_version = 100005;
min_json_version = 100200;
}
if (version >= min_fts_version)
if (version >= min_json_version)
{
NebulaLog::log("ONE", Log::INFO, "\tFTS enabled");
NebulaLog::log("ONE", Log::INFO, "\tJSON queries enabled");
}
else
{
NebulaLog::log("ONE", Log::INFO, "\tFTS disabled");
NebulaLog::log("ONE", Log::INFO, "\tJSON queries disabled");
}
features =
{
{SqlFeature::MULTIPLE_VALUE, true},
{SqlFeature::LIMIT, true},
{SqlFeature::FTS, version >= min_fts_version},
{SqlFeature::JSON_QUERY, version >= min_json_version},
{SqlFeature::COMPARE_BINARY, one_util::icasecmp(_compare_binary, "YES")}
};
}

4
src/sql/OneDB.cc
View File

@ -58,13 +58,13 @@ namespace one_db
const char * vm_db_names =
"oid, name, body, uid, gid, state, lcm_state, "
"owner_u, group_u, other_u, short_body, search_token";
"owner_u, group_u, other_u, short_body, body_json";
const char * vm_db_bootstrap = "CREATE TABLE IF NOT EXISTS "
"vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, "
"uid INTEGER, gid INTEGER, state INTEGER, "
"lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, "
"short_body MEDIUMTEXT, search_token MEDIUMTEXT";
"short_body MEDIUMTEXT, body_json JSON)";
const char * vm_monitor_table = "vm_monitoring";

2
src/sql/PostgreSqlDB.cc
View File

@ -92,7 +92,7 @@ PostgreSqlDB::PostgreSqlDB(const string& _server,
{
{SqlFeature::MULTIPLE_VALUE, PQlibVersion() > 80200},
{SqlFeature::LIMIT, false},
{SqlFeature::FTS, false},
{SqlFeature::JSON_QUERY, false},
{SqlFeature::COMPARE_BINARY, false}
};
}

2
src/sql/SqliteDB.cc
View File

@ -65,7 +65,7 @@ SqliteDB::SqliteDB(const string& db_name, int timeout)
{
{SqlFeature::MULTIPLE_VALUE, false},
{SqlFeature::LIMIT, enable_limit == 1},
{SqlFeature::FTS, false},
{SqlFeature::JSON_QUERY, false},
{SqlFeature::COMPARE_BINARY, false}
};
}

19
src/template/Template.cc
View File

@ -414,6 +414,9 @@ string& Template::to_xml(string& xml, const string& extra) const
string& Template::to_json(string& json) const
{
// List of attributes that should be an Array (even with just 1 element)
static const std::set<string> ARRAY_ATTRS = {"DISK", "NIC"};
ostringstream oss;
bool is_first = true;
@ -433,7 +436,8 @@ string& Template::to_json(string& json) const
oss << "\"" << it->first << "\": ";
if ( attributes.count(it->first) == 1 )
if ( attributes.count(it->first) == 1 &&
ARRAY_ATTRS.count(it->first) == 0)
{
it->second->to_json(oss);
@ -471,19 +475,6 @@ string& Template::to_json(string& json) const
return json;
}
string& Template::to_token(string& str) const
{
ostringstream os;
for ( auto it = attributes.begin(); it!=attributes.end(); it++)
{
it->second->to_token(os);
}
str = os.str();
return str;
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */

20
src/vm/History.cc
View File

@ -372,26 +372,6 @@ string& History::to_json(string& json) const
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
string& History::to_token(string& text) const
{
ostringstream oss;
oss << "HOSTNAME=";
one_util::escape_token(hostname, oss);
oss << "\n";
oss << "HID=" << hid << "\n" <<
"CID=" << cid << "\n" <<
"DS_ID=" << ds_id << "\n";
text = oss.str();
return text;
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
string& History::to_xml_short(string& xml) const
{
ostringstream oss;

46
src/vm/VirtualMachine.cc
View File

@ -642,15 +642,6 @@ int VirtualMachine::bootstrap(SqlDB * db)
oss_vm << one_db::vm_db_bootstrap;
if (db->supports(SqlDB::SqlFeature::FTS))
{
oss_vm << ", FULLTEXT ftidx(search_token))";
}
else
{
oss_vm << ")";
}
ostringstream oss_monit(one_db::vm_monitor_db_bootstrap);
ostringstream oss_hist(one_db::history_db_bootstrap);
ostringstream oss_showback(one_db::vm_showback_db_bootstrap);
@ -1952,7 +1943,7 @@ int VirtualMachine::insert_replace(SqlDB *db, bool replace, string& error_str)
goto error_xml_short;
}
sql_text = db->escape_str(to_token(text));
sql_text = db->escape_str(to_json(text));
if ( sql_text == 0 )
{
@ -2034,7 +2025,7 @@ int VirtualMachine::update_search(SqlDB * db)
std::ostringstream oss;
std::string text;
char * sql_text = db->escape_str(to_token(text));
char * sql_text = db->escape_str(to_json(text));
if ( sql_text == 0 )
{
@ -2043,7 +2034,7 @@ int VirtualMachine::update_search(SqlDB * db)
}
oss << "UPDATE " << one_db::vm_table << " SET "
<< "search_token = '" << sql_text << "' "
<< "body_json = '" << sql_text << "' "
<< "WHERE oid = " << oid;
db->free_str(sql_text);
@ -2683,37 +2674,6 @@ string& VirtualMachine::to_json(string& json) const
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
string& VirtualMachine::to_token(string& text) const
{
string template_text;
string user_template_text;
string history_text;
ostringstream oss;
oss << "UNAME="<< uname << "\n"
<< "GNAME="<< gname << "\n"
<< obj_template->to_token(template_text) << "\n"
<< user_obj_template->to_token(user_template_text)
<< "NAME=";
one_util::escape_token(name, oss);
oss << "\n";
if ( hasHistory() )
{
oss << "\n" << history->to_token(history_text);
}
text = oss.str();
return text;
}
/* -------------------------------------------------------------------------- */
/* -------------------------------------------------------------------------- */
string& VirtualMachine::to_xml_short(string& xml)
{
string disks_xml, user_template_xml, history_xml, nics_xml;