diff --git a/include/Attribute.h b/include/Attribute.h index 59beaa71f9..da9e757590 100644 --- a/include/Attribute.h +++ b/include/Attribute.h @@ -79,8 +79,6 @@ public: virtual void to_json(std::ostringstream& s) const = 0; - virtual void to_token(std::ostringstream& s) const = 0; - virtual void to_xml(std::ostringstream& s, const std::map> &hidden) const = 0; @@ -203,19 +201,6 @@ public: one_util::escape_json(attribute_value, s); } - void to_token(std::ostringstream& s) const override - { - if (attribute_name.empty() || attribute_value.empty()) - { - return; - } - - one_util::escape_token(attribute_name, s); - s << "="; - one_util::escape_token(attribute_value, s); - s << std::endl; - } - /** * Builds a new attribute from a string. */ @@ -456,8 +441,6 @@ public: void to_json(std::ostringstream& s) const override; - void to_token(std::ostringstream& s) const override; - /** * Builds a new attribute from a string of the form: * "VAL_NAME_1=VAL_VALUE_1,...,VAL_NAME_N=VAL_VALUE_N". diff --git a/include/ExtendedAttribute.h b/include/ExtendedAttribute.h index d5dce7cf68..fd41c11dda 100644 --- a/include/ExtendedAttribute.h +++ b/include/ExtendedAttribute.h @@ -93,11 +93,6 @@ public: return va->to_json(s); }; - void to_token(std::ostringstream& s) const override - { - return va->to_token(s); - }; - void to_xml(std::ostringstream& s, const std::map> &hidden) const override { diff --git a/include/History.h b/include/History.h index 75f7849937..eccef5da53 100644 --- a/include/History.h +++ b/include/History.h @@ -192,8 +192,6 @@ private: std::string& to_json(std::string& json) const; - std::string& to_token(std::string& text) const; - /** * Rebuilds the object from an xml node * @param node The xml node pointer diff --git a/include/SqlDB.h b/include/SqlDB.h index 4c0b612069..9bef671264 100644 --- a/include/SqlDB.h +++ b/include/SqlDB.h @@ -45,7 +45,7 @@ public: { MULTIPLE_VALUE, // syntax INSERT VALUES (data), (data), (data) LIMIT, // LIMIT in queries with DELETE and UPDATE - FTS, // Full Text Search + JSON_QUERY, // JSON queries Search COMPARE_BINARY // Use BINARY for comparing name in DB }; @@ -198,7 +198,7 @@ protected: { {SqlFeature::MULTIPLE_VALUE, false}, {SqlFeature::LIMIT, false}, - {SqlFeature::FTS, false}, + {SqlFeature::JSON_QUERY, false}, {SqlFeature::COMPARE_BINARY, false} }; diff --git a/include/Template.h b/include/Template.h index 748c705ea5..de699d827d 100644 --- a/include/Template.h +++ b/include/Template.h @@ -199,8 +199,6 @@ public: std::string& to_json(std::string& xml) const; - std::string& to_token(std::string& xml) const; - /** * Writes the template in a plain text string * @param str string that hold the template representation diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index deadd685c7..c613d7f995 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -1965,8 +1965,6 @@ private: std::string& to_json(std::string& json) const; - std::string& to_token(std::string& text) const; - // ------------------------------------------------------------------------- // Attribute Parser // ------------------------------------------------------------------------- diff --git a/src/cli/one_helper/onevm_helper.rb b/src/cli/one_helper/onevm_helper.rb index b3c27e059e..4bcce265c3 100644 --- a/src/cli/one_helper/onevm_helper.rb +++ b/src/cli/one_helper/onevm_helper.rb @@ -160,7 +160,8 @@ class OneVMHelper < OpenNebulaHelper::OneHelper SEARCH = { :name => 'search', :large => '--search search', - :description => 'query in KEY=VALUE format', + :description => 'Query in PATH=VALUE format. For example: ' \ + 'onevm list --search "VM.NAME=abc&VM.TEMPLATE.DISK[*].IMAGE=db1"', :format => String } diff --git a/src/common/Attribute.cc b/src/common/Attribute.cc index 28ef8608b2..c2fc83f9a8 100644 --- a/src/common/Attribute.cc +++ b/src/common/Attribute.cc @@ -147,22 +147,6 @@ void VectorAttribute::to_json(std::ostringstream& s) const s << "}"; } -void VectorAttribute::to_token(std::ostringstream& s) const -{ - for (auto it=attribute_value.begin(); it!=attribute_value.end(); it++) - { - if (it->first.empty() || it->second.empty()) - { - continue; - } - - one_util::escape_token(it->first, s); - s << "="; - one_util::escape_token(it->second, s); - s << std::endl; - } -} - /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ diff --git a/src/host/HostMonitoringTemplate.cc b/src/host/HostMonitoringTemplate.cc index d764c3c3e2..165d0b32a2 100644 --- a/src/host/HostMonitoringTemplate.cc +++ b/src/host/HostMonitoringTemplate.cc @@ -127,24 +127,18 @@ int NUMAMonitoring::from_template(const Template &tmpl) unsigned long size; unsigned long fr; - ostringstream oss; - if (page->vector_value("NODE_ID", node_id) != 0) { - page->to_token(oss); - NebulaLog::warn("HMM", "Hugepage doesn't contain node ID: " - + oss.str()); + + page->marshall(",")); continue; } if (page->vector_value("SIZE", size) != 0) { - page->to_token(oss); - NebulaLog::warn("HMM", "Hugepage doesn't contain size: " - + oss.str()); + + page->marshall(",")); continue; } @@ -165,14 +159,10 @@ int NUMAMonitoring::from_template(const Template &tmpl) unsigned long used; unsigned long fr; - ostringstream oss; - if (mem->vector_value("NODE_ID", node_id) != 0) { - mem->to_token(oss); - NebulaLog::warn("HMM", "Memory node doesn't contain node ID: " - + oss.str()); + + mem->marshall(",")); continue; } diff --git a/src/onedb/database_schema.rb b/src/onedb/database_schema.rb index bb73dda492..02cf07e42d 100644 --- a/src/onedb/database_schema.rb +++ b/src/onedb/database_schema.rb @@ -106,7 +106,6 @@ class OneDBBacKEnd "body MEDIUMTEXT, uid INTEGER, gid INTEGER," << "owner_u INTEGER, group_u INTEGER, other_u INTEGER", index_sql: [{ :name =>'state_oid_idx', :table => 'vm_pool', :columns => '(state, oid)' }, - { :name =>'ftidx', :table => 'vm_pool', :columns => '(search_token)', :type => 'FULLTEXT' }, { :name =>'applied_idx', :table => 'logdb', :columns => '(applied)' }, { :name =>'fed_index_idx', :table => 'logdb', :columns => '(fed_index)' }], diff --git a/src/onedb/onedb b/src/onedb/onedb index 6e8cf69abb..e4ef206196 100755 --- a/src/onedb/onedb +++ b/src/onedb/onedb @@ -104,12 +104,6 @@ FEDERATED = { :description => 'Limit backup/restore to federated tables' } -RECREATE = { - :name => 'recreate', - :large => '--recreate', - :description => 'Delete FTS index and create a new one' -} - ############################################################################### # SQLite options ############################################################################### @@ -891,23 +885,4 @@ CommandParser::CmdParser.new(ARGV) do 0 # exit code end - - ########################################################################### - # Create and recreate FTS index in vmpool - ########################################################################### - fts_desc = <<-EOT.unindent - Create or recreate FTS index in vm pool to be able to search VMs - EOT - - command :'create-index', fts_desc, :options => [RECREATE] do - begin - helper = OneDB.new(options) - helper.fts_index(options.key?(:recreate)) - rescue StandardError => e - STDERR.puts e.message - [-1, e.message] - end - - 0 - end end diff --git a/src/onedb/onedb.rb b/src/onedb/onedb.rb index 26f50042bd..a995931205 100644 --- a/src/onedb/onedb.rb +++ b/src/onedb/onedb.rb @@ -709,17 +709,6 @@ class OneDB end end - # Create or recreate FTS index on vm_pool search_token column - # - # @param recreate [Boolean] True to delete the index and create it again - def fts_index(recreate = false) - if backend.is_a? BackEndSQLite - raise 'This is operation is not supported for sqlite backend' - end - - backend.fts_index(recreate) - end - private def one_not_running() diff --git a/src/onedb/onedb_backend.rb b/src/onedb/onedb_backend.rb index cde26ac480..1226aa6c40 100644 --- a/src/onedb/onedb_backend.rb +++ b/src/onedb/onedb_backend.rb @@ -386,23 +386,6 @@ class BackEndMySQL < OneDBBacKEnd puts "MySQL DB #{@db_name} at #{@server} restored." end - # Create or recreate FTS index on vm_pool search_token column - # - # @param recreate [Boolean] True to delete the index and create it again - def fts_index(recreate = false) - connect_db - - if recreate - @db.alter_table(:vm_pool) do - drop_index :search_token, name: 'ftidx' - end - end - - @db.alter_table(:vm_pool) do - add_full_text_index :search_token, name: 'ftidx' - end - end - def idx?(idx) query = "SHOW INDEX FROM #{idx[:table]} WHERE KEY_NAME = '#{idx[:name]}'" !@db.fetch(query).first.nil? @@ -800,16 +783,12 @@ class BackEndPostgreSQL < OneDBBacKEnd puts "PostgreSQL DB #{@db_name} at #{@server} restored." end - def fts_index(recreate = false) - raise "FTS index not supported for PostgreSQL." - end - def idx?(idx) query = "SELECT * FROM pg_indexes WHERE indexname = '#{idx[:name]}'" !@db.fetch(query).first.nil? end - def create_idx(version = nil) + def create_idx(version = nil) schema = get_schema(:index_sqlite, version) schema.each do |idx| @@ -830,7 +809,7 @@ class BackEndPostgreSQL < OneDBBacKEnd schema.each do |idx| next unless idx? idx - + query = "DROP INDEX #{idx[:name]};" @db.run query diff --git a/src/rm/RequestManagerPoolInfoFilter.cc b/src/rm/RequestManagerPoolInfoFilter.cc index 93322d7685..072444e8d2 100644 --- a/src/rm/RequestManagerPoolInfoFilter.cc +++ b/src/rm/RequestManagerPoolInfoFilter.cc @@ -295,15 +295,15 @@ void VirtualMachinePoolInfo::request_execute( int end_id = xmlrpc_c::value_int(paramList.getInt(3)); int state = xmlrpc_c::value_int(paramList.getInt(4)); - std::string fts_query; + std::string json_query; if (paramList.size() > 5) { - fts_query = xmlrpc_c::value_string(paramList.getString(5)); + json_query = xmlrpc_c::value_string(paramList.getString(5)); - if (!fts_query.empty() && !pool->supports(SqlDB::SqlFeature::FTS)) + if (!json_query.empty() && !pool->supports(SqlDB::SqlFeature::JSON_QUERY)) { - att.resp_msg = "Full text search is not supported by the SQL backend"; + att.resp_msg = "JSON query search is not supported by the SQL backend"; failure_response(INTERNAL, att); return; @@ -335,11 +335,11 @@ void VirtualMachinePoolInfo::request_execute( break; } - if (!fts_query.empty()) + if (!json_query.empty()) { - char * _fts_query = pool->escape_str(fts_query); + char * _json_query = pool->escape_str(json_query); - if ( _fts_query == 0 ) + if ( _json_query == 0 ) { att.resp_msg = "Error building search query"; @@ -352,11 +352,33 @@ void VirtualMachinePoolInfo::request_execute( and_filter << " AND "; } - and_filter << "MATCH(search_token) AGAINST ('+\""; - one_util::escape_token(_fts_query, and_filter); - and_filter << "\"' in boolean mode)"; + vector keys; - pool->free_str(_fts_query); + one_util::split(_json_query, '&', keys); + + for(const auto& key: keys) + { + vector kv; + + one_util::split(key, '=', kv); + + if (kv.size() == 1) + { + // No key specified, search whole json body + kv.push_back(kv[0]); + kv[0] = "*"; + } + + and_filter << "JSON_UNQUOTE(JSON_EXTRACT(body_json, '$." << kv[0] << "'))"; + and_filter << " LIKE '%" << kv[1] << "%'"; + + if (key != keys.back()) + { + and_filter << " AND "; + } + } + + pool->free_str(_json_query); } dump(att, filter_flag, start_id, end_id, and_filter.str(), ""); diff --git a/src/sql/MySqlDB.cc b/src/sql/MySqlDB.cc index a30e42cd45..12ba063e3a 100644 --- a/src/sql/MySqlDB.cc +++ b/src/sql/MySqlDB.cc @@ -252,10 +252,10 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, oss.clear(); //-------------------------------------------------------------------------- - // Get server information and FTS support & features + // Get server information and JSON query support & features //-------------------------------------------------------------------------- unsigned long version; - unsigned long min_fts_version; + unsigned long min_json_version; string server_info = mysql_get_server_info(db_escape_connect); @@ -271,27 +271,27 @@ MySqlDB::MySqlDB(const string& s, int p, const string& u, const string& _p, if (server_info.find("mariadb") == std::string::npos) { - min_fts_version = 50600; + min_json_version = 50708; } else { - min_fts_version = 100005; + min_json_version = 100200; } - if (version >= min_fts_version) + if (version >= min_json_version) { - NebulaLog::log("ONE", Log::INFO, "\tFTS enabled"); + NebulaLog::log("ONE", Log::INFO, "\tJSON queries enabled"); } else { - NebulaLog::log("ONE", Log::INFO, "\tFTS disabled"); + NebulaLog::log("ONE", Log::INFO, "\tJSON queries disabled"); } features = { {SqlFeature::MULTIPLE_VALUE, true}, {SqlFeature::LIMIT, true}, - {SqlFeature::FTS, version >= min_fts_version}, + {SqlFeature::JSON_QUERY, version >= min_json_version}, {SqlFeature::COMPARE_BINARY, one_util::icasecmp(_compare_binary, "YES")} }; } diff --git a/src/sql/OneDB.cc b/src/sql/OneDB.cc index e223f9444a..4609a8dad8 100644 --- a/src/sql/OneDB.cc +++ b/src/sql/OneDB.cc @@ -58,13 +58,13 @@ namespace one_db const char * vm_db_names = "oid, name, body, uid, gid, state, lcm_state, " - "owner_u, group_u, other_u, short_body, search_token"; + "owner_u, group_u, other_u, short_body, body_json"; const char * vm_db_bootstrap = "CREATE TABLE IF NOT EXISTS " "vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, " "uid INTEGER, gid INTEGER, state INTEGER, " "lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, " - "short_body MEDIUMTEXT, search_token MEDIUMTEXT"; + "short_body MEDIUMTEXT, body_json JSON)"; const char * vm_monitor_table = "vm_monitoring"; diff --git a/src/sql/PostgreSqlDB.cc b/src/sql/PostgreSqlDB.cc index 15f8acf5bd..3ff1698ae1 100644 --- a/src/sql/PostgreSqlDB.cc +++ b/src/sql/PostgreSqlDB.cc @@ -92,7 +92,7 @@ PostgreSqlDB::PostgreSqlDB(const string& _server, { {SqlFeature::MULTIPLE_VALUE, PQlibVersion() > 80200}, {SqlFeature::LIMIT, false}, - {SqlFeature::FTS, false}, + {SqlFeature::JSON_QUERY, false}, {SqlFeature::COMPARE_BINARY, false} }; } diff --git a/src/sql/SqliteDB.cc b/src/sql/SqliteDB.cc index de605b5a28..27a579bbf1 100644 --- a/src/sql/SqliteDB.cc +++ b/src/sql/SqliteDB.cc @@ -65,7 +65,7 @@ SqliteDB::SqliteDB(const string& db_name, int timeout) { {SqlFeature::MULTIPLE_VALUE, false}, {SqlFeature::LIMIT, enable_limit == 1}, - {SqlFeature::FTS, false}, + {SqlFeature::JSON_QUERY, false}, {SqlFeature::COMPARE_BINARY, false} }; } diff --git a/src/template/Template.cc b/src/template/Template.cc index 2ecdee2f38..88d572b995 100644 --- a/src/template/Template.cc +++ b/src/template/Template.cc @@ -414,6 +414,9 @@ string& Template::to_xml(string& xml, const string& extra) const string& Template::to_json(string& json) const { + // List of attributes that should be an Array (even with just 1 element) + static const std::set ARRAY_ATTRS = {"DISK", "NIC"}; + ostringstream oss; bool is_first = true; @@ -433,7 +436,8 @@ string& Template::to_json(string& json) const oss << "\"" << it->first << "\": "; - if ( attributes.count(it->first) == 1 ) + if ( attributes.count(it->first) == 1 && + ARRAY_ATTRS.count(it->first) == 0) { it->second->to_json(oss); @@ -471,19 +475,6 @@ string& Template::to_json(string& json) const return json; } -string& Template::to_token(string& str) const -{ - ostringstream os; - - for ( auto it = attributes.begin(); it!=attributes.end(); it++) - { - it->second->to_token(os); - } - - str = os.str(); - return str; -} - /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ diff --git a/src/vm/History.cc b/src/vm/History.cc index cd49f9e628..38e904efc3 100644 --- a/src/vm/History.cc +++ b/src/vm/History.cc @@ -372,26 +372,6 @@ string& History::to_json(string& json) const /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -string& History::to_token(string& text) const -{ - ostringstream oss; - - oss << "HOSTNAME="; - one_util::escape_token(hostname, oss); - oss << "\n"; - - oss << "HID=" << hid << "\n" << - "CID=" << cid << "\n" << - "DS_ID=" << ds_id << "\n"; - - text = oss.str(); - - return text; -} - -/* -------------------------------------------------------------------------- */ -/* -------------------------------------------------------------------------- */ - string& History::to_xml_short(string& xml) const { ostringstream oss; diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 7468d224c8..fe3d332993 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -642,15 +642,6 @@ int VirtualMachine::bootstrap(SqlDB * db) oss_vm << one_db::vm_db_bootstrap; - if (db->supports(SqlDB::SqlFeature::FTS)) - { - oss_vm << ", FULLTEXT ftidx(search_token))"; - } - else - { - oss_vm << ")"; - } - ostringstream oss_monit(one_db::vm_monitor_db_bootstrap); ostringstream oss_hist(one_db::history_db_bootstrap); ostringstream oss_showback(one_db::vm_showback_db_bootstrap); @@ -1952,7 +1943,7 @@ int VirtualMachine::insert_replace(SqlDB *db, bool replace, string& error_str) goto error_xml_short; } - sql_text = db->escape_str(to_token(text)); + sql_text = db->escape_str(to_json(text)); if ( sql_text == 0 ) { @@ -2034,7 +2025,7 @@ int VirtualMachine::update_search(SqlDB * db) std::ostringstream oss; std::string text; - char * sql_text = db->escape_str(to_token(text)); + char * sql_text = db->escape_str(to_json(text)); if ( sql_text == 0 ) { @@ -2043,7 +2034,7 @@ int VirtualMachine::update_search(SqlDB * db) } oss << "UPDATE " << one_db::vm_table << " SET " - << "search_token = '" << sql_text << "' " + << "body_json = '" << sql_text << "' " << "WHERE oid = " << oid; db->free_str(sql_text); @@ -2683,37 +2674,6 @@ string& VirtualMachine::to_json(string& json) const /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -string& VirtualMachine::to_token(string& text) const -{ - string template_text; - string user_template_text; - string history_text; - - ostringstream oss; - - oss << "UNAME="<< uname << "\n" - << "GNAME="<< gname << "\n" - << obj_template->to_token(template_text) << "\n" - << user_obj_template->to_token(user_template_text) - << "NAME="; - - one_util::escape_token(name, oss); - - oss << "\n"; - - if ( hasHistory() ) - { - oss << "\n" << history->to_token(history_text); - } - - text = oss.str(); - - return text; -} - -/* -------------------------------------------------------------------------- */ -/* -------------------------------------------------------------------------- */ - string& VirtualMachine::to_xml_short(string& xml) { string disks_xml, user_template_xml, history_xml, nics_xml;