From 431cba56a3fdc1141b6899349621010b5b8478a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Gonz=C3=A1lez?= Date: Tue, 19 Feb 2019 20:31:51 +0100 Subject: [PATCH] F #2966 Add applied field to logdb table (#2967) --- src/onedb/database_schema.rb | 6 ++- src/onedb/local/5.6.0_to_5.7.80.rb | 24 ++++++++++-- src/sql/LogDB.cc | 61 ++++++++++++++++++++---------- 3 files changed, 64 insertions(+), 27 deletions(-) diff --git a/src/onedb/database_schema.rb b/src/onedb/database_schema.rb index af6e9a0c7e..10ea4b8934 100644 --- a/src/onedb/database_schema.rb +++ b/src/onedb/database_schema.rb @@ -99,9 +99,11 @@ class OneDBBacKEnd "owner_u INTEGER, group_u INTEGER, other_u INTEGER", index_sql: ["CREATE INDEX state_oid_idx ON vm_pool (state, oid);", - "CREATE FULLTEXT INDEX ftidx ON vm_pool(search_token)"], + "CREATE FULLTEXT INDEX ftidx ON vm_pool(search_token);", + "CREATE INDEX applied_idx ON logdb (applied);"], - index_sqlite: ["CREATE INDEX state_oid_idx ON vm_pool (state, oid);"] + index_sqlite: ["CREATE INDEX state_oid_idx ON vm_pool (state, oid);", + "CREATE INDEX applied_idx ON logdb (applied);"] } } diff --git a/src/onedb/local/5.6.0_to_5.7.80.rb b/src/onedb/local/5.6.0_to_5.7.80.rb index 0781aa6109..eb16cba0f3 100644 --- a/src/onedb/local/5.6.0_to_5.7.80.rb +++ b/src/onedb/local/5.6.0_to_5.7.80.rb @@ -40,7 +40,8 @@ module Migrator feature_2253 feature_2489_2671 feature_826 - create_idxs + feature_2966 + create_idxs #MUST be the last one true end @@ -275,9 +276,9 @@ module Migrator } xml.MONITORING { - xml.CPU body.root.xpath('MONITORING/CPU').text - xml.MEMORY body.root.xpath('MONITORING/MEMORY').text - xml.STATE body.root.xpath('MONITORING/STATE').text + xml.CPU body.root.xpath('MONITORING/CPU').text unless body.root.xpath('MONITORING/CPU').text.empty? + xml.MEMORY body.root.xpath('MONITORING/MEMORY').text unless body.root.xpath('MONITORING/MEMORY').text.empty? + xml.STATE body.root.xpath('MONITORING/STATE').text unless body.root.xpath('MONITORING/STATE').text.empty? } xml.USER_TEMPLATE { @@ -402,4 +403,19 @@ module Migrator create_idx(:index_sql, db_version) end end + + def feature_2966 + @db.run "ALTER TABLE logdb ADD applied BOOLEAN DEFAULT 1;" + @db.run "UPDATE logdb SET applied = 0 WHERE timestamp = 0;" + + if @db.adapter_scheme == :sqlite + @db.run "DROP INDEX IF EXISTS timestamp_idx" + else + begin + @db.run "ALTER TABLE logdb DROP INDEX timestamp_idx;" + rescue Sequel::DatabaseError + end + @db.run "ALTER TABLE logdb ALTER COLUMN applied DROP DEFAULT" + end + end end diff --git a/src/sql/LogDB.cc b/src/sql/LogDB.cc index 3a07a41d1a..44c4050859 100644 --- a/src/sql/LogDB.cc +++ b/src/sql/LogDB.cc @@ -25,11 +25,11 @@ const char * LogDB::table = "logdb"; -const char * LogDB::db_names = "log_index, term, sqlcmd, timestamp, fed_index"; +const char * LogDB::db_names = "log_index, term, sqlcmd, timestamp, fed_index, applied"; const char * LogDB::db_bootstrap = "CREATE TABLE IF NOT EXISTS " "logdb (log_index INTEGER PRIMARY KEY, term INTEGER, sqlcmd MEDIUMTEXT, " - "timestamp INTEGER, fed_index INTEGER)"; + "timestamp INTEGER, fed_index INTEGER, applied BOOLEAN)"; /* -------------------------------------------------------------------------- */ @@ -46,7 +46,7 @@ int LogDB::bootstrap(SqlDB *_db) rc += _db->exec_local_wr(oss); - oss.str("CREATE INDEX timestamp_idx on logdb (timestamp);"); + oss.str("CREATE INDEX applied_idx on logdb (applied);"); rc += _db->exec_local_wr(oss); @@ -319,9 +319,11 @@ int LogDB::insert(int index, int term, const std::string& sql, time_t tstamp, oss << "INSERT"; } + bool applied = tstamp != 0; + oss << " INTO " << table << " ("<< db_names <<") VALUES (" << index << "," << term << "," << "'" << sql_db << "'," << tstamp - << "," << fed_index << ")"; + << "," << fed_index << "," << applied << ")"; int rc = db->exec_wr(oss); @@ -590,8 +592,10 @@ int LogDB::purge_log() empty_cb cb; multiple_cb cb_info; + single_cb cb_min_idx; std::vector maxmin_i; std::vector maxmin_e; + string min_idx; int rc = 0; int frc = 0; @@ -611,17 +615,25 @@ int LogDB::purge_log() /* ---------------------------------------------------------------------- */ /* Non-federated records. Keep last log_retention records */ /* ---------------------------------------------------------------------- */ + + oss.str(""); + oss << " SELECT MIN(i.log_index) FROM (" + << " SELECT log_index FROM logdb WHERE fed_index = -1 AND" + << " applied = 1 AND log_index >= 0 " + << " ORDER BY log_index DESC LIMIT " << log_retention + << " ) AS i"; + + cb_min_idx.set_callback(&min_idx); + + db->exec_rd(oss, &cb_min_idx); + + cb_min_idx.unset_callback(); + cb.set_affected_rows(0); oss.str(""); - oss << "DELETE FROM logdb WHERE timestamp > 0 AND log_index >= 0 " - << "AND fed_index = -1 AND log_index < (" - << " SELECT MIN(i.log_index) FROM (" - << " SELECT log_index FROM logdb WHERE fed_index = -1 AND" - << " timestamp > 0 AND log_index >= 0 " - << " ORDER BY log_index DESC LIMIT " << log_retention - << " ) AS i" - << ")"; + oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 " + << "AND fed_index = -1 AND log_index < " << min_idx; if ( db->limit_support() ) { @@ -645,7 +657,7 @@ int LogDB::purge_log() cb_info.unset_callback(); oss.str(""); - oss << "Purging obsolete LogDB records: " << rc << " records purged. Log state: " + oss << "Purging obsolete LogDB records: " << rc << " records purged. Log state: " << maxmin_i[0] << "," << maxmin_i[1] << " - " << maxmin_e[0] << "," << maxmin_e[1]; NebulaLog::log("DBM", Log::INFO, oss); @@ -667,17 +679,24 @@ int LogDB::purge_log() return rc; } + oss.str(""); + oss << " SELECT MIN(i.log_index) FROM (" + << " SELECT log_index FROM logdb WHERE fed_index != -1 AND" + << " applied = 1 AND log_index >= 0 " + << " ORDER BY log_index DESC LIMIT " << log_retention + << " ) AS i"; + + cb_min_idx.set_callback(&min_idx); + + db->exec_rd(oss, &cb_min_idx); + + cb_min_idx.unset_callback(); + cb.set_affected_rows(0); oss.str(""); - oss << "DELETE FROM logdb WHERE timestamp > 0 AND log_index >= 0 " - << "AND fed_index != -1 AND log_index < (" - << " SELECT MIN(i.log_index) FROM (" - << " SELECT log_index FROM logdb WHERE fed_index != -1 AND" - << " timestamp > 0 AND log_index >= 0 " - << " ORDER BY log_index DESC LIMIT " << log_retention - << " ) AS i" - << ")"; + oss << "DELETE FROM logdb WHERE applied = 1 AND log_index >= 0 " + << "AND fed_index != -1 AND log_index < " << min_idx; if ( db->limit_support() ) {