Upgrade Database to 5.6.0. Better migrator for SNAPSHOTS

include/Nebula.h

@@ -366,7 +366,7 @@ public:
      */
     static string shared_db_version()
     {
-        return "5.5.80";
+        return "5.6.0";
     }
 
     /**
@@ -375,7 +375,7 @@ public:
      */
     static string local_db_version()
     {
-        return "5.5.80";
+        return "5.6.0";
     }
 
     /**

install.sh

@@ -1313,7 +1313,8 @@ ONEDB_SHARED_MIGRATOR_FILES="src/onedb/shared/2.0_to_2.9.80.rb \
                              src/onedb/shared/5.2.0_to_5.3.80.rb \
                              src/onedb/shared/5.3.80_to_5.4.0.rb \
                              src/onedb/shared/5.4.0_to_5.4.1.rb \
-                             src/onedb/shared/5.4.1_to_5.5.80.rb"
+                             src/onedb/shared/5.4.1_to_5.5.80.rb \
+                             src/onedb/shared/5.5.80_to_5.6.0.rb"
 
 ONEDB_LOCAL_MIGRATOR_FILES="src/onedb/local/4.5.80_to_4.7.80.rb \
                             src/onedb/local/4.7.80_to_4.9.80.rb \
@@ -1325,8 +1326,8 @@ ONEDB_LOCAL_MIGRATOR_FILES="src/onedb/local/4.5.80_to_4.7.80.rb \
                             src/onedb/local/4.90.0_to_5.3.80.rb \
                             src/onedb/local/5.3.80_to_5.4.0.rb \
                             src/onedb/local/5.4.0_to_5.4.1.rb \
-                            src/onedb/local/5.4.1_to_5.5.80.rb"
+                            src/onedb/local/5.4.1_to_5.5.80.rb \
-
+                            src/onedb/local/5.5.80_to_5.6.0.rb"
 
 ONEDB_PATCH_FILES="src/onedb/patches/4.14_monitoring.rb \
                    src/onedb/patches/history_times.rb"

src/onedb/database_schema.rb

@@ -21,27 +21,40 @@ class OneDBBacKEnd
         cluster_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
             "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " <<
             "group_u INTEGER, other_u INTEGER, UNIQUE(name)",
+
         cluster_datastore_relation: "cid INTEGER, oid INTEGER, " <<
             "PRIMARY KEY(cid, oid)",
+
         cluster_network_relation: "cid INTEGER, oid INTEGER, " <<
             "PRIMARY KEY(cid, oid)",
+
         datastore_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
             "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " <<
             "group_u INTEGER, other_u INTEGER",
+
         cluster_vnc_bitmap: "id INTEGER, map LONGTEXT, PRIMARY KEY(id)",
+
         host_pool: "oid INTEGER PRIMARY KEY, " <<
             "name VARCHAR(128), body MEDIUMTEXT, state INTEGER, " <<
             "last_mon_time INTEGER, uid INTEGER, gid INTEGER, " <<
             "owner_u INTEGER, group_u INTEGER, other_u INTEGER, " <<
             "cid INTEGER",
+
         image_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
             "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " <<
             "group_u INTEGER, other_u INTEGER, UNIQUE(name,uid)",
+
         network_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
             "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " <<
             "group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid)",
+
         user_quotas: "user_oid INTEGER PRIMARY KEY, body MEDIUMTEXT",
-        group_quotas: "group_oid INTEGER PRIMARY KEY, body MEDIUMTEXT"
+
+        group_quotas: "group_oid INTEGER PRIMARY KEY, body MEDIUMTEXT",
+
+        document_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " << 
+            "body MEDIUMTEXT, type INTEGER, uid INTEGER, gid INTEGER, " << 
+            "owner_u INTEGER, group_u INTEGER, other_u INTEGER"
     }
 
     VERSION_SCHEMA = {
@@ -50,27 +63,33 @@ class OneDBBacKEnd
                 "body MEDIUMTEXT, uid INTEGER, gid INTEGER, " <<
                 "owner_u INTEGER, group_u INTEGER, other_u INTEGER, " <<
                 "UNIQUE(name,uid)",
+
             host_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
                 "body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, " <<
                 "uid INTEGER, gid INTEGER, owner_u INTEGER, " <<
                 "group_u INTEGER, other_u INTEGER, cid INTEGER",
+
             vm_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
                 "body MEDIUMTEXT, uid INTEGER, gid INTEGER, " <<
                 "last_poll INTEGER, state INTEGER, lcm_state INTEGER, " <<
                 "owner_u INTEGER, group_u INTEGER, other_u INTEGER",
+
             logdb: "log_index INTEGER PRIMARY KEY, term INTEGER, " <<
                 "sqlcmd MEDIUMTEXT, timestamp INTEGER, fed_index INTEGER",
+
             history: "vid INTEGER, seq INTEGER, body MEDIUMTEXT, " <<
                      "stime INTEGER, etime INTEGER, PRIMARY KEY(vid,seq)",
+
             zone_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " <<
                        "body MEDIUMTEXT, uid INTEGER, gid INTEGER, " <<
                        "owner_u INTEGER, group_u INTEGER, other_u INTEGER, " <<
                        "UNIQUE(name)"
         },
-        "5.4.0" => {}
+        "5.4.0" => {},
+        "5.6.0" => {}
     }
 
-    LATEST_DB_VERSION = "5.4.0"
+    LATEST_DB_VERSION = "5.6.0"
 
     def get_schema(type, version = nil)
         if !version

src/onedb/local/5.4.1_to_5.5.80.rb

@@ -46,13 +46,32 @@ module Migrator
 
     private
 
+    def xpath(doc, sxpath)
+        element = doc.root.at_xpath(sxpath)
+        if !element.nil?
+            element.text
+        else
+            ""
+        end
+    end
+
+    def delete_element(doc, element)
+        doc.search("//#{element}").each do |node|
+            node.remove
+        end
+    end
+
     def feature_1377()
+        @db.run "DROP TABLE IF EXISTS old_document_pool;"
         @db.run "ALTER TABLE document_pool RENAME TO old_document_pool;"
-        @db.run "CREATE TABLE document_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, type INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
+
+        create_table(:document_pool)
 
         @db.transaction do
             @db.fetch("SELECT * FROM old_document_pool") do |row|
-                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
+                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING) { |c| 
+                    c.default_xml.noblanks
+                }
 
                 delete_element(doc, "LOCK")
 
@@ -71,22 +90,6 @@ module Migrator
         end
 
       @db.run "DROP TABLE old_document_pool;"
-
-    end
-
-    def xpath(doc, sxpath)
-        element = doc.root.at_xpath(sxpath)
-        if !element.nil?
-            element.text
-        else
-            ""
-        end
-    end
-
-    def delete_element(doc, element)
-        doc.search("//#{element}").each do |node|
-            node.remove
-        end
     end
 
     def feature_1709()
@@ -117,7 +120,8 @@ module Migrator
         regions = az_conf["regions"]
 
         if !regions
-            STDERR.puts "  > Regions not found in Az config file, skipping migration"
+            STDERR.puts "  > Regions not found in Az config file, " << 
+                "skipping migration"
             return
         end
 
@@ -167,20 +171,34 @@ module Migrator
     end
 
     def bug_2189()
+        @db.run "DROP TABLE IF EXISTS old_image_pool;"
         @db.run "ALTER TABLE image_pool RENAME TO old_image_pool;"
-        @db.run "CREATE TABLE image_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid) );"
+
+        create_table(:image_pool)
 
         @db.transaction do
             @db.fetch("SELECT * FROM old_image_pool") do |row|
-                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
+                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){
+                    |c| c.default_xml.noblanks
+                }
 
-                if ( !doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max.nil? )
+                max = doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max
-                    next_snapshot = doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max.text.to_i + 1
+
+                if max
+                    next_snapshot = max.text.to_i + 1
                 else
                     next_snapshot = 0
                 end
 
-                doc.xpath("//SNAPSHOTS").first.add_child(doc.create_element("NEXT_SNAPSHOT")).content = next_snapshot
+                sxml = doc.xpath("//SNAPSHOTS")
+
+                if !sxml
+                    ns = doc.create_element("NEXT_SNAPSHOT")
+
+                    ns.content = next_snapshot
+
+                    sxml = sxml.first.add_child(ns)
+                end
 
                 @db[:image_pool].insert(
                     :oid      => row[:oid],
@@ -196,20 +214,34 @@ module Migrator
 
         @db.run "DROP TABLE old_image_pool;"
 
+        @db.run "DROP TABLE IF EXISTS old_vm_pool;"
         @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;"
-        @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);"
+
+        create_table(:vm_pool)
 
         @db.transaction do
             @db.fetch("SELECT * FROM old_vm_pool") do |row|
-                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks}
+                doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){ |c| 
+                    c.default_xml.noblanks
+                }
 
-                if ( !doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max.nil? )
+                max = doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max
-                    next_snapshot = doc.xpath("//SNAPSHOTS/SNAPSHOT/ID").max.text.to_i + 1
+
+                if max
+                    next_snapshot = max.text.to_i + 1
                 else
                     next_snapshot = 0
                 end
 
-                doc.xpath("//SNAPSHOTS").first.add_child(doc.create_element("NEXT_SNAPSHOT")).content = next_snapshot
+                sxml = doc.xpath("//SNAPSHOTS")
+
+                if !sxml
+                    ns = doc.create_element("NEXT_SNAPSHOT")
+
+                    ns.content = next_snapshot
+
+                    sxml = sxml.first.add_child(ns)
+                end
 
                 @db[:vm_pool].insert(
                     :oid        => row[:oid],
@@ -227,6 +259,5 @@ module Migrator
         end
 
         @db.run "DROP TABLE old_vm_pool;"
-
     end
 end

src/onedb/local/5.5.80_to_5.6.0.rb

@@ -0,0 +1,41 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2018, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+
+require 'set'
+require 'base64'
+require 'zlib'
+require 'pathname'
+require 'yaml'
+require 'opennebula'
+
+$: << File.dirname(__FILE__)
+
+include OpenNebula
+
+module Migrator
+    def db_version
+        "5.6.0"
+    end
+
+    def one_version
+        "OpenNebula 5.6.0"
+    end
+
+    def up
+        return true
+    end
+end

src/onedb/shared/5.5.80_to_5.6.0.rb

@@ -0,0 +1,29 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2018, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+module Migrator
+    def db_version
+        "5.6.0"
+    end
+
+    def one_version
+        "OpenNebula 5.6.0"
+    end
+
+    def up
+        return true
+    end
+end