Feature #2743: onedb upgrade now has two set of migrator files, for shared and local tables

install.sh

@@ -227,6 +227,8 @@ LIB_DIRS="$LIB_LOCATION/ruby \
           $LIB_LOCATION/ruby/cloud/marketplace \
           $LIB_LOCATION/ruby/cloud/CloudAuth \
           $LIB_LOCATION/ruby/onedb \
+          $LIB_LOCATION/ruby/onedb/shared \
+          $LIB_LOCATION/ruby/onedb/local \
           $LIB_LOCATION/ruby/vendors \
           $LIB_LOCATION/ruby/vendors/rbvmomi \
           $LIB_LOCATION/ruby/vendors/rbvmomi/lib \
@@ -394,7 +396,9 @@ INSTALL_FILES=(
     MAD_RUBY_LIB_FILES:$VAR_LOCATION/remotes
     MAD_SH_LIB_FILES:$LIB_LOCATION/sh
     MAD_SH_LIB_FILES:$VAR_LOCATION/remotes
-    ONEDB_MIGRATOR_FILES:$LIB_LOCATION/ruby/onedb
+    ONEDB_FILES:$LIB_LOCATION/ruby/onedb
+    ONEDB_SHARED_MIGRATOR_FILES:$LIB_LOCATION/ruby/onedb/shared
+    ONEDB_LOCAL_MIGRATOR_FILES:$LIB_LOCATION/ruby/onedb/local
     MADS_LIB_FILES:$LIB_LOCATION/mads
     IM_PROBES_FILES:$VAR_LOCATION/remotes/im
     IM_PROBES_KVM_FILES:$VAR_LOCATION/remotes/im/kvm.d
@@ -1041,43 +1045,48 @@ DATASTORE_DRIVER_CEPH_SCRIPTS="src/datastore_mad/remotes/ceph/cp \
 #-------------------------------------------------------------------------------
 # Migration scripts for onedb command, to be installed under $LIB_LOCATION
 #-------------------------------------------------------------------------------
-ONEDB_MIGRATOR_FILES="src/onedb/2.0_to_2.9.80.rb \
-                      src/onedb/2.9.80_to_2.9.85.rb \
-                      src/onedb/2.9.85_to_2.9.90.rb \
-                      src/onedb/2.9.90_to_3.0.0.rb \
-                      src/onedb/3.0.0_to_3.1.0.rb \
-                      src/onedb/3.1.0_to_3.1.80.rb \
-                      src/onedb/3.1.80_to_3.2.0.rb \
-                      src/onedb/3.2.0_to_3.2.1.rb \
-                      src/onedb/3.2.1_to_3.3.0.rb \
-                      src/onedb/3.3.0_to_3.3.80.rb \
-                      src/onedb/3.3.80_to_3.4.0.rb \
-                      src/onedb/3.4.0_to_3.4.1.rb \
-                      src/onedb/3.4.1_to_3.5.80.rb \
-                      src/onedb/3.5.80_to_3.6.0.rb \
-                      src/onedb/3.6.0_to_3.7.80.rb \
-                      src/onedb/3.7.80_to_3.8.0.rb \
-                      src/onedb/3.8.0_to_3.8.1.rb \
-                      src/onedb/3.8.1_to_3.8.2.rb \
-                      src/onedb/3.8.2_to_3.8.3.rb \
-                      src/onedb/3.8.3_to_3.8.4.rb \
-                      src/onedb/3.8.4_to_3.8.5.rb \
-                      src/onedb/3.8.5_to_3.9.80.rb \
-                      src/onedb/3.9.80_to_3.9.90.rb \
-                      src/onedb/3.9.90_to_4.0.0.rb \
-                      src/onedb/4.0.0_to_4.0.1.rb \
-                      src/onedb/4.0.1_to_4.1.80.rb \
-                      src/onedb/4.1.80_to_4.2.0.rb \
-                      src/onedb/4.2.0_to_4.3.80.rb \
-                      src/onedb/4.3.80_to_4.3.85.rb \
-                      src/onedb/4.3.85_to_4.3.90.rb \
-                      src/onedb/4.3.90_to_4.4.0.rb \
-                      src/onedb/4.4.0_to_4.4.1.rb \
-                      src/onedb/4.4.1_to_4.5.80.rb \
-                      src/onedb/fsck.rb \
-                      src/onedb/import_slave.rb \
-                      src/onedb/onedb.rb \
-                      src/onedb/onedb_backend.rb"
+
+
+ONEDB_FILES="src/onedb/fsck.rb \
+            src/onedb/import_slave.rb \
+            src/onedb/onedb.rb \
+            src/onedb/onedb_backend.rb"
+
+ONEDB_SHARED_MIGRATOR_FILES="src/onedb/shared/2.0_to_2.9.80.rb \
+                             src/onedb/shared/2.9.80_to_2.9.85.rb \
+                             src/onedb/shared/2.9.85_to_2.9.90.rb \
+                             src/onedb/shared/2.9.90_to_3.0.0.rb \
+                             src/onedb/shared/3.0.0_to_3.1.0.rb \
+                             src/onedb/shared/3.1.0_to_3.1.80.rb \
+                             src/onedb/shared/3.1.80_to_3.2.0.rb \
+                             src/onedb/shared/3.2.0_to_3.2.1.rb \
+                             src/onedb/shared/3.2.1_to_3.3.0.rb \
+                             src/onedb/shared/3.3.0_to_3.3.80.rb \
+                             src/onedb/shared/3.3.80_to_3.4.0.rb \
+                             src/onedb/shared/3.4.0_to_3.4.1.rb \
+                             src/onedb/shared/3.4.1_to_3.5.80.rb \
+                             src/onedb/shared/3.5.80_to_3.6.0.rb \
+                             src/onedb/shared/3.6.0_to_3.7.80.rb \
+                             src/onedb/shared/3.7.80_to_3.8.0.rb \
+                             src/onedb/shared/3.8.0_to_3.8.1.rb \
+                             src/onedb/shared/3.8.1_to_3.8.2.rb \
+                             src/onedb/shared/3.8.2_to_3.8.3.rb \
+                             src/onedb/shared/3.8.3_to_3.8.4.rb \
+                             src/onedb/shared/3.8.4_to_3.8.5.rb \
+                             src/onedb/shared/3.8.5_to_3.9.80.rb \
+                             src/onedb/shared/3.9.80_to_3.9.90.rb \
+                             src/onedb/shared/3.9.90_to_4.0.0.rb \
+                             src/onedb/shared/4.0.0_to_4.0.1.rb \
+                             src/onedb/shared/4.0.1_to_4.1.80.rb \
+                             src/onedb/shared/4.1.80_to_4.2.0.rb \
+                             src/onedb/shared/4.2.0_to_4.3.80.rb \
+                             src/onedb/shared/4.3.80_to_4.3.85.rb \
+                             src/onedb/shared/4.3.85_to_4.3.90.rb \
+                             src/onedb/shared/4.3.90_to_4.4.0.rb \
+                             src/onedb/shared/4.4.0_to_4.4.1.rb \
+                             src/onedb/shared/4.4.1_to_4.5.80.rb"
+
+ONEDB_LOCAL_MIGRATOR_FILES=""
 
 #-------------------------------------------------------------------------------
 # Configuration files for OpenNebula, to be installed under $ETC_LOCATION

src/onedb/onedb.rb

@@ -133,64 +133,52 @@ class OneDB
     # max_version is ignored for now, as this is the first onedb release.
     # May be used in next releases
     def upgrade(max_version, ops)
-        ret = @backend.read_db_version
+        db_version = @backend.read_db_version
 
         if ops[:verbose]
-            pretty_print_db_version(ret)
+            pretty_print_db_version(db_version)
 
             puts ""
         end
 
-        # TODO: different upgrade path for slave/master database tables
-
-        matches = Dir.glob("#{RUBY_LIB_LOCATION}/onedb/#{ret[:version]}_to_*.rb")
-
-        if ( matches.size > 0 )
-            # At least one upgrade will be executed, make DB backup
-            backup(ops[:backup], ops)
-        end
+        backup(ops[:backup], ops)
 
         begin
-            result = nil
-            i = 0
-
             timea = Time.now
 
-            while ( matches.size > 0 )
-                if ( matches.size > 1 )
-                    raise "There are more than one file that match \
-                            \"#{RUBY_LIB_LOCATION}/onedb/#{ret[:version]}_to_*.rb\""
+            # Upgrade shared (federation) tables, only for standalone and master
+            if !db_version[:is_slave]
+                puts
+                puts ">>> Running migrators for shared tables"
+
+                dir_prefix = "#{RUBY_LIB_LOCATION}/onedb/shared"
+
+                result = apply_migrators(dir_prefix, db_version[:version], ops)
+
+                # Modify db_versioning table
+                if result != nil
+                    @backend.update_db_version(db_version[:version])
+                else
+                    puts "Database already uses version #{db_version[:version]}"
                 end
-
-                file = matches[0]
-
-                puts "  > Running migrator #{file}" if ops[:verbose]
-
-                time0 = Time.now
-
-                load(file)
-                @backend.extend Migrator
-                result = @backend.up
-
-                time1 = Time.now
-
-                if !result
-                    raise "Error while upgrading from #{ret[:version]} to " <<
-                          " #{@backend.db_version}"
-                end
-
-                puts "  > Done in #{"%0.02f" % (time1 - time0).to_s}s" if ops[:verbose]
-                puts "" if ops[:verbose]
-
-                matches = Dir.glob(
-                    "#{RUBY_LIB_LOCATION}/onedb/#{@backend.db_version}_to_*.rb")
             end
 
+            db_version = @backend.read_db_version
+
+            # Upgrade local tables, for standalone, master, and slave
+
+            puts
+            puts ">>> Running migrators for local tables"
+
+            dir_prefix = "#{RUBY_LIB_LOCATION}/onedb/local"
+
+            result = apply_migrators(dir_prefix, db_version[:local_version], ops)
+
             # Modify db_versioning table
             if result != nil
-                @backend.update_db_version(ret[:version])
+                @backend.update_local_db_version(db_version[:local_version])
             else
-                puts "Database already uses version #{ret[:version]}"
+                puts "Database already uses version #{db_version[:local_version]}"
             end
 
             timeb = Time.now
@@ -213,6 +201,45 @@ class OneDB
         end
     end
 
+    def apply_migrators(prefix, db_version, ops)
+        result = nil
+        i = 0
+
+        matches = Dir.glob("#{prefix}/#{db_version}_to_*.rb")
+
+        while ( matches.size > 0 )
+            if ( matches.size > 1 )
+                raise "There are more than one file that match \
+                        \"#{prefix}/#{db_version}_to_*.rb\""
+            end
+
+            file = matches[0]
+
+            puts "  > Running migrator #{file}" if ops[:verbose]
+
+            time0 = Time.now
+
+            load(file)
+            @backend.extend Migrator
+            result = @backend.up
+
+            time1 = Time.now
+
+            if !result
+                raise "Error while upgrading from #{db_version} to " <<
+                      " #{@backend.db_version}"
+            end
+
+            puts "  > Done in #{"%0.02f" % (time1 - time0).to_s}s" if ops[:verbose]
+            puts "" if ops[:verbose]
+
+            matches = Dir.glob(
+                "#{prefix}/#{@backend.db_version}_to_*.rb")
+        end
+
+        return result
+    end
+
     def fsck(ops)
         ret = @backend.read_db_version
 

src/onedb/onedb_backend.rb

@@ -42,14 +42,19 @@ class OneDBBacKEnd
                 ret[:comment]   = row[:comment]
             end
 
+            ret[:local_version]   = ret[:version]
+            ret[:local_timestamp] = ret[:timestamp]
+            ret[:local_comment]   = ret[:comment]
+            ret[:is_slave]        = false
+
             begin
                @db.fetch("SELECT version, timestamp, comment, is_slave FROM "+
                         "local_db_versioning WHERE oid=(SELECT MAX(oid) "+
                         "FROM local_db_versioning)") do |row|
-                   ret[:local_version]   = row[:version]
-                   ret[:local_timestamp] = row[:timestamp]
-                   ret[:local_comment]   = row[:comment]
-                   ret[:is_slave]        = row[:is_slave]
+                    ret[:local_version]   = row[:version]
+                    ret[:local_timestamp] = row[:timestamp]
+                    ret[:local_comment]   = row[:comment]
+                    ret[:is_slave]        = row[:is_slave]
                end 
             rescue Exception => e
                 if e.class == Sequel::DatabaseConnectionError
@@ -127,6 +132,37 @@ class OneDBBacKEnd
         puts comment
     end
 
+    def update_local_db_version(version)
+        comment = "Database migrated from #{version} to #{db_version}"+
+                  " (#{one_version}) by onedb command."
+
+        max_oid = nil
+        @db.fetch("SELECT MAX(oid) FROM local_db_versioning") do |row|
+            max_oid = row[:"MAX(oid)"].to_i
+        end
+
+        max_oid = 0 if max_oid.nil?
+
+        is_slave = 0
+
+        @db.fetch("SELECT is_slave FROM local_db_versioning "<<
+                  "WHERE oid=#{max_oid}") do |row|
+            is_slave = row[:is_slave]
+        end
+
+        @db.run(
+            "INSERT INTO local_db_versioning (oid, version, timestamp, comment, is_slave) "<<
+            "VALUES ("                                                     <<
+                "#{max_oid+1}, "                                           <<
+                "'#{db_version}', "                                        <<
+                "#{Time.new.to_i}, "                                       <<
+                "'#{comment}',"                                            <<
+                "#{is_slave})"
+        )
+
+        puts comment
+    end
+
     def db()
         return @db
     end