diff --git a/install.sh b/install.sh index 422f501fdf..1038958f39 100755 --- a/install.sh +++ b/install.sh @@ -511,6 +511,8 @@ INSTALL_FILES=( MAD_SH_LIB_FILES:$VAR_LOCATION/remotes ONEDB_FILES:$LIB_LOCATION/ruby/onedb ONEDB_PATCH_FILES:$LIB_LOCATION/ruby/onedb/patches + ONEDB_SHARED_MIGRATOR_FILES:$LIB_LOCATION/ruby/onedb/shared + ONEDB_LOCAL_MIGRATOR_FILES:$LIB_LOCATION/ruby/onedb/local MADS_LIB_FILES:$LIB_LOCATION/mads IM_PROBES_FILES:$VAR_LOCATION/remotes/im IM_PROBES_LIB_FILES:$VAR_LOCATION/remotes/im/lib @@ -1898,6 +1900,68 @@ ONEDB_FILES="src/onedb/fsck.rb \ ONEDB_PATCH_FILES="src/onedb/patches/4.14_monitoring.rb \ src/onedb/patches/history_times.rb" +ONEDB_SHARED_MIGRATOR_FILES="src/onedb/shared/2.0_to_2.9.80.rb \ + src/onedb/shared/2.9.80_to_2.9.85.rb \ + src/onedb/shared/2.9.85_to_2.9.90.rb \ + src/onedb/shared/2.9.90_to_3.0.0.rb \ + src/onedb/shared/3.0.0_to_3.1.0.rb \ + src/onedb/shared/3.1.0_to_3.1.80.rb \ + src/onedb/shared/3.1.80_to_3.2.0.rb \ + src/onedb/shared/3.2.0_to_3.2.1.rb \ + src/onedb/shared/3.2.1_to_3.3.0.rb \ + src/onedb/shared/3.3.0_to_3.3.80.rb \ + src/onedb/shared/3.3.80_to_3.4.0.rb \ + src/onedb/shared/3.4.0_to_3.4.1.rb \ + src/onedb/shared/3.4.1_to_3.5.80.rb \ + src/onedb/shared/3.5.80_to_3.6.0.rb \ + src/onedb/shared/3.6.0_to_3.7.80.rb \ + src/onedb/shared/3.7.80_to_3.8.0.rb \ + src/onedb/shared/3.8.0_to_3.8.1.rb \ + src/onedb/shared/3.8.1_to_3.8.2.rb \ + src/onedb/shared/3.8.2_to_3.8.3.rb \ + src/onedb/shared/3.8.3_to_3.8.4.rb \ + src/onedb/shared/3.8.4_to_3.8.5.rb \ + src/onedb/shared/3.8.5_to_3.9.80.rb \ + src/onedb/shared/3.9.80_to_3.9.90.rb \ + src/onedb/shared/3.9.90_to_4.0.0.rb \ + src/onedb/shared/4.0.0_to_4.0.1.rb \ + src/onedb/shared/4.0.1_to_4.1.80.rb \ + src/onedb/shared/4.1.80_to_4.2.0.rb \ + src/onedb/shared/4.2.0_to_4.3.80.rb \ + src/onedb/shared/4.3.80_to_4.3.85.rb \ + src/onedb/shared/4.3.85_to_4.3.90.rb \ + src/onedb/shared/4.3.90_to_4.4.0.rb \ + src/onedb/shared/4.4.0_to_4.4.1.rb \ + src/onedb/shared/4.4.1_to_4.5.80.rb\ + src/onedb/shared/4.5.80_to_4.6.0.rb \ + src/onedb/shared/4.6.0_to_4.11.80.rb \ + src/onedb/shared/4.11.80_to_4.90.0.rb \ + src/onedb/shared/4.90.0_to_5.2.0.rb \ + src/onedb/shared/5.2.0_to_5.3.80.rb \ + src/onedb/shared/5.3.80_to_5.4.0.rb \ + src/onedb/shared/5.4.0_to_5.4.1.rb \ + src/onedb/shared/5.4.1_to_5.5.80.rb \ + src/onedb/shared/5.5.80_to_5.6.0.rb \ + src/onedb/shared/5.6.0_to_5.10.0.rb \ + src/onedb/shared/5.10.0_to_5.12.0.rb" + +ONEDB_LOCAL_MIGRATOR_FILES="src/onedb/local/4.5.80_to_4.7.80.rb \ + src/onedb/local/4.7.80_to_4.9.80.rb \ + src/onedb/local/4.9.80_to_4.10.3.rb \ + src/onedb/local/4.10.3_to_4.11.80.rb \ + src/onedb/local/4.11.80_to_4.13.80.rb \ + src/onedb/local/4.13.80_to_4.13.85.rb \ + src/onedb/local/4.13.85_to_4.90.0.rb \ + src/onedb/local/4.90.0_to_5.3.80.rb \ + src/onedb/local/5.3.80_to_5.4.0.rb \ + src/onedb/local/5.4.0_to_5.4.1.rb \ + src/onedb/local/5.4.1_to_5.5.80.rb \ + src/onedb/local/5.5.80_to_5.6.0.rb \ + src/onedb/local/5.6.0_to_5.7.80.rb \ + src/onedb/local/5.7.80_to_5.8.0.rb \ + src/onedb/local/5.8.0_to_5.10.0.rb \ + src/onedb/local/5.10.0_to_5.12.0.rb" + #------------------------------------------------------------------------------- # Configuration files for OpenNebula, to be installed under $ETC_LOCATION #------------------------------------------------------------------------------- diff --git a/src/onedb/local/4.10.3_to_4.11.80.rb b/src/onedb/local/4.10.3_to_4.11.80.rb new file mode 100644 index 0000000000..4ff3f1d314 --- /dev/null +++ b/src/onedb/local/4.10.3_to_4.11.80.rb @@ -0,0 +1,183 @@ +# -------------------------------------------------------------------------- # +# Copyright 2019-2020, OpenNebula Systems S.L. # +# # +# Licensed under the OpenNebula Software License # +# (the "License"); you may not use this file except in compliance with # +# the License. You may obtain a copy of the License as part of the software # +# distribution. # +# # +# See https://github.com/OpenNebula/one/blob/master/LICENSE.onsla # +# (or copy bundled with OpenNebula in /usr/share/doc/one/). # +# # +# Unless agreed to in writing, software distributed under the License is # +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # +# KIND, either express or implied. See the License for the specific language # +# governing permissions and limitations under the License. # +# -------------------------------------------------------------------------- # + +require 'nokogiri' + +module Migrator + def db_version + "4.11.80" + end + + def one_version + "OpenNebula 4.11.80" + end + + def up + + init_log_time() + + ######################################################################## + # Showback + ######################################################################## + + @db.run "CREATE TABLE vm_showback (vmid INTEGER, year INTEGER, month INTEGER, body MEDIUMTEXT, PRIMARY KEY(vmid, year, month));" + + log_time() + + ######################################################################## + # Security Groups + ######################################################################## + + oneadmin_uname = nil + + @db.fetch("SELECT name FROM user_pool WHERE oid=0") do |row| + oneadmin_uname = row[:name] + end + + if oneadmin_uname == nil + puts "Error trying to read oneadmin's user name ('SELECT name FROM user_pool WHERE oid=0')" + return false + end + + oneadmin_gname = nil + + @db.fetch("SELECT name FROM group_pool WHERE oid=0") do |row| + oneadmin_gname = row[:name] + end + + if oneadmin_gname == nil + puts "Error trying to read oneadmin's group name ('SELECT name FROM group_pool WHERE oid=0')" + return false + end + + @db.run "CREATE TABLE secgroup_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid));" + @db.run "INSERT INTO secgroup_pool VALUES(0,'default','000#{oneadmin_uname}#{oneadmin_gname}default111100100',0,0,1,1,1);" + + @db.run "INSERT INTO pool_control VALUES('secgroup_pool',99);" + + + @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;" + @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER, pid INTEGER, UNIQUE(name,uid));" + + @db.transaction do + @db.fetch("SELECT * FROM old_network_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_network_pool') + + template = doc.root.at_xpath("TEMPLATE") + + # The cleaner doc.create_cdata(txt) is not supported in + # old versions of nokogiri + template.add_child(doc.create_element("SECURITY_GROUPS")). + add_child(Nokogiri::XML::CDATA.new(doc,"0")) + + @db[:network_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :cid => row[:cid], + :pid => row[:pid]) + end + end + + @db.run "DROP TABLE old_network_pool;" + + log_time() + + ######################################################################## + # Datastore status + ######################################################################## + + @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;" + @db.run "CREATE TABLE datastore_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER, UNIQUE(name));" + + @db.transaction do + @db.fetch("SELECT * FROM old_datastore_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_datastore_pool') + + doc.root.add_child(doc.create_element("STATE")).content = "0" + + @db[:datastore_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :cid => row[:cid]) + end + end + + @db.run "DROP TABLE old_datastore_pool;" + + log_time() + + ######################################################################## + # VM previous status + ######################################################################## + + @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;" + @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + @db.run "INSERT INTO vm_pool SELECT * FROM old_vm_pool WHERE state = 6;" + + log_time() + + @db.transaction do + @db.fetch("SELECT * FROM old_vm_pool WHERE state<>6") do |row| + doc = nokogiri_doc(row[:body], 'old_vm_pool') + + ["STATE", "LCM_STATE"].each do |ename| + prev_elem = doc.root.at_xpath("PREV_#{ename}") + + if prev_elem.nil? + prev_elem = doc.root.add_child( + doc.create_element("PREV_#{ename}")) + + prev_elem.content = doc.root.at_xpath(ename).text + end + end + + @db[:vm_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :last_poll => row[:last_poll], + :state => row[:state], + :lcm_state => row[:lcm_state], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + + end + end + + @db.run "DROP TABLE old_vm_pool;" + + log_time() + + return true + end +end diff --git a/src/onedb/local/4.11.80_to_4.13.80.rb b/src/onedb/local/4.11.80_to_4.13.80.rb new file mode 100644 index 0000000000..322003ded8 --- /dev/null +++ b/src/onedb/local/4.11.80_to_4.13.80.rb @@ -0,0 +1,257 @@ +# -------------------------------------------------------------------------- # +# Copyright 2019-2020, OpenNebula Systems S.L. # +# # +# Licensed under the OpenNebula Software License # +# (the "License"); you may not use this file except in compliance with # +# the License. You may obtain a copy of the License as part of the software # +# distribution. # +# # +# See https://github.com/OpenNebula/one/blob/master/LICENSE.onsla # +# (or copy bundled with OpenNebula in /usr/share/doc/one/). # +# # +# Unless agreed to in writing, software distributed under the License is # +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # +# KIND, either express or implied. See the License for the specific language # +# governing permissions and limitations under the License. # +# -------------------------------------------------------------------------- # + +module Migrator + def db_version + "4.13.80" + end + + def one_version + "OpenNebula 4.13.80" + end + + def up + #3654 + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts "#{one_version} improves the management of FAILED VMs " + puts "Please remove (onevm delete) any FAILED VM before continuing. " + puts + + #2742 + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts + puts "The scheduler (and oned) has been update to enforce access " + puts "rights on system datastores. This new version also checks that" + puts "the user can access the System DS." + puts "This *may require* to update system DS rights of your cloud" + puts + printf "Do you want to proceed ? [y/N]" + + ans = STDIN.gets.strip.downcase + + return false if ans != "y" + + init_log_time() + + # 3805 + @db.run "ALTER TABLE document_pool RENAME TO old_document_pool;" + @db.run "CREATE TABLE document_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, type INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + @db.transaction do + @db.fetch("SELECT * FROM old_document_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_document_pool') + + lock_elem = doc.create_element("LOCK") + lock_elem.add_child(doc.create_element("LOCKED")).content = "0" + lock_elem.add_child(doc.create_element("OWNER")).content = "" + lock_elem.add_child(doc.create_element("EXPIRES")).content = "0" + + doc.root.add_child(lock_elem) + + @db[:document_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :type => row[:type], + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_document_pool;" + + log_time() + + # 3718 + + # Move monitoring attributes in VM pool table. VMs in the DONE state + # will be processed by the onedb patch command. + + @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;" + @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + @db.run "INSERT INTO vm_pool SELECT * FROM old_vm_pool WHERE state=6;" + + log_time() + + @db.transaction do + #@db.fetch("SELECT * FROM old_vm_pool WHERE state<>6") do |row| + @db.fetch("SELECT * FROM old_vm_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_vm_pool') + + update_monitoring(doc.root.at_xpath("/VM")) + + @db[:vm_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :last_poll => row[:last_poll], + :state => row[:state], + :lcm_state => row[:lcm_state], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_vm_pool;" + + log_time() + + # Move monitoring attributes in the history table. Closed records + # will be processed by the onedb patch command. + + @db.run "ALTER TABLE history RENAME TO old_history;" + @db.run "CREATE TABLE history (vid INTEGER, seq INTEGER, body MEDIUMTEXT, stime INTEGER, etime INTEGER,PRIMARY KEY(vid,seq));" + + @db.run "INSERT INTO history SELECT * FROM old_history WHERE etime<>0;" + + log_time() + + @db.transaction do + #@db.fetch("SELECT * FROM old_history WHERE etime=0") do |row| + @db.fetch("SELECT * FROM old_history") do |row| + doc = nokogiri_doc(row[:body], 'old_history') + + elem = doc.root.at_xpath("/HISTORY/VM") + if !elem.nil? + update_monitoring(elem) + end + + @db[:history].insert( + :vid => row[:vid], + :seq => row[:seq], + :body => doc.root.to_s, + :stime => row[:stime], + :etime => row[:etime]) + end + end + + @db.run "DROP TABLE old_history;" + + log_time() + + # 3782 + + @db.run "ALTER TABLE image_pool RENAME TO old_image_pool;" + @db.run "CREATE TABLE image_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid) );" + + @db.transaction do + @db.fetch("SELECT * FROM old_image_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_image_pool') + + doc.root.add_child(doc.create_element("TARGET_SNAPSHOT")).content = "-1" + doc.root.add_child(doc.create_element("SNAPSHOTS")) + + @db[:image_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_image_pool;" + + log_time() + + return true + end + + def mv_monitoring(vm_elem, prev_name, new_name) + elem = vm_elem.at_xpath(prev_name) + + if (!elem.nil?) + vm_elem.at_xpath("MONITORING").add_child( + vm_elem.document.create_element(new_name)).content = elem.text + + elem.remove + end + end + + def update_monitoring(vm_elem) + vm_elem.add_child(vm_elem.document.create_element("MONITORING")) + + mv_monitoring(vm_elem, "CPU", "CPU") + mv_monitoring(vm_elem, "MEMORY", "MEMORY") + mv_monitoring(vm_elem, "NET_RX", "NETRX") + mv_monitoring(vm_elem, "NET_TX", "NETTX") + + mv_monitoring(vm_elem, "AZ_AVAILABILITY_SET_NAME", "AZ_AVAILABILITY_SET_NAME") + mv_monitoring(vm_elem, "AZ_CLOUD_SERVICE_NAME", "AZ_CLOUD_SERVICE_NAME") + mv_monitoring(vm_elem, "AZ_DATA_DISKS", "AZ_DATA_DISKS") + mv_monitoring(vm_elem, "AZ_DEPLOYMENT_NAME", "AZ_DEPLOYMENT_NAME") + mv_monitoring(vm_elem, "AZ_DISK_NAME", "AZ_DISK_NAME") + mv_monitoring(vm_elem, "AZ_HOSTNAME", "AZ_HOSTNAME") + mv_monitoring(vm_elem, "AZ_IMAGE", "AZ_IMAGE") + mv_monitoring(vm_elem, "AZ_IPADDRESS", "AZ_IPADDRESS") + mv_monitoring(vm_elem, "AZ_MEDIA_LINK", "AZ_MEDIA_LINK") + mv_monitoring(vm_elem, "AZ_OS_TYPE", "AZ_OS_TYPE") + mv_monitoring(vm_elem, "AZ_ROLE_SIZE", "AZ_ROLE_SIZE") + mv_monitoring(vm_elem, "AZ_TCP_ENDPOINTS", "AZ_TCP_ENDPOINTS") + mv_monitoring(vm_elem, "AZ_UDP_ENDPOINTS", "AZ_UDP_ENDPOINTS") + mv_monitoring(vm_elem, "AZ_VIRTUAL_NETWORK_NAME", "AZ_VIRTUAL_NETWORK_NAME") + + mv_monitoring(vm_elem, "SL_CRED_PASSWORD", "SL_CRED_PASSWORD") + mv_monitoring(vm_elem, "SL_CRED_USER", "SL_CRED_USER") + mv_monitoring(vm_elem, "SL_DOMAIN", "SL_DOMAIN") + mv_monitoring(vm_elem, "SL_FULLYQUALIFIEDDOMAINNAME", "SL_FULLYQUALIFIEDDOMAINNAME") + mv_monitoring(vm_elem, "SL_GLOBALIDENTIFIER", "SL_GLOBALIDENTIFIER") + mv_monitoring(vm_elem, "SL_HOSTNAME", "SL_HOSTNAME") + mv_monitoring(vm_elem, "SL_ID", "SL_ID") + mv_monitoring(vm_elem, "SL_MAXCPU", "SL_MAXCPU") + mv_monitoring(vm_elem, "SL_MAXMEMORY", "SL_MAXMEMORY") + mv_monitoring(vm_elem, "SL_PRIMARYBACKENDIPADDRESS", "SL_PRIMARYBACKENDIPADDRESS") + mv_monitoring(vm_elem, "SL_PRIMARYIPADDRESS", "SL_PRIMARYIPADDRESS") + mv_monitoring(vm_elem, "SL_STARTCPUS", "SL_STARTCPUS") + mv_monitoring(vm_elem, "SL_UUID", "SL_UUID") + + mv_monitoring(vm_elem, "AWS_DNS_NAME", "AWS_DNS_NAME") + mv_monitoring(vm_elem, "AWS_PRIVATE_DNS_NAME", "AWS_PRIVATE_DNS_NAME") + mv_monitoring(vm_elem, "AWS_KEY_NAME", "AWS_KEY_NAME") + mv_monitoring(vm_elem, "AWS_AVAILABILITY_ZONE", "AWS_AVAILABILITY_ZONE") + mv_monitoring(vm_elem, "AWS_PLATFORM", "AWS_PLATFORM") + mv_monitoring(vm_elem, "AWS_VPC_ID", "AWS_VPC_ID") + mv_monitoring(vm_elem, "AWS_PRIVATE_IP_ADDRESS", "AWS_PRIVATE_IP_ADDRESS") + mv_monitoring(vm_elem, "AWS_IP_ADDRESS", "AWS_IP_ADDRESS") + mv_monitoring(vm_elem, "AWS_SUBNET_ID", "AWS_SUBNET_ID") + mv_monitoring(vm_elem, "AWS_SECURITY_GROUPS", "AWS_SECURITY_GROUPS") + mv_monitoring(vm_elem, "AWS_INSTANCE_TYPE", "AWS_INSTANCE_TYPE") + + mv_monitoring(vm_elem, "ESX_HOST", "ESX_HOST") + mv_monitoring(vm_elem, "GUEST_IP", "GUEST_IP") + mv_monitoring(vm_elem, "GUEST_STATE", "GUEST_STATE") + mv_monitoring(vm_elem, "VMWARETOOLS_RUNNING_STATUS", "VMWARETOOLS_RUNNING_STATUS") + mv_monitoring(vm_elem, "VMWARETOOLS_VERSION", "VMWARETOOLS_VERSION") + mv_monitoring(vm_elem, "VMWARETOOLS_VERSION_STATUS", "VMWARETOOLS_VERSION_STATUS") + end +end diff --git a/src/onedb/local/4.13.80_to_4.13.85.rb b/src/onedb/local/4.13.80_to_4.13.85.rb new file mode 100644 index 0000000000..945e15bb7a --- /dev/null +++ b/src/onedb/local/4.13.80_to_4.13.85.rb @@ -0,0 +1,239 @@ +# -------------------------------------------------------------------------- # +# Copyright 2019-2020, OpenNebula Systems S.L. # +# # +# Licensed under the OpenNebula Software License # +# (the "License"); you may not use this file except in compliance with # +# the License. You may obtain a copy of the License as part of the software # +# distribution. # +# # +# See https://github.com/OpenNebula/one/blob/master/LICENSE.onsla # +# (or copy bundled with OpenNebula in /usr/share/doc/one/). # +# # +# Unless agreed to in writing, software distributed under the License is # +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # +# KIND, either express or implied. See the License for the specific language # +# governing permissions and limitations under the License. # +# -------------------------------------------------------------------------- # + +module Migrator + def db_version + "4.13.85" + end + + def one_version + "OpenNebula 4.13.85" + end + + def up + init_log_time() + + # 1727 + + @db.run "ALTER TABLE user_quotas RENAME TO old_user_quotas;" + @db.run "CREATE TABLE user_quotas (user_oid INTEGER PRIMARY KEY, body MEDIUMTEXT);" + + @db.transaction do + # oneadmin does not have quotas + @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid=0") do |row| + @db[:user_quotas].insert(row) + end + + @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid>0") do |row| + doc = nokogiri_doc(row[:body], 'old_user_quotas') + + calculate_quotas(doc, "uid=#{row[:user_oid]}", "User") + + @db[:user_quotas].insert( + :user_oid => row[:user_oid], + :body => doc.root.to_s) + end + end + + @db.run "DROP TABLE old_user_quotas;" + + log_time() + + @db.run "ALTER TABLE group_quotas RENAME TO old_group_quotas;" + @db.run "CREATE TABLE group_quotas (group_oid INTEGER PRIMARY KEY, body MEDIUMTEXT);" + + @db.transaction do + # oneadmin does not have quotas + @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid=0") do |row| + @db[:group_quotas].insert(row) + end + + @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid>0") do |row| + doc = nokogiri_doc(row[:body], 'old_group_quotas') + + calculate_quotas(doc, "gid=#{row[:group_oid]}", "Group") + + @db[:group_quotas].insert( + :group_oid => row[:group_oid], + :body => doc.root.to_s) + end + end + + @db.run "DROP TABLE old_group_quotas;" + + log_time() + + default_user_quotas = nil + default_group_quotas = nil + + @db.fetch("SELECT * FROM system_attributes WHERE name = 'DEFAULT_USER_QUOTAS'") do |row| + default_user_quotas = nokogiri_doc(row[:body], 'system_attributes') + + vm_elem = default_user_quotas.root.at_xpath("VM_QUOTA/VM") + + if !vm_elem.nil? + vm_elem.at_xpath("VOLATILE_SIZE").name = "SYSTEM_DISK_SIZE" + vm_elem.at_xpath("VOLATILE_SIZE_USED").name = "SYSTEM_DISK_SIZE_USED" + end + end + + @db.fetch("SELECT * FROM system_attributes WHERE name = 'DEFAULT_GROUP_QUOTAS'") do |row| + default_group_quotas = nokogiri_doc(row[:body], 'system_attributes') + + vm_elem = default_group_quotas.root.at_xpath("VM_QUOTA/VM") + + if !vm_elem.nil? + vm_elem.at_xpath("VOLATILE_SIZE").name = "SYSTEM_DISK_SIZE" + vm_elem.at_xpath("VOLATILE_SIZE_USED").name = "SYSTEM_DISK_SIZE_USED" + end + end + + if !default_user_quotas.nil? + @db[:system_attributes].where(:name => "DEFAULT_USER_QUOTAS").update( + :body => default_user_quotas.root.to_s) + end + + if !default_group_quotas.nil? + @db[:system_attributes].where(:name => "DEFAULT_GROUP_QUOTAS").update( + :body => default_group_quotas.root.to_s) + end + + log_time() + + @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;" + @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);" + + @db.transaction do + @db.fetch("SELECT * FROM old_host_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_host_pool') + + doc.root.at_xpath("HOST_SHARE").add_child(doc.create_element("PCI_DEVICES")) + + @db[:host_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :state => row[:state], + :last_mon_time => row[:last_mon_time], + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :cid => row[:cid]) + end + end + + @db.run "DROP TABLE old_host_pool;" + + log_time() + + @db.transaction do + @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;" + @db.run "CREATE TABLE datastore_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);" + + @db.run "INSERT INTO datastore_pool SELECT * FROM old_datastore_pool;" + + @db.run "DROP TABLE old_datastore_pool;" + end + + log_time() + + return true + end + + # Copied from fsck file, this method only recalculates the SYSTEM_DISK quotas + def calculate_quotas(doc, where_filter, resource) + + oid = doc.root.at_xpath("ID").text.to_i + + sys_used = 0 + + @db.fetch("SELECT body FROM vm_pool WHERE #{where_filter} AND state<>6") do |vm_row| + vmdoc = nokogiri_doc(row[:body], 'vm_pool') + + vmdoc.root.xpath("TEMPLATE/DISK").each { |e| + type = "" + + e.xpath("TYPE").each { |t_elem| + type = t_elem.text.upcase + } + + size = 0 + + if !e.at_xpath("SIZE").nil? + size = e.at_xpath("SIZE").text.to_i + end + + if ( type == "SWAP" || type == "FS") + sys_used += size + else + if !e.at_xpath("CLONE").nil? + clone = (e.at_xpath("CLONE").text.upcase == "YES") + + target = nil + + if clone + target = e.at_xpath("CLONE_TARGET").text if !e.at_xpath("CLONE_TARGET").nil? + else + target = e.at_xpath("LN_TARGET").text if !e.at_xpath("LN_TARGET").nil? + end + + if !target.nil? && target != "NONE" # self or system + sys_used += size + + if !e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").nil? + sys_used += e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").text.to_i + end + end + end + end + } + end + + vm_elem = doc.root.at_xpath("VM_QUOTA/VM") + + if !vm_elem.nil? + vm_elem.at_xpath("VOLATILE_SIZE").name = "SYSTEM_DISK_SIZE" + vm_elem.at_xpath("VOLATILE_SIZE_USED").name = "SYSTEM_DISK_SIZE_USED" + else + doc.root.xpath("VM_QUOTA").each { |e| e.remove } + + vm_quota = doc.root.add_child(doc.create_element("VM_QUOTA")) + vm_elem = vm_quota.add_child(doc.create_element("VM")) + + vm_elem.add_child(doc.create_element("CPU")).content = "-1" + vm_elem.add_child(doc.create_element("CPU_USED")).content = "0" + + vm_elem.add_child(doc.create_element("MEMORY")).content = "-1" + vm_elem.add_child(doc.create_element("MEMORY_USED")).content = "0" + + vm_elem.add_child(doc.create_element("VMS")).content = "-1" + vm_elem.add_child(doc.create_element("VMS_USED")).content = "0" + + vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE")).content = "-1" + vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE_USED")).content = "0" + end + + vm_elem.xpath("SYSTEM_DISK_SIZE_USED").each { |e| + if e.text != sys_used.to_s + #puts("#{resource} #{oid} quotas: SYSTEM_DISK_SIZE_USED has #{e.text} \tis\t#{sys_used}") + e.content = sys_used.to_s + end + } + end +end diff --git a/src/onedb/local/4.13.85_to_4.90.0.rb b/src/onedb/local/4.13.85_to_4.90.0.rb new file mode 100644 index 0000000000..d06f38101f --- /dev/null +++ b/src/onedb/local/4.13.85_to_4.90.0.rb @@ -0,0 +1,972 @@ +# -------------------------------------------------------------------------- # +# Copyright 2019-2020, OpenNebula Systems S.L. # +# # +# Licensed under the OpenNebula Software License # +# (the "License"); you may not use this file except in compliance with # +# the License. You may obtain a copy of the License as part of the software # +# distribution. # +# # +# See https://github.com/OpenNebula/one/blob/master/LICENSE.onsla # +# (or copy bundled with OpenNebula in /usr/share/doc/one/). # +# # +# Unless agreed to in writing, software distributed under the License is # +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # +# KIND, either express or implied. See the License for the specific language # +# governing permissions and limitations under the License. # +# -------------------------------------------------------------------------- # + +require 'set' +require 'base64' +require 'zlib' +require 'pathname' + +require 'opennebula' + +include OpenNebula + +module Migrator + def db_version + "4.90.0" + end + + def one_version + "OpenNebula 4.90.0" + end + + TEMPLATE_TRANSFORM_ATTRS = { + 'SUNSTONE_NETWORK_SELECT' => 'NETWORK_SELECT' + } + + def up + init_log_time() + + ############################################################################ + # 4369 + ############################################################################ + + @db.run "CREATE TABLE cluster_datastore_relation (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));" + @db.run "CREATE TABLE cluster_network_relation (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));" + + + @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;" + @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);" + + host_ids = [] + + @db.transaction do + @db.fetch("SELECT * FROM old_host_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_host_pool') + + cid_elem = doc.root.at_xpath("CLUSTER_ID") + cid = cid_elem.text.to_i + + if (cid == -1) + cid = 0 + cid_elem.content = "0" + doc.root.at_xpath("CLUSTER").content = "default" + + host_ids << row[:oid] + end + + @db[:host_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :state => row[:state], + :last_mon_time => row[:last_mon_time], + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :cid => cid) + end + end + + @db.run "DROP TABLE old_host_pool;" + + log_time() + + @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;" + @db.run "CREATE TABLE datastore_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + ds_ids = [] + + @db.transaction do + @db.fetch("SELECT * FROM old_datastore_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_datastore_pool') + + doc.root.at_xpath("CLUSTER").remove + + cid_elem = doc.root.at_xpath("CLUSTER_ID") + cid = cid_elem.text.to_i + + cid_elem.remove + + if (cid == -1) + cid = 0 + ds_ids << row[:oid] + end + + cluster_ids_elem = doc.create_element("CLUSTERS") + cluster_ids_elem.add_child(doc.create_element("ID")).content = cid.to_s + + doc.root.add_child(cluster_ids_elem) + + @db[:datastore_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + + @db[:cluster_datastore_relation].insert( + :cid => cid, + :oid => row[:oid]) + end + end + + @db.run "DROP TABLE old_datastore_pool;" + + log_time() + + @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;" + @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));" + + vnet_ids = [] + + @db.transaction do + @db.fetch("SELECT * FROM old_network_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_network_pool') + + doc.root.at_xpath("CLUSTER").remove + + cid_elem = doc.root.at_xpath("CLUSTER_ID") + cid = cid_elem.text.to_i + + cid_elem.remove + + if (cid == -1) + cid = 0 + vnet_ids << row[:oid] + end + + cluster_ids_elem = doc.create_element("CLUSTERS") + cluster_ids_elem.add_child(doc.create_element("ID")).content = cid.to_s + + doc.root.add_child(cluster_ids_elem) + + @db[:network_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :pid => row[:pid]) + + @db[:cluster_network_relation].insert( + :cid => cid, + :oid => row[:oid]) + end + end + + @db.run "DROP TABLE old_network_pool;" + + log_time() + + @db.run "ALTER TABLE vm_pool RENAME TO old_vm_pool;" + @db.run "CREATE TABLE vm_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, last_poll INTEGER, state INTEGER, lcm_state INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER)" + + @db.transaction do + @db.fetch("SELECT * FROM old_vm_pool") do |row| + if row[:state] != 6 + doc = nokogiri_doc(row[:body], 'old_vm_pool') + + cid = doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").text.to_i rescue nil + + if cid == -1 + doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").content = 0 + end + + # Bug #4467 + + elem = doc.at_xpath("/VM/USER_TEMPLATE/EC2") + + if (!elem.nil?) + elem.name = "PUBLIC_CLOUD" + + if elem.at_xpath("TYPE").nil? + elem.add_child(doc.create_element("TYPE")).content = "ec2" + end + end + + row[:body] = doc.root.to_s + end + + @db[:vm_pool].insert(row) + end + end + + @db.run "DROP TABLE old_vm_pool;" + + log_time() + + default_cl_xml = '0default' + doc = Nokogiri::XML(default_cl_xml,nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + hosts_elem = doc.root.at_xpath("HOSTS") + host_ids.each { |id| + hosts_elem.add_child(doc.create_element("ID")).content = id.to_s + } + + ds_elem = doc.root.at_xpath("DATASTORES") + ds_ids.each { |id| + ds_elem.add_child(doc.create_element("ID")).content = id.to_s + } + + vnets_elem = doc.root.at_xpath("VNETS") + vnet_ids.each { |id| + vnets_elem.add_child(doc.create_element("ID")).content = id.to_s + } + + @db[:cluster_pool].insert( + :oid => 0, + :name => 'default', + :body => doc.root.to_s, + :uid => 0, + :gid => 0, + :owner_u => 1, + :group_u => 0, + :other_u => 0) + + log_time() + + ############################################################################ + # 4215 + ############################################################################ + + @db.run "CREATE TABLE vrouter_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;" + @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));" + + @db.transaction do + @db.fetch("SELECT * FROM old_network_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_network_pool') + + doc.root.add_child(doc.create_element("VROUTERS")) + + @db[:network_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u], + :pid => row[:pid]) + end + end + + @db.run "DROP TABLE old_network_pool;" + + log_time() + + @db.run "ALTER TABLE template_pool RENAME TO old_template_pool;" + @db.run "CREATE TABLE template_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" + + @db.transaction do + @db.fetch("SELECT * FROM old_template_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_template_pool') + + # Feature #3671 + + TEMPLATE_TRANSFORM_ATTRS.each do |old_name, new_name| + elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/#{old_name}") + + if (!elem.nil?) + elem.remove + + elem.name = new_name + + if (doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE").nil?) + doc.at_xpath("/VMTEMPLATE/TEMPLATE").add_child( + doc.create_element("SUNSTONE")) + end + + doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE").add_child(elem) + end + end + + # Feature #4317 + + elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/SUNSTONE_CAPACITY_SELECT") + + if elem.nil? + capacity_edit = true + else + elem.remove + capacity_edit = (elem.text != "NO") + end + + if !capacity_edit + cpu_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/CPU") + memory_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/MEMORY") + vcpu_e = doc.at_xpath("/VMTEMPLATE/TEMPLATE/VCPU") + + cpu = cpu_e != nil ? cpu_e.text : "" + memory = memory_e != nil ? memory_e.text : "" + vcpu = vcpu_e != nil ? vcpu_e.text : "" + + user_inputs = doc.at_xpath("/VMTEMPLATE/TEMPLATE/USER_INPUTS") + + if user_inputs.nil? + user_inputs = doc.create_element("USER_INPUTS") + doc.at_xpath("/VMTEMPLATE/TEMPLATE").add_child(user_inputs) + end + + user_inputs.add_child(doc.create_element("CPU")).content = "O|fixed|||#{cpu}" + user_inputs.add_child(doc.create_element("MEMORY")).content = "O|fixed|||#{memory}" + user_inputs.add_child(doc.create_element("VCPU")).content = "O|fixed|||#{vcpu}" + end + + # Bug #4467 + + elem = doc.at_xpath("/VMTEMPLATE/TEMPLATE/EC2") + + if (!elem.nil?) + elem.name = "PUBLIC_CLOUD" + + if elem.at_xpath("TYPE").nil? + elem.add_child(doc.create_element("TYPE")).content = "ec2" + end + end + + @db[:template_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_template_pool;" + + log_time() + + ############################################################################ + # Feature #4217 + ############################################################################ + + @db.run "ALTER TABLE image_pool RENAME TO old_image_pool;" + @db.run "CREATE TABLE image_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid) );" + + @db.transaction do + @db.fetch("SELECT * FROM old_image_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_image_pool') + + doc.root.add_child(doc.create_element("APP_CLONES")) + + @db[:image_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_image_pool;" + + log_time() + + ############################################################################ + # Feature #3204 + ############################################################################ + + @db.run "ALTER TABLE secgroup_pool RENAME TO old_secgroup_pool;" + @db.run "CREATE TABLE secgroup_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid));" + + @db.transaction do + @db.fetch("SELECT * FROM old_secgroup_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_secgroup_pool') + + doc.root.at_xpath("VMS").name = "UPDATED_VMS" + doc.root.add_child(doc.create_element("OUTDATED_VMS")) + doc.root.add_child(doc.create_element("UPDATING_VMS")) + doc.root.add_child(doc.create_element("ERROR_VMS")) + + @db[:secgroup_pool].insert( + :oid => row[:oid], + :name => row[:name], + :body => doc.root.to_s, + :uid => row[:uid], + :gid => row[:gid], + :owner_u => row[:owner_u], + :group_u => row[:group_u], + :other_u => row[:other_u]) + end + end + + @db.run "DROP TABLE old_secgroup_pool;" + + log_time() + + # Bug #4248 - Remove Firewall Drivers + + vms_with_fw = [] + @db.transaction do + @db.fetch("SELECT * FROM vm_pool WHERE state != 6") do |row| + doc = nokogiri_doc(row[:body], 'vm_pool') + + has_fw_attrs = !doc.root.xpath("TEMPLATE/NIC[ICMP|WHITE_PORTS_TCP|WHITE_PORTS_UDP|BLACK_PORTS_TCP|BLACK_PORTS_UDP]").empty? + + vms_with_fw << row[:oid].to_i if has_fw_attrs + end + end + + if !vms_with_fw.empty? + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts "The old driver 'fw' has been removed from OpenNebula. It was " + puts "deprecated in 4.12: " + puts "http://docs.opennebula.org/4.12/release_notes/release_notes/compatibility.html" + puts + puts "We have detected that you still have active VMs with these " + puts "attributes: ICMP, WHITE_PORTS_TCP, WHITE_PORTS_UDP, " + puts "BLACK_PORTS_TCP, BLACK_PORTS_UDP. " + puts + puts "The list of affected VMs is: " + vms_with_fw.each{|vm| puts "- #{vm}"} + puts + puts "Please note that OpenNebula will not modify the current " + puts "iptables rules, so you will need to manually clean them when " + puts "any of VMs is removed." + puts + puts "Please consider switching to Security Groups " + end + + log_time() + + ############################################################################ + # Remove Xen, VMware and SoftLayer Drivers + ############################################################################ + + @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;" + + @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);" + + has_xen_hosts = false + has_vmware_hosts = false + has_sl_hosts = false + + @db.transaction do + @db.fetch("SELECT * FROM old_host_pool") do |row| + do_disable = false + + doc = nokogiri_doc(row[:body], 'old_host_pool') + + vm_mad = doc.root.at_xpath("VM_MAD").text + im_mad = doc.root.at_xpath("IM_MAD").text + + if vm_mad.match(/xen/) || im_mad.match(/xen/) + do_disable = true + has_xen_hosts = true + end + + if vm_mad.match(/vmware/) || im_mad.match(/vmware/) + do_disable = true + has_vmware_hosts = true + end + + + if vm_mad.match(/sl/) || im_mad.match(/sl/) + do_disable = true + has_sl_hosts = true + end + + if do_disable + doc.root.at_xpath('STATE').content = 4 + + row[:state] = 4 + row[:body] = doc.root.to_s + end + + @db[:host_pool].insert(row) + end + end + + @db.run "DROP TABLE old_host_pool;" + + if has_xen_hosts + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts "Xen is no longer included in the core distribution. It is" + puts "however available as an addon which must be manually installed:" + puts "https://github.com/OpenNebula/addon-xen" + puts + puts "Note that the host has been automatically disabled. After installing" + puts "the addon you can manually enable it." + puts + end + + if has_vmware_hosts + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts "VMware is no longer supported. You are encouraged to migrate" + puts "to the vCenter driver:" + puts "http://docs.opennebula.org/stable/administration/virtualization/vcenterg.html" + puts + puts "Note that the host has been automatically disabled, but not removed." + puts + end + + if has_sl_hosts + puts "**************************************************************" + puts "* WARNING WARNING WARNING WARNING WARNING WARNING WARNING *" + puts "**************************************************************" + puts + puts "SoftLayer is no longer included in the core distribution. It is" + puts "however available as an addon which must be manually installed:" + puts "https://github.com/OpenNebula/addon-softlayer" + puts + puts "Note that the host has been automatically disabled. After installing" + puts "the addon you can manually enable it." + puts + end + + log_time() + + ############################################################################ + # Move HOST/VN_MAD --> VNET/VN_MAD + ############################################################################ + + # Build net_vnmad + net_vnmad = {} + @db.transaction do + @db.fetch("SELECT * FROM vm_pool WHERE state != 6") do |row| + doc = nokogiri_doc(row[:body], 'vm_pool') + state = row[:state].to_i + + vnmads = Set.new + doc.root.xpath("HISTORY_RECORDS/HISTORY/VNMMAD").collect{|v| vnmads << v.text } + + doc.root.xpath("TEMPLATE/NIC/NETWORK_ID").each do |net_id| + net_id = net_id.text.to_i + + net_vnmad[net_id] ||= Set.new + net_vnmad[net_id] += vnmads + end + end + end + + # Build cluster_vnmad and fix hosts + @db.run "ALTER TABLE host_pool RENAME TO old_host_pool;" + @db.run "CREATE TABLE host_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, state INTEGER, last_mon_time INTEGER, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, cid INTEGER);" + + cluster_vnmad = {} + @db.transaction do + @db.fetch("SELECT * FROM old_host_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_host_pool') + + # Get cluster + cluster_id = doc.root.xpath('CLUSTER_ID').text.to_i + + # Store VN_MAD + vnmad = doc.root.xpath('VN_MAD').text + + cluster_vnmad[cluster_id] ||= Set.new + cluster_vnmad[cluster_id] << vnmad + + # Remove VN_MAD + doc.root.xpath('//VN_MAD').remove + + row[:body] = doc.root.to_s + @db[:host_pool].insert(row) + end + end + @db.run "DROP TABLE old_host_pool;" + + # Fix Networks + + # So far we have two hashes: net_vnmad, which lists the specific vnmad found + # for each network, based on the nics of the VMs, and cluster_vnmad, with + # the vnmad found in the hosts. + # + # We will report a warning if either the cluster or the network has more + # than one vnmad. We will automatically choose one vnmad from the network + # list, or if empty from the cluster list. + # + # That vnmad may be changed through onedb patch. + # + # It could happen that no network is found in the net or in the cluster, in + # which case the admin **must** run the onedb patch, it's not optional any + # more. + + @db.run "ALTER TABLE network_pool RENAME TO old_network_pool;" + @db.run "CREATE TABLE network_pool (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));" + + reserved_vlan_ids = Set.new + final_net_vnmad = {} + manual_intervention = false + @db.transaction do + @db.fetch("SELECT * FROM old_network_pool") do |row| + doc = nokogiri_doc(row[:body], 'old_network_pool') + + net_id = row[:oid] + net_name = row[:name] + + cluster_id = doc.root.xpath('CLUSTERS/ID').first.text.to_i + + # Get possible VN_MADs + net_vnmad_len = net_vnmad[net_id].length rescue 0 + cluster_vnmad_len = cluster_vnmad[cluster_id].length rescue 0 + + # Check if the network has VLAN=NO + vlan = doc.root.xpath('VLAN').text rescue nil + vlan_no = (vlan == "0") + + # Remove the VLAN attributes + doc.root.xpath('//VLAN').remove + + vnmad = nil + other_vnmads = nil + + # Get vnmad + # + # net_vnmad_len == 1 => that one + # net_vnmad_len > 1 => interactive + # net_vnmad_len == 0 && cluster_vnmad_len == 1 => that one + # net_vnmad_len == 0 && cluster_vnmad_len > 1 => interactive + # net_vnmad_len == 0 && cluster_vnmad_len == 0 => interactive + + if net_vnmad_len == 1 + vnmad = net_vnmad[net_id].first + elsif net_vnmad_len > 1 + other_vnmads = net_vnmad[net_id] + elsif net_vnmad_len == 0 && cluster_vnmad_len == 1 + vnmad = cluster_vnmad[cluster_id].first + elsif net_vnmad_len == 0 && cluster_vnmad_len > 1 + other_vnmads = cluster_vnmad[cluster_id] + end + + # Ambiguous vnmad, require user input (TODO) + if vnmad.nil? + + if !manual_intervention + manual_intervention = true + puts + puts "Manual Intervention required. Please input the VN_MAD " << + " for the following networks:" + puts + end + + suggested = if other_vnmads + " (suggested: [#{other_vnmads.to_a.join(', ')}])" + else + "" + end + + input = "" + while (input.empty?) do + puts "* Net ##{net_id} (#{net_name}) VN_MAD#{suggested}:" + input = STDIN.gets.chomp.strip + + if input.match(/[^\w\-.]/) + puts "Invalid char found." + input = "" + end + end + + vnmad = input + end + + # If VLAN = NO => don't use isolated VN_MADs + if vlan_no && vnmad && ["802.1q", "ovswitch", "vxlan", "ebtables"].include?(vnmad.downcase) + input = "" + while (input.empty?) do + puts "Net ##{net_id} (#{net_name}) has VN_MAD='#{vnmad}' but it also has VLAN=NO. Change to 'fw'? (y/n)" + + input = STDIN.gets.chomp.strip + + case input + when 'y' + vnmad = 'fw' + when 'n' + else + puts "Invalid value." + input = "" + end + end + end + + if vnmad.nil? + STDERR.puts "Error getting VN_MAD for Network #{net_id}." + exit 1 + end + + # Create the VN_MAD element: + final_net_vnmad[net_id] = vnmad + doc.root.add_child(doc.create_element("VN_MAD")).content = vnmad + + # Create/Move the VLAN_ID and VLAN_ID_AUTOMATIC attributes: + # + # Manual => VLAN_ID exists + # Automatic => VLAN_ID does not exist + # + # If VLAN has been set automatically, + # + # top-level + # top-level 1 + # remove VLAN_ID from