shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-03-16 22:50:10 +03:00
Code Releases Wiki Activity

B #4165: fix encoding for fsck (#4166)

Browse Source 
Co-authored-by: Christian González <cgonzalez@opennebula.systems>
(cherry picked from commit 944f12446d639b1c3a725ab102d1daa8fcfeb9bd)
This commit is contained in:
Alejandro Huertas Herrero 2020-02-07 21:15:34 +01:00 committed by Ruben S. Montero
parent aef6407620
commit 2b89a19c62
No known key found for this signature in database
GPG Key ID: A0CEA6FA880A1D87
7 changed files with 149 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/onedb/fsck.rb
View File

@ -109,8 +109,14 @@ EOT
FEDERATED_TABLES
end
def nokogiri_doc(body)
Nokogiri::XML(body, nil, NOKOGIRI_ENCODING) do |c|
def nokogiri_doc(body, table = nil)
nk_enconding = NOKOGIRI_ENCODING
unless table.nil?
nk_enconding = get_table_enconding(table)
end
Nokogiri::XML(body, nil, nk_enconding) do |c|
c.default_xml.noblanks
end
end

51
src/onedb/fsck/cluster.rb
View File

@ -24,7 +24,8 @@ module OneDBFsck
@db.transaction do
@db.fetch('SELECT * from cluster_pool') do |row|
cluster_id = row[:oid]
doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'cluster_pool')
# Hosts
process_hosts(doc, cluster_id, cluster[cluster_id][:hosts])
@ -53,21 +54,25 @@ module OneDBFsck
# @param cid [Integer] Cluster ID
# @param hosts [Array] Hosts to process
def process_hosts(doc, cid, hosts)
hosts_elem = doc.root.elements.delete('HOSTS')
hosts_new_elem = doc.root.add_element('HOSTS')
hosts_elem = doc.xpath('//HOSTS').remove
hosts_new_elem = doc.create_element('HOSTS')
doc.root.add_child(hosts_new_elem)
hosts.each do |id|
id_elem = hosts_elem.elements.delete("ID[.=#{id}]")
id_elem = hosts_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
if id_elem.empty?
log_error("Host #{id} is missing from cluster " \
"#{cid} host id list")
end
hosts_new_elem.add_element('ID').text = id.to_s
aux = doc.create_element('ID')
aux.content = id.to_s
hosts_new_elem.add_child(aux)
end
hosts_elem.each_element('ID') do |id|
hosts_elem.children.each do |id|
id = id.text
log_error("Host #{id} is in cluster #{cid} " \
@ -81,18 +86,22 @@ module OneDBFsck
# @param cid [Integer] Cluster ID
# @param datastores [Array] Datastores to process
def process_ds(doc, cid, datastores)
ds_elem = doc.root.elements.delete('DATASTORES')
ds_new_elem = doc.root.add_element('DATASTORES')
ds_elem = doc.xpath('//DATASTORES').remove
ds_new_elem = doc.create_element('DATASTORES')
doc.root.add_child(ds_new_elem)
datastores.each do |id|
id_elem = ds_elem.elements.delete("ID[.=#{id}]")
id_elem = ds_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
if id_elem.empty?
log_error("Datastore #{id} is missing from cluster " \
"#{cid} datastore id list")
end
ds_new_elem.add_element('ID').text = id.to_s
aux = doc.create_element('ID')
aux.content = id.to_s
ds_new_elem.add_child(aux)
next unless @db.fetch('SELECT * FROM cluster_datastore_relation ' \
"WHERE cid=#{cid} AND oid=#{id}").empty?
@ -103,7 +112,7 @@ module OneDBFsck
@db[:cluster_datastore_relation].insert(:cid => cid, :oid => id)
end
ds_elem.each_element('ID') do |id|
ds_elem.children.each do |id|
id = id.text
log_error("Datastore #{id} is in cluster #{cid} datastore id " \
@ -117,18 +126,22 @@ module OneDBFsck
# @param cid [Integer] Cluster ID
# @param vnets [Array] VNets to process
def process_vnets(doc, cid, vnets)
vnets_elem = doc.root.elements.delete('VNETS')
vnets_new_elem = doc.root.add_element('VNETS')
vnets_elem = doc.xpath('//VNETS').remove
vnets_new_elem = doc.create_element('VNETS')
doc.root.add_child(vnets_new_elem)
vnets.each do |id|
id_elem = vnets_elem.elements.delete("ID[.=#{id}]")
id_elem = vnets_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
if id_elem.empty?
log_error("VNet #{id} is missing from cluster #{cid} " \
'vnet id list')
end
vnets_new_elem.add_element('ID').text = id.to_s
aux = doc.create_element('ID')
aux.content = id.to_s
vnets_new_elem.add_child(aux)
next unless @db.fetch('SELECT * FROM cluster_network_relation ' \
"WHERE cid=#{cid} AND oid=#{id}").empty?
@ -139,7 +152,7 @@ module OneDBFsck
@db[:cluster_network_relation].insert(:cid => cid, :oid => id)
end
vnets_elem.each_element('ID') do |id|
vnets_elem.children.each do |id|
id = id.text
log_error("VNet #{id} is in cluster #{cid} vnet id list, " \

27
src/onedb/fsck/datastore.rb
View File

@ -19,12 +19,14 @@ module OneDBFsck
@db.transaction do
@db.fetch('SELECT * from datastore_pool') do |row|
ds_id = row[:oid]
doc = Document.new(row[:body])
images_elem = doc.root.elements.delete('IMAGES')
images_new_elem = doc.root.add_element('IMAGES')
doc = nokogiri_doc(row[:body], 'datastore_pool')
images_elem = doc.root.xpath('IMAGES').remove
images_new_elem = doc.create_element('IMAGES')
doc.root.add_child(images_new_elem)
datastore[ds_id][:images].each do |id|
id_elem = images_elem.elements.delete("ID[.=#{id}]")
id_elem = images_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
log_error(
@ -33,10 +35,11 @@ module OneDBFsck
)
end
images_new_elem.add_element('ID').text = id.to_s
i_e = doc.create_element('ID')
images_new_elem.add_child(i_e).content = id.to_s
end
images_elem.each_element('ID') do |id_elem|
images_elem.children.each do |id_elem|
log_error(
"Image #{id_elem.text} is in Datastore #{ds_id} " \
'image id list, but it should not'
@ -61,7 +64,7 @@ module OneDBFsck
@fixes_datastore_cluster = {}
@db.fetch('SELECT oid,body FROM datastore_pool') do |row|
doc = nokogiri_doc(row[:body])
doc = nokogiri_doc(row[:body], 'datastore_pool')
oid = row[:oid]
doc.root.xpath('CLUSTERS/ID').each do |e|
@ -88,9 +91,9 @@ module OneDBFsck
@fixes_datastore_image = {}
@db.fetch('SELECT oid,body FROM image_pool') do |row|
doc = Document.new(row[:body])
ds_id = doc.root.get_text('DATASTORE_ID').to_s.to_i
ds_name = doc.root.get_text('DATASTORE')
doc = nokogiri_doc(row[:body], 'datastore_pool')
ds_id = doc.root.xpath('DATASTORE_ID').text.to_i
ds_name = doc.root.xpath('DATASTORE').text
if ds_id != -1
ds_entry = datastore[ds_id]
@ -100,7 +103,7 @@ module OneDBFsck
'but it does not exist. The image is probably ' \
"unusable, and needs to be deleted manually:\n" \
' * The image contents should be deleted ' \
"manually:\n #{doc.root.get_text('SOURCE')}\n" \
"manually:\n #{doc.root.xpath('SOURCE').text}\n" \
' * The DB entry can be then deleted with ' \
"the command:\n" \
' DELETE FROM image_pool WHERE ' \
@ -111,7 +114,7 @@ module OneDBFsck
"for datastore #{ds_id}, #{ds_name}. " \
"It will be changed to #{ds_entry[:name]}")
doc.root.each_element('DATASTORE') do |e|
doc.root.xpath('DATASTORE').each do |e|
e.text = ds_entry[:name]
end

13
src/onedb/fsck/host.rb
View File

@ -19,9 +19,10 @@ module OneDBFsck
hosts_fix = @fixes_host_cluster = {}
@db.fetch('SELECT oid,body,cid FROM host_pool') do |row|
doc = Document.new(row[:body])
cid = doc.root.get_text('CLUSTER_ID').to_s.to_i
cname = doc.root.get_text('CLUSTER')
doc = nokogiri_doc(row[:body], 'host_pool')
cid = doc.root.xpath("CLUSTER_ID").text.to_i
cname = doc.root.xpath("CLUSTER").text
if cid != row[:cid]
log_error("Host #{row[:oid]} is in cluster #{cid}, but cid " \
@ -38,11 +39,11 @@ module OneDBFsck
log_error("Host #{row[:oid]} is in cluster #{cid}, " \
'but it does not exist')
doc.root.each_element('CLUSTER_ID') do |e|
doc.root.xpath('CLUSTER_ID').each do |e|
e.text = '-1'
end
doc.root.each_element('CLUSTER') do |e|
doc.root.xpath('CLUSTER').each do |e|
e.text = ''
end
@ -55,7 +56,7 @@ module OneDBFsck
"cluster #{cid}, #{cname}. " \
"It will be changed to #{new_cluster}")
doc.root.each_element('CLUSTER') do |e|
doc.root.xpath('CLUSTER').each do |e|
e.text = new_cluster
end

52
src/onedb/fsck/image.rb
View File

@ -11,9 +11,9 @@ module OneDBFsck
}
end
doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'image_pool')
doc.root.each_element("CLONING_ID") do |e|
doc.root.xpath("CLONING_ID").each do |e|
img_id = e.text.to_i
if counters[:image][img_id].nil?
@ -34,12 +34,12 @@ module OneDBFsck
@db.transaction do
@db[:image_pool].each do |row|
doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'image_pool')
oid = row[:oid]
persistent = ( doc.root.get_text('PERSISTENT').to_s == "1" )
current_state = doc.root.get_text('STATE').to_s.to_i
persistent = ( doc.root.xpath('PERSISTENT').text == "1" )
current_state = doc.root.xpath('STATE').text.to_i
counters_img = counters[:image][oid]
@ -48,7 +48,7 @@ module OneDBFsck
# DATA: CHECK: running vm counter with this image
# rewrite running_vms
doc.root.each_element("RUNNING_VMS") {|e|
doc.root.xpath("RUNNING_VMS") {|e|
if e.text != rvms.to_s
log_error("Image #{oid} RUNNING_VMS has #{e.text} \tis\t#{rvms}")
e.text = rvms
@ -56,22 +56,24 @@ module OneDBFsck
}
# re-do list of VM IDs
vms_elem = doc.root.elements.delete("VMS")
vms_elem = doc.root.xpath("VMS").remove
vms_new_elem = doc.root.add_element("VMS")
vms_new_elem = doc.create_element("VMS")
doc.root.add_child(vms_new_elem)
# DATA: CHECK: running vm list with this image
counters_img[:vms].each do |id|
id_elem = vms_elem.elements.delete("ID[.=#{id}]")
id_elem = vms_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
log_error("VM #{id} is missing from Image #{oid} VM id list")
end
vms_new_elem.add_element("ID").text = id.to_s
i_e = doc.create_element('ID')
vms_new_elem.add_child(i_e).content = id.to_s
end
vms_elem.each_element("ID") do |id_elem|
vms_elem.children.each do |id_elem|
log_error("VM #{id_elem.text} is in Image #{oid} VM id list, but it should not")
end
@ -83,7 +85,7 @@ module OneDBFsck
end
# DATA: CHECK: Check number of clones
doc.root.each_element("CLONING_OPS") { |e|
doc.root.xpath("CLONING_OPS") { |e|
if e.text != n_cloning_ops.to_s
log_error("Image #{oid} CLONING_OPS has #{e.text} \tis\t#{n_cloning_ops}")
e.text = n_cloning_ops
@ -91,43 +93,47 @@ module OneDBFsck
}
# re-do list of Images cloning this one
clones_elem = doc.root.elements.delete("CLONES")
clones_elem = doc.root.xpath("CLONES").remove
clones_new_elem = doc.root.add_element("CLONES")
clones_new_elem = doc.create_element("CLONES")
doc.root.add_child(clones_new_elem)
# DATA: CHECK: image clones (is it used?)
counters_img[:clones].each do |id|
id_elem = clones_elem.elements.delete("ID[.=#{id}]")
id_elem = clones_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
log_error("Image #{id} is missing from Image #{oid} CLONES id list")
end
clones_new_elem.add_element("ID").text = id.to_s
i_e = doc.create_element('ID')
clones_new_elem.add_child(i_e).content = id.to_s
end
clones_elem.each_element("ID") do |id_elem|
clones_elem.children.each do |id_elem|
log_error("Image #{id_elem.text} is in Image #{oid} CLONES id list, but it should not")
end
# re-do list of Apps cloning this one
clones_elem = doc.root.elements.delete("APP_CLONES")
clones_elem = doc.root.xpath("APP_CLONES").remove
clones_new_elem = doc.root.add_element("APP_CLONES")
clones_new_elem = doc.create_element("APP_CLONES")
doc.root.add_child(clones_new_elem)
# DATA: CHECK: check app clones
# DATA: TODO: understand app clones and image clones
counters_img[:app_clones].each do |id|
id_elem = clones_elem.elements.delete("ID[.=#{id}]")
id_elem = clones_elem.xpath("ID[.=#{id}]").remove
if id_elem.nil?
log_error("Marketplace App #{id} is missing from Image #{oid} APP_CLONES id list")
end
clones_new_elem.add_element("ID").text = id.to_s
h_e = doc.create_element('ID')
clones_new_elem.add_child(i_e).content = id.to_s
end
clones_elem.each_element("ID") do |id_elem|
clones_elem.children.each do |id_elem|
log_error("Marketplace App #{id_elem.text} is in Image #{oid} APP_CLONES id list, but it should not")
end
@ -159,7 +165,7 @@ module OneDBFsck
end
end
doc.root.each_element("STATE") { |e|
doc.root.xpath("STATE") { |e|
if e.text != state.to_s
log_error("Image #{oid} has STATE " <<
OpenNebula::Image::IMAGE_STATES[e.text.to_i] <<

12
src/onedb/fsck/marketplaceapp.rb
View File

@ -22,10 +22,10 @@ module OneDBFsck
# DATA: go through all apps
@db.fetch("SELECT oid,body FROM marketplaceapp_pool") do |row|
doc = Document.new(row[:body])
doc = nokogiri_doc(row[:body], 'marketplaceapp_pool')
market_id = doc.root.get_text('MARKETPLACE_ID').to_s.to_i
market_name = doc.root.get_text('MARKETPLACE')
market_id = doc.root.xpath('MARKETPLACE_ID').text.to_i
market_name = doc.root.xpath('MARKETPLACE').text
####################################################################
# DATA: TODO, BUG: this code will only work for a standalone oned.
@ -34,8 +34,8 @@ module OneDBFsck
####################################################################
# DATA: get image origin id. Does it work?
origin_id = doc.root.get_text('ORIGIN_ID').to_s.to_i
if origin_id >= 0 && doc.root.get_text('STATE').to_s.to_i == 2 # LOCKED
origin_id = doc.root.xpath('ORIGIN_ID').text.to_i
if origin_id >= 0 && doc.root.xpath('STATE').text.to_i == 2 # LOCKED
counters[:image][origin_id][:app_clones].add(row[:oid])
end
@ -56,7 +56,7 @@ module OneDBFsck
if market_name != market_entry[:name]
log_error("Marketplace App #{row[:oid]} has a wrong name for marketplace #{market_id}, #{market_name}. It will be changed to #{market_entry[:name]}")
doc.root.each_element('MARKETPLACE') do |e|
doc.root.xpath('MARKETPLACE').each do |e|
e.text = market_entry[:name]
end

52
src/onedb/onedb_backend.rb
View File

@ -319,6 +319,54 @@ class BackEndMySQL < OneDBBacKEnd
@encoding = table_enc
end
def get_table_enconding(table)
enconding = nil
@db.fetch(
'select CCSA.character_set_name FROM information_schema.' \
'`TABLES` T, information_schema.' \
'`COLLATION_CHARACTER_SET_APPLICABILITY` CCSA WHERE ' \
'CCSA.collation_name = T.table_collation AND ' \
"T.table_schema = '#{@db_name}' AND "\
"T.table_name = '#{table}';"
) do |row|
enconding = row[:character_set_name]
end
table_to_nk(enconding)
end
def table_to_nk(enconding)
case(enconding)
when 'utf8mb4'
'UTF-8'
when 'utf16le'
'UTF16LE'
when 'utf16'
'UTF16BE'
when 'ucs2'
'UCS2'
when 'latin1'
'ISO-8859-1'
when 'latin2'
'ISO-8859-2'
when 'greek'
'ISO-8859-7'
when 'hebrew'
'ISO-8859-8'
when 'latin5'
'ISO-8859-9'
when 'sjis'
'SHIFT-JIS'
when 'ujis'
'EUC-JP'
when 'ascii'
'ASCII'
else
'NONE'
end
end
def restore(bck_file, force=nil, federated=false)
connect_db
@ -465,6 +513,10 @@ class BackEndSQLite < OneDBBacKEnd
puts "Use 'onedb restore' to restore the DB."
end
def get_table_enconding(table = nil)
'UTF-8'
end
def restore(bck_file, force=nil, federated=false)
if !federated
if File.exists?(@sqlite_file) && !force