shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-02-21 13:57:56 +03:00
Code Releases Wiki Activity

B #6171: Improve LinuxContainers Marketplace queries (#2603)

Browse Source 
Instead of traversing links in https://images.linuxcontainers.org/ (which takes
a lot of time), now the LXC image information is extracted parsing the file
https://images.linuxcontainers.org/streams/v1/images.json

(cherry picked from commit 5748f96c17982482d98f041ebe3d8ea88ed33ee4)
(cherry picked from commit e4343eb05171285f4780b9269d96ac2c411820a5)
This commit is contained in:
brodriguez-opennebula 2023-05-17 12:02:33 +02:00 committed by Ruben S. Montero
parent 420cd10c50
commit 6d09265b61
No known key found for this signature in database
GPG Key ID: A0CEA6FA880A1D87
1 changed files with 94 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

203
src/market_mad/remotes/linuxcontainers/monitor
View File

@ -44,18 +44,18 @@ class LinuxContainersMarket
# :privileged container idmap
#---------------------------------------------------------------------------
DEFAULTS = {
:url => 'https://images.linuxcontainers.org',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:tested_apps => ['alpine', 'alt', 'centos', 'rockylinux', 'almalinux', 'oracle', 'debian',
'ubuntu', 'fedora', 'devuan', 'opensuse', 'amazonlinux'],
:url => 'https://images.linuxcontainers.org/',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:tested_apps => %w[alpine alt centos rockylinux almalinux oracle debian
ubuntu fedora devuan opensuse amazonlinux],
:skip_untested => 'yes',
:memory => '768',
:cpu => 1,
:vcpu => 2,
:privileged => true
:memory => '768',
:cpu => 1,
:vcpu => 2,
:privileged => true
}
def initialize(options = {})
@ -64,7 +64,7 @@ class LinuxContainersMarket
version_path = File.dirname(__FILE__) + '/../../VERSION'
@options[:agent] = "OpenNebula #{File.read(version_path)}" \
if File.exist? version_path
if File.exist? version_path
@http_client = nil
end
@ -83,100 +83,93 @@ class LinuxContainersMarket
[response.code.to_i, response.msg]
end
# Get the list of appliances
# Relevant image.json format fields are
#
# {...,
# "products": {
# $distro:$version:$arch:$variant: {
# ...
# "versions": {
# $release_date: {
# "items": {
# $container_type: {
# ...
# "size": int
# "path": $path
# "sha256": $sha256
# }
# }
# }
# }
# }
# }
# }
def fetch_appliances
first_level = '/images/'
image_list_address = '/streams/v1/images.json'
open_connection
rc, body = get(first_level)
rc, body = get(image_list_address)
return rc, body if rc != 0
distros = body.scan(%r{a href="([a-z].*/)">})
distros.map! {|d| d.join.chomp('/') }
if @options[:skip_untested] == 'yes'
distros.delete_if {|d| !@options[:tested_apps].include?(d) }
end
tree = {}
distros.each do |distro|
d_suf = "#{distro}/"
rc, body = get(first_level + d_suf)
next if rc != 0
versions = body.scan(%r{a href="(.*/)">})
versions.shift # Remove first entry ("Parent Directory")
version_path = {}
versions.each do |version|
path = "#{first_level}#{d_suf}#{version[0]}amd64/default/"
rc, body = get(path)
next if rc != 0
release_dates = body.scan(%r{a href="(.*/)">})
# Previous release_dates array leaves a point in the html page
release_date = release_dates.last[0]
version_path[version[0]] = "#{path}#{release_date}rootfs.tar.xz"
end
tree[d_suf] = version_path
end
image_list = JSON.parse(body)
appstr = ''
#-----------------------------------------------------------------------
# Generate the container app information
#-----------------------------------------------------------------------
tree.each do |distro, value|
value.each do |version, path|
path = URI.decode_www_form_component(path)
image_list['products'].each do |img, img_data|
distro, version, arch, variant = img.split(':')
description = "Downloaded from #{@options[:url]}"
regtime = app_time(path).to_s
next if variant != 'default' || arch != 'amd64'
next if @options[:skip_untested] == 'yes' && !@options[:tested_apps].include?(distro)
data = {
'NAME' => "#{distro[0...-1]}_#{version[0...-1]} - LXD",
'SIZE' => @options[:sizemb],
'PUBLISHER' => @options[:url],
'FORMAT' => @options[:format],
'REGTIME' => regtime,
'SOURCE' => app_url(path).to_s,
'MD5' => md5(regtime),
'IMPORT_ID' => '-1',
'ORIGIN_ID' => '-1',
'VERSION' => '1.0',
'TYPE' => 'IMAGE',
'DESCRIPTION' => description,
'TAGS' => '',
'LINK' => "#{@options[:url]}#{path}"
.gsub('/rootfs.tar.xz', '')
}
# Use the last version for each container
regtime = img_data['versions'].to_a.last[0]
tmpl = ''
data.each {|key, val| print_var(tmpl, key, val) }
tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')
data = { 'APPTEMPLATE64' => tmpl64,
'VMTEMPLATE64' => LXDMarket.template(@options) }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end
appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
begin
path = img_data['versions'][regtime]['items']['root.tar.xz']['path']
rescue StandardError
next
end
builddate = DateTime.strptime(regtime, '%Y%m%d_%H:%M').to_time.to_i
data = {
'NAME' => "#{distro}_#{version} - LXD",
'SIZE' => @options[:sizemb],
'PUBLISHER' => @options[:url],
'FORMAT' => @options[:format],
'REGTIME' => builddate,
'SOURCE' => app_url(path).to_s,
'MD5' => md5(regtime),
'IMPORT_ID' => '-1',
'ORIGIN_ID' => '-1',
'VERSION' => '1.0',
'TYPE' => 'IMAGE',
'DESCRIPTION' => "Downloaded from #{@options[:url]}",
'TAGS' => '',
'LINK' => "#{@options[:url]}#{path}"
}
tmpl = ''
data.each { |key, val| print_var(tmpl, key, val) }
tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')
data = { 'APPTEMPLATE64' => tmpl64,
'VMTEMPLATE64' => LXDMarket.template(@options) }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end
appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
end
appstr
ensure
@http_client.finish if @http_client
@http_client&.finish if @http_client
end
private
@ -190,14 +183,6 @@ class LinuxContainersMarket
"&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
end
# Returns build date based on image path
def app_time(path)
buildate = path.split('/')[6]
buildate = DateTime.strptime(buildate, '%Y%m%d_%H:%M')
buildate.to_time.to_i
end
# Print variable in an APP template
def print_var(str, name, val)
return if val.nil?
@ -213,7 +198,7 @@ class LinuxContainersMarket
# Opens a TCP connection to @options[:url]
def open_connection
@http_client.finish if @http_client
@http_client&.finish if @http_client
http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY'] # for ruby 1.9.3
@ -244,7 +229,7 @@ class LinuxContainersMarket
end
################################################################################
# Main Program. Outpust the list of marketplace appliances
# Main Program. Its output is the list of marketplace appliances
################################################################################
def set_option(option, doc, name, path)
option[name] = doc.elements[path].text if doc.elements[path]
@ -258,18 +243,18 @@ begin
pre = 'MARKETPLACE/TEMPLATE'
data = {
:url => "#{pre}/ENDPOINT",
:sizemb => "#{pre}/IMAGE_SIZE_MB",
:fs => "#{pre}/FILESYSTEM",
:format => "#{pre}/FORMAT",
:url => "#{pre}/ENDPOINT",
:sizemb => "#{pre}/IMAGE_SIZE_MB",
:fs => "#{pre}/FILESYSTEM",
:format => "#{pre}/FORMAT",
:skip_untested => "#{pre}/SKIP_UNTESTED",
:memory => "#{pre}/MEMORY",
:cpu => "#{pre}/CPU",
:vcpu => "#{pre}/VCPU",
:privileged => "#{pre}/PRIVILEGED"
:memory => "#{pre}/MEMORY",
:cpu => "#{pre}/CPU",
:vcpu => "#{pre}/VCPU",
:privileged => "#{pre}/PRIVILEGED"
}
data.each {|key, value| set_option(options, doc, key, value) }
data.each { |key, value| set_option(options, doc, key, value) }
str = LinuxContainersMarket.new(options).fetch_appliances
puts str