shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-01-21 18:03:38 +03:00
Code Releases Wiki Activity

F #5411: add support for Docker Private Registry (#1248)

Browse Source
This commit is contained in:
Alejandro Huertas Herrero 2021-05-31 13:27:53 +02:00 committed by GitHub
parent 05848db905
commit 8b26f099ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 389 additions and 96 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
install.sh
View File

@ -493,6 +493,7 @@ VAR_DIRS="$VAR_LOCATION/remotes \
$VAR_LOCATION/remotes/market/linuxcontainers \
$VAR_LOCATION/remotes/market/turnkeylinux \
$VAR_LOCATION/remotes/market/dockerhub \
$VAR_LOCATION/remotes/market/docker_registry \
$VAR_LOCATION/remotes/datastore/iscsi_libvirt \
$VAR_LOCATION/remotes/auth \
$VAR_LOCATION/remotes/auth/plain \
@ -713,6 +714,7 @@ INSTALL_FILES=(
MARKETPLACE_DRIVER_LXC_SCRIPTS:$VAR_LOCATION/remotes/market/linuxcontainers
MARKETPLACE_DRIVER_TK_SCRIPTS:$VAR_LOCATION/remotes/market/turnkeylinux
MARKETPLACE_DRIVER_DH_SCRIPTS:$VAR_LOCATION/remotes/market/dockerhub
MARKETPLACE_DRIVER_REGISTRY_SCRIPTS:$VAR_LOCATION/remotes/market/docker_registry
IPAM_DRIVER_DUMMY_SCRIPTS:$VAR_LOCATION/remotes/ipam/dummy
IPAM_DRIVER_PACKET_SCRIPTS:$VAR_LOCATION/remotes/ipam/packet
IPAM_DRIVER_VULTR_SCRIPTS:$VAR_LOCATION/remotes/ipam/vultr
@ -2148,7 +2150,8 @@ MARKETPLACE_DRIVER_S3_SCRIPTS="src/market_mad/remotes/s3/import \
src/market_mad/remotes/s3/monitor \
src/market_mad/remotes/s3/S3.rb"
MARKETPLACE_DRIVER_COMMON_SCRIPTS="src/market_mad/remotes/common/lxd.rb"
MARKETPLACE_DRIVER_COMMON_SCRIPTS="src/market_mad/remotes/common/lxd.rb \
src/market_mad/remotes/common/docker.rb"
MARKETPLACE_DRIVER_LXC_SCRIPTS="src/market_mad/remotes/linuxcontainers/import \
src/market_mad/remotes/linuxcontainers/delete \
@ -2162,6 +2165,10 @@ MARKETPLACE_DRIVER_DH_SCRIPTS="src/market_mad/remotes/dockerhub/import \
src/market_mad/remotes/dockerhub/delete \
src/market_mad/remotes/dockerhub/monitor"
MARKETPLACE_DRIVER_REGISTRY_SCRIPTS="src/market_mad/remotes/docker_registry/import \
src/market_mad/remotes/docker_registry/delete \
src/market_mad/remotes/docker_registry/monitor"
#-------------------------------------------------------------------------------
# Migration scripts for onedb command, to be installed under $LIB_LOCATION
#-------------------------------------------------------------------------------

9
share/etc/oned.conf
View File

@ -650,7 +650,7 @@ DATASTORE_MAD = [
MARKET_MAD = [
EXECUTABLE = "one_market",
ARGUMENTS = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux,dockerhub"
ARGUMENTS = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux,dockerhub,docker_registry"
]
#*******************************************************************************
@ -1297,6 +1297,13 @@ MARKET_MAD_CONF = [
PUBLIC = "yes"
]
MARKET_MAD_CONF = [
NAME = "docker_registry",
SUNSTONE_NAME = "DockerRegistry",
REQUIRED_ATTRS = "BASE_URL",
APP_ACTIONS = "monitor"
]
#*******************************************************************************
# Authentication Driver Behavior Definition
#*******************************************************************************

147
src/market_mad/remotes/common/docker.rb Normal file
View File

@ -0,0 +1,147 @@
#!/usr/bin/ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
require 'base64'
require 'digest/md5'
require 'net/http'
require 'uri'
# Utilities for Docker based marketplaces
module DockerMarket
class << self
# TODO: Make configurable
# Returns template to append to all applications
def template
unindent(<<-EOS)
CPU = \"1\"
MEMORY = \"768\"
GRAPHICS = [
LISTEN =\"0.0.0.0\",
TYPE =\"vnc\"
]
CONTEXT = [
NETWORK =\"YES\",
SSH_PUBLIC_KEY =\"$USER[SSH_PUBLIC_KEY]\",
SET_HOSTNAME =\"$NAME\"
]
OS = [
KERNEL_CMD=\"console=ttyS0 reboot=k panic=1\"
]"
EOS
end
# Makes text unindent
#
# @param str [String] String to use
#
# @return [String]
def unindent(str)
m = str.match(/^(\s*)/)
spaces = m[1].size
str.gsub!(/^ {#{spaces}}/, '')
end
# Fetch data from path
#
# @param options [Hash] Class options
# @param path [String] URL to fetch
# @param ssl [Boolean] True to use SSL
#
# @return [0/rc, body]
def get(options, path, ssl = true)
# Get proxy params (needed for ruby 1.9.3)
http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
p_host = nil
p_port = nil
if http_proxy
p_uri = URI(http_proxy)
p_host = p_uri.host
p_port = p_uri.port
end
uri = URI(path)
req = Net::HTTP::Get.new(uri.request_uri)
req['User-Agent'] = options[:agent] if options[:agent]
opts = { :use_ssl => true } if ssl
rc = Net::HTTP.start(uri.hostname,
uri.port,
p_host,
p_port,
opts) do |http|
http.request(req)
end
return [rc.code.to_i, rc.msg] unless rc.is_a? Net::HTTPSuccess
[0, rc.body]
end
# Generates app template
#
# @param app [Hash] App information
#
# @returns [String] App information in base64
def gen_template(app)
tmpl = ''
tmpl64 = ''
app.each {|key, val| print_var(tmpl, key, val) }
print_var(tmpl64, 'DRIVER', 'raw')
print_var(tmpl64, 'DEV_PREFIX', 'vd')
data = { 'APPTEMPLATE64' => tmpl64, 'VMTEMPLATE64' => template }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end
"APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
end
# Prints variable
#
# @param str [String] String to print variable to
# @param name [String] KEY
# @param val [String] Value
#
# @return [String] KEY=VALUE string
def print_var(str, name, val)
return if val.nil?
return if val.class == String && val.empty?
str << "#{name}=\"#{val}\"\n"
end
# Returns an md5 from a combination of the @option hash and an input str
#
# @param options [Hash] Options to use
# @param string [String] String to combine
def md5(options, string)
Digest::MD5.hexdigest("#{options} #{string}")
end
end
end

29
src/market_mad/remotes/docker_registry/delete Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
error_message "Cannot import app into Docker Registry marketplace"
exit -1

29
src/market_mad/remotes/docker_registry/import Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
error_message "Cannot import app into Docker Registry marketplace"
exit -1

146
src/market_mad/remotes/docker_registry/monitor Executable file
View File

@ -0,0 +1,146 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2021, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
require 'json'
require 'rexml/document'
require 'time'
require_relative '../common/docker'
#-------------------------------------------------------------------------------
# Class to interact and monitor Docker Private Registry
#-------------------------------------------------------------------------------
class DockerRegistryMarket
#---------------------------------------------------------------------------
# Default Configuration parameters for the Driver
#---------------------------------------------------------------------------
DEFAULTS = {
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:ssl => false
}
#---------------------------------------------------------------------------
# Configuration varibales
#
# :sizemb -> default size for container images
# :fs -> filesystem for the image file
# :format -> for the image file, qcow2, raw
# :agent -> for HTTP client
# :ssl -> true if registry is behind SSL
#---------------------------------------------------------------------------
def initialize(options = {})
@options = DEFAULTS
version_path = File.dirname(__FILE__) + '/../../VERSION'
@options.merge!(options)
return unless File.exist?(version_path)
@options[:agent] = "OpenNebula #{File.read(version_path)}"
end
#---------------------------------------------------------------------------
# Get appliance list
#---------------------------------------------------------------------------
#
# @param url [String] Docker registry URL
def appliances(url)
appstr = ''
url = "#{url}/" unless url[-1] == '/'
rc, body = DockerMarket.get(@options,
"#{url}v2/_catalog",
@options[:ssl])
return [rc, body] if rc != 0
JSON.parse(body)['repositories'].each do |app|
regt = ''
if !app['last_updated'].nil?
time = app['last_updated'].split('T')[0].split('-')
regt = Time.new(time[0], time[1], time[2]).to_i
end
data = {
'NAME' => app,
'SOURCE' => app_url(url, app),
'IMPORT_ID' => -1,
'ORIGIN_ID' => -1,
'TYPE' => 'IMAGE',
'PUBLISHER' => 'hub.docker.com',
'MD5' => DockerMarket.md5(@options, regt),
'FORMAT' => 'raw',
'VERSION' => '1.0',
'TAGS' => '',
'REGTIME' => regt,
'SIZE' => @options[:sizemb],
'DESCRIPTION' => '',
'LINK' => "#{url}/v2/#{app}/tags/list"
}
appstr << DockerMarket.gen_template(data)
end
[0, appstr]
end
# Get app URL
#
# @param url [String] Docker registry URL
# @param app [String] App name
def app_url(url, app)
url = url.gsub('http://', '')
"docker://#{url}#{app}?size=#{@options[:sizemb]}" \
"&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
end
end
################################################################################
# Main Program. Outpust the list of marketplace appliances
################################################################################
def set_option(opt, doc, name, path)
opt[name] = doc.elements[path].text if doc.elements[path]
end
begin
options = {}
drv_message = Base64.decode64(ARGV[0])
doc = REXML::Document.new(drv_message).root
set_option(options, doc, :url, 'MARKETPLACE/TEMPLATE/BASE_URL')
set_option(options, doc, :url, 'MARKETPLACE/TEMPLATE/SSL')
set_option(options, doc, :sizemb, 'MARKETPLACE/TEMPLATE/IMAGE_SIZE_MB')
set_option(options, doc, :fs, 'MARKETPLACE/TEMPLATE/FILESYSTEM')
set_option(options, doc, :format, 'MARKETPLACE/TEMPLATE/FORMAT')
rc, str = DockerRegistryMarket.new(options).appliances(options[:url])
if rc != 0
STDERR.puts str
exit(-1)
end
puts str
end

116
src/market_mad/remotes/dockerhub/monitor
View File

@ -16,16 +16,14 @@
# limitations under the License. #
# -------------------------------------------------------------------------- #
require 'net/http'
require 'uri'
require 'json'
require 'base64'
require 'rexml/document'
require 'time'
require 'digest/md5'
require_relative '../common/docker'
#-------------------------------------------------------------------------------
#
# Class to interact and monitor DockerHub
#-------------------------------------------------------------------------------
class DockerHubMarket
@ -33,78 +31,32 @@ class DockerHubMarket
# Default Configuration parameters for the Driver
#---------------------------------------------------------------------------
DEFAULTS = {
:url => 'https://hub.docker.com/v2/repositories/library/',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:url => 'https://hub.docker.com/v2/repositories/library/',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:page_size => 100
}
# TODO: Make configurable
TEMPLATE = "
CPU = \"1\"
MEMORY = \"768\"
GRAPHICS = [
LISTEN =\"0.0.0.0\",
TYPE =\"vnc\"
]
CONTEXT = [
NETWORK =\"YES\",
SSH_PUBLIC_KEY =\"$USER[SSH_PUBLIC_KEY]\",
SET_HOSTNAME =\"$NAME\"
]
OS = [
KERNEL_CMD=\"console=ttyS0 reboot=k panic=1\"
]"
#---------------------------------------------------------------------------
# Configuration varibales
# :url of linuxcontainers market place
# :sizemb default size for container images
# :fs filesystem for the image file
# :format for the image file, qcow2, raw
# :agent for HTTP client
#
# :url -> of DockerHub market place
# :sizemb -> default size for container images
# :fs -> filesystem for the image file
# :format -> for the image file, qcow2, raw
# :agent -> for HTTP client
#---------------------------------------------------------------------------
def initialize(options = {})
@options = DEFAULTS
@options = DEFAULTS
version_path = File.dirname(__FILE__) + '/../../VERSION'
@options.merge!(options)
version_path = File.dirname(__FILE__) + '/../../VERSION'
@options[:agent] = "OpenNebula #{File.read(version_path)}" \
if File.exist? version_path
end
return unless File.exist?(version_path)
#---------------------------------------------------------------------------
# Fetch URL
#---------------------------------------------------------------------------
def get(path)
# Get proxy params (needed for ruby 1.9.3)
http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
p_host = nil
p_port = nil
if http_proxy
p_uri = URI(http_proxy)
p_host = p_uri.host
p_port = p_uri.port
end
uri = URI(path)
req = Net::HTTP::Get.new(uri.request_uri)
req['User-Agent'] = @options[:agent] if @options[:agent]
opts = { :use_ssl => true }
rc = Net::HTTP.start(uri.hostname, uri.port,
p_host, p_port, opts) do |http|
http.request(req)
end
return [rc.code.to_i, rc.msg] unless rc.is_a? Net::HTTPSuccess
[0, rc.body]
@options[:agent] = "OpenNebula #{File.read(version_path)}"
end
#---------------------------------------------------------------------------
@ -117,7 +69,7 @@ OS = [
next_query = "#{@options[:url]}?page_size=#{@options[:page_size]}"
loop do
rc, body = get(next_query)
rc, body = DockerMarket.get(@options, next_query)
return [rc, body] if rc != 0
@ -167,7 +119,7 @@ OS = [
'ORIGIN_ID' => -1,
'TYPE' => 'IMAGE',
'PUBLISHER' => 'hub.docker.com',
'MD5' => md5(regt),
'MD5' => DockerMarket.md5(@options, regt),
'FORMAT' => 'raw',
'VERSION' => '1.0',
'TAGS' => '',
@ -177,19 +129,7 @@ OS = [
'LINK' => "https://hub.docker.com/_/#{app['name']}"
}
tmpl = ''
data.each {|key, val| print_var(tmpl, key, val) }
tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')
print_var(tmpl64, 'DEV_PREFIX', 'vd')
data = { 'APPTEMPLATE64' => tmpl64, 'VMTEMPLATE64' => TEMPLATE }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end
appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
appstr << DockerMarket.gen_template(data)
end
[0, appstr]
@ -200,18 +140,6 @@ OS = [
"&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
end
def print_var(str, name, val)
return if val.nil?
return if val.class == String && val.empty?
str << "#{name}=\"#{val}\"\n"
end
# Returns an md5 from a combination of the @option hash and an input string
def md5(string)
Digest::MD5.hexdigest("#{@options} #{string}")
end
end
################################################################################