shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2024-12-23 17:33:56 +03:00
Code Releases Wiki Activity

F #4089: Integrate DockerHub marketplace (#4684)

Browse Source
This commit is contained in:
Christian González 2020-05-10 20:14:20 +02:00 committed by GitHub
parent 6eae04f57b
commit 30b1425f2c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 374 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
install.sh
View File

@ -419,6 +419,7 @@ VAR_DIRS="$VAR_LOCATION/remotes \
$VAR_LOCATION/remotes/market/s3 \
$VAR_LOCATION/remotes/market/linuxcontainers \
$VAR_LOCATION/remotes/market/turnkeylinux \
$VAR_LOCATION/remotes/market/dockerhub \
$VAR_LOCATION/remotes/datastore/iscsi_libvirt \
$VAR_LOCATION/remotes/auth \
$VAR_LOCATION/remotes/auth/plain \
@ -623,6 +624,7 @@ INSTALL_FILES=(
MARKETPLACE_DRIVER_S3_SCRIPTS:$VAR_LOCATION/remotes/market/s3
MARKETPLACE_DRIVER_LXC_SCRIPTS:$VAR_LOCATION/remotes/market/linuxcontainers
MARKETPLACE_DRIVER_TK_SCRIPTS:$VAR_LOCATION/remotes/market/turnkeylinux
MARKETPLACE_DRIVER_DH_SCRIPTS:$VAR_LOCATION/remotes/market/dockerhub
IPAM_DRIVER_DUMMY_SCRIPTS:$VAR_LOCATION/remotes/ipam/dummy
IPAM_DRIVER_PACKET_SCRIPTS:$VAR_LOCATION/remotes/ipam/packet
NETWORK_FILES:$VAR_LOCATION/remotes/vnm
@ -1887,6 +1889,10 @@ MARKETPLACE_DRIVER_TK_SCRIPTS="src/market_mad/remotes/turnkeylinux/import \
src/market_mad/remotes/turnkeylinux/delete \
src/market_mad/remotes/turnkeylinux/monitor"
MARKETPLACE_DRIVER_DH_SCRIPTS="src/market_mad/remotes/dockerhub/import \
src/market_mad/remotes/dockerhub/delete \
src/market_mad/remotes/dockerhub/monitor"
#-------------------------------------------------------------------------------
# Migration scripts for onedb command, to be installed under $LIB_LOCATION
#-------------------------------------------------------------------------------

10
share/etc/oned.conf
View File

@ -656,7 +656,7 @@ DATASTORE_MAD = [
MARKET_MAD = [
EXECUTABLE = "one_market",
ARGUMENTS = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux"
ARGUMENTS = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux,dockerhub"
]
#*******************************************************************************
@ -1280,6 +1280,14 @@ MARKET_MAD_CONF = [
PUBLIC = "yes"
]
MARKET_MAD_CONF = [
NAME = "dockerhub",
SUNSTONE_NAME = "DockerHub",
REQUIRED_ATTRS = "",
APP_ACTIONS = "monitor",
PUBLIC = "yes"
]
#*******************************************************************************
# Authentication Driver Behavior Definition
#*******************************************************************************

15
src/cli/onemarketapp
View File

@ -75,6 +75,13 @@ CommandParser::CmdParser.new(ARGV) do
:description => 'lock all actions'
}
TAG = {
:name => 'tag',
:large => '--tag tag',
:format => String,
:description => 'DockerHub image tag (default latest)'
}
########################################################################
# Global Options
########################################################################
@ -87,7 +94,8 @@ CommandParser::CmdParser.new(ARGV) do
CREATE_OPTIONS = [OneMarketPlaceHelper::MARKETPLACE]
EXPORT_OPTIONS = [OneDatastoreHelper::DATASTORE,
OneMarketPlaceAppHelper::VMNAME]
OneMarketPlaceAppHelper::VMNAME,
TAG]
########################################################################
# Formatters for arguments
@ -172,10 +180,13 @@ CommandParser::CmdParser.new(ARGV) do
command :export, export_desc, :appid, :name, :options => EXPORT_OPTIONS do
helper.perform_action(args[0], options, 'exported') do |obj|
tag ="tag=#{options[:tag]}" if options[:tag]
rc = obj.export(
:dsid => options[:datastore],
:name => args[1],
:vmtemplate_name => options[:vmname]
:vmtemplate_name => options[:vmname],
:url_args => tag
)
next rc if OpenNebula.is_error?(rc)

27
src/datastore_mad/remotes/docker_downloader.sh
View File

@ -114,7 +114,6 @@ id=`uuidgen`
sid=`echo $id | cut -d '-' -f 1`
url=`echo $MARKET_URL | grep -oP "^"docker://"\K.*"`
docker_hub=`echo $url | cut -d '?' -f 1`
arguments=`echo $url | cut -d '?' -f 2`
selected_tag=`get_tag_name`
@ -126,6 +125,11 @@ for p in ${arguments//&/ }; do
[ -n "$k" -a -n "$v" ] && eval $k=$v;
done
if [ -z $tag ]; then
tag="latest"
fi
docker_hub="`echo $url | cut -d '?' -f 1`:${tag}"
docker_image=`echo $docker_hub | cut -f1 -d':'``echo $id |cut -f1 -d'-'`
dockerdir=$TMP_DIR/$id
@ -141,6 +145,23 @@ trap clean EXIT
mkdir -p $dockerdir
mkdir -p $dockerdir/mnt
# Check distro
distro=`docker run --rm \
-e "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
$docker_hub cat /etc/os-release | grep "ID_LIKE=" | cut -d= -f 2 || true`
if [ -z $distro ]; then
distro=`docker run --rm \
-e "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
$docker_hub cat /etc/os-release | grep "^ID=.*\n" | cut -d= -f 2 || true`
fi
if [ -z $distro ]; then
echo "Cannot identified $docker_hub distribution" 1>&2
exit 1
fi
#-------------------------------------------------------------------------------
# Create a DockerFile
#-------------------------------------------------------------------------------
@ -221,10 +242,6 @@ RUN rc-update add sshd default && \
rc-update add networking default
RUN echo 'rc_sys=""' >> /etc/rc.conf
RUN sed -e '159a dev_context=/dev/vdb' \
-e '169s/.*/\t\tmount -o ro \/dev\/vdb \${MOUNT_DIR} 2\>\/dev\/null/' \
-i /usr/sbin/one-contextd
EOC
)
;;

9
src/datastore_mad/remotes/fs/cp
View File

@ -63,7 +63,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CONVERT \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/DRIVER \
/DS_DRIVER_ACTION_DATA/IMAGE/TYPE)
/DS_DRIVER_ACTION_DATA/IMAGE/TYPE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/URL_ARGS)
unset i
@ -81,6 +82,7 @@ LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}"
CONVERT="${XPATH_ELEMENTS[i++]:-yes}"
DRIVER="${XPATH_ELEMENTS[i++]}"
IMAGE_TYPE="${XPATH_ELEMENTS[i++]}"
URL_ARGS="${XPATH_ELEMENTS[i++]}"
DST=`generate_image_path`
IMAGE_HASH=`basename $DST`
@ -98,6 +100,11 @@ if [ "$IMAGE_TYPE" = "1" ] || [ "$TYPE" = "2" ]; then
CONVERT=no
fi
# Append URL args to SRC url
if [ ! -z $URL_ARGS ]; then
SRC+="&$URL_ARGS"
fi
if [ -n "$BRIDGE_LIST" ]; then
DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
else

20
src/market/MarketPlacePool.cc
View File

@ -51,6 +51,12 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
"DESCRIPTION=\"TurnKey linux is a free software repository"
" based on Debian images hosted at turnkeylinux.org\"";
string dh_market =
"NAME=\"DockerHub\"\n"
"MARKET_MAD=dockerhub\n"
"DESCRIPTION=\"DockerHub is the world's largest library and"
" community for container images hosted at hub.docker.com/\"";
Nebula& nd = Nebula::instance();
UserPool * upool = nd.get_upool();
User * oneadmin = upool->get_ro(0);
@ -60,12 +66,14 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
MarketPlaceTemplate * default_tmpl = new MarketPlaceTemplate;
MarketPlaceTemplate * lxc_tmpl = new MarketPlaceTemplate;
MarketPlaceTemplate * tk_tmpl = new MarketPlaceTemplate;
MarketPlaceTemplate * dh_tmpl = new MarketPlaceTemplate;
char * error_parse;
default_tmpl->parse(default_market, &error_parse);
lxc_tmpl->parse(lxc_market, &error_parse);
tk_tmpl->parse(tk_market, &error_parse);
dh_tmpl->parse(dh_market, &error_parse);
MarketPlace * marketplace = new MarketPlace(
oneadmin->get_uid(),
@ -91,24 +99,36 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
oneadmin->get_umask(),
tk_tmpl);
MarketPlace * dh_marketplace = new MarketPlace(
oneadmin->get_uid(),
oneadmin->get_gid(),
oneadmin->get_uname(),
oneadmin->get_gname(),
oneadmin->get_umask(),
dh_tmpl);
oneadmin->unlock();
marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
lxc_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
tk_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
dh_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
marketplace->zone_id = Nebula::instance().get_zone_id();
lxc_marketplace->zone_id = Nebula::instance().get_zone_id();
tk_marketplace->zone_id = Nebula::instance().get_zone_id();
dh_marketplace->zone_id = Nebula::instance().get_zone_id();
marketplace->parse_template(error);
lxc_marketplace->parse_template(error);
tk_marketplace->parse_template(error);
dh_marketplace->parse_template(error);
int rc = PoolSQL::allocate(marketplace, error);
rc += PoolSQL::allocate(lxc_marketplace, error);
rc += PoolSQL::allocate(tk_marketplace, error);
rc += PoolSQL::allocate(dh_marketplace, error);
if (rc < 0)
{

29
src/market_mad/remotes/dockerhub/delete Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
error_message "Cannot delete app from DockerHub marketplace"
exit -1

29
src/market_mad/remotes/dockerhub/import Executable file
View File

@ -0,0 +1,29 @@
#!/bin/bash
# -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
if [ -z "${ONE_LOCATION}" ]; then
LIB_LOCATION=/usr/lib/one
else
LIB_LOCATION=$ONE_LOCATION/lib
fi
. $LIB_LOCATION/sh/scripts_common.sh
error_message "Cannot import app into DockerHub marketplace"
exit -1

234
src/market_mad/remotes/dockerhub/monitor Executable file
View File

@ -0,0 +1,234 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
require 'net/http'
require 'uri'
require 'json'
require 'base64'
require 'rexml/document'
require 'time'
require 'digest/md5'
#-------------------------------------------------------------------------------
#
#-------------------------------------------------------------------------------
class DockerHubMarket
#---------------------------------------------------------------------------
# Default Configuration parameters for the Driver
#---------------------------------------------------------------------------
DEFAULTS = {
:url => 'https://hub.docker.com/v2/repositories/library/',
:sizemb => 2048,
:fs => 'ext4',
:format => 'raw',
:agent => 'OpenNebula',
:page_size => 100
}
# TODO: Make configurable
TEMPLATE = "
CPU = \"1\"
MEMORY = \"768\"
GRAPHICS = [
LISTEN =\"0.0.0.0\",
TYPE =\"vnc\"
]
CONTEXT = [
NETWORK =\"YES\",
SSH_PUBLIC_KEY =\"$USER[SSH_PUBLIC_KEY]\",
SET_HOSTNAME =\"$NAME\"
]"
#---------------------------------------------------------------------------
# Configuration varibales
# :url of linuxcontainers market place
# :sizemb default size for container images
# :fs filesystem for the image file
# :format for the image file, qcow2, raw
# :agent for HTTP client
#---------------------------------------------------------------------------
def initialize(options = {})
@options = DEFAULTS
@options.merge!(options)
version_path = File.dirname(__FILE__) + '/../../VERSION'
@options[:agent] = "OpenNebula #{File.read(version_path)}" \
if File.exist? version_path
end
#---------------------------------------------------------------------------
# Fetch URL
#---------------------------------------------------------------------------
def get(path)
# Get proxy params (needed for ruby 1.9.3)
http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
p_host = nil
p_port = nil
if http_proxy
p_uri = URI(http_proxy)
p_host = p_uri.host
p_port = p_uri.port
end
uri = URI(path)
req = Net::HTTP::Get.new(uri.request_uri)
req['User-Agent'] = @options[:agent] if @options[:agent]
opts = { :use_ssl => true }
rc = Net::HTTP.start(uri.hostname, uri.port,
p_host, p_port, opts) do |http|
http.request(req)
end
return [rc.code.to_i, rc.msg] unless rc.is_a? Net::HTTPSuccess
[0, rc.body]
end
#---------------------------------------------------------------------------
# Get appliance list
#---------------------------------------------------------------------------
def appliances
try = 0
appstr = ''
apps = []
next_query = "#{@options[:url]}?page_size=#{@options[:page_size]}"
loop do
rc, body = get(next_query)
return [rc, body] if rc != 0
parsed = JSON.parse(body)
if parsed['results'].empty?
try += 1
return [0, ''] if try > 5
end
# reset try for new query
try = 0
apps.concat(parsed['results'])
next_query = parsed['next']
break if next_query.nil? || next_query.empty?
end
# App JSON example:
# {
# "user": "library",
# "name": "busybox",
# "namespace": "library",
# "repository_type": "image",
# "status": 1,
# "description": "Busybox base image.",
# "is_private": false,
# "is_automated": false,
# "can_edit": false,
# "star_count": 1878,
# "pull_count": 2147483647,
# "last_updated": "2020-04-16T08:59:03.768123Z",
# "is_migrated": false
# }
apps[0..-1].each do |app|
time = app['last_updated'].split('T')[0].split('-')
regt = Time.new(time[0], time[1], time[2]).to_i
data = {
'NAME' => app['name'],
'SOURCE' => app_url(app),
'IMPORT_ID' => -1,
'ORIGIN_ID' => -1,
'TYPE' => 'IMAGE',
'PUBLISHER' => 'hub.docker.com',
'MD5' => md5(regt),
'FORMAT' => 'raw',
'VERSION' => '1.0',
'TAGS' => '',
'REGTIME' => regt,
'SIZE' => @options[:sizemb],
'DESCRIPTION' => app['description'].delete('\\"')
}
tmpl = ''
data.each {|key, val| print_var(tmpl, key, val) }
tmpl64 = ''
print_var(tmpl64, 'DRIVER', 'raw')
data = { 'APPTEMPLATE64' => tmpl64, 'VMTEMPLATE64' => TEMPLATE }
data.each do |key, val|
print_var(tmpl, key, Base64.strict_encode64(val))
end
appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
end
[0, appstr]
end
def app_url(app)
"docker://#{app['name']}?size=#{@options[:sizemb]}" \
"&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
end
def print_var(str, name, val)
return if val.nil?
return if val.class == String && val.empty?
str << "#{name}=\"#{val}\"\n"
end
# Returns an md5 from a combination of the @option hash and an input string
def md5(string)
Digest::MD5.hexdigest("#{@options} #{string}")
end
end
################################################################################
# Main Program. Outpust the list of marketplace appliances
################################################################################
def set_option(opt, doc, name, path)
opt[name] = doc.elements[path].text if doc.elements[path]
end
begin
options = {}
drv_message = Base64.decode64(ARGV[0])
doc = REXML::Document.new(drv_message).root
set_option(options, doc, :url, 'MARKETPLACE/TEMPLATE/ENDPOINT')
set_option(options, doc, :sizemb, 'MARKETPLACE/TEMPLATE/IMAGE_SIZE_MB')
set_option(options, doc, :fs, 'MARKETPLACE/TEMPLATE/FILESYSTEM')
set_option(options, doc, :format, 'MARKETPLACE/TEMPLATE/FORMAT')
rc, str = DockerHubMarket.new(options).appliances
if rc != 0
STDERR.puts str
exit(-1)
end
puts str
end

2
src/market_mad/remotes/turnkeylinux/monitor
View File

@ -152,7 +152,7 @@ class TurnkeyLinux
# Line example
# debian-9-turnkey-zurmo_15.2-1_amd64.tar.gz.hash 2018-11-29 09:45 2.2K
apps[1..-1].each do |l|
apps[0..-1].each do |l|
f = l.split
next if f[0] =~ /\.hash$/

5
src/oca/ruby/opennebula/marketplaceapp.rb
View File

@ -192,8 +192,9 @@ module OpenNebula
options[:vmtemplate_name] = name unless options[:vmtemplate_name]
tmpl << "\n"
tmpl << "NAME=\"" << name << "\"\n"
tmpl << "FROM_APP=\"" << self['ID'] << "\"\n"
tmpl << 'NAME="' << name << "\"\n"
tmpl << 'FROM_APP="' << self['ID'] << "\"\n"
tmpl << 'URL_ARGS="' << options[:url_args] << "\"\n" if options[:url_args]
case type_str
when 'IMAGE'