F #4089: Integrate DockerHub marketplace (#4684)

install.sh

@@ -419,6 +419,7 @@ VAR_DIRS="$VAR_LOCATION/remotes \
           $VAR_LOCATION/remotes/market/s3 \
           $VAR_LOCATION/remotes/market/linuxcontainers \
           $VAR_LOCATION/remotes/market/turnkeylinux \
+          $VAR_LOCATION/remotes/market/dockerhub \
           $VAR_LOCATION/remotes/datastore/iscsi_libvirt \
           $VAR_LOCATION/remotes/auth \
           $VAR_LOCATION/remotes/auth/plain \
@@ -623,6 +624,7 @@ INSTALL_FILES=(
     MARKETPLACE_DRIVER_S3_SCRIPTS:$VAR_LOCATION/remotes/market/s3
     MARKETPLACE_DRIVER_LXC_SCRIPTS:$VAR_LOCATION/remotes/market/linuxcontainers
     MARKETPLACE_DRIVER_TK_SCRIPTS:$VAR_LOCATION/remotes/market/turnkeylinux
+    MARKETPLACE_DRIVER_DH_SCRIPTS:$VAR_LOCATION/remotes/market/dockerhub
     IPAM_DRIVER_DUMMY_SCRIPTS:$VAR_LOCATION/remotes/ipam/dummy
     IPAM_DRIVER_PACKET_SCRIPTS:$VAR_LOCATION/remotes/ipam/packet
     NETWORK_FILES:$VAR_LOCATION/remotes/vnm
@@ -1887,6 +1889,10 @@ MARKETPLACE_DRIVER_TK_SCRIPTS="src/market_mad/remotes/turnkeylinux/import \
             src/market_mad/remotes/turnkeylinux/delete \
             src/market_mad/remotes/turnkeylinux/monitor"
 
+MARKETPLACE_DRIVER_DH_SCRIPTS="src/market_mad/remotes/dockerhub/import \
+            src/market_mad/remotes/dockerhub/delete \
+            src/market_mad/remotes/dockerhub/monitor"
+
 #-------------------------------------------------------------------------------
 # Migration scripts for onedb command, to be installed under $LIB_LOCATION
 #-------------------------------------------------------------------------------

share/etc/oned.conf

@@ -656,7 +656,7 @@ DATASTORE_MAD = [
 
 MARKET_MAD = [
     EXECUTABLE = "one_market",
-    ARGUMENTS  = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux"
+    ARGUMENTS  = "-t 15 -m http,s3,one,linuxcontainers,turnkeylinux,dockerhub"
 ]
 
 #*******************************************************************************
@@ -1280,6 +1280,14 @@ MARKET_MAD_CONF = [
     PUBLIC = "yes"
 ]
 
+MARKET_MAD_CONF = [
+    NAME = "dockerhub",
+    SUNSTONE_NAME = "DockerHub",
+    REQUIRED_ATTRS = "",
+    APP_ACTIONS = "monitor",
+    PUBLIC = "yes"
+]
+
 #*******************************************************************************
 # Authentication Driver Behavior Definition
 #*******************************************************************************

src/cli/onemarketapp

@@ -75,6 +75,13 @@ CommandParser::CmdParser.new(ARGV) do
         :description => 'lock all actions'
     }
 
+    TAG = {
+        :name => 'tag',
+        :large => '--tag tag',
+        :format => String,
+        :description => 'DockerHub image tag (default latest)'
+    }
+
     ########################################################################
     # Global Options
     ########################################################################
@@ -87,7 +94,8 @@ CommandParser::CmdParser.new(ARGV) do
 
     CREATE_OPTIONS = [OneMarketPlaceHelper::MARKETPLACE]
     EXPORT_OPTIONS = [OneDatastoreHelper::DATASTORE,
-                      OneMarketPlaceAppHelper::VMNAME]
+                      OneMarketPlaceAppHelper::VMNAME,
+                      TAG]
 
     ########################################################################
     # Formatters for arguments
@@ -172,10 +180,13 @@ CommandParser::CmdParser.new(ARGV) do
 
     command :export, export_desc, :appid, :name, :options => EXPORT_OPTIONS do
         helper.perform_action(args[0], options, 'exported') do |obj|
+            tag ="tag=#{options[:tag]}" if options[:tag]
+
             rc = obj.export(
                 :dsid => options[:datastore],
                 :name => args[1],
-                :vmtemplate_name => options[:vmname]
+                :vmtemplate_name => options[:vmname],
+                :url_args => tag
             )
 
             next rc if OpenNebula.is_error?(rc)

src/datastore_mad/remotes/docker_downloader.sh

@@ -114,7 +114,6 @@ id=`uuidgen`
 sid=`echo $id | cut -d '-' -f 1`
 
 url=`echo $MARKET_URL | grep -oP "^"docker://"\K.*"`
-docker_hub=`echo $url | cut -d '?' -f 1`
 arguments=`echo $url | cut -d '?' -f 2`
 
 selected_tag=`get_tag_name`
@@ -126,6 +125,11 @@ for p in ${arguments//&/ }; do
     [ -n "$k" -a -n "$v" ] && eval $k=$v;
 done
 
+if [ -z $tag ]; then
+    tag="latest"
+fi
+
+docker_hub="`echo $url | cut -d '?' -f 1`:${tag}"
 docker_image=`echo $docker_hub | cut -f1 -d':'``echo $id |cut -f1 -d'-'`
 
 dockerdir=$TMP_DIR/$id
@@ -141,6 +145,23 @@ trap clean EXIT
 mkdir -p $dockerdir
 mkdir -p $dockerdir/mnt
 
+# Check distro
+
+distro=`docker run --rm \
+        -e "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+        $docker_hub cat /etc/os-release | grep "ID_LIKE=" | cut -d= -f 2 || true`
+
+if [ -z $distro ]; then
+    distro=`docker run --rm \
+            -e "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" \
+            $docker_hub cat /etc/os-release | grep "^ID=.*\n" | cut -d= -f 2 || true`
+fi
+
+if [ -z $distro ]; then
+    echo "Cannot identified $docker_hub distribution" 1>&2
+    exit 1
+fi
+
 #-------------------------------------------------------------------------------
 # Create a DockerFile
 #-------------------------------------------------------------------------------
@@ -221,10 +242,6 @@ RUN rc-update add sshd default && \
     rc-update add networking default
 
 RUN echo 'rc_sys=""' >> /etc/rc.conf
-
-RUN sed -e '159a dev_context=/dev/vdb' \
-        -e '169s/.*/\t\tmount -o ro \/dev\/vdb \${MOUNT_DIR} 2\>\/dev\/null/' \
-        -i /usr/sbin/one-contextd
 EOC
 )
     ;;

src/datastore_mad/remotes/fs/cp

@@ -63,7 +63,8 @@ done < <($XPATH     /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
                     /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \
                     /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CONVERT \
                     /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/DRIVER \
-                    /DS_DRIVER_ACTION_DATA/IMAGE/TYPE)
+                    /DS_DRIVER_ACTION_DATA/IMAGE/TYPE \
+                    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/URL_ARGS)
 
 unset i
 
@@ -81,6 +82,7 @@ LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}"
 CONVERT="${XPATH_ELEMENTS[i++]:-yes}"
 DRIVER="${XPATH_ELEMENTS[i++]}"
 IMAGE_TYPE="${XPATH_ELEMENTS[i++]}"
+URL_ARGS="${XPATH_ELEMENTS[i++]}"
 
 DST=`generate_image_path`
 IMAGE_HASH=`basename $DST`
@@ -98,6 +100,11 @@ if [ "$IMAGE_TYPE" = "1" ] || [ "$TYPE" = "2" ]; then
     CONVERT=no
 fi
 
+# Append URL args to SRC url
+if [ ! -z $URL_ARGS ]; then
+    SRC+="&$URL_ARGS"
+fi
+
 if [ -n "$BRIDGE_LIST" ]; then
     DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
 else

src/market/MarketPlacePool.cc

@@ -51,6 +51,12 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
             "DESCRIPTION=\"TurnKey linux is a free software repository"
             " based on Debian images hosted at turnkeylinux.org\"";
 
+        string dh_market =
+            "NAME=\"DockerHub\"\n"
+            "MARKET_MAD=dockerhub\n"
+            "DESCRIPTION=\"DockerHub is the world's largest library and"
+            "  community for container images hosted at hub.docker.com/\"";
+
         Nebula& nd         = Nebula::instance();
         UserPool * upool   = nd.get_upool();
         User *    oneadmin = upool->get_ro(0);
@@ -60,12 +66,14 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
         MarketPlaceTemplate * default_tmpl = new MarketPlaceTemplate;
         MarketPlaceTemplate * lxc_tmpl     = new MarketPlaceTemplate;
         MarketPlaceTemplate * tk_tmpl      = new MarketPlaceTemplate;
+        MarketPlaceTemplate * dh_tmpl      = new MarketPlaceTemplate;
 
         char * error_parse;
 
         default_tmpl->parse(default_market, &error_parse);
         lxc_tmpl->parse(lxc_market, &error_parse);
         tk_tmpl->parse(tk_market, &error_parse);
+        dh_tmpl->parse(dh_market, &error_parse);
 
         MarketPlace * marketplace = new MarketPlace(
                 oneadmin->get_uid(),
@@ -91,24 +99,36 @@ MarketPlacePool::MarketPlacePool(SqlDB * db, bool is_federation_slave)
                 oneadmin->get_umask(),
                 tk_tmpl);
 
+        MarketPlace * dh_marketplace = new MarketPlace(
+                oneadmin->get_uid(),
+                oneadmin->get_gid(),
+                oneadmin->get_uname(),
+                oneadmin->get_gname(),
+                oneadmin->get_umask(),
+                dh_tmpl);
+
         oneadmin->unlock();
 
         marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
         lxc_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
         tk_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
+        dh_marketplace->set_permissions(1,1,1, 1,0,0, 1,0,0, error);
 
         marketplace->zone_id = Nebula::instance().get_zone_id();
         lxc_marketplace->zone_id = Nebula::instance().get_zone_id();
         tk_marketplace->zone_id = Nebula::instance().get_zone_id();
+        dh_marketplace->zone_id = Nebula::instance().get_zone_id();
 
         marketplace->parse_template(error);
         lxc_marketplace->parse_template(error);
         tk_marketplace->parse_template(error);
+        dh_marketplace->parse_template(error);
 
         int rc = PoolSQL::allocate(marketplace, error);
 
         rc += PoolSQL::allocate(lxc_marketplace, error);
         rc += PoolSQL::allocate(tk_marketplace, error);
+        rc += PoolSQL::allocate(dh_marketplace, error);
 
         if (rc < 0)
         {

src/market_mad/remotes/dockerhub/delete

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+error_message "Cannot delete app from DockerHub marketplace"
+
+exit -1

src/market_mad/remotes/dockerhub/import

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+error_message "Cannot import app into DockerHub marketplace"
+
+exit -1

src/market_mad/remotes/dockerhub/monitor

@@ -0,0 +1,234 @@
+#!/usr/bin/env ruby
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'base64'
+require 'rexml/document'
+require 'time'
+require 'digest/md5'
+
+#-------------------------------------------------------------------------------
+#
+#-------------------------------------------------------------------------------
+class DockerHubMarket
+
+    #---------------------------------------------------------------------------
+    # Default Configuration parameters for the Driver
+    #---------------------------------------------------------------------------
+    DEFAULTS = {
+        :url => 'https://hub.docker.com/v2/repositories/library/',
+        :sizemb => 2048,
+        :fs => 'ext4',
+        :format => 'raw',
+        :agent => 'OpenNebula',
+        :page_size => 100
+    }
+
+    # TODO: Make configurable
+    TEMPLATE = "
+CPU = \"1\"
+MEMORY = \"768\"
+GRAPHICS = [
+    LISTEN  =\"0.0.0.0\",
+    TYPE  =\"vnc\"
+]
+CONTEXT = [
+    NETWORK  =\"YES\",
+    SSH_PUBLIC_KEY  =\"$USER[SSH_PUBLIC_KEY]\",
+    SET_HOSTNAME  =\"$NAME\"
+]"
+
+    #---------------------------------------------------------------------------
+    # Configuration varibales
+    #   :url of linuxcontainers market place
+    #   :sizemb default size for container images
+    #   :fs filesystem for the image file
+    #   :format for the image file, qcow2, raw
+    #   :agent for HTTP client
+    #---------------------------------------------------------------------------
+    def initialize(options = {})
+        @options = DEFAULTS
+        @options.merge!(options)
+
+        version_path = File.dirname(__FILE__) + '/../../VERSION'
+        @options[:agent] = "OpenNebula #{File.read(version_path)}" \
+            if File.exist? version_path
+    end
+
+    #---------------------------------------------------------------------------
+    # Fetch URL
+    #---------------------------------------------------------------------------
+    def get(path)
+        # Get proxy params (needed for ruby 1.9.3)
+        http_proxy = ENV['http_proxy'] || ENV['HTTP_PROXY']
+
+        p_host  = nil
+        p_port  = nil
+
+        if http_proxy
+            p_uri   = URI(http_proxy)
+            p_host  = p_uri.host
+            p_port  = p_uri.port
+        end
+
+        uri = URI(path)
+        req = Net::HTTP::Get.new(uri.request_uri)
+
+        req['User-Agent'] = @options[:agent] if @options[:agent]
+
+        opts = { :use_ssl => true }
+        rc = Net::HTTP.start(uri.hostname, uri.port,
+                             p_host, p_port, opts) do |http|
+            http.request(req)
+        end
+
+        return [rc.code.to_i, rc.msg] unless rc.is_a? Net::HTTPSuccess
+
+        [0, rc.body]
+    end
+
+    #---------------------------------------------------------------------------
+    # Get appliance list
+    #---------------------------------------------------------------------------
+    def appliances
+        try    = 0
+        appstr = ''
+        apps   = []
+        next_query = "#{@options[:url]}?page_size=#{@options[:page_size]}"
+
+        loop do
+            rc, body = get(next_query)
+
+            return [rc, body] if rc != 0
+
+            parsed = JSON.parse(body)
+
+            if parsed['results'].empty?
+                try += 1
+                return [0, ''] if try > 5
+            end
+
+            # reset try for new query
+            try = 0
+            apps.concat(parsed['results'])
+            next_query = parsed['next']
+
+            break if next_query.nil? || next_query.empty?
+        end
+
+        # App JSON example:
+        # {
+        #    "user": "library",
+        #    "name": "busybox",
+        #    "namespace": "library",
+        #    "repository_type": "image",
+        #    "status": 1,
+        #    "description": "Busybox base image.",
+        #    "is_private": false,
+        #    "is_automated": false,
+        #    "can_edit": false,
+        #    "star_count": 1878,
+        #    "pull_count": 2147483647,
+        #    "last_updated": "2020-04-16T08:59:03.768123Z",
+        #    "is_migrated": false
+        # }
+        apps[0..-1].each do |app|
+            time = app['last_updated'].split('T')[0].split('-')
+            regt = Time.new(time[0], time[1], time[2]).to_i
+
+            data = {
+                'NAME'        => app['name'],
+                'SOURCE'      => app_url(app),
+                'IMPORT_ID'   => -1,
+                'ORIGIN_ID'   => -1,
+                'TYPE'        => 'IMAGE',
+                'PUBLISHER'   => 'hub.docker.com',
+                'MD5'         => md5(regt),
+                'FORMAT'      => 'raw',
+                'VERSION'     => '1.0',
+                'TAGS'        => '',
+                'REGTIME'     => regt,
+                'SIZE'        => @options[:sizemb],
+                'DESCRIPTION' => app['description'].delete('\\"')
+            }
+
+            tmpl = ''
+            data.each {|key, val| print_var(tmpl, key, val) }
+
+            tmpl64 = ''
+            print_var(tmpl64, 'DRIVER', 'raw')
+
+            data = { 'APPTEMPLATE64' => tmpl64, 'VMTEMPLATE64' => TEMPLATE }
+            data.each do |key, val|
+                print_var(tmpl, key, Base64.strict_encode64(val))
+            end
+
+            appstr << "APP=\"#{Base64.strict_encode64(tmpl)}\"\n"
+        end
+
+        [0, appstr]
+    end
+
+    def app_url(app)
+        "docker://#{app['name']}?size=#{@options[:sizemb]}" \
+        "&filesystem=#{@options[:fs]}&format=#{@options[:format]}"
+    end
+
+    def print_var(str, name, val)
+        return if val.nil?
+        return if val.class == String && val.empty?
+
+        str << "#{name}=\"#{val}\"\n"
+    end
+
+    # Returns an md5 from a combination of the @option hash and an input string
+    def md5(string)
+        Digest::MD5.hexdigest("#{@options} #{string}")
+    end
+
+end
+
+################################################################################
+# Main Program. Outpust the list of marketplace appliances
+################################################################################
+def set_option(opt, doc, name, path)
+    opt[name] = doc.elements[path].text if doc.elements[path]
+end
+
+begin
+    options     = {}
+    drv_message = Base64.decode64(ARGV[0])
+
+    doc = REXML::Document.new(drv_message).root
+
+    set_option(options, doc, :url, 'MARKETPLACE/TEMPLATE/ENDPOINT')
+    set_option(options, doc, :sizemb, 'MARKETPLACE/TEMPLATE/IMAGE_SIZE_MB')
+    set_option(options, doc, :fs, 'MARKETPLACE/TEMPLATE/FILESYSTEM')
+    set_option(options, doc, :format, 'MARKETPLACE/TEMPLATE/FORMAT')
+
+    rc, str = DockerHubMarket.new(options).appliances
+
+    if rc != 0
+        STDERR.puts str
+        exit(-1)
+    end
+
+    puts str
+end

src/market_mad/remotes/turnkeylinux/monitor

@@ -152,7 +152,7 @@ class TurnkeyLinux
 
         # Line example
         # debian-9-turnkey-zurmo_15.2-1_amd64.tar.gz.hash 2018-11-29 09:45  2.2K
-        apps[1..-1].each do |l|
+        apps[0..-1].each do |l|
             f = l.split
 
             next if f[0] =~ /\.hash$/

src/oca/ruby/opennebula/marketplaceapp.rb

@@ -192,8 +192,9 @@ module OpenNebula
             options[:vmtemplate_name] = name unless options[:vmtemplate_name]
 
             tmpl << "\n"
-            tmpl << "NAME=\"" << name << "\"\n"
+            tmpl << 'NAME="' << name << "\"\n"
-            tmpl << "FROM_APP=\"" << self['ID'] << "\"\n"
+            tmpl << 'FROM_APP="' << self['ID'] << "\"\n"
+            tmpl << 'URL_ARGS="' << options[:url_args] << "\"\n" if options[:url_args]
 
             case type_str
             when 'IMAGE'