Feature #4217: Add ceph/export and rbd handling in downloader.sh

install.sh

@@ -1160,6 +1160,7 @@ TM_ISCSI_FILES="src/tm_mad/iscsi/clone \
 
 DATASTORE_DRIVER_COMMON_SCRIPTS="src/datastore_mad/remotes/xpath.rb \
                              src/datastore_mad/remotes/downloader.sh \
+                             src/datastore_mad/remotes/url.rb \
                              src/datastore_mad/remotes/libfs.sh"
 
 DATASTORE_DRIVER_DUMMY_SCRIPTS="src/datastore_mad/remotes/dummy/cp \

src/datastore_mad/remotes/ceph/export

@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+###############################################################################
+# This script is used to export an image to qcow2 file
+###############################################################################
+
+# ------------ Set up the environment to source common tools ------------
+
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+source ${DRIVER_PATH}/../libfs.sh
+source ${DRIVER_PATH}/ceph.conf
+
+# -------- Get rm and datastore arguments from OpenNebula core ------------
+
+DRV_ACTION=$1
+ID=$2
+
+XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
+
+unset i XPATH_ELEMENTS
+
+while IFS= read -r -d '' element; do
+    XPATH_ELEMENTS[i++]="$element"
+done < <($XPATH     /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
+                    /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
+                    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
+                    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
+                    /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/FORMAT \
+                    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
+                    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
+                    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \
+                    /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF)
+
+unset i
+
+SRC="${XPATH_ELEMENTS[i++]}"
+SIZE="${XPATH_ELEMENTS[i++]}"
+MD5="${XPATH_ELEMENTS[i++]}"
+SHA1="${XPATH_ELEMENTS[i++]}"
+FORMAT="${XPATH_ELEMENTS[i++]:-raw}"
+BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
+POOL_NAME="${XPATH_ELEMENTS[i++]:-$POOL_NAME}"
+CEPH_USER="${XPATH_ELEMENTS[i++]}"
+CEPH_CONF="${XPATH_ELEMENTS[i++]}"
+
+DST_HOST=`get_destination_host $ID`
+
+if [ -z "$DST_HOST" ]; then
+    error_message "Datastore template missing 'BRIDGE_LIST' attribute."
+    exit -1
+fi
+
+IMPORT_SOURCE="rbd://$DST_HOST/$SRC"
+
+if [ -n "$CEPH_USER" ]; then
+    RBD="$RBD --id ${CEPH_USER}"
+    IMPORT_SOURCE="${IMPORT_SOURCE}?CEPH_USER=${CEPH_USER}"
+fi
+
+if [ -n "$CEPH_CONF" ]; then
+    RBD="$RBD --conf ${CEPH_CONF}"
+
+    if [ -n "$CEPH_USER" ]; then
+        IMPORT_SOURCE="${IMPORT_SOURCE}&"
+    else
+        IMPORT_SOURCE="${IMPORT_SOURCE}?"
+    fi
+
+    IMPORT_SOURCE="${IMPORT_SOURCE}CEPH_CONF=${CEPH_CONF}"
+fi
+
+#-------------------------------------------------------------------------------
+# Output image source and format
+#-------------------------------------------------------------------------------
+
+INFO_SCRIPT=$(cat <<EOF
+if [ -z "$MD5" ]; then
+    CHECKSUM=\$(
+        $RBD export ${SRC} - | ${MD5SUM} | cut -f1 -d' '
+        ps=\$PIPESTATUS
+
+        if [ "\$ps" != "0" ]; then
+            exit \$ps
+        fi
+    )
+
+    status=\$?
+
+    [ "\$status" != "0" ] && exit \$status
+else
+    CHECKSUM="$MD5"
+fi
+
+cat <<EOT
+<MD5>\$CHECKSUM</MD5>
+<SIZE>$SIZE</SIZE>
+<FORMAT>${FORMAT}</FORMAT>
+EOT
+EOF
+)
+
+INFO=$(ssh_monitor_and_log "$DST_HOST" "$INFO_SCRIPT" "Image info script" 2>&1)
+INFO_STATUS=$?
+
+if [ "$INFO_STATUS" != "0" ]; then
+    echo "$INFO"
+    exit $INFO_STATUS
+fi
+
+cat <<EOF
+<IMPORT_INFO>
+    <IMPORT_SOURCE>$IMPORT_SOURCE</IMPORT_SOURCE>
+    $INFO
+    <DISPOSE>NO</DISPOSE>
+</IMPORT_INFO>"
+EOF
+

src/datastore_mad/remotes/downloader.sh

@@ -16,6 +16,16 @@
 # limitations under the License.                                             #
 #--------------------------------------------------------------------------- #
 
+if [ -z "${ONE_LOCATION}" ]; then
+    LIB_LOCATION=/usr/lib/one
+else
+    LIB_LOCATION=$ONE_LOCATION/lib
+fi
+
+. $LIB_LOCATION/sh/scripts_common.sh
+
+DRIVER_PATH=$(dirname $0)
+
 # Execute a command (first parameter) and use the first kb of stdout
 # to determine the file type
 function get_type
@@ -136,6 +146,49 @@ function s3_request
          " https://${BUCKET}.s3.amazonaws.com/${OBJECT}"
 }
 
+function get_rbd_cmd
+{
+    local i j URL_ELEMENTS
+
+    FROM="$1"
+
+    URL_RB="$DRIVER_PATH/url.rb"
+
+    while IFS= read -r -d '' element; do
+        URL_ELEMENTS[i++]="$element"
+    done < <($URL_RB    $FROM \
+                        USER \
+                        HOST \
+                        SOURCE \
+                        PARAM_DS \
+                        PARAM_CEPH_USER \
+                        PARAM_CEPH_CONF)
+
+    USER="${URL_ELEMENTS[j++]}"
+    DST_HOST="${URL_ELEMENTS[j++]}"
+    SOURCE="${URL_ELEMENTS[j++]}"
+    DS="${URL_ELEMENTS[j++]}"
+    CEPH_USER="${URL_ELEMENTS[j++]}"
+    CEPH_CONF="${URL_ELEMENTS[j++]}"
+
+    # Remove leading '/'
+    SOURCE="${SOURCE#/}"
+
+    if [ -n "$USER" ]; then
+        DST_HOST="$USER@$DST_HOST"
+    fi
+
+    if [ -n "$CEPH_USER" ]; then
+        RBD="$RBD --id ${CEPH_USER}"
+    fi
+
+    if [ -n "$CEPH_CONF" ]; then
+        RBD="$RBD --conf ${CEPH_CONF}"
+    fi
+
+    echo "ssh $DST_HOST $RBD export $SOURCE -"
+}
+
 TEMP=`getopt -o m:s:l:n -l md5:,sha1:,limit:,nodecomp -- "$@"`
 
 if [ $? != 0 ] ; then
@@ -214,6 +267,9 @@ s3://*)
     curl_args="$(s3_request $FROM)"
     command="curl $curl_args"
     ;;
+rbd://*)
+    command="$(get_rbd_cmd $FROM)"
+    ;;
 *)
     if [ ! -r $FROM ]; then
         echo "Cannot read from $FROM" >&2
@@ -228,7 +284,7 @@ decompressor=$(get_decompressor "$file_type")
 
 eval "$command" | tee >( hasher $HASH_TYPE) | decompress "$decompressor" "$TO"
 
-if [ "$?" != "0" ]; then
+if [ "$?" != "0" -o "$PIPESTATUS" != "0" ]; then
     echo "Error copying" >&2
     exit -1
 fi

src/datastore_mad/remotes/url.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+# -------------------------------------------------------------------------- */
+# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems                #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    */
+# not use this file except in compliance with the License. You may obtain    */
+# a copy of the License at                                                   */
+#                                                                            */
+# http://www.apache.org/licenses/LICENSE-2.0                                 */
+#                                                                            */
+# Unless required by applicable law or agreed to in writing, software        */
+# distributed under the License is distributed on an "AS IS" BASIS,          */
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+# See the License for the specific language governing permissions and        */
+# limitations under the License.                                             */
+# -------------------------------------------------------------------------- */
+
+#
+# Simple command to parse an URLs more elements (refer by keywords or params)
+#
+
+require 'uri'
+require 'cgi'
+require 'pp'
+
+url = ARGV.shift
+
+u = URI.parse(url)
+
+SPECIAL_KEYS = {
+    "PROTOCOL" => u.scheme,
+    "USER"     => u.user,
+    "PASSWORD" => u.password,
+    "HOST"     => u.host,
+    "PORT"     => u.port,
+    "SOURCE"   => u.path
+}
+
+PARAMS = CGI.parse(u.query)
+
+values = ""
+ARGV.each do |key|
+    key = key.to_s
+    if key.downcase.start_with?("param_")
+        param = key.gsub(/^param_/i,"")
+        value = PARAMS[param]
+        value = [value].flatten.first || ""
+        values << value
+    else
+        value = SPECIAL_KEYS[key] || ""
+        values << value
+    end
+    values << "\0"
+end
+
+puts values
+
+exit 0