diff --git a/src/datastore_mad/remotes/common/clone b/src/datastore_mad/remotes/common/clone index e6fa699ceb..078985bdc5 100755 --- a/src/datastore_mad/remotes/common/clone +++ b/src/datastore_mad/remotes/common/clone @@ -46,20 +46,29 @@ unset i XPATH_ELEMENTS while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH) -BASE_PATH="${XPATH_ELEMENTS[0]}" -SRC="${XPATH_ELEMENTS[1]}" +unset i + +BASE_PATH="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +SRC="${XPATH_ELEMENTS[i++]}" -mkdir -p "$BASE_PATH" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" DST=`generate_image_path` # ------------ Copy the image to the repository ------------- -log "Copying local image $SRC to the image repository" - -exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST" +if [ -n "$BRIDGE_LIST" ]; then + log "Copying remotely local image $SRC to the image repository" + DST_HOST=`get_destination_host $ID` + ssh_exec_and_log "$DST_HOST" "mkdir -p $BASE_PATH; cp -f $SRC $DST" "Error copying $SRC to $DST in $DST_HOST" +else + log "Copying local image $SRC to the image repository" + mkdir -p "$BASE_PATH" + exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST" +fi echo "$DST" diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index a84e4c8f8b..9e1a87f4b1 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -50,41 +50,41 @@ while IFS= read -r -d '' element; do done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/STAGING_DIR \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW) -BASE_PATH="${XPATH_ELEMENTS[0]}" -RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" -SAFE_DIRS="${XPATH_ELEMENTS[2]}" -SRC="${XPATH_ELEMENTS[3]}" -MD5="${XPATH_ELEMENTS[4]}" -SHA1="${XPATH_ELEMENTS[5]}" -NO_DECOMPRESS="${XPATH_ELEMENTS[6]}" -LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[7]}" +unset i -mkdir -p "$BASE_PATH" +BASE_PATH="${XPATH_ELEMENTS[i++]}" +RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}" +SAFE_DIRS="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +STAGING_DIR="${XPATH_ELEMENTS[i++]:-/var/tmp}" +SRC="${XPATH_ELEMENTS[i++]}" +MD5="${XPATH_ELEMENTS[i++]}" +SHA1="${XPATH_ELEMENTS[i++]}" +NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" +LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" + +DST=`generate_image_path` +IMAGE_HASH=`basename $DST` set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" -DST=`generate_image_path` - -DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"` +if [ -n "$BRIDGE_LIST" ]; then + DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -` +else + DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"` +fi COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS" -# ------------ Copy the image to the repository ------------- - -case $SRC in -http://*|https://*) - log "Downloading $SRC to the image repository" - - exec_and_log "$COPY_COMMAND" "Error downloading $SRC" - ;; - -*) +if echo "$SRC" | grep -vq '^https\?://'; then if [ `check_restricted $SRC` -eq 1 ]; then log_error "Not allowed to copy images from $RESTRICTED_DIRS" error_message "Not allowed to copy image file $SRC" @@ -92,9 +92,22 @@ http://*|https://*) fi log "Copying local image $SRC to the image repository" +else + log "Downloading image $SRC to the image repository" +fi +if [ -n "$BRIDGE_LIST" ]; then + DST_HOST=`get_destination_host $ID` + TMP_DST="$STAGING_DIR/$IMAGE_HASH" + + exec_and_log "eval $COPY_COMMAND | $SSH $DST_HOST $DD of=$TMP_DST bs=64k" \ + "Error dumping $SRC to $DST_HOST:$TMP_DST" + + ssh_exec_and_log "$DST_HOST" "mkdir -p $BASE_PATH; mv -f $TMP_DST $DST" \ + "Error moving $TMP_DST to $DST in $DST_HOST" +else + mkdir -p "$BASE_PATH" exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST" - ;; -esac +fi echo "$DST" diff --git a/src/datastore_mad/remotes/fs/mkfs b/src/datastore_mad/remotes/fs/mkfs index a3e46daba1..2d181763e1 100755 --- a/src/datastore_mad/remotes/fs/mkfs +++ b/src/datastore_mad/remotes/fs/mkfs @@ -48,17 +48,19 @@ while IFS= read -r -d '' element; do done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/IMAGE/FSTYPE \ /DS_DRIVER_ACTION_DATA/IMAGE/SIZE) +unset i -BASE_PATH="${XPATH_ELEMENTS[0]}" -RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" -SAFE_DIRS="${XPATH_ELEMENTS[2]}" -FSTYPE="${XPATH_ELEMENTS[3]}" -SIZE="${XPATH_ELEMENTS[4]}" +BASE_PATH="${XPATH_ELEMENTS[i++]}" +RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}" +SAFE_DIRS="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +FSTYPE="${XPATH_ELEMENTS[i++]}" +SIZE="${XPATH_ELEMENTS[i++]}" -mkdir -p "$BASE_PATH" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" DST=`generate_image_path` @@ -74,9 +76,29 @@ fi MKFS_CMD=`mkfs_command $DST $FSTYPE $SIZE` -exec_and_log "$DD if=/dev/zero of=$DST bs=1 count=1 seek=${SIZE}M" \ - "Could not create image $DST" -exec_and_log "$MKFS_CMD" \ - "Unable to create filesystem $FSTYPE in $DST" +REMOTE_REGISTER_CMD=$(cat </dev/null | cut -f1` -TOTAL_MB=`df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print $2}'` -FREE_MB=`df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print $4}'` +USED_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$3}') +TOTAL_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$2}') +FREE_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$4}') -if [ -z "$USED_MB" -o -z "$TOTAL_MB" -o -z "$FREE_MB"]; then - msg="Empty monitoring data" - error_message "$msg: USED_MB=$USED_MB, TOTAL_MB=$TOTAL_MB, FREE_MB=$FREE_MB" +if [ -z "\$USED_MB" -o -z "\$TOTAL_MB" -o -z "\$FREE_MB" ]; then + msg="Empty value found" + echo "\$msg: USED_MB=\$USED_MB, TOTAL_MB=\$TOTAL_MB, FREE_MB=\$FREE_MB" exit 1 fi -MONITOR_DATA=$(cat <&1) +else + MONITOR_DATA=$(monitor_and_log "$MONITOR_SCRIPT" "Monitor script" 2>&1) +fi + +MONITOR_STATUS=$? + +if [ "$MONITOR_STATUS" = "0" ]; then + echo "$MONITOR_DATA" | tr ' ' '\n' + exit 0 +else + echo "$MONITOR_DATA" + exit $MONITOR_STATUS +fi diff --git a/src/datastore_mad/remotes/fs/rm b/src/datastore_mad/remotes/fs/rm index f967902b3e..46e86b7e6c 100755 --- a/src/datastore_mad/remotes/fs/rm +++ b/src/datastore_mad/remotes/fs/rm @@ -45,22 +45,32 @@ unset i XPATH_ELEMENTS while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" done < <($XPATH /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH) -SRC="${XPATH_ELEMENTS[0]}" -BASE_PATH="${XPATH_ELEMENTS[1]}" +unset i -BASENAME_SRC=`basename "${SRC##$BASE_PATH}"` +SRC="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +BASE_PATH="${XPATH_ELEMENTS[i++]}" # ------------ Remove the image from the repository ------------ -if [ -f "$SRC" -a `dirname "$SRC"` = "$BASE_PATH" -a -n "$BASENAME_SRC" ] -then - log "Removing $SRC from the image repository" +if [ -n "$BRIDGE_LIST" ]; then + DST_HOST=`get_destination_host $ID` - exec_and_log "rm -f $SRC" \ - "Error deleting $SRC" + ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -f $SRC" \ + "Error deleting $SRC in $DST_HOST" else - log_error "Bad formed or unavailable Image source: ${SRC}" - exit 1 + BASENAME_SRC=`basename "${SRC##$REMOTE_RM_CMD}"` + if [ -f "$SRC" -a `dirname "$SRC"` = "$BASE_PATH" -a -n "$BASENAME_SRC" ] + then + log "Removing $SRC from the image repository" + + exec_and_log "rm -f $SRC" \ + "Error deleting $SRC" + else + log_error "Bad formed or unavailable Image source: ${SRC}" + exit 1 + fi fi diff --git a/src/datastore_mad/remotes/libfs.sh b/src/datastore_mad/remotes/libfs.sh index 2f4f28bd3c..9ed6103ab9 100644 --- a/src/datastore_mad/remotes/libfs.sh +++ b/src/datastore_mad/remotes/libfs.sh @@ -154,7 +154,7 @@ function gzip_file_size { function fs_size { case $1 in - http://*) + http://*|https://*) HEADERS=`curl -LIk --max-time 60 $1 2>&1` if echo "$HEADERS" | grep -q "OpenNebula-AppMarket-Size"; then diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index 7288e0e9e0..2fa24db3c0 100644 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -149,6 +149,33 @@ function exec_and_log fi } +# This function executes $1 and returns stdout +# If a second parameter is present it is used as the error message when +# the command fails +function monitor_and_log +{ + EXEC_OUT=`bash -s 2>/dev/null </dev/null <