shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-03-26 06:50:09 +03:00
Code Releases Wiki Activity

F #5058: improve sparse files image management (#240)

Browse Source 
Co-authored-by: Vlastimil Holer <vholer@opennebula.io>
This commit is contained in:
Christian González 2020-11-12 16:48:25 +01:00 committed by GitHub
parent 3da0fb41c1
commit 4db78e267a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 157 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/datastore_mad/remotes/downloader.sh
View File

@ -418,6 +418,8 @@ if [ "$TO" != "-" ]; then
convert_image
fi
fallocate -d "$TO"
elif [ -n "$CONVERT" ]; then
convert_image
fi

4
src/datastore_mad/remotes/fs/cp
View File

@ -145,7 +145,7 @@ if [ -n "$BRIDGE_LIST" ]; then
multiline_exec_and_log "set -e -o pipefail; $COPY_COMMAND | $SSH $DST_HOST $DD of=$TMP_DST bs=${DD_BLOCK_SIZE:-64k}" \
"Error dumping $SRC to $DST_HOST:$TMP_DST"
ssh_exec_and_log "$DST_HOST" "set -e -o pipefail; mkdir -p $BASE_PATH; mv -f $TMP_DST $DST" \
ssh_exec_and_log "$DST_HOST" "set -e -o pipefail; mkdir -p $BASE_PATH; mv -f $TMP_DST $DST; fallocate -d $DST" \
"Error moving $TMP_DST to $DST in $DST_HOST"
if [ "x$CONVERT" = 'xyes' ] && [ -n "$DRIVER" ]; then
@ -162,6 +162,8 @@ else
multiline_exec_and_log "$CONVERT_CMD" "Error converting $DST"
fi
fallocate -d "$DST"
FORMAT=$($QEMU_IMG info $DST | grep "^file format:" | awk '{print $3}' || :)
fi

51
src/mad/sh/scripts_common.sh
View File

@ -45,8 +45,6 @@ RBD=${RBD:-rbd}
READLINK=${READLINK:-readlink}
RM=${RM:-rm}
CP=${CP:-cp}
SCP=${SCP:-scp}
SCP_FWD=${SCP_FWD:-scp -o ForwardAgent=yes -o ControlMaster=no -o ControlPath=none}
SED=${SED:-sed}
SSH=${SSH:-ssh}
SSH_FWD=${SSH_FWD:-ssh -o ForwardAgent=yes -o ControlMaster=no -o ControlPath=none}
@ -78,7 +76,7 @@ SCRIPT_NAME=`basename $0`
# /some//path///somewhere/ -> /some/path/somewhere
function fix_dir_slashes
{
dirname "$1/file" | $SED 's/\/+/\//g'
dirname "$1/file" | $SED 's/\/\+/\//g'
}
# ------------------------------------------------------------------------------
@ -197,20 +195,22 @@ function retry_if
# Executes a command, if it fails returns error message and exits
# If a second parameter is present it is used as the error message when
# the command fails
# args: <command> [<error message>]
function exec_and_log
{
message=$2
_cmd="$1"
_msg="$2"
EXEC_LOG_ERR=`$1 2>&1 1>/dev/null`
EXEC_LOG_ERR=$(${_cmd} 2>&1 1>/dev/null)
EXEC_LOG_RC=$?
if [ $EXEC_LOG_RC -ne 0 ]; then
log_error "Command \"$1\" failed: $EXEC_LOG_ERR"
log_error "Command \"${_cmd}\" failed: $EXEC_LOG_ERR"
if [ -n "$2" ]; then
error_message "$2"
if [ -n "${_msg}" ]; then
error_message "${_msg}"
else
error_message "Error executing $1: $EXEC_LOG_ERR"
error_message "Error executing \"${_cmd}\": $EXEC_LOG_ERR"
fi
exit $EXEC_LOG_RC
fi
@ -293,6 +293,32 @@ function exec_and_set_error
fi
}
# Like exec_and_log but does not exit on failure. Just sets the variable
# ERROR to the error message.
function multiline_exec_and_set_error
{
message=$2
EXEC_LOG_ERR=`bash -s 2>&1 1>/dev/null <<EOF
export LANG=C
export LC_ALL=C
$1
EOF`
EXEC_LOG_RC=$?
export ERROR=""
if [ $EXEC_LOG_RC -ne 0 ]; then
log_error "Command \"$1\" failed: $EXEC_LOG_ERR"
if [ -n "$2" ]; then
export ERROR="$2"
else
export ERROR="Error executing $1: $EXEC_LOG_ERR"
fi
fi
}
# Like exec_and_log but the first argument is the number of seconds
# before here is timeout and kills the command
#
@ -437,21 +463,16 @@ function mkfs_command {
}
# This function will accept command as an argument for which it will override
# the env. variables SSH and SCP with their agent forwarding alternative
# the env. variables SSH with their agent forwarding alternative
ssh_forward()
{
_ssh_cmd_saved="$SSH"
_scp_cmd_saved="$SCP"
SSH="$SSH_FWD"
SCP="$SCP_FWD"
"$@"
_ssh_forward_result=$?
SSH="$_ssh_cmd_saved"
SCP="$_scp_cmd_saved"
return $_ssh_forward_result
}

2
src/market_mad/remotes/http/import
View File

@ -81,6 +81,8 @@ if [ -n "$BRIDGE_LIST" ]; then
multiline_exec_and_log "set -e -o pipefail; $CP_CMD | $SSH ${DST_HOST} $DD of=${DST_PATH} bs=${DD_BLOCK_SIZE:-64k}" \
"Error dumping ${IMPORT_SOURCE} to ${DST_HOST}:${DST_PATH}"
ssh_exec_and_log "$DST_HOST" "fallocate -d $DST_PATH" "Error running: fallocate -d $DST_PATH"
#if [ "${DISPOSE}" = "YES" ]; then
#TODO This should call a ds operation
#fi

15
src/tm_mad/common/context
View File

@ -51,6 +51,7 @@ function exit_error
DST_PATH=`arg_path $DST`
DST_HOST=`arg_host $DST`
DST_DIR=`dirname $DST_PATH`
DST_FILE=`basename $DST_PATH`
#-------------------------------------------------------------------------------
# Create DST path
@ -65,7 +66,8 @@ log "Generating context block device at $DST"
VM_ID=`basename $DST_DIR`
ISO_DIR="$DS_DIR/.isofiles/$VM_ID"
ISO_FILE="$ISO_DIR/$VM_ID.iso"
ISO_FILE="$VM_ID.iso"
ISO_PATH="$ISO_DIR/$ISO_FILE"
exec_and_set_error "mkdir -p $ISO_DIR" \
"Could not create tmp dir to make context dev"
@ -93,11 +95,18 @@ for f in "${SRC[@]}"; do
[ -n "$ERROR" ] && exit_error
done
exec_and_set_error "$MKISOFS -o $ISO_FILE -V CONTEXT -J -R $ISO_DIR" \
exec_and_set_error "$MKISOFS -o $ISO_PATH -V CONTEXT -J -R $ISO_DIR" \
"Error creating iso fs"
[ -n "$ERROR" ] && exit_error
exec_and_set_error "$SCP $ISO_FILE $DST" "Error copying context ISO to $DST"
COPY_CMD=$(cat <<EOF
set -e -o pipefail
$TAR -C $ISO_DIR --transform="flags=r;s|$ISO_FILE|$DST_FILE|" -cSf - $ISO_FILE | \
$SSH $DST_HOST "$TAR -xSf - -C $DST_DIR"
EOF
)
multiline_exec_and_set_error "$COPY_CMD" "Error copying context ISO to $DST"
[ -n "$ERROR" ] && exit_error
rm -rf $ISO_DIR > /dev/null 2>&1

37
src/tm_mad/ssh/clone
View File

@ -48,16 +48,20 @@ XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
# Set dst path and dir
#-------------------------------------------------------------------------------
SRC_PATH=`arg_path $SRC`
DST_PATH=`arg_path $DST`
SRC_PATH="$(arg_path $SRC)"
DST_PATH="$(arg_path $DST)"
SRC_PATH_SNAP="${SRC_PATH}.snap"
DST_PATH_SNAP="${DST_PATH}.snap"
SRC_HOST=`arg_host $SRC`
DST_HOST=`arg_host $DST`
SRC_HOST="$(arg_host $SRC)"
DST_HOST="$(arg_host $DST)"
DST_DIR=`dirname $DST_PATH`
SRC_FILE="$(basename $SRC_PATH)"
DST_FILE="$(basename $DST_PATH)"
SRC_DIR="$(dirname $SRC_PATH)"
DST_DIR="$(dirname $DST_PATH)"
ssh_make_path $DST_HOST $DST_DIR "ssh"
@ -93,15 +97,24 @@ fi
#-------------------------------------------------------------------------------
# Copy files to the remote host
#-------------------------------------------------------------------------------
log "Cloning $SRC in $DST_PATH"
ssh_forward exec_and_log "$SCP -r $SRC $DST" "Error copying $SRC to $DST"
log "Cloning $SRC_PATH in $DST_PATH"
if [ -d "${SRC_PATH_SNAP}" ]; then
ssh_forward exec_and_log "$SCP -r $SRC_HOST:${SRC_PATH_SNAP} $DST_HOST:${DST_PATH_SNAP}" \
"Error copying $SRC to $DST"
fi
COPY_CMD=$(cat <<EOF
set -e -o pipefail
if [ -n "$ORIGINAL_SIZE" -a "$SIZE" -gt "$ORIGINAL_SIZE" ]; then
if [ -d "${SRC_PATH_SNAP}" ]; then
SRC_SNAP="${SRC_FILE}.snap"
fi
$TAR -C $SRC_DIR --transform="flags=r;s|$SRC_FILE|$DST_FILE|" -cSf - $SRC_FILE \$SRC_SNAP | \
$SSH $DST_HOST "$TAR -xSf - -C $DST_DIR"
EOF
)
ssh_forward ssh_exec_and_log "$SRC_HOST" "$COPY_CMD" \
"Error copying $SRC_PATH to $DST"
if [ -n "$ORIGINAL_SIZE" ] && [ "$SIZE" -gt "$ORIGINAL_SIZE" ]; then
RESIZE_CMD="qemu-img resize ${DST_PATH} ${SIZE}M"
ssh_exec_and_log "$DST_HOST" "$RESIZE_CMD" \
"Error resizing image $DST"

51
src/tm_mad/ssh/cpds
View File

@ -16,7 +16,7 @@
# limitations under the License. #
#--------------------------------------------------------------------------- #
# cpds host:remote_system_ds/disk.i fe:SOURCE snapid vmid dsid
# cpds host:remote_system_ds/disk.i fe:SOURCE|SOURCE snapid vmid dsid
# - fe is the front-end hostname
# - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
# - host is the target host to deploy the VM
@ -41,6 +41,21 @@ DRIVER_PATH=$(dirname $0)
. $TMCOMMON
. ${DRIVER_PATH}/../../etc/vmm/kvm/kvmrc
function set_src_dst_vars
{
SRC_PATH="$(arg_path "$SRC")"
DST_PATH="$(arg_path "$DST")"
SRC_HOST="$(arg_host "$SRC")"
DST_HOST="$(arg_host "$DST")"
SRC_FILE="$(basename "$SRC_PATH")"
DST_FILE="$(basename "$DST_PATH")"
SRC_DIR="$(dirname "$SRC_PATH")"
DST_DIR="$(dirname "$DST_PATH")"
}
#-------------------------------------------------------------------------------
# Get Image information
#-------------------------------------------------------------------------------
@ -66,14 +81,14 @@ DISK_TARGET="${XPATH_ELEMENTS[j++]}"
# Set src path
#-------------------------------------------------------------------------------
SRC_PATH=`arg_path $SRC`
SRC_HOST=`arg_host $SRC`
SRC_TEMP_PATH=$(mktemp -u ${SRC_PATH}.XXXXXXXX)
if [ "${SNAP_ID}" != "-1" ]; then
SRC="${SRC}.snap/${SNAP_ID}"
fi
set_src_dst_vars
SRC_TEMP_PATH="$(mktemp -u "${SRC_PATH}.XXXXXXXX")"
#-------------------------------------------------------------------------------
# Move the image back to the datastore
#-------------------------------------------------------------------------------
@ -114,13 +129,35 @@ EOF
if ssh_exec_and_log_no_error $SRC_HOST "$CPDS_CMD_EXPORT" "$CPDS_CMD_ERR"; then
# Export creation succeeded, point SRC there
SRC="${SRC_HOST}:${SRC_TEMP_PATH}"
set_src_dst_vars
else
log_error "Exporting failed"
exit 1
fi
fi
log "Moving $SRC to datastore as $DST"
ssh_forward exec_and_log "$SCP -r $SRC $DST" "Error copying $SRC to $DST"
log "Moving $SRC to datastore as $DST_PATH"
# Copy from possibly remote host to possible remote host
if [ "$(fix_dir_slashes $DST_HOST)" != "$DST_PATH" ]; then
COPY_CMD=$(cat <<EOF
set -e -o pipefail
$TAR -C $SRC_DIR --transform="flags=r;s|$SRC_FILE|$DST_FILE|" -cSf - $SRC_FILE | \
$SSH $DST_HOST "$TAR -xSf - -C $DST_DIR"
EOF
)
ssh_forward ssh_exec_and_log "$SRC_HOST" "$COPY_CMD" \
"Error copying $SRC to $DST_HOST:$DST_PATH"
else # Copy from possibly remote host to local
REMOTE_CMD=$(cat <<EOF
set -e -o pipefail
$SSH $SRC_HOST "$TAR -C $SRC_DIR --transform=\"flags=r;s|$SRC_FILE|$DST_FILE|\" -cSf - $SRC_FILE" | \
$TAR -xSf - -C $DST_DIR
EOF
)
multiline_exec_and_log "$REMOTE_CMD" "Error copying $SRC to $DST_PATH"
fi
exit 0

36
src/tm_mad/ssh/mvds
View File

@ -16,7 +16,7 @@
# limitations under the License. #
#--------------------------------------------------------------------------- #
# mvds host:remote_system_ds/disk.i fe:SOURCE vmid dsid
# mvds host:remote_system_ds/disk.i fe:SOURCE|SOURCE vmid dsid
# - fe is the front-end hostname
# - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk
# - host is the target host to deploy the VM
@ -39,17 +39,47 @@ fi
. $TMCOMMON
SRC_PATH="$(arg_path $SRC)"
DST_PATH="$(arg_path $DST)"
SRC_HOST="$(arg_host $SRC)"
DST_HOST="$(arg_host $DST)"
SRC_PATH_SNAP="${SRC_PATH}.snap"
DST_SNAP="${DST}.snap"
SRC_FILE="$(basename $SRC_PATH)"
DST_FILE="$(basename $DST_PATH)"
DST_DIR="$(dirname $DST_PATH)"
SRC_DIR="$(dirname $SRC_PATH)"
#-------------------------------------------------------------------------------
# Move the image back to the datastore
#-------------------------------------------------------------------------------
log "Moving $SRC to datastore as $DST"
ssh_forward exec_and_log "$SCP -r $SRC $DST" "Error copying $SRC to $DST"
log "Moving $SRC to datastore as $DST_PATH"
# Copy from possibly remote host to possible remote host
if [ "$(fix_dir_slashes $DST_HOST)" != "$DST_PATH" ]; then
COPY_CMD=$(cat <<EOF
set -e -o pipefail
$TAR -C $SRC_DIR --transform="flags=r;s|$SRC_FILE|$DST_FILE|" -cSf - $SRC_FILE | \
$SSH $DST_HOST "$TAR -xSf - -C $DST_DIR"
EOF
)
ssh_forward ssh_exec_and_log "$SRC_HOST" "$COPY_CMD" \
"Error copying $SRC to $DST_HOST:$DST_PATH"
else # Copy from possibly remote host to local
REMOTE_CMD=$(cat <<EOF
set -e -o pipefail
$SSH $SRC_HOST "$TAR -C $SRC_DIR --transform=\"flags=r;s|$SRC_FILE|$DST_FILE|\" -cSf - $SRC_FILE" | \
$TAR -xSf - -C $DST_DIR
EOF
)
multiline_exec_and_log "$REMOTE_CMD" "Error copying $SRC to $DST_PATH"
fi
if $SSH $SRC_HOST ls ${SRC_PATH_SNAP} >/dev/null 2>&1; then
exec_and_log "rsync -r --delete ${SRC_HOST}:${SRC_PATH_SNAP}/ ${DST_SNAP}"