From 94689fae98274aae4222acc86ceae91d1ffbf025 Mon Sep 17 00:00:00 2001
From: "Ruben S. Montero" <rsmontero@opennebula.org>
Date: Sat, 10 Nov 2012 00:36:15 +0100
Subject: [PATCH] bug #1649: Add option LIMIT_TRANSFER_BW to datastores, to
limit the transfer rate when downloading http/https images.
---
src/datastore_mad/remotes/downloader.sh | 15 +++++++++++++--
src/datastore_mad/remotes/fs/cp | 6 ++++--
src/datastore_mad/remotes/iscsi/cp | 6 ++++--
src/datastore_mad/remotes/libfs.sh | 11 ++++++++---
src/datastore_mad/remotes/lvm/cp | 6 ++++--
src/datastore_mad/remotes/vmfs/cp | 6 ++++--
src/datastore_mad/remotes/vmware/cp | 6 ++++--
7 files changed, 41 insertions(+), 15 deletions(-)
diff --git a/src/datastore_mad/remotes/downloader.sh b/src/datastore_mad/remotes/downloader.sh
index 237e632b9b..bca92689b4 100755
--- a/src/datastore_mad/remotes/downloader.sh
+++ b/src/datastore_mad/remotes/downloader.sh
@@ -117,7 +117,7 @@ function unarchive
fi
}
-TEMP=`getopt -o m:s:n -l md5:,sha1:,nodecomp -- "$@"`
+TEMP=`getopt -o m:s:l:n -l md5:,sha1:,limit:,nodecomp -- "$@"`
if [ $? != 0 ] ; then
echo "Arguments error"
@@ -142,6 +142,10 @@ while true; do
export NO_DECOMPRESS="yes"
shift
;;
+ -l|--limit)
+ export LIMIT_RATE="$2"
+ shift 2
+ ;;
--)
shift
break
@@ -163,7 +167,14 @@ http://*|https://*)
# -k so it does not check the certificate
# -L to follow redirects
# -sS to hide output except on failure
- command="curl -sS -k -L $FROM"
+ # --limit_rate to limit the bw
+ curl_args="-sS -k -L $FROM"
+
+ if [ -n "$LIMIT_RATE" ]; then
+ curl_args="--limit-rate $LIMIT_RATE $curl_args"
+ fi
+
+ command="curl $curl_args"
;;
*)
command="cat $FROM"
diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp
index 840ac28fd8..61a2f6bf5e 100755
--- a/src/datastore_mad/remotes/fs/cp
+++ b/src/datastore_mad/remotes/fs/cp
@@ -54,7 +54,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
- /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS)
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@@ -64,6 +65,7 @@ SRC="${XPATH_ELEMENTS[4]}"
MD5="${XPATH_ELEMENTS[5]}"
SHA1="${XPATH_ELEMENTS[6]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[7]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[8]}"
mkdir -p "$BASE_PATH"
@@ -71,7 +73,7 @@ set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
DST=`generate_image_path`
-DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$SRC" "$DST"`
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
diff --git a/src/datastore_mad/remotes/iscsi/cp b/src/datastore_mad/remotes/iscsi/cp
index c99b970c16..ed5b02bbd6 100755
--- a/src/datastore_mad/remotes/iscsi/cp
+++ b/src/datastore_mad/remotes/iscsi/cp
@@ -59,7 +59,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
- /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS)
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@@ -73,6 +74,7 @@ SIZE="${XPATH_ELEMENTS[8]}"
MD5="${XPATH_ELEMENTS[9]}"
SHA1="${XPATH_ELEMENTS[10]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[11]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[12]}"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
@@ -95,7 +97,7 @@ REGISTER_CMD=$(cat <<EOF
EOF
)
-DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$SRC" -`
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
diff --git a/src/datastore_mad/remotes/libfs.sh b/src/datastore_mad/remotes/libfs.sh
index 340d808d50..1cacf89dce 100644
--- a/src/datastore_mad/remotes/libfs.sh
+++ b/src/datastore_mad/remotes/libfs.sh
@@ -88,8 +88,9 @@ EOF
# @param $1 - MD5 string
# @param $2 - SHA1 string
# @param $3 - NO_DECOMPRESS
-# @param $4 - SRC
-# @param $5 - DST
+# @param $4 - BW LIMIT
+# @param $5 - SRC
+# @param $6 - DST
# @return downloader.sh util arguments
#-------------------------------------------------------------------------------
function set_downloader_args {
@@ -107,7 +108,11 @@ function set_downloader_args {
HASHES="$HASHES --nodecomp"
fi
- echo "$HASHES $4 $5"
+ if [ -n "$4" ]; then
+ HASHES="$HASHES --limit $4"
+ fi
+
+ echo "$HASHES $5 $6"
}
#-------------------------------------------------------------------------------
diff --git a/src/datastore_mad/remotes/lvm/cp b/src/datastore_mad/remotes/lvm/cp
index 2865f68c39..28d2a533e7 100755
--- a/src/datastore_mad/remotes/lvm/cp
+++ b/src/datastore_mad/remotes/lvm/cp
@@ -58,7 +58,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
- /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS)
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@@ -71,6 +72,7 @@ SIZE="${XPATH_ELEMENTS[7]}"
MD5="${XPATH_ELEMENTS[8]}"
SHA1="${XPATH_ELEMENTS[9]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[10]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[11]}"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
@@ -84,7 +86,7 @@ REGISTER_CMD=$(cat <<EOF
EOF
)
-DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$SRC" -`
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
diff --git a/src/datastore_mad/remotes/vmfs/cp b/src/datastore_mad/remotes/vmfs/cp
index 0d28add4dc..90a8a254be 100755
--- a/src/datastore_mad/remotes/vmfs/cp
+++ b/src/datastore_mad/remotes/vmfs/cp
@@ -62,7 +62,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
/DS_DRIVER_ACTION_DATA/IMAGE/ID \
- /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS)
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW)
BASE_PATH="${XPATH_ELEMENTS[0]}"
DS_ID="${XPATH_ELEMENTS[1]}"
@@ -77,6 +78,7 @@ MD5="${XPATH_ELEMENTS[9]}"
SHA1="${XPATH_ELEMENTS[10]}"
IMAGE_ID="${XPATH_ELEMENTS[11]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[12]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[13]}"
FE_DST=`generate_image_path`
DST_FOLDER=`basename $FE_DST`
@@ -98,7 +100,7 @@ if [ ! -d $TMP_DIR ]; then
mkdir -p $TMP_DIR
fi
-DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$SRC" "$TMP_DIR/$DST_FOLDER"`
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$TMP_DIR/$DST_FOLDER"`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
diff --git a/src/datastore_mad/remotes/vmware/cp b/src/datastore_mad/remotes/vmware/cp
index f3294c3063..7f22712a75 100755
--- a/src/datastore_mad/remotes/vmware/cp
+++ b/src/datastore_mad/remotes/vmware/cp
@@ -54,7 +54,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
- /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS)
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
+ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@@ -64,13 +65,14 @@ SRC="${XPATH_ELEMENTS[4]}"
MD5="${XPATH_ELEMENTS[5]}"
SHA1="${XPATH_ELEMENTS[6]}"
NO_DECOMPRESS="${XPATH_ELEMENTS[7]}"
+LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[8]}"
mkdir -p "$BASE_PATH"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
DST=`generate_image_path`
-DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$SRC" "$DST"`
+DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"`
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"