diff --git a/src/datastore_mad/remotes/downloader.sh b/src/datastore_mad/remotes/downloader.sh index 24fc8715e4..2c656b6ce8 100755 --- a/src/datastore_mad/remotes/downloader.sh +++ b/src/datastore_mad/remotes/downloader.sh @@ -148,18 +148,52 @@ function s3_env XPATH_ELEMENTS[i++]="$element" done < <($XPATH /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ACCESS_KEY_ID \ /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY \ + /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/REGION \ + /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/AWS \ /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ENDPOINT) S3_ACCESS_KEY_ID="${XPATH_ELEMENTS[j++]}" S3_SECRET_ACCESS_KEY="${XPATH_ELEMENTS[j++]}" + S3_REGION="${XPATH_ELEMENTS[j++]}" + S3_AWS="${XPATH_ELEMENTS[j++]}" S3_ENDPOINT="${XPATH_ELEMENTS[j++]}" + + CURRENT_DATE_DAY="$(date -u '+%Y%m%d')" + CURRENT_DATE_ISO8601="${CURRENT_DATE_DAY}T$(date -u '+%H%M%S')Z" +} + +# Create an SHA-256 hash in hexadecimal. +# Usage: +# hash_sha256 +function hash_sha256 { + printf "${1}" | openssl dgst -sha256 | sed 's/^.* //' +} + +# Create an SHA-256 hmac in hexadecimal. +# Usage: +# hmac_sha256 +function hmac_sha256 { + printf "${2}" | openssl dgst -sha256 -mac HMAC -macopt "${1}" | sed 's/^.* //' +} + +# Create the signature. +# Usage: +# create_signature +function create_signature { + stringToSign="AWS4-HMAC-SHA256\n${CURRENT_DATE_ISO8601}\n${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request\n$(hash_sha256 "${HTTP_CANONICAL_REQUEST}")" + dateKey=$(hmac_sha256 key:"AWS4${S3_SECRET_ACCESS_KEY}" "${CURRENT_DATE_DAY}") + regionKey=$(hmac_sha256 hexkey:"${dateKey}" "${S3_REGION}") + serviceKey=$(hmac_sha256 hexkey:"${regionKey}" "s3") + signingKey=$(hmac_sha256 hexkey:"${serviceKey}" "aws4_request") + + printf "${stringToSign}" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"${signingKey}" | sed 's/(stdin)= //' } function s3_curl_args { FROM="$1" - ENDPOINT=${S3_ENDPOINT:-https://s3.amazonaws.com} + ENDPOINT="$S3_ENDPOINT" OBJECT=$(basename "$FROM") BUCKET=$(basename $(dirname "$FROM")) @@ -175,6 +209,43 @@ function s3_curl_args " '$(esc_sq "${ENDPOINT}/${BUCKET}/${OBJECT}")'" } +function s3_curl_args_aws +{ + FROM="$1" + + OBJECT=$(basename "$FROM") + BUCKET=$(basename "$(dirname "$FROM")") + + ENDPOINT="$BUCKET.s3.amazonaws.com" + + AWS_S3_PATH="$(echo $OBJECT | sed 's;^\([^/]\);/\1;')" + + HTTP_REQUEST_PAYLOAD_HASH="$(echo "" | openssl dgst -sha256 | sed 's/^.* //')" + HTTP_CANONICAL_REQUEST_URI="${AWS_S3_PATH}" + HTTP_REQUEST_CONTENT_TYPE='application/octet-stream' + + HTTP_CANONICAL_REQUEST_HEADERS="content-type:${HTTP_REQUEST_CONTENT_TYPE} +host:${ENDPOINT} +x-amz-content-sha256:${HTTP_REQUEST_PAYLOAD_HASH} +x-amz-date:${CURRENT_DATE_ISO8601}" + + HTTP_REQUEST_SIGNED_HEADERS="content-type;host;x-amz-content-sha256;x-amz-date" +HTTP_CANONICAL_REQUEST="GET +${HTTP_CANONICAL_REQUEST_URI}\n +${HTTP_CANONICAL_REQUEST_HEADERS}\n +${HTTP_REQUEST_SIGNED_HEADERS} +${HTTP_REQUEST_PAYLOAD_HASH}" + + SIGNATURE="$(create_signature)" + HTTP_REQUEST_AUTHORIZATION_HEADER="AWS4-HMAC-SHA256 Credential=${S3_ACCESS_KEY_ID}/${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request, SignedHeaders=${HTTP_REQUEST_SIGNED_HEADERS}, Signature=${SIGNATURE}" + + echo " -H \"Authorization: ${HTTP_REQUEST_AUTHORIZATION_HEADER}\"" \ + " -H \"content-type: ${HTTP_REQUEST_CONTENT_TYPE}\"" \ + " -H \"x-amz-content-sha256: ${HTTP_REQUEST_PAYLOAD_HASH}\"" \ + " -H \"x-amz-date: ${CURRENT_DATE_ISO8601}\"" \ + " \"https://${ENDPOINT}${HTTP_CANONICAL_REQUEST_URI}\"" +} + function get_rbd_cmd { local i j URL_ELEMENTS @@ -317,7 +388,13 @@ s3://*) exit -1 fi - curl_args="$(s3_curl_args "$FROM")" + curl_args="" + + if [[ "$S3_AWS" =~ (no|NO) ]]; then + curl_args="$(s3_curl_args "$FROM")" + else + curl_args="$(s3_curl_args_aws "$FROM")" + fi command="curl $GLOBAL_CURL_ARGS $curl_args" ;; diff --git a/src/datastore_mad/remotes/vcenter/cp b/src/datastore_mad/remotes/vcenter/cp index 66836acef6..3b7272b95c 100755 --- a/src/datastore_mad/remotes/vcenter/cp +++ b/src/datastore_mad/remotes/vcenter/cp @@ -104,7 +104,7 @@ if VCenterDriver::FileHelper.remote_or_needs_unpack?(img_path) temp_folder = File.join(VAR_LOCATION, "vcenter/#{target_path}") temp_file = File.join(temp_folder, File.basename(img_path)) # if the original file doesnt have the vmdk extension, add it - if !temp_file.match(/\.vmdk$/) && !temp_file.match(/\.iso$/) + if !temp_file.match(/\.vmdk$/) && !temp_file.match(/\.iso$/) && !VCenterDriver::FileHelper.from_s3?(img_path) temp_file += '.vmdk' end @@ -117,7 +117,9 @@ if VCenterDriver::FileHelper.remote_or_needs_unpack?(img_path) downsh_args << "--sha1 #{sha1} " if sha1 && !sha1.empty? downsh_args << '--nodecomp ' if nodecomp && !nodecomp.empty? downsh_args << "--limit #{limit_bw} " if limit_bw && !limit_bw.empty? - downsh_args << '--convert vmdk' + unless VCenterDriver::FileHelper.from_s3?(img_path) + downsh_args << '--convert vmdk' + end downloader = "#{File.dirname(__FILE__)}/../downloader.sh #{downsh_args}" b64 = Base64.encode64(drv_action.to_xml).gsub!("\n", '') diff --git a/src/vmm_mad/remotes/lib/vcenter_driver/file_helper.rb b/src/vmm_mad/remotes/lib/vcenter_driver/file_helper.rb index e06e0402a0..0d65828db5 100644 --- a/src/vmm_mad/remotes/lib/vcenter_driver/file_helper.rb +++ b/src/vmm_mad/remotes/lib/vcenter_driver/file_helper.rb @@ -81,6 +81,10 @@ module VCenterDriver file.match(%r{^https?://}) || file.match(%r{^s3?://}) end + def self.from_s3?(file) + file.match(%r{^s3?://}) + end + def self.vmdk?(file) type = `file #{file}`