diff --git a/install.sh b/install.sh index 013c368902..1aeca0c404 100755 --- a/install.sh +++ b/install.sh @@ -868,6 +868,7 @@ TM_LVM_FILES="src/tm_mad/lvm/clone \ #------------------------------------------------------------------------------- DATASTORE_DRIVER_COMMON_SCRIPTS="src/datastore_mad/remotes/xpath.rb \ + src/datastore_mad/remotes/downloader.rb \ src/datastore_mad/remotes/libfs.sh" DATASTORE_DRIVER_DUMMY_SCRIPTS="src/datastore_mad/remotes/dummy/cp \ diff --git a/src/datastore_mad/remotes/downloader.rb b/src/datastore_mad/remotes/downloader.rb new file mode 100755 index 0000000000..4153b58e26 --- /dev/null +++ b/src/datastore_mad/remotes/downloader.rb @@ -0,0 +1,286 @@ +#!/usr/bin/env ruby + +# -------------------------------------------------------------------------- # +# Copyright 2002-2012, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'open3' +require 'net/http' +require 'uri' +require 'optparse' +require 'digest' +require 'fileutils' + +require 'pp' + + +class Stream + BLOCK_SIZE=1024*64 + + TYPES={ + "application/x-gzip" => "gunzip -c -", + "application/x-bzip2" => "bunzip2 -c -", + } + + POSTPROCESS_TYPE={ + "application/x-tar" => "tar -xf #IN# -C #OUT#", + "application/zip" => "unzip -d #OUT# #IN#" + } + + DIGEST={ + "md5" => lambda { Digest::MD5.new }, + "sha1" => lambda { Digest::SHA1.new }, + "sha256" => lambda { Digest::SHA2.new(256) }, + "sha384" => lambda { Digest::SHA2.new(384) }, + "sha512" => lambda { Digest::SHA2.new(512) }, + } + + def initialize(from, to, options={}) + @from=from + @to=to + @options=options + + @digests={} + + @uncompress_proc=nil + @compr_in=nil + @compr_out=nil + + @writer_thread=nil + + prepare_digests + end + + def open_output_file + if @to=='-' + @output_file=STDOUT.dup + else + begin + @output_file=File.open(@to, "w") + rescue + STDERR.puts "Error opening output file '#{@to}" + exit(-1) + end + end + end + + def prepare_digests + @options.each do |key, value| + if DIGEST.has_key?(key) + @digests[key]={ + :hash => value, + :digest => DIGEST[key].call + } + end + end + end + + def process(data) + begin + try=5 + begin + @compr_in.write(data) + @compr_in.flush + rescue + if try>0 + try-=1 + sleep 0.1 + retry + end + end + rescue + STDERR.puts "Error uncompressing image." + exit(-1) + end + + @digests.each do |name, algo| + algo[:digest] << data + end + end + + def type(header) + io=Open3.popen3("file -b --mime-type -") + io[0].write(header) + io[0].close + out=io[1].read.strip + io[1].close + out + end + + def set_compress(header) + t=type(header) + + compr=TYPES[t]||"cat" + + compr="#{compr} >&#{@output_file.fileno}" + + @uncompress_proc=Open3.popen3(compr) + + @compr_in=@uncompress_proc[0] + @compr_out=@uncompress_proc[1] + end + + def start_file_writer + @writer_thread=Thread.new do + while(!@compr_out.eof?) + data=@compr_out.read(BLOCK_SIZE) + if data + #STDERR.puts "Compr reader #{data.length}" + @output_file.write(data) + #STDERR.puts "File writer #{data.length}" + else + #STDERR.puts "Data is empty!" + end + end + end + end + + def wget_downloader(url) + @popen=Open3.popen3("wget -O - '#{url}'") + @popen[0].close + @popen[1] + end + + def check_hashes + fail=false + + @digests.each do |name, d| + given=d[:hash].downcase + computed=d[:digest].hexdigest.downcase + if given!=computed + fail=true + STDERR.puts "Digest #{name} does not match. "<< + "#{given}!=#{computed}" + end + end + + exit(-1) if fail + end + + def download + io=nil + + begin + case @from + when '-' + io=STDIN + when /^https?:\/\// + io=wget_downloader(@from) + when /^file:\/\/(.*)$/ + name=$1 + io=open(name, 'r') + else + io=open(@from, 'r') + end + rescue # Errno::ENOENT + STDERR.puts "File not found" + exit(-1) + end + + header=io.read(BLOCK_SIZE) + + if !header + if @popen + STDERR.puts @popen[2].read.strip.split("\n").last + exit(-1) + end + end + + open_output_file + + set_compress(header) + + download_stderr="" + + process(header) + + while(!io.eof?) + if defined?(@popen) + @popen[2].read_nonblock(BLOCK_SIZE, download_stderr) + end + data=io.read(BLOCK_SIZE) + process(data) + end + + @finished=true + + @compr_in.close_write + + check_hashes + + postprocess if @to!='-' + end + + def postprocess + f=open(@to) + header=f.read(BLOCK_SIZE) + f.close + + t=type(header) + + if POSTPROCESS_TYPE.has_key?(t) + if @to[0,1]=='/' + to=@to + else + to=ENV['PWD']+'/'+@to + end + + tmp_file="#{to}.tmp" + FileUtils.mv(to, tmp_file) + + FileUtils.mkdir(to) + + cmd=POSTPROCESS_TYPE[t] + cmd.gsub!('#IN#', tmp_file) + cmd.gsub!('#OUT#', @to) + + system(cmd) + status=$? + + if !status.success? + STDERR.puts "Error uncompressing archive" + exit(-1) + end + + FileUtils.rm(tmp_file) + end + end +end + + +options={} + +OptionParser.new do |opts| + opts.banner="Usage: download_helper " + + Stream::DIGEST.each do |name, value| + opts.on("--#{name} HASH", "Check #{name} hash") do |v| + options[name]=v + end + end +end.parse! + +if ARGV.length<2 + STDERR.puts "You need to specify source and destination" + exit(-1) +end + +input=ARGV[0] +output=ARGV[1] + + +s=Stream.new(input, output, options) +s.download + diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index fbb80dce23..a3f92f7503 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -39,7 +39,9 @@ source ${DRIVER_PATH}/../libfs.sh DRV_ACTION=$1 ID=$2 -XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION" +UTILS_PATH="${DRIVER_PATH}/.." + +XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION" unset i XPATH_ELEMENTS @@ -49,26 +51,42 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/UMASK \ - /DS_DRIVER_ACTION_DATA/IMAGE/PATH) + /DS_DRIVER_ACTION_DATA/IMAGE/PATH \ + /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \ + /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1) BASE_PATH="${XPATH_ELEMENTS[0]}" RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" SAFE_DIRS="${XPATH_ELEMENTS[2]}" UMASK="${XPATH_ELEMENTS[3]}" SRC="${XPATH_ELEMENTS[4]}" +MD5="${XPATH_ELEMENTS[5]}" +SHA1="${XPATH_ELEMENTS[6]}" mkdir -p "$BASE_PATH" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK" DST=`generate_image_path` +HASHES="" + +if [ -n "$MD5" ]; then + HASHES="$HASHES --md5 $MD5" +fi + +if [ -n "$SHA1" ]; then + HASHES="$HASHES --sha1 $SHA1" +fi + +COPY_COMMAND="$UTILS_PATH/downloader.rb $HASHES $SRC $DST" + # ------------ Copy the image to the repository ------------- case $SRC in http://*) log "Downloading $SRC to the image repository" - exec_and_log "$WGET -O $DST $SRC" "Error downloading $SRC" + exec_and_log "$COPY_COMMAND" "Error downloading $SRC" ;; *) @@ -80,7 +98,7 @@ http://*) log "Copying local image $SRC to the image repository" - exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST" + exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST" ;; esac diff --git a/src/datastore_mad/remotes/iscsi/cp b/src/datastore_mad/remotes/iscsi/cp index 7399f0acc1..8a7f451512 100755 --- a/src/datastore_mad/remotes/iscsi/cp +++ b/src/datastore_mad/remotes/iscsi/cp @@ -40,7 +40,9 @@ source ${DRIVER_PATH}/iscsi.conf DRV_ACTION=$1 ID=$2 -XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION" +UTILS_PATH="${DRIVER_PATH}/.." + +XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION" unset i XPATH_ELEMENTS @@ -55,7 +57,9 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BASE_IQN \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BASE_TID \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH \ - /DS_DRIVER_ACTION_DATA/IMAGE/SIZE) + /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \ + /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \ + /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1) BASE_PATH="${XPATH_ELEMENTS[0]}" RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" @@ -67,6 +71,8 @@ BASE_IQN="${XPATH_ELEMENTS[6]:-$BASE_IQN}" BASE_TID="${XPATH_ELEMENTS[7]:-$BASE_TID}" SRC="${XPATH_ELEMENTS[8]}" SIZE="${XPATH_ELEMENTS[9]}" +MD5="${XPATH_ELEMENTS[10]}" +SHA1="${XPATH_ELEMENTS[11]}" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK" @@ -85,11 +91,23 @@ REGISTER_CMD=$(cat <MD5='+md5+''; + $("#custom_var_image_box").append(option); + } + + var sha1 = response['files'][0]['checksum']['sha1'] + if ( sha1 ) { + option = ''; + $("#custom_var_image_box").append(option); + } + popUpCreateImageDialog(); }, error: function(response)