1
0
mirror of https://github.com/OpenNebula/one.git synced 2025-03-20 10:50:08 +03:00

Merge branch 'feature-1303'

Conflicts:
	src/datastore_mad/remotes/iscsi/cp
	src/datastore_mad/remotes/lvm/cp
	src/datastore_mad/remotes/vmware/cp
This commit is contained in:
Ruben S. Montero 2012-06-19 23:31:31 +02:00
commit 24fe649e5d
7 changed files with 405 additions and 22 deletions

View File

@ -868,6 +868,7 @@ TM_LVM_FILES="src/tm_mad/lvm/clone \
#-------------------------------------------------------------------------------
DATASTORE_DRIVER_COMMON_SCRIPTS="src/datastore_mad/remotes/xpath.rb \
src/datastore_mad/remotes/downloader.rb \
src/datastore_mad/remotes/libfs.sh"
DATASTORE_DRIVER_DUMMY_SCRIPTS="src/datastore_mad/remotes/dummy/cp \

View File

@ -0,0 +1,286 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2012, OpenNebula Project Leads (OpenNebula.org) #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
require 'open3'
require 'net/http'
require 'uri'
require 'optparse'
require 'digest'
require 'fileutils'
require 'pp'
class Stream
BLOCK_SIZE=1024*64
TYPES={
"application/x-gzip" => "gunzip -c -",
"application/x-bzip2" => "bunzip2 -c -",
}
POSTPROCESS_TYPE={
"application/x-tar" => "tar -xf #IN# -C #OUT#",
"application/zip" => "unzip -d #OUT# #IN#"
}
DIGEST={
"md5" => lambda { Digest::MD5.new },
"sha1" => lambda { Digest::SHA1.new },
"sha256" => lambda { Digest::SHA2.new(256) },
"sha384" => lambda { Digest::SHA2.new(384) },
"sha512" => lambda { Digest::SHA2.new(512) },
}
def initialize(from, to, options={})
@from=from
@to=to
@options=options
@digests={}
@uncompress_proc=nil
@compr_in=nil
@compr_out=nil
@writer_thread=nil
prepare_digests
end
def open_output_file
if @to=='-'
@output_file=STDOUT.dup
else
begin
@output_file=File.open(@to, "w")
rescue
STDERR.puts "Error opening output file '#{@to}"
exit(-1)
end
end
end
def prepare_digests
@options.each do |key, value|
if DIGEST.has_key?(key)
@digests[key]={
:hash => value,
:digest => DIGEST[key].call
}
end
end
end
def process(data)
begin
try=5
begin
@compr_in.write(data)
@compr_in.flush
rescue
if try>0
try-=1
sleep 0.1
retry
end
end
rescue
STDERR.puts "Error uncompressing image."
exit(-1)
end
@digests.each do |name, algo|
algo[:digest] << data
end
end
def type(header)
io=Open3.popen3("file -b --mime-type -")
io[0].write(header)
io[0].close
out=io[1].read.strip
io[1].close
out
end
def set_compress(header)
t=type(header)
compr=TYPES[t]||"cat"
compr="#{compr} >&#{@output_file.fileno}"
@uncompress_proc=Open3.popen3(compr)
@compr_in=@uncompress_proc[0]
@compr_out=@uncompress_proc[1]
end
def start_file_writer
@writer_thread=Thread.new do
while(!@compr_out.eof?)
data=@compr_out.read(BLOCK_SIZE)
if data
#STDERR.puts "Compr reader #{data.length}"
@output_file.write(data)
#STDERR.puts "File writer #{data.length}"
else
#STDERR.puts "Data is empty!"
end
end
end
end
def wget_downloader(url)
@popen=Open3.popen3("wget -O - '#{url}'")
@popen[0].close
@popen[1]
end
def check_hashes
fail=false
@digests.each do |name, d|
given=d[:hash].downcase
computed=d[:digest].hexdigest.downcase
if given!=computed
fail=true
STDERR.puts "Digest #{name} does not match. "<<
"#{given}!=#{computed}"
end
end
exit(-1) if fail
end
def download
io=nil
begin
case @from
when '-'
io=STDIN
when /^https?:\/\//
io=wget_downloader(@from)
when /^file:\/\/(.*)$/
name=$1
io=open(name, 'r')
else
io=open(@from, 'r')
end
rescue # Errno::ENOENT
STDERR.puts "File not found"
exit(-1)
end
header=io.read(BLOCK_SIZE)
if !header
if @popen
STDERR.puts @popen[2].read.strip.split("\n").last
exit(-1)
end
end
open_output_file
set_compress(header)
download_stderr=""
process(header)
while(!io.eof?)
if defined?(@popen)
@popen[2].read_nonblock(BLOCK_SIZE, download_stderr)
end
data=io.read(BLOCK_SIZE)
process(data)
end
@finished=true
@compr_in.close_write
check_hashes
postprocess if @to!='-'
end
def postprocess
f=open(@to)
header=f.read(BLOCK_SIZE)
f.close
t=type(header)
if POSTPROCESS_TYPE.has_key?(t)
if @to[0,1]=='/'
to=@to
else
to=ENV['PWD']+'/'+@to
end
tmp_file="#{to}.tmp"
FileUtils.mv(to, tmp_file)
FileUtils.mkdir(to)
cmd=POSTPROCESS_TYPE[t]
cmd.gsub!('#IN#', tmp_file)
cmd.gsub!('#OUT#', @to)
system(cmd)
status=$?
if !status.success?
STDERR.puts "Error uncompressing archive"
exit(-1)
end
FileUtils.rm(tmp_file)
end
end
end
options={}
OptionParser.new do |opts|
opts.banner="Usage: download_helper <source> <destination>"
Stream::DIGEST.each do |name, value|
opts.on("--#{name} HASH", "Check #{name} hash") do |v|
options[name]=v
end
end
end.parse!
if ARGV.length<2
STDERR.puts "You need to specify source and destination"
exit(-1)
end
input=ARGV[0]
output=ARGV[1]
s=Stream.new(input, output, options)
s.download

View File

@ -39,7 +39,9 @@ source ${DRIVER_PATH}/../libfs.sh
DRV_ACTION=$1
ID=$2
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
UTILS_PATH="${DRIVER_PATH}/.."
XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
unset i XPATH_ELEMENTS
@ -49,26 +51,42 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/UMASK \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH)
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
SAFE_DIRS="${XPATH_ELEMENTS[2]}"
UMASK="${XPATH_ELEMENTS[3]}"
SRC="${XPATH_ELEMENTS[4]}"
MD5="${XPATH_ELEMENTS[5]}"
SHA1="${XPATH_ELEMENTS[6]}"
mkdir -p "$BASE_PATH"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
DST=`generate_image_path`
HASHES=""
if [ -n "$MD5" ]; then
HASHES="$HASHES --md5 $MD5"
fi
if [ -n "$SHA1" ]; then
HASHES="$HASHES --sha1 $SHA1"
fi
COPY_COMMAND="$UTILS_PATH/downloader.rb $HASHES $SRC $DST"
# ------------ Copy the image to the repository -------------
case $SRC in
http://*)
log "Downloading $SRC to the image repository"
exec_and_log "$WGET -O $DST $SRC" "Error downloading $SRC"
exec_and_log "$COPY_COMMAND" "Error downloading $SRC"
;;
*)
@ -80,7 +98,7 @@ http://*)
log "Copying local image $SRC to the image repository"
exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST"
exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST"
;;
esac

View File

@ -40,7 +40,9 @@ source ${DRIVER_PATH}/iscsi.conf
DRV_ACTION=$1
ID=$2
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
UTILS_PATH="${DRIVER_PATH}/.."
XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
unset i XPATH_ELEMENTS
@ -55,7 +57,9 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BASE_IQN \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BASE_TID \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE)
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@ -67,6 +71,8 @@ BASE_IQN="${XPATH_ELEMENTS[6]:-$BASE_IQN}"
BASE_TID="${XPATH_ELEMENTS[7]:-$BASE_TID}"
SRC="${XPATH_ELEMENTS[8]}"
SIZE="${XPATH_ELEMENTS[9]}"
MD5="${XPATH_ELEMENTS[10]}"
SHA1="${XPATH_ELEMENTS[11]}"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
@ -85,11 +91,23 @@ REGISTER_CMD=$(cat <<EOF
EOF
)
HASHES=""
if [ -n "$MD5" ]; then
HASHES="$HASHES --md5 $MD5"
fi
if [ -n "$SHA1" ]; then
HASHES="$HASHES --sha1 $SHA1"
fi
COPY_COMMAND="$UTILS_PATH/downloader.rb $HASHES $SRC -"
case $SRC in
http://*)
log "Downloading $SRC to the image repository"
DUMP="$WGET $SRC -O-"
DUMP="$COPY_COMMAND"
;;
*)
@ -101,7 +119,7 @@ http://*)
log "Copying local image $SRC to the image repository"
DUMP="cat $SRC"
DUMP="$COPY_COMMAND"
;;
esac

View File

@ -53,7 +53,9 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/HOST \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VG_NAME \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE)
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
@ -63,6 +65,8 @@ DST_HOST="${XPATH_ELEMENTS[4]:-$HOST}"
VG_NAME="${XPATH_ELEMENTS[5]:-$VG_NAME}"
SRC="${XPATH_ELEMENTS[6]}"
SIZE="${XPATH_ELEMENTS[7]}"
MD5="${XPATH_ELEMENTS[8]}"
SHA1="${XPATH_ELEMENTS[9]}"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
@ -76,11 +80,23 @@ REGISTER_CMD=$(cat <<EOF
EOF
)
HASHES=""
if [ -n "$MD5" ]; then
HASHES="$HASHES --md5 $MD5"
fi
if [ -n "$SHA1" ]; then
HASHES="$HASHES --sha1 $SHA1"
fi
COPY_COMMAND="$UTILS_PATH/downloader.rb $HASHES $SRC -"
case $SRC in
http://*)
log "Downloading $SRC to the image repository"
DUMP="$WGET $SRC -O-"
DUMP="$COPY_COMMAND"
;;
*)
@ -92,7 +108,7 @@ http://*)
log "Copying local image $SRC to the image repository"
DUMP="cat $SRC"
DUMP="$COPY_COMMAND"
;;
esac

View File

@ -39,7 +39,9 @@ source ${DRIVER_PATH}/../libfs.sh
DRV_ACTION=$1
ID=$2
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
UTILS_PATH="${DRIVER_PATH}/.."
XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
unset i XPATH_ELEMENTS
@ -49,22 +51,44 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/UMASK \
/DS_DRIVER_ACTION_DATA/IMAGE/PATH)
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1)
BASE_PATH="${XPATH_ELEMENTS[0]}"
RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}"
SAFE_DIRS="${XPATH_ELEMENTS[2]}"
UMASK="${XPATH_ELEMENTS[3]}"
SRC="${XPATH_ELEMENTS[4]}"
MD5="${XPATH_ELEMENTS[5]}"
SHA1="${XPATH_ELEMENTS[6]}"
mkdir -p "$BASE_PATH"
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" "$UMASK"
DST=`generate_image_path`
HASHES=""
if [ -n "$MD5" ]; then
HASHES="$HASHES --md5 $MD5"
fi
if [ -n "$SHA1" ]; then
HASHES="$HASHES --sha1 $SHA1"
fi
COPY_COMMAND="$UTILS_PATH/downloader.rb $HASHES $SRC $DST"
# ------------ Copy the image to the repository -------------
case $SRC in
http://*)
log "Downloading $SRC to the image repository"
exec_and_log "$COPY_COMMAND" "Error downloading $SRC"
;;
*)
if [ `check_restricted $SRC` -eq 1 ]; then
log_error "Not allowed to copy images from $RESTRICTED_DIRS"
@ -74,17 +98,24 @@ case $SRC in
log "Copying local disk folder $SRC to the image repository"
exec_and_log "cp -rf $SRC $DST" "Error copying $SRC to $DST"
if [ ! -f $DST/disk.vmdk ]; then
BASE_DISK_FILE=`ls $DST | grep -v '\-\(flat\|delta\|s[0-9]*\)\.vmdk$'`
exec_and_log "mv -f $DST/$BASE_DISK_FILE $DST/disk.vmdk" \
"Error renaming disk file $BASE_DISK_FILE to disk.vmdk"
if [ -d $SRC ]; then
exec_and_log "cp -rf $SRC $DST" "Error copying $SRC to $DST"
else
exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST"
fi
;;
esac
if [ ! -f $DST/disk.vmdk ]; then
BASE_DISK_FILE=`ls $DST | grep -v '\-\(flat\|delta\|s[0-9]*\)\.vmdk$'`
exec_and_log "mv -f $DST/$BASE_DISK_FILE $DST/disk.vmdk" \
"Error renaming disk file $BASE_DISK_FILE to disk.vmdk"
fi
# ---------------- Get the size of the image ------------
SIZE=`fs_du $DST`
echo "$DST"

View File

@ -90,6 +90,19 @@ var market_actions = {
success: function(response){
document.getElementById("img_name").value = response['name'];
document.getElementById("img_path").value = response['links']['download']['href'];
var md5 = response['files'][0]['checksum']['md5']
if ( md5 ) {
option = '<option value=\''+md5+'\' name="MD5">MD5='+md5+'</option>';
$("#custom_var_image_box").append(option);
}
var sha1 = response['files'][0]['checksum']['sha1']
if ( sha1 ) {
option = '<option value=\''+sha1+'\' name="SHA1">SHA1='+sha1+'</option>';
$("#custom_var_image_box").append(option);
}
popUpCreateImageDialog();
},
error: function(response)