From 0f3dbf58db011665cabcd6f2d3b288431203eab8 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Wed, 9 Mar 2016 18:33:03 +0100 Subject: [PATCH] Feature #4217: Use Sunstone to download images and marketplaces This commit changes entirely the way images are downloaded. Instead of downloading them by running 'downloader.sh' in the local machine, it will do it on the Sunstone server, and it will in turn stream the response to the client. This commit implements the server and the CLI. --- src/cli/one_helper.rb | 107 ++++++++++++++++++++++ src/cli/oneimage | 7 +- src/cli/onemarketapp | 6 +- src/oca/ruby/opennebula/image.rb | 50 ---------- src/oca/ruby/opennebula/marketplaceapp.rb | 39 -------- src/sunstone/models/SunstoneServer.rb | 76 +++++++++++++++ src/sunstone/sunstone-server.rb | 43 ++++++++- 7 files changed, 234 insertions(+), 94 deletions(-) diff --git a/src/cli/one_helper.rb b/src/cli/one_helper.rb index a6d0cf8896..2952b09985 100644 --- a/src/cli/one_helper.rb +++ b/src/cli/one_helper.rb @@ -332,6 +332,12 @@ EOT } ] + FORCE={ + :name => 'force', + :large => '--force', + :description => 'Overwrite the file' + } + TEMPLATE_OPTIONS_VM=[TEMPLATE_NAME_VM]+TEMPLATE_OPTIONS+[DRY] CAPACITY_OPTIONS_VM=[TEMPLATE_OPTIONS[0],TEMPLATE_OPTIONS[1],TEMPLATE_OPTIONS[3]] @@ -1046,4 +1052,105 @@ EOT # in options hash (template_options-options.keys)!=template_options end + + def self.sunstone_url + if (one_sunstone = ENV['ONE_SUNSTONE']) + one_sunstone + elsif (one_xmlrpc = ENV['ONE_XMLRPC']) + uri = URI(one_xmlrpc) + "#{uri.scheme}://#{uri.host}:9869" + else + "http://localhost:9869" + end + end + + def self.download_resource_sunstone(kind, id, path, force) + client = OneHelper.client + user, password = client.one_auth.split(":", 2) + + # Step 1: Build Session to get Cookie + uri = URI(File.join(sunstone_url,"login")) + + req = Net::HTTP::Post.new(uri) + req.basic_auth user, password + + begin + res = Net::HTTP.start(uri.hostname, uri.port) do |http| + http.request(req) + end + rescue + return OpenNebula::Error.new("Error connecting to '#{uri}'.") + end + + cookie = res.response['set-cookie'].split('; ')[0] + + if cookie.nil? + return OpenNebula::Error.new("Unable to get Cookie. Is OpenNebula running?") + end + + # Step 2: Open '/' to get the csrftoken + uri = URI(sunstone_url) + + req = Net::HTTP::Get.new(uri) + req['Cookie'] = cookie + + begin + res = Net::HTTP.start(uri.hostname, uri.port) do |http| + http.request(req) + end + rescue + return OpenNebula::Error.new("Error connecting to '#{uri}'.") + end + + m = res.body.match(/var csrftoken = '(.*)';/) + csrftoken = m[1] rescue nil + + if csrftoken.nil? + return OpenNebula::Error.new("Unable to get csrftoken.") + end + + # Step 3: Download resource + uri = URI(File.join(sunstone_url, + kind.to_s, + id.to_s, + "download?csrftoken=#{csrftoken}")) + + req = Net::HTTP::Get.new(uri) + + req['Cookie'] = cookie + req['User-Agent'] = "OpenNebula CLI" + + begin + File.open(path, 'wb') do |f| + Net::HTTP.start(uri.hostname, uri.port) do |http| + http.request(req) do |res| + res.read_body do |chunk| + f.write(chunk) + end + end + end + end + rescue Errno::EACCES + return OpenNebula::Error.new("Target file not writable.") + end + + error_message = nil + + File.open(path, 'rb') do |f| + begin + f.seek(-1024, IO::SEEK_END) + rescue Errno::EINVAL + end + + tail = f.read + + m = tail.match(/@\^_\^@ (.*) @\^_\^@/m) + error_message = m[1] if m + end + + if error_message + File.unlink(path) + return OpenNebula::Error.new("Remote server error: #{error_message}") + end + end end diff --git a/src/cli/oneimage b/src/cli/oneimage index 3822d5222a..ce432ef57a 100755 --- a/src/cli/oneimage +++ b/src/cli/oneimage @@ -180,9 +180,12 @@ cmd=CommandParser::CmdParser.new(ARGV) do Downloads an image to a file EOT - command :download, download_desc, :imageid, :path do + command :download, download_desc, :imageid, :path, + :options => [OpenNebulaHelper::FORCE] do + helper.perform_action(args[0],options,"downloaded") do |image| - image.download(args[1], helper.client) + download_args = [:image, args[0], args[1], options[:force]] + OpenNebulaHelper.download_resource_sunstone(*download_args) end end diff --git a/src/cli/onemarketapp b/src/cli/onemarketapp index 4451e3b07c..84cade3758 100755 --- a/src/cli/onemarketapp +++ b/src/cli/onemarketapp @@ -131,9 +131,11 @@ CommandParser::CmdParser.new(ARGV) do Downloads a MarketApp to a file EOT - command :download, download_desc, :appid, :path do + command :download, download_desc, :appid, :path, + :options => [OpenNebulaHelper::FORCE] do helper.perform_action(args[0],options,"downloaded") do |app| - app.download(args[1], helper.client) + download_args = [:marketplaceapp, args[0], args[1], options[:force]] + OpenNebulaHelper.download_resource_sunstone(*download_args) end end diff --git a/src/oca/ruby/opennebula/image.rb b/src/oca/ruby/opennebula/image.rb index 275bb43e82..c11b8ba726 100644 --- a/src/oca/ruby/opennebula/image.rb +++ b/src/oca/ruby/opennebula/image.rb @@ -253,56 +253,6 @@ module OpenNebula return call(IMAGE_METHODS[:snapshotflatten], @pe_id, snap_id) end - ####################################################################### - # OCA specific methods - ####################################################################### - - # Invokes 'download.sh' to copy the image to the specified path - # It calls the /export action and downloader.sh - # - # @param path The destination of the downloader.sh action - # @param client The request client - # - # @return [nil, OpenNebula::Error] nil in case of success or Error - def download(path, client) - rc = info - return rc if OpenNebula.is_error?(rc) - - ds_id = self['DATASTORE_ID'] - ds = Datastore.new(Datastore.build_xml(ds_id), client) - - rc = ds.info - return rc if OpenNebula.is_error?(rc) - - drv_message = "" << - "#{to_xml}#{ds.to_xml}" << - "" - - drv_message_64 = Base64::strict_encode64(drv_message) - - export = "#{VAR_LOCATION}/remotes/datastore/#{ds['DS_MAD']}/export" - - export_stdout = `#{export} #{drv_message_64} #{id}` - doc = REXML::Document.new(export_stdout).root - - import_source = doc.elements['IMPORT_SOURCE'].text rescue nil - - if import_source.nil? || import_source.empty? - return OpenNebula::Error.new("Cannot find image source.") - end - - download_cmd = "#{VAR_LOCATION}/remotes/datastore/downloader.sh " << - "#{import_source} #{path}" - - system(download_cmd) - - if (status = $?.exitstatus) != 0 - error_msg = "Error executing '#{download_cmd}'. " << - "Exit status: #{status}" - return OpenNebula::Error.new(error_msg) - end - end - ####################################################################### # Helpers to get Image information ####################################################################### diff --git a/src/oca/ruby/opennebula/marketplaceapp.rb b/src/oca/ruby/opennebula/marketplaceapp.rb index d3edf056eb..d5750e8601 100644 --- a/src/oca/ruby/opennebula/marketplaceapp.rb +++ b/src/oca/ruby/opennebula/marketplaceapp.rb @@ -214,45 +214,6 @@ module OpenNebula end end - # Invokes 'download.sh' to download the app to the specified path - # - # @param path The destination of the downloader.sh action - # @param client The request client - # - # @return [nil, OpenNebula::Error] nil in case of success or Error - def download(path, client) - rc = info - return rc if OpenNebula.is_error?(rc) - - market_id = self['MARKETPLACE_ID'] - market = MarketPlace.new(MarketPlace.build_xml(market_id), client) - - rc = market.info - return rc if OpenNebula.is_error?(rc) - - # This 'drv_message' is missing some elements, compared to the one - # sent by the core. However, 'downloader.sh' only requires the - # MarketPlace information. - drv_message = "" << - "#{market.to_xml}" << - "" - - drv_message_64 = Base64::strict_encode64(drv_message) - - ENV['DRV_ACTION'] = drv_message_64 - - download_cmd = "#{VAR_LOCATION}/remotes/datastore/downloader.sh " << - "#{self['SOURCE']} #{path}" - - system(download_cmd) - - if (status = $?.exitstatus) != 0 - error_msg = "Error executing '#{download_cmd}'. " << - "Exit status: #{status}" - return OpenNebula::Error.new(error_msg) - end - end - # Enables this app def enable return call(MARKETPLACEAPP_METHODS[:enable], @pe_id, true) diff --git a/src/sunstone/models/SunstoneServer.rb b/src/sunstone/models/SunstoneServer.rb index fe9ad56ec2..929c38f483 100644 --- a/src/sunstone/models/SunstoneServer.rb +++ b/src/sunstone/models/SunstoneServer.rb @@ -214,6 +214,82 @@ class SunstoneServer < CloudServer end end + ############################################################################ + # + ############################################################################ + def download_resource(kind, id) + case kind + when "image" + # Get Image + image = Image.new(Image.build_xml(id.to_i), @client) + rc = image.info + + return [500, rc.message] if OpenNebula.is_error?(rc) + + # Get Datastore + ds_id = image['DATASTORE_ID'] + + ds = Datastore.new(Datastore.build_xml(ds_id), @client) + rc = ds.info + + return [500, rc.message] if OpenNebula.is_error?(rc) + + # Build Driver message + drv_message = "" << + "#{image.to_xml}#{ds.to_xml}" << + "" + + drv_message_64 = Base64::strict_encode64(drv_message) + + begin + export = "#{VAR_LOCATION}/remotes/datastore/#{ds['DS_MAD']}/export" + + export_stdout = `#{export} #{drv_message_64} #{id}` + + doc = REXML::Document.new(export_stdout).root + + source = doc.elements['IMPORT_SOURCE'].text + rescue Exception => e + return [500, "#{e.message}\n#{e.backtrace}"] + end + when "marketplaceapp" + # Get MarketPlaceApp + marketapp = MarketPlaceApp.new(MarketPlaceApp.build_xml(id.to_i), @client) + + rc = marketapp.info + return [500, rc.message] if OpenNebula.is_error?(rc) + + # Get Datastore + market_id = marketapp['MARKETPLACE_ID'] + + market = MarketPlace.new(MarketPlace.build_xml(market_id), @client) + rc = market.info + + return [500, rc.message] if OpenNebula.is_error?(rc) + + # Build Driver message + drv_message = "" << + "#{market.to_xml}" << + "" + + drv_message_64 = Base64::strict_encode64(drv_message) + + source = marketapp['SOURCE'] + + else + return [404, "Unknown resource."] + end + + download_cmd = "DRV_ACTION=#{drv_message_64} "<< + "#{VAR_LOCATION}/remotes/datastore/downloader.sh " << + "#{source} -" + + + filename = "one-#{kind}-#{id}" + + return [download_cmd, filename] + end + ############################################################################ # Unused ############################################################################ diff --git a/src/sunstone/sunstone-server.rb b/src/sunstone/sunstone-server.rb index cc84a3a673..d3abe7c9ca 100755 --- a/src/sunstone/sunstone-server.rb +++ b/src/sunstone/sunstone-server.rb @@ -92,6 +92,8 @@ require 'yaml' require 'securerandom' require 'tmpdir' require 'fileutils' +require 'base64' +require 'rexml/document' require 'CloudAuth' require 'SunstoneServer' @@ -324,7 +326,7 @@ before do request.body.rewind unless %w(/ /login /vnc /spice).include?(request.path) - halt 401 unless authorized? && valid_csrftoken? + halt [401, "csrftoken"] unless authorized? && valid_csrftoken? end if env['HTTP_ZONE_NAME'] @@ -717,6 +719,45 @@ post '/upload_chunk' do "" end +############################################################################## +# Download image or marketapp +############################################################################## +READ_LENGTH = 10*1024*1024 + +get '/:resource/:id/download' do + dl_resource = @SunstoneServer.download_resource(params[:resource], params[:id]) + + # If the first element of dl_resource is a number, it is the exit_code after + # an error happend, so return it. + return dl_resource if dl_resource[0].kind_of?(Fixnum) + + download_cmd, filename = dl_resource + + # Send headers + headers['Cache-Control'] = "no-transform" # Do not use Rack::Deflater + headers['Content-Disposition'] = "attachment; filename=\"#{filename}\"" + content_type :'application/octet-stream' + + # Start stream + stream do |out| + Open3.popen3(download_cmd) do |_,o,e,w| + + until o.eof? + out << o.read(READ_LENGTH) + end + + if !w.value.success? + error_message = "downloader.sh: " << e.read + logger.error { error_message } + + if request.user_agent == "OpenNebula CLI" + out << "@^_^@ #{error_message} @^_^@" + end + end + end + end +end + ############################################################################## # Create a new Resource ##############################################################################