From f4f280ce685b5c3f96393c631224bb2970d47a78 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Wed, 9 Mar 2016 19:06:55 +0100 Subject: [PATCH] Feature #4217: Add support for READ_LENGTH --- src/market_mad/remotes/s3/import | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/market_mad/remotes/s3/import b/src/market_mad/remotes/s3/import index 2f1da80db5..10718bec47 100755 --- a/src/market_mad/remotes/s3/import +++ b/src/market_mad/remotes/s3/import @@ -68,14 +68,11 @@ region = xpath(xml, 'MARKETPLACE/TEMPLATE/REGION') signature_version = xpath(xml, 'MARKETPLACE/TEMPLATE/SIGNATURE_VERSION') endpoint = xpath(xml, 'MARKETPLACE/TEMPLATE/ENDPOINT') force_path_style = xpath(xml, 'MARKETPLACE/TEMPLATE/FORCE_PATH_STYLE') +read_length = xpath(xml, 'MARKETPLACE/TEMPLATE/READx_LENGTH') name = "marketapp-#{id}" source = "s3://#{bucket}/#{name}" -# Upload - -READ_LENGTH = 10*1024*1024 # Read in chunks of 10MB - s3_config = { :region => region, :access_key_id => access_key_id, @@ -86,6 +83,14 @@ s3_config[:signature_version] = signature_version if !signature_version.to_s.emp s3_config[:endpoint] = endpoint if !endpoint.to_s.empty? s3_config[:force_path_style] = true if force_path_style.to_s.downcase == "yes" +DEFAULT_READ_LENGTH = 32*1024*1024 # Read in chunks of 32MB + +if read_length.to_s.empty? + read_length = DEFAULT_READ_LENGTH +else + read_length = 1024*1024*read_length.to_i +end + s3 = S3.new(s3_config) s3.name = name @@ -99,7 +104,7 @@ end cmd = "#{UTILS_PATH}/downloader.sh #{import_source} -" Open3.popen3(cmd) do |_, o, _, _| - body = o.read(READ_LENGTH) + body = o.read(read_length) if o.eof? s3.put_object(body) @@ -110,7 +115,7 @@ Open3.popen3(cmd) do |_, o, _, _| s3.upload_part(body) until o.eof? - body = o.read(READ_LENGTH) + body = o.read(read_length) s3.upload_part(body) end