shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-01-24 02:03:52 +03:00
Code Releases Wiki Activity

Feature #4217: Add market_mad drivers for S3

Browse Source
This commit is contained in:
Jaime Melis 2016-01-25 16:42:59 +01:00
parent 49b43ecc91
commit 44111d5f1f
5 changed files with 411 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
install.sh
View File

@ -428,6 +428,7 @@ INSTALL_FILES=(
DATASTORE_DRIVER_DEV_SCRIPTS:$VAR_LOCATION/remotes/datastore/dev
MARKETPLACE_DRIVER_HTTP_SCRIPTS:$VAR_LOCATION/remotes/market/http
MARKETPLACE_DRIVER_ONE_SCRIPTS:$VAR_LOCATION/remotes/market/one
MARKETPLACE_DRIVER_S3_SCRIPTS:$VAR_LOCATION/remotes/market/s3
NETWORK_FILES:$VAR_LOCATION/remotes/vnm
NETWORK_8021Q_FILES:$VAR_LOCATION/remotes/vnm/802.1Q
NETWORK_VXLAN_FILES:$VAR_LOCATION/remotes/vnm/vxlan
@ -1221,11 +1222,15 @@ MARKETPLACE_DRIVER_ONE_SCRIPTS="src/market_mad/remotes/one/import \
src/market_mad/remotes/one/delete \
src/market_mad/remotes/one/monitor"
MARKETPLACE_DRIVER_S3_SCRIPTS="src/market_mad/remotes/s3/import \
src/market_mad/remotes/s3/delete \
src/market_mad/remotes/s3/monitor \
src/market_mad/remotes/s3/S3.rb"
#-------------------------------------------------------------------------------
# Migration scripts for onedb command, to be installed under $LIB_LOCATION
#-------------------------------------------------------------------------------
ONEDB_FILES="src/onedb/fsck.rb \
src/onedb/import_slave.rb \
src/onedb/onedb.rb \

123
src/market_mad/remotes/s3/S3.rb Normal file
View File

@ -0,0 +1,123 @@
# -------------------------------------------------------------------------- #
# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
# This class is a generic wrapper to the s3 upload and delete facilities.
# It can either handle simple or multipart uploads, but the logic to decide
# which uploader to use is not included in this class.
class S3
def initialize(h)
@name = h[:name]
@config = {
:bucket => h[:bucket],
:md5 => h[:md5],
:region => h[:region],
:access_key_id => h[:access_key_id],
:secret_access_key => h[:secret_access_key]
}
@client = Aws::S3::Client.new({
:region => @config[:region],
:access_key_id => @config[:access_key_id],
:secret_access_key => @config[:secret_access_key]
})
@parts = []
@part_number = 1
end
def create_multipart_upload
resp = @client.create_multipart_upload({
:bucket => @config[:bucket],
:key => @name
})
@upload_id = resp.upload_id
end
def complete_multipart_upload
@client.complete_multipart_upload({
:bucket => @config[:bucket],
:key => @name,
:upload_id => @upload_id,
:multipart_upload => {:parts => @parts}
})
end
def abort_multipart_upload
@client.abort_multipart_upload({
:upload_id => @upload_id,
:key => @name,
:bucket => @config[:bucket]
})
end
def upload_part(body)
resp = @client.upload_part({
:body => body,
:upload_id => @upload_id,
:part_number => @part_number,
:key => @name,
:bucket => @config[:bucket]
})
@parts << {
:etag => resp.etag,
:part_number => @part_number
}
@part_number += 1
end
def put_object(body)
@client.put_object({
:body => body,
:bucket => @config[:bucket],
:key => @name
})
end
def delete_object
@client.delete_object({
:bucket => @config[:bucket],
:key => @name
})
end
def exists?
begin
!!@client.head_object({
:bucket => @config[:bucket],
:key => @name
})
rescue Aws::S3::Errors::NotFound
false
end
end
def get_bucket_size
resp = @client.list_objects({
bucket: @config[:bucket]
})
size = 0
resp.contents.each do |o|
size += o.size
end
return size
end
end

75
src/market_mad/remotes/s3/delete Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
###############################################################################
# This script is used to import a file into the marketplace. The source file
# is an opaque representation of an OpenNebula object, like a image file or a
# tar.gz with several vm template or flow disk images
###############################################################################
ONE_LOCATION = ENV["ONE_LOCATION"]
if !ONE_LOCATION
RUBY_LIB_LOCATION = "/usr/lib/one/ruby"
else
RUBY_LIB_LOCATION = ONE_LOCATION + "/lib/ruby"
end
UTILS_PATH = File.join(File.dirname(__FILE__), '../../datastore')
$: << RUBY_LIB_LOCATION
$: << RUBY_LIB_LOCATION+"/cli"
$: << File.dirname(__FILE__)
require 'base64'
require 'rexml/document'
require 'getoptlong'
require 'aws-sdk'
require 'pp'
require 'S3'
def xpath(xml, xpath)
xml.elements[xpath].text.to_s rescue nil
end
xml = REXML::Document.new(Base64::decode64(ARGV[0])).root
source = xpath(xml, 'MARKETPLACEAPP/SOURCE')
access_key_id = xpath(xml, 'MARKETPLACE/TEMPLATE/ACCESS_KEY_ID')
secret_access_key = xpath(xml, 'MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY')
bucket = xpath(xml, 'MARKETPLACE/TEMPLATE/BUCKET')
region = xpath(xml, 'MARKETPLACE/TEMPLATE/REGION')
name = File.basename(source)
s3 = S3.new(
:name => name,
:bucket => bucket,
:region => region,
:access_key_id => access_key_id,
:secret_access_key => secret_access_key
)
if !s3.exists?
STDERR.puts "Object '#{name}' does not exist."
exit 1
end
s3.delete_object

132
src/market_mad/remotes/s3/import Executable file
View File

@ -0,0 +1,132 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
###############################################################################
# This script is used to import a file into the marketplace. The source file
# is an opaque representation of an OpenNebula object, like a image file or a
# tar.gz with several vm template or flow disk images
###############################################################################
ONE_LOCATION = ENV["ONE_LOCATION"]
if !ONE_LOCATION
RUBY_LIB_LOCATION = "/usr/lib/one/ruby"
else
RUBY_LIB_LOCATION = ONE_LOCATION + "/lib/ruby"
end
UTILS_PATH = File.join(File.dirname(__FILE__), '../../datastore')
$: << RUBY_LIB_LOCATION
$: << RUBY_LIB_LOCATION+"/cli"
$: << File.dirname(__FILE__)
require 'base64'
require 'rexml/document'
require 'getoptlong'
require 'aws-sdk'
require 'open3'
require 'pp'
require 'S3'
def xpath(xml, xpath)
xml.elements[xpath].text.to_s rescue nil
end
xml = REXML::Document.new(Base64::decode64(ARGV[0])).root
import_source = xpath(xml, 'IMPORT_SOURCE')
format = xpath(xml, 'FORMAT')
dispose = xpath(xml, 'DISPOSE')
size = xpath(xml, 'SIZE')
md5 = xpath(xml, 'MD5')
id = xpath(xml, 'MARKETPLACEAPP/ID')
access_key_id = xpath(xml, 'MARKETPLACE/TEMPLATE/ACCESS_KEY_ID')
secret_access_key = xpath(xml, 'MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY')
bucket = xpath(xml, 'MARKETPLACE/TEMPLATE/BUCKET')
region = xpath(xml, 'MARKETPLACE/TEMPLATE/REGION')
name = "marketplaceapp-#{id}"
source = "s3://#{bucket}/#{name}"
# Upload
READ_LENGTH = 10*1024*1024 # Read in chunks of 10MB
s3 = S3.new(
:name => name,
:md5 => md5,
:bucket => bucket,
:region => region,
:access_key_id => access_key_id,
:secret_access_key => secret_access_key
)
if s3.exists?
STDERR.puts "Object '#{name}' already exists."
exit 1
end
cmd = "#{UTILS_PATH}/downloader.sh #{import_source} -"
Open3.popen3(cmd) do |_, o, _, _|
body = o.read(READ_LENGTH)
if o.eof?
s3.put_object(body)
else
s3.create_multipart_upload
begin
s3.upload_part(body)
until o.eof?
body = o.read(READ_LENGTH)
s3.upload_part(body)
end
s3.complete_multipart_upload
rescue Exception => e
STDERR.puts(e.message)
STDERR.puts(e.backtrace)
resp = s3.abort_multipart_upload
STDERR.puts(resp.inspect)
exit 1
end
end
end
# Result
marketplaceapp = {
:source => source,
:md5 => md5,
:size => size,
:format => format
}
marketplaceapp.each do |k,v|
puts "#{k.upcase}=\"#{v}\""
end
exit 0

75
src/market_mad/remotes/s3/monitor Executable file
View File

@ -0,0 +1,75 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2015, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
# -------------------------------------------------------------------------- #
###############################################################################
# This script is used to import a file into the marketplace. The source file
# is an opaque representation of an OpenNebula object, like a image file or a
# tar.gz with several vm template or flow disk images
###############################################################################
ONE_LOCATION = ENV["ONE_LOCATION"]
if !ONE_LOCATION
RUBY_LIB_LOCATION = "/usr/lib/one/ruby"
else
RUBY_LIB_LOCATION = ONE_LOCATION + "/lib/ruby"
end
UTILS_PATH = File.join(File.dirname(__FILE__), '../../datastore')
$: << RUBY_LIB_LOCATION
$: << RUBY_LIB_LOCATION+"/cli"
$: << File.dirname(__FILE__)
TOTAL_MB_DEFAULT = 1048576 # Default maximum 1TB
require 'base64'
require 'rexml/document'
require 'getoptlong'
require 'aws-sdk'
require 'pp'
require 'S3'
def xpath(xml, xpath)
xml.elements[xpath].text.to_s rescue nil
end
xml = REXML::Document.new(Base64::decode64(ARGV[0])).root
access_key_id = xpath(xml, 'MARKETPLACE/TEMPLATE/ACCESS_KEY_ID')
secret_access_key = xpath(xml, 'MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY')
bucket = xpath(xml, 'MARKETPLACE/TEMPLATE/BUCKET')
region = xpath(xml, 'MARKETPLACE/TEMPLATE/REGION')
total_mb = xpath(xml, 'MARKETPLACE/TEMPLATE/TOTAL_MB') || TOTAL_MB_DEFAULT
s3 = S3.new(
:bucket => bucket,
:region => region,
:access_key_id => access_key_id,
:secret_access_key => secret_access_key
)
used_mb = (s3.get_bucket_size.to_f/1024/1024).ceil
free_mb = total_mb - used_mb
puts <<EOF
USED_MB="#{used_mb}"
FREE_MB="#{free_mb}"
TOTAL_MB="#{total_mb}"
EOF