From f75fd6028474a454f2304cbbea0d939b03af0855 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 31 Aug 2011 00:51:48 +0200 Subject: [PATCH 01/24] feature #788: Quota files from #754 branch --- src/authm_mad/remotes/quota/authorize | 62 +++++++++ src/authm_mad/remotes/quota/one_usage.rb | 124 +++++++++++++++++ src/authm_mad/remotes/quota/quota.rb | 164 +++++++++++++++++++++++ 3 files changed, 350 insertions(+) create mode 100755 src/authm_mad/remotes/quota/authorize create mode 100644 src/authm_mad/remotes/quota/one_usage.rb create mode 100644 src/authm_mad/remotes/quota/quota.rb diff --git a/src/authm_mad/remotes/quota/authorize b/src/authm_mad/remotes/quota/authorize new file mode 100755 index 0000000000..19c0ca02f4 --- /dev/null +++ b/src/authm_mad/remotes/quota/authorize @@ -0,0 +1,62 @@ +#!/usr/bin/env ruby + +# -------------------------------------------------------------------------- # +# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +ONE_LOCATION=ENV["ONE_LOCATION"] + +if !ONE_LOCATION + RUBY_LIB_LOCATION="/usr/lib/one/ruby" + ETC_LOCATION="/etc/one/" +else + RUBY_LIB_LOCATION=ONE_LOCATION+"/lib/ruby" + ETC_LOCATION=ONE_LOCATION+"/etc/" +end + +$: << RUBY_LIB_LOCATION + +require 'scripts_common' +require 'quota' + +user_id = ARGV.shift + +overall_evalutation = ARGV.pop +exit -1 if overall_evalutation == 0 + +quota = Quota.new + +#q = { +# :cpu => 10, +# :memory => 2048, +# :storage => 100000, +# :num_vms => 5 +#} +# +#quota.set(1, q) +#OpenNebula.log_debug("quotas: #{quota.get(1)}") + +ARGV.each {|request| + rc = quota.check_request(user_id, request) + + if rc + OpenNebula.error_message rc + exit -1 + end +} + +#OpenNebula.log_debug("AUTHORIZE ARGS: #{ARGV.join(' ')}") + +exit 0 \ No newline at end of file diff --git a/src/authm_mad/remotes/quota/one_usage.rb b/src/authm_mad/remotes/quota/one_usage.rb new file mode 100644 index 0000000000..0d83e684d2 --- /dev/null +++ b/src/authm_mad/remotes/quota/one_usage.rb @@ -0,0 +1,124 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'OpenNebula' + +# This class retrieves and caches vms and its consuption grouped +# by users. 'update_user' method should be called to fill data for +# a user before any calculation is made +class OneUsage + VM_USAGE = { + :cpu => { + :proc_info => lambda {|template| template['CPU']}, + :xpath => 'TEMPLATE/CPU' + }, + :memory => { + :proc_info => lambda {|template| template['MEMORY']}, + :xpath => 'TEMPLATE/MEMORY' + }, + :num_vms => { + :proc_info => lambda {|template| 1 }, + :xpath => 'ID', + :count => true + } + } + + IMAGE_USAGE = { + :storage => { + :proc_info => lambda {|template| File.size(template['PATH']) }, + :proc_total => 'TEMPLATE/SIZE' + } + } + + RESOURCES = ["VM", "IMAGE"] + + def initialize() + @client = OpenNebula::Client.new + @usage = Hash.new + end + + def total(user_id, resource=nil, force=false) + usage = Hash.new + + if force + resources = [resource] if RESOURCES.include?(resource) + + resources.each{ |res| + pool = get_pool(res, user_id) + + base_xpath = "/#{res}_POOL/#{resource}" + OneUsage.const_get("#{res}_USAGE".to_sym).each { |key, params| + usage[key] ||= 0 + pool.each_xpath("#{base_xpath}/#{params[:xpath]}") { |elem| + usage[key] += params[:count] ? 1 : elem.to_i + } + } + + @usage[:user_id] ||= Hash.new + @usage[:user_id].merge!(usage) + } + else + usage = get_usage(user_id) + end + + usage + end + + # Retrieve the useful information of the template for the specified + # kind of resource + def get_resources(resource, xml_template) + template = OpenNebula::XMLElement.new + template.initialize_xml(xml_template, 'TEMPLATE') + + info = Hash.new + + self.class.const_get("#{resource}_USAGE").each { |key, params| + info[key] = params[:proc_info].call(template).to_i + } + + info + end + + private + + def get_usage(user_id) + usage = @usage[:user_id] + + unless usage + usage = Hash.new + + keys = VM_USAGE.keys + IMAGE_USAGE.keys + keys.each { |key| + usage[key] = 0 + } + + @usage[:user_id] = usage + end + + usage + end + + # Returns a an Array than contains the elements of the resource Pool + def get_pool(resource, user_id) + pool = case resource + when "VM" then OpenNebula::VirtualMachinePool.new(@client, user_id) + when "IMAGE" then OpenNebula::ImagePool.new(@client, user_id) + end + + rc = pool.info + return pool + end +end diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb new file mode 100644 index 0000000000..028c32ca93 --- /dev/null +++ b/src/authm_mad/remotes/quota/quota.rb @@ -0,0 +1,164 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +require 'one_usage' +require 'sequel' +require 'base64' + +# Quota functionality for auth driver. Stores in database limits for each +# user and using OneUsage is able to retrieve resource usage from +# OpenNebula daemon and check if it is below limits +class Quota + attr_accessor :defaults + ########################################################################### + #Constants with paths to relevant files and defaults + ########################################################################### + if !ENV["ONE_LOCATION"] + VAR_LOCATION = "/var/lib/one" + else + VAR_LOCATION = ENV["ONE_LOCATION"] + "/var" + end + + CONF = { + :db => "sqlite://#{VAR_LOCATION}/onequota.db", + :defaults => { + :cpu => nil, + :memory => nil, + :num_vms => nil, + :storage => nil + } + } + + TABLE_NAME = :quotas + + DB_QUOTA_SCHEMA = { + :cpu => Float, + :memory => Integer, + :num_vms => Integer, + :storage => Integer + } + + + # 'db' is a Sequel database where to store user limits and client + # is OpenNebula::Client used to connect to OpenNebula daemon + def initialize(conf={}) + # TBD merge with the conf file + @conf=CONF + + @defaults=@conf[:defaults] + + @db=Sequel.connect(@conf[:db]) + + create_table + @table=@db[TABLE_NAME] + + @one_usage=OneUsage.new + end + + ########################################################################### + # DB handling + ########################################################################### + + # Creates database quota table if it does not exist + def create_table + @db.create_table?(TABLE_NAME) do + Integer :uid + + DB_QUOTA_SCHEMA.each { |key,value| + column key, value + } + + primary_key :uid + index :uid + end + end + + # Adds new user limits + def set(uid, quota={}) + data=quota.delete_if{|key,value| !DB_QUOTA_SCHEMA.keys.include?(key)} + + quotas=@table.filter(:uid => uid) + + if quotas.first + quotas.update(data) + else + @table.insert(data.merge!(:uid => uid)) + end + end + + # Gets user limits + def get(uid) + limit=@table.filter(:uid => uid).first + if limit + limit + else + @conf[:defaults] + end + end + + + ########################################################################### + # Authorization + ########################################################################### + + def check_request(user_id, request) + obj, template_or_id, op, owner, pub, acl_eval = request.split(':') + + if acl_eval == 0 + return "ACL evaluation denied" + end + + # Check if this op needs to check the quota + return false unless with_quota?(obj, op) + + # If the object is a template the info should be retrived from the + # VM pool. + obj = "VM" if obj == "TEMPLATE" + template = Base64::decode64(template_or_id) + + check_quotas(user_id.to_i, obj, template) + end + + def check_quotas(user_id, obj, template) + info = @one_usage.get_resources(obj, template) + total = @one_usage.total(obj, user_id) + quota = get(user_id) + + msg = "" + info.each { |quota_name, quota_requested| + spent = total[quota_name].to_i + quota_requested.to_i + if quota[quota_name] && spent > quota[quota_name].to_i + msg << " #{quota_name.to_s.upcase} quota exceeded " + msg << "(Quota: #{quota[quota_name].to_i}, " + msg << "Used: #{spent.to_i}, " + msg << "Asked: #{quota_requested.to_i})." + end + } + + if msg=="" + return false + else + return msg.strip + end + end + + def with_quota?(obj, op) + return (obj == "VM" && op == "CREATE") || + (obj == "IMAGE" && op == "CREATE") || + (obj == "TEMPLATE" && op == "INSTANTIATE") + end +end + From 261a347d62383e4cdd14e8c17aa19c695273e8f4 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 14:01:52 +0200 Subject: [PATCH 02/24] feature #788: Return all the quotas if the user is not specified --- src/authm_mad/remotes/quota/quota.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb index 028c32ca93..0a0027b88d 100644 --- a/src/authm_mad/remotes/quota/quota.rb +++ b/src/authm_mad/remotes/quota/quota.rb @@ -100,12 +100,16 @@ class Quota end # Gets user limits - def get(uid) + def get(uid=nil) + if uid limit=@table.filter(:uid => uid).first - if limit - limit + if limit + limit + else + @conf[:defaults] + end else - @conf[:defaults] + @table.all end end From 4df59c631029b400e63aaa65d9d9d842493e8733 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 14:02:14 +0200 Subject: [PATCH 03/24] feature #788: Improve error messages in the cli_helper --- src/cli/cli_helper.rb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/cli/cli_helper.rb b/src/cli/cli_helper.rb index dc97027f28..416d756099 100644 --- a/src/cli/cli_helper.rb +++ b/src/cli/cli_helper.rb @@ -127,6 +127,7 @@ module CLIHelper end def default(*args) + args.map!{|a| a.to_sym } @default_columns=args end @@ -194,7 +195,7 @@ module CLIHelper size=@columns[field][:size] return "%#{minus}#{size}.#{size}s" % [ data.to_s ] else - exit -1, "Column not defined" + exit -1, "Column #{field} not defined." end end @@ -222,7 +223,7 @@ module CLIHelper if @columns[c] format_str(c, c.to_s) else - puts "Column not defined" + puts "Column #{c} not defined." exit -1 end }.compact.join(' ') From c8f44ede9aab94b28abd733654fb3592cc112409 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 31 Aug 2011 16:21:18 +0200 Subject: [PATCH 04/24] feature #788: Compute image size when moving an image to the repository. Drivers now return SUCESS IMAGE_ID SOURCE SIZE. The fs based image repository drivers has been updates, oneimage utility has been also updated to show the size (list and show operations). --- include/Image.h | 26 ++++++++--- src/cli/etc/oneimage.yaml | 7 ++- src/cli/one_helper/oneimage_helper.rb | 16 ++++--- src/image/Image.cc | 3 ++ src/image/ImageManagerDriver.cc | 65 ++++++++++++--------------- src/image_mad/remotes/fs/cp | 5 ++- src/image_mad/remotes/fs/fsrc | 13 ++++++ src/image_mad/remotes/fs/mkfs | 5 ++- src/image_mad/remotes/fs/mv | 5 ++- src/mad/sh/scripts_common.sh | 1 + 10 files changed, 94 insertions(+), 52 deletions(-) diff --git a/include/Image.h b/include/Image.h index 9804de2a57..5f5defe5ea 100644 --- a/include/Image.h +++ b/include/Image.h @@ -90,14 +90,21 @@ public: } /** - * Returns the source path of the image - * @return source of image + * Sets the source path of the image */ void set_source(const string& _source) { source = _source; } + /** + * Sets the size for the image + */ + void set_size(unsigned int _size_mb) + { + size_mb = _size_mb; + } + /** * Returns the type of the image * @return type @@ -285,27 +292,32 @@ private: /** * Type of the Image */ - ImageType type; + ImageType type; /** * Persistency of the Image */ - int persistent_img; + int persistent_img; /** * Registration time */ - time_t regtime; + time_t regtime; /** * Path to the image */ - string source; + string source; + + /** + * Size of the image in MB + */ + unsigned int size_mb; /** * Image state */ - ImageState state; + ImageState state; /** * Number of VMs using the image diff --git a/src/cli/etc/oneimage.yaml b/src/cli/etc/oneimage.yaml index c632dd5669..baf79e95ea 100644 --- a/src/cli/etc/oneimage.yaml +++ b/src/cli/etc/oneimage.yaml @@ -5,7 +5,7 @@ :NAME: :desc: Name of the Image - :size: 16 + :size: 12 :left: true :USER: @@ -18,6 +18,10 @@ :size: 8 :left: true +:SIZE: + :desc: Size of the Image + :size: 7 + :TYPE: :desc: Type of the Image :size: 4 @@ -47,6 +51,7 @@ - :USER - :GROUP - :NAME +- :SIZE - :TYPE - :REGTIME - :PUBLIC diff --git a/src/cli/one_helper/oneimage_helper.rb b/src/cli/one_helper/oneimage_helper.rb index c79b67f72b..2af923b7f0 100644 --- a/src/cli/one_helper/oneimage_helper.rb +++ b/src/cli/one_helper/oneimage_helper.rb @@ -57,10 +57,10 @@ class OneImageHelper < OpenNebulaHelper::OneHelper str_h1="%-80s" CLIHelper.print_header(str_h1 % "IMAGE #{image['ID']} INFORMATION") - puts str % ["ID", image.id.to_s] + puts str % ["ID", image.id.to_s] puts str % ["NAME", image.name] puts str % ["USER", image['UNAME']] - puts str % ["GROUP", image['GNAME']] + puts str % ["GROUP",image['GNAME']] puts str % ["TYPE", image.type_str] puts str % ["REGISTER TIME", OpenNebulaHelper.time_to_str(image['REGTIME'])] @@ -68,7 +68,8 @@ class OneImageHelper < OpenNebulaHelper::OneHelper OpenNebulaHelper.boolean_to_str(image['PUBLIC'])] puts str % ["PERSISTENT", OpenNebulaHelper.boolean_to_str(image["PERSISTENT"])] - puts str % ["SOURCE", image['SOURCE']] + puts str % ["SOURCE",image['SOURCE']] + puts str % ["SIZE", image['SIZE']] puts str % ["STATE", image.short_state_str] puts str % ["RUNNING_VMS", image['RUNNING_VMS']] puts @@ -85,7 +86,7 @@ class OneImageHelper < OpenNebulaHelper::OneHelper d["ID"] end - column :NAME, "Name of the Image", :left, :size=>15 do |d| + column :NAME, "Name of the Image", :left, :size=>12 do |d| d["NAME"] end @@ -127,7 +128,12 @@ class OneImageHelper < OpenNebulaHelper::OneHelper d['RUNNING_VMS'] end - default :ID, :USER, :GROUP, :NAME, :TYPE, :REGTIME, :PUBLIC, + column :SIZE, "Size of the image", + :size=>7 do |d| + OpenNebulaHelper.unit_to_str(d['SIZE'].to_i,options) + end + + default :ID, :USER, :GROUP, :NAME, :SIZE, :TYPE, :REGTIME, :PUBLIC, :PERSISTENT , :STAT, :RVMS end diff --git a/src/image/Image.cc b/src/image/Image.cc index b02a3f7d8a..a66af4935c 100644 --- a/src/image/Image.cc +++ b/src/image/Image.cc @@ -43,6 +43,7 @@ Image::Image(int _uid, type(OS), regtime(time(0)), source("-"), + size_mb(0), state(INIT), running_vms(0) { @@ -334,6 +335,7 @@ string& Image::to_xml(string& xml) const "" << persistent_img << "" << "" << regtime << "" << "" << source << "" << + "" << size_mb << "" << "" << state << "" << "" << running_vms << "" << obj_template->to_xml(template_xml) << @@ -374,6 +376,7 @@ int Image::from_xml(const string& xml) rc += xpath(regtime, "/IMAGE/REGTIME", 0); rc += xpath(source, "/IMAGE/SOURCE", "not_found"); + rc += xpath(size_mb, "/IMAGE/SIZE", 0); rc += xpath(int_state, "/IMAGE/STATE", 0); rc += xpath(running_vms, "/IMAGE/RUNNING_VMS", -1); diff --git a/src/image/ImageManagerDriver.cc b/src/image/ImageManagerDriver.cc index b1723b2e57..a7663799e1 100644 --- a/src/image/ImageManagerDriver.cc +++ b/src/image/ImageManagerDriver.cc @@ -91,6 +91,7 @@ void ImageManagerDriver::protocol( int id; Image * image; string source; + unsigned int size_mb; string info; @@ -130,6 +131,27 @@ void ImageManagerDriver::protocol( else return; + // Parse driver message for CP, MV and MKFS + // SUCESS IMAGE_ID SOURCE SIZE + if ( (result == "SUCCESS") && (action != "RM") ) + { + if ( is.good() ) + { + is >> source >> ws; + } + + if ( is.good() ) + { + is >> size_mb >> ws; + } + + if ( is.fail() ) + { + result = "FAILURE"; + } + } + + // Get the image from the pool image = ipool->get(id,true); @@ -143,19 +165,9 @@ void ImageManagerDriver::protocol( { if ( result == "SUCCESS" ) { - string source; - - if ( is.good() ) - { - is >> source >> ws; - } - - if ( is.fail() ) - { - goto error_cp; - } - image->set_source(source); + image->set_size(size_mb); + image->set_state(Image::READY); ipool->update(image); @@ -173,22 +185,13 @@ void ImageManagerDriver::protocol( { if (image->get_source() == "-") { - string source; - - if ( is.good() ) - { - is >> source >> ws; - } - - if ( is.fail() ) - { - goto error_mv; - } - image->set_source(source); } + image->set_size(size_mb); + image->set_state(Image::READY); + ipool->update(image); NebulaLog::log("ImM", Log::INFO, "Image saved and ready to use."); @@ -202,19 +205,9 @@ void ImageManagerDriver::protocol( { if ( result == "SUCCESS" ) { - string source; - - if ( is.good() ) - { - is >> source >> ws; - } - - if ( is.fail() ) - { - goto error_mkfs; - } - image->set_source(source); + image->set_size(size_mb); + image->set_state(Image::READY); ipool->update(image); diff --git a/src/image_mad/remotes/fs/cp b/src/image_mad/remotes/fs/cp index 24da5fbaff..86276ec30a 100755 --- a/src/image_mad/remotes/fs/cp +++ b/src/image_mad/remotes/fs/cp @@ -52,6 +52,9 @@ http://*) ;; esac +# ---------------- Get the size of the image & fix perms ------------ exec_and_log "chmod 0660 $DST" -echo "$DST" +SIZE=`fs_du $DST` + +echo "$DST $SIZE" diff --git a/src/image_mad/remotes/fs/fsrc b/src/image_mad/remotes/fs/fsrc index 57e87cdfe0..423f27e26b 100644 --- a/src/image_mad/remotes/fs/fsrc +++ b/src/image_mad/remotes/fs/fsrc @@ -39,3 +39,16 @@ EOF echo "$IMAGE_REPOSITORY_PATH/`echo $CANONICAL_MD5 | cut -d ' ' -f1`" } + +function fs_du { + +SIZE=`$DU -m $1 2>/dev/null` + +if [ $? -ne 0 ]; then + SIZE=0 +else + SIZE=`echo $SIZE | cut -f1 -d' '` +fi + +echo "$SIZE" +} diff --git a/src/image_mad/remotes/fs/mkfs b/src/image_mad/remotes/fs/mkfs index 4747d12084..d4e8c61157 100755 --- a/src/image_mad/remotes/fs/mkfs +++ b/src/image_mad/remotes/fs/mkfs @@ -65,4 +65,7 @@ exec_and_log "$MKFS -t $FSTYPE $OPTS $DST" \ exec_and_log "chmod 0660 $DST" -echo "$DST" +# ---------------- Get the size of the image ------------ +SIZE=`fs_du $DST` + +echo "$DST $SIZE" diff --git a/src/image_mad/remotes/fs/mv b/src/image_mad/remotes/fs/mv index 4b9a4fed4a..121bdf9e11 100755 --- a/src/image_mad/remotes/fs/mv +++ b/src/image_mad/remotes/fs/mv @@ -64,4 +64,7 @@ esac exec_and_log "chmod 0660 $DST" -echo "$DST" +# ---------------- Get the size of the image ------------ +SIZE=`fs_du $DST` + +echo "$DST $SIZE" diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index 6cfaec0477..221c997959 100755 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -20,6 +20,7 @@ BASH=/bin/bash CUT=cut DATE=/bin/date DD=/bin/dd +DU=/bin/du LVCREATE=/sbin/lvcreate LVREMOVE=/sbin/lvremove LVS=/sbin/lvs From d488cffddcfdfac5c05d7668047c2d75756ebe37 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 31 Aug 2011 16:29:43 +0200 Subject: [PATCH 05/24] feature #788: Optional base unit for unit_to_str method --- src/cli/one_helper.rb | 4 ++-- src/cli/one_helper/oneimage_helper.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/one_helper.rb b/src/cli/one_helper.rb index 0faa621144..d908eb100e 100644 --- a/src/cli/one_helper.rb +++ b/src/cli/one_helper.rb @@ -363,11 +363,11 @@ EOT BinarySufix = ["K", "M", "G", "T" ] - def OpenNebulaHelper.unit_to_str(value, options) + def OpenNebulaHelper.unit_to_str(value, options, unit="K") if options[:kilobytes] value else - i=0 + i=BinarySufix.index(unit).to_i while value > 1024 && i < 3 do value /= 1024.0 diff --git a/src/cli/one_helper/oneimage_helper.rb b/src/cli/one_helper/oneimage_helper.rb index 2af923b7f0..cf3a00059f 100644 --- a/src/cli/one_helper/oneimage_helper.rb +++ b/src/cli/one_helper/oneimage_helper.rb @@ -130,7 +130,7 @@ class OneImageHelper < OpenNebulaHelper::OneHelper column :SIZE, "Size of the image", :size=>7 do |d| - OpenNebulaHelper.unit_to_str(d['SIZE'].to_i,options) + OpenNebulaHelper.unit_to_str(d['SIZE'].to_i,options,"M") end default :ID, :USER, :GROUP, :NAME, :SIZE, :TYPE, :REGTIME, :PUBLIC, From d9792bb1f58eefe44d01b6ca3ade5249a5e234f3 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 31 Aug 2011 17:00:19 +0200 Subject: [PATCH 06/24] feature #720: Fix ruby 1.9 problem in watch_client --- src/acct/watch_client.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/acct/watch_client.rb b/src/acct/watch_client.rb index 6f65d07b48..1b1feb6a5c 100644 --- a/src/acct/watch_client.rb +++ b/src/acct/watch_client.rb @@ -98,14 +98,14 @@ module OneWatchClient max_per_vm = rsql. group(:id, :last_poll). - select(:last_poll, :MAX[mr.to_sym].as(:max_mr)) + select{[:last_poll, max(mr.to_sym).as(:max_mr)]} # SUM the monitoring resource for each last_poll value last_poll_and_sum = max_per_vm. from_self. group(:last_poll). - select(:last_poll, :SUM[:max_mr].as(:sum_mr)) + select{[:last_poll, sum(:max_mr).as(:sum_mr)]} # Retrieve the information in an Array a = Array.new @@ -113,7 +113,7 @@ module OneWatchClient if row[:last_poll] && row[:last_poll] != 0 a << [row[:last_poll], row[:sum_mr].to_i] end - end + end a end @@ -229,4 +229,4 @@ module OneWatchClient end end end -end \ No newline at end of file +end From a328fd796a4b58e1bd9517eddfcfdb3ed25103d1 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 17:44:03 +0200 Subject: [PATCH 07/24] feature #788: Refactor Quota class --- src/authm_mad/remotes/quota/authorize | 3 +- src/authm_mad/remotes/quota/one_usage.rb | 124 ----------------- src/authm_mad/remotes/quota/quota.rb | 162 +++++++++++++++++------ 3 files changed, 127 insertions(+), 162 deletions(-) delete mode 100644 src/authm_mad/remotes/quota/one_usage.rb diff --git a/src/authm_mad/remotes/quota/authorize b/src/authm_mad/remotes/quota/authorize index 19c0ca02f4..fe8644c176 100755 --- a/src/authm_mad/remotes/quota/authorize +++ b/src/authm_mad/remotes/quota/authorize @@ -29,6 +29,7 @@ end $: << RUBY_LIB_LOCATION require 'scripts_common' +require 'opennebula' require 'quota' user_id = ARGV.shift @@ -49,7 +50,7 @@ quota = Quota.new #OpenNebula.log_debug("quotas: #{quota.get(1)}") ARGV.each {|request| - rc = quota.check_request(user_id, request) + rc = quota.authorize(user_id, request) if rc OpenNebula.error_message rc diff --git a/src/authm_mad/remotes/quota/one_usage.rb b/src/authm_mad/remotes/quota/one_usage.rb deleted file mode 100644 index 0d83e684d2..0000000000 --- a/src/authm_mad/remotes/quota/one_usage.rb +++ /dev/null @@ -1,124 +0,0 @@ -# -------------------------------------------------------------------------- # -# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # -# # -# Licensed under the Apache License, Version 2.0 (the "License"); you may # -# not use this file except in compliance with the License. You may obtain # -# a copy of the License at # -# # -# http://www.apache.org/licenses/LICENSE-2.0 # -# # -# Unless required by applicable law or agreed to in writing, software # -# distributed under the License is distributed on an "AS IS" BASIS, # -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # -# See the License for the specific language governing permissions and # -# limitations under the License. # -#--------------------------------------------------------------------------- # - -require 'OpenNebula' - -# This class retrieves and caches vms and its consuption grouped -# by users. 'update_user' method should be called to fill data for -# a user before any calculation is made -class OneUsage - VM_USAGE = { - :cpu => { - :proc_info => lambda {|template| template['CPU']}, - :xpath => 'TEMPLATE/CPU' - }, - :memory => { - :proc_info => lambda {|template| template['MEMORY']}, - :xpath => 'TEMPLATE/MEMORY' - }, - :num_vms => { - :proc_info => lambda {|template| 1 }, - :xpath => 'ID', - :count => true - } - } - - IMAGE_USAGE = { - :storage => { - :proc_info => lambda {|template| File.size(template['PATH']) }, - :proc_total => 'TEMPLATE/SIZE' - } - } - - RESOURCES = ["VM", "IMAGE"] - - def initialize() - @client = OpenNebula::Client.new - @usage = Hash.new - end - - def total(user_id, resource=nil, force=false) - usage = Hash.new - - if force - resources = [resource] if RESOURCES.include?(resource) - - resources.each{ |res| - pool = get_pool(res, user_id) - - base_xpath = "/#{res}_POOL/#{resource}" - OneUsage.const_get("#{res}_USAGE".to_sym).each { |key, params| - usage[key] ||= 0 - pool.each_xpath("#{base_xpath}/#{params[:xpath]}") { |elem| - usage[key] += params[:count] ? 1 : elem.to_i - } - } - - @usage[:user_id] ||= Hash.new - @usage[:user_id].merge!(usage) - } - else - usage = get_usage(user_id) - end - - usage - end - - # Retrieve the useful information of the template for the specified - # kind of resource - def get_resources(resource, xml_template) - template = OpenNebula::XMLElement.new - template.initialize_xml(xml_template, 'TEMPLATE') - - info = Hash.new - - self.class.const_get("#{resource}_USAGE").each { |key, params| - info[key] = params[:proc_info].call(template).to_i - } - - info - end - - private - - def get_usage(user_id) - usage = @usage[:user_id] - - unless usage - usage = Hash.new - - keys = VM_USAGE.keys + IMAGE_USAGE.keys - keys.each { |key| - usage[key] = 0 - } - - @usage[:user_id] = usage - end - - usage - end - - # Returns a an Array than contains the elements of the resource Pool - def get_pool(resource, user_id) - pool = case resource - when "VM" then OpenNebula::VirtualMachinePool.new(@client, user_id) - when "IMAGE" then OpenNebula::ImagePool.new(@client, user_id) - end - - rc = pool.info - return pool - end -end diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb index 0a0027b88d..93014848c1 100644 --- a/src/authm_mad/remotes/quota/quota.rb +++ b/src/authm_mad/remotes/quota/quota.rb @@ -14,22 +14,17 @@ # limitations under the License. # #--------------------------------------------------------------------------- # -require 'one_usage' require 'sequel' require 'base64' -# Quota functionality for auth driver. Stores in database limits for each -# user and using OneUsage is able to retrieve resource usage from -# OpenNebula daemon and check if it is below limits class Quota - attr_accessor :defaults ########################################################################### - #Constants with paths to relevant files and defaults + # Constants with paths to relevant files and defaults ########################################################################### if !ENV["ONE_LOCATION"] - VAR_LOCATION = "/var/lib/one" + VAR_LOCATION = "/var/lib/one" else - VAR_LOCATION = ENV["ONE_LOCATION"] + "/var" + VAR_LOCATION = ENV["ONE_LOCATION"] + "/var" end CONF = { @@ -41,9 +36,10 @@ class Quota :storage => nil } } - - TABLE_NAME = :quotas + ########################################################################### + # Schema for the USAGE and QUOTA tables + ########################################################################### DB_QUOTA_SCHEMA = { :cpu => Float, :memory => Integer, @@ -51,30 +47,55 @@ class Quota :storage => Integer } + QUOTA_TABLE = :quotas + USAGE_TABLE = :usage - # 'db' is a Sequel database where to store user limits and client - # is OpenNebula::Client used to connect to OpenNebula daemon - def initialize(conf={}) - # TBD merge with the conf file - @conf=CONF + ########################################################################### + # Usage params to calculate each quota + ########################################################################### + VM_USAGE = { + :cpu => { + :proc_info => lambda {|template| template['CPU']}, + :xpath => 'TEMPLATE/CPU' + }, + :memory => { + :proc_info => lambda {|template| template['MEMORY']}, + :xpath => 'TEMPLATE/MEMORY' + }, + :num_vms => { + :proc_info => lambda {|template| 1 }, + :xpath => 'ID', + :count => true + } + } - @defaults=@conf[:defaults] + IMAGE_USAGE = { + :storage => { + :proc_info => lambda {|template| File.size(template['PATH']) }, + :xpath => 'SIZE' + } + } - @db=Sequel.connect(@conf[:db]) - - create_table - @table=@db[TABLE_NAME] - - @one_usage=OneUsage.new - end + RESOURCES = ["VM", "IMAGE"] ########################################################################### # DB handling ########################################################################### + def initialize(conf={}) + # TBD merge with the conf file + @conf=CONF + + @client = OpenNebula::Client.new + + @db=Sequel.connect(@conf[:db]) + + create_table(QUOTA_TABLE) + create_table(USAGE_TABLE) + end # Creates database quota table if it does not exist - def create_table - @db.create_table?(TABLE_NAME) do + def create_table(table) + @db.create_table?(table) do Integer :uid DB_QUOTA_SCHEMA.each { |key,value| @@ -87,38 +108,47 @@ class Quota end # Adds new user limits - def set(uid, quota={}) + def set(table, uid, quota={}) data=quota.delete_if{|key,value| !DB_QUOTA_SCHEMA.keys.include?(key)} - quotas=@table.filter(:uid => uid) + quotas=@db[table].filter(:uid => uid) if quotas.first quotas.update(data) else - @table.insert(data.merge!(:uid => uid)) + @db[table].insert(data.merge!(:uid => uid)) end end # Gets user limits - def get(uid=nil) + def get(table, uid=nil) if uid - limit=@table.filter(:uid => uid).first + limit=@db[table].filter(:uid => uid).first if limit limit else @conf[:defaults] end else - @table.all + @db[table].all end end + ########################################################################### + # Quota Client + ########################################################################### + def set_quota(uid, quota={}) + set(QUOTA_TABLE, uid, quota) + end + + def get_quota(uid=nil) + get(QUOTA_TABLE, uid) + end ########################################################################### # Authorization ########################################################################### - - def check_request(user_id, request) + def authorize(user_id, request) obj, template_or_id, op, owner, pub, acl_eval = request.split(':') if acl_eval == 0 @@ -137,9 +167,9 @@ class Quota end def check_quotas(user_id, obj, template) - info = @one_usage.get_resources(obj, template) - total = @one_usage.total(obj, user_id) - quota = get(user_id) + info = get_resources(obj, template) + total = get_usage(obj, user_id) + quota = get_quota(user_id) msg = "" info.each { |quota_name, quota_requested| @@ -164,5 +194,63 @@ class Quota (obj == "IMAGE" && op == "CREATE") || (obj == "TEMPLATE" && op == "INSTANTIATE") end -end + ########################################################################### + # Usage + ########################################################################### + def get_usage(user_id, resource=nil, force=false) + usage = Hash.new + + if force + if RESOURCES.include?(resource) + resources = [resource] + else + resources = RESOURCES + end + + resources.each{ |res| + pool = get_pool(res, user_id) + + base_xpath = "/#{res}_POOL/#{resource}" + Quota.const_get("#{res}_USAGE".to_sym).each { |key, params| + usage[key] ||= 0 + pool.each_xpath("#{base_xpath}/#{params[:xpath]}") { |elem| + usage[key] += params[:count] ? 1 : elem.to_i + } + } + + set(USAGE_TABLE, user_id, usage) + } + else + usage = get(USAGE_TABLE, user_id) + end + + usage + end + + # Retrieve the useful information of the template for the specified + # kind of resource + def get_resources(resource, xml_template) + template = OpenNebula::XMLElement.new + template.initialize_xml(xml_template, 'TEMPLATE') + + info = Hash.new + + self.class.const_get("#{resource}_USAGE").each { |key, params| + info[key] = params[:proc_info].call(template).to_i + } + + info + end + + # Returns a an Array than contains the elements of the resource Pool + def get_pool(resource, user_id) + pool = case resource + when "VM" then OpenNebula::VirtualMachinePool.new(@client, user_id) + when "IMAGE" then OpenNebula::ImagePool.new(@client, user_id) + end + + rc = pool.info + return pool + end +end From 547e5573fb7c5ca9d1187858983c0929d321eb6a Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 17:44:40 +0200 Subject: [PATCH 08/24] feature #788: Add onequota command --- src/authm_mad/remotes/quota/onequota | 158 +++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 src/authm_mad/remotes/quota/onequota diff --git a/src/authm_mad/remotes/quota/onequota b/src/authm_mad/remotes/quota/onequota new file mode 100755 index 0000000000..dd0705cd60 --- /dev/null +++ b/src/authm_mad/remotes/quota/onequota @@ -0,0 +1,158 @@ +#!/usr/bin/env ruby + +# -------------------------------------------------------------------------- # +# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +ONE_LOCATION=ENV["ONE_LOCATION"] + +if !ONE_LOCATION + RUBY_LIB_LOCATION="/usr/lib/one/ruby" +else + RUBY_LIB_LOCATION=ONE_LOCATION+"/lib/ruby" +end + +$: << RUBY_LIB_LOCATION +$: << RUBY_LIB_LOCATION+"/cli" + +require 'command_parser' +require 'cli_helper' +require 'one_helper' +require 'quota' + +require 'pp' + +QUOTA_KEYS = Quota::DB_QUOTA_SCHEMA.keys + +def show_table(values, usage=nil) + size = usage ? 12 : 8 + + values.sort!{|v1, v2| v1[:uid]<=>v2[:uid] } + + table=CLIHelper::ShowTable.new(nil, self) do + column :uid, "ONE identifier for the User", :size=>4 do |d| + d[:uid] + end + + QUOTA_KEYS.each { |key| + column key, "#{key} quota", :size=>size do |d| + if usage + "#{usage[key].to_i}/#{d[key].to_i}" + else + "#{d[key].to_i}" + end + end + } + + default :uid, *QUOTA_KEYS + end + + table.show(values) +end + +cmd=CommandParser::CmdParser.new(ARGV) do + usage "`onequota` [] []" + version OpenNebulaHelper::ONE_VERSION + + quota = Quota.new + + ######################################################################## + # Global Options + ######################################################################## + set :option, CommandParser::OPTIONS + + ######################################################################## + # Argument Formats + ######################################################################## + quota_list_desc = <<-EOT.unindent + List of quota keys, comma separated. + Available quotas: #{QUOTA_KEYS.join(', ')} + EOT + + set :format, :quota_list, quota_list_desc do |arg| + arg_list = arg.split(',') + + rc = nil + arg_list.each do |elem| + if !QUOTA_KEYS.include?(elem.to_sym) + rc = -1, "#{elem} is not a valid quota" + end + end + + rc ? rc : [0, arg_list] + end + + set :format, :value_list, "List of quota values, comma separated." do |arg| + arg_list = arg.split(',') + arg_list.map! {|a| a.to_i } + [0, arg_list] + end + + set :format, :userid, OpenNebulaHelper.rname_to_id_desc("USER") do |arg| + OpenNebulaHelper.rname_to_id(arg, "USER") + end + + ######################################################################## + # Commands + ######################################################################## + set_desc = <<-EOT.unindent + Set a quota for a given user. + Examples: + onequota set 3 cpu 12 + onequota set 4 cpu,memory,storage 8,4096,10000 + EOT + + command :set, set_desc, :userid, :quota_list, :value_list do + user_id, keys, values = args + + if keys.length != values.length + exit_with_code -1, "The number of keys and values does not match" + end + + values_hash = Hash.new + keys.each_with_index { |k,i| + values_hash[k.to_sym] = values[i] + } + + quota.set_quota(user_id, values_hash) + exit_with_code 0 + end + + ######################################################################## + show_desc = "Show the user's quota and usage. (usage/quota)" + + FORCE={ + :name => "force", + :short => "-f", + :large => "--force", + :description => "Force the usage calculation instead of using the cache" + } + + command :show, show_desc, :userid, :options=>[FORCE] do + user_usage = quota.get_usage(args[0],nil,options[:force]) + user_quota = quota.get_quota(args[0]) + + show_table([user_quota], user_usage) + exit_with_code 0 + end + + ######################################################################## + list_desc = "List the defined quotas for all the users" + + command :list, list_desc do + show_table(quota.get_quota) + exit_with_code 0 + end +end \ No newline at end of file From f308f38fc5fe998d137421f0de29096c2f5d9ffd Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 18:09:24 +0200 Subject: [PATCH 09/24] feature #788: Add delete quota functionality --- src/authm_mad/remotes/quota/onequota | 8 ++++++++ src/authm_mad/remotes/quota/quota.rb | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/authm_mad/remotes/quota/onequota b/src/authm_mad/remotes/quota/onequota index dd0705cd60..a6fd24e669 100755 --- a/src/authm_mad/remotes/quota/onequota +++ b/src/authm_mad/remotes/quota/onequota @@ -130,6 +130,14 @@ cmd=CommandParser::CmdParser.new(ARGV) do exit_with_code 0 end + ######################################################################## + delete_desc = "Delete the defined quotas for the given user" + + command :delete, delete_desc, :userid do + quota.delete_quota(args[0]) + exit_with_code 0 + end + ######################################################################## show_desc = "Show the user's quota and usage. (usage/quota)" diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb index 93014848c1..9e259e271d 100644 --- a/src/authm_mad/remotes/quota/quota.rb +++ b/src/authm_mad/remotes/quota/quota.rb @@ -127,13 +127,22 @@ class Quota if limit limit else - @conf[:defaults] + @conf[:defaults].merge!(:uid => uid) end else @db[table].all end end + # Delete user limits + def delete(table, uid) + quotas=@db[table].filter(:uid => uid) + + if quotas.first + quotas.delete + end + end + ########################################################################### # Quota Client ########################################################################### @@ -145,6 +154,10 @@ class Quota get(QUOTA_TABLE, uid) end + def delete_quota(uid) + delete(QUOTA_TABLE, uid) + end + ########################################################################### # Authorization ########################################################################### From d4d4184cd774a8cc0a4434289ea5d89889bd7660 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Wed, 31 Aug 2011 18:36:33 +0200 Subject: [PATCH 10/24] feature #788: Add quota conf file --- src/authm_mad/remotes/quota/quota.conf | 27 +++++++++++++++++++++ src/authm_mad/remotes/quota/quota.rb | 33 +++++++++++++++----------- 2 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 src/authm_mad/remotes/quota/quota.conf diff --git a/src/authm_mad/remotes/quota/quota.conf b/src/authm_mad/remotes/quota/quota.conf new file mode 100644 index 0000000000..04cbc0f8a3 --- /dev/null +++ b/src/authm_mad/remotes/quota/quota.conf @@ -0,0 +1,27 @@ +# -------------------------------------------------------------------------- # +# Copyright 2002-2011, OpenNebula Project Leads (OpenNebula.org) # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#--------------------------------------------------------------------------- # + +# Database URI +#:db: sqlite:///var/one/onequota.db + +#------------------------------------------------------------------------------- +# Default quotas +#------------------------------------------------------------------------------- +:defaults: + :cpu: nil, + :memory: nil, + :num_vms: nil, + :storage: nil diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb index 9e259e271d..64a8d3ae8e 100644 --- a/src/authm_mad/remotes/quota/quota.rb +++ b/src/authm_mad/remotes/quota/quota.rb @@ -16,17 +16,24 @@ require 'sequel' require 'base64' +require 'yaml' class Quota ########################################################################### # Constants with paths to relevant files and defaults ########################################################################### - if !ENV["ONE_LOCATION"] + ONE_LOCATION=ENV["ONE_LOCATION"] + + if !ONE_LOCATION VAR_LOCATION = "/var/lib/one" + ETC_LOCATION = "/etc/one" else - VAR_LOCATION = ENV["ONE_LOCATION"] + "/var" + VAR_LOCATION = ONE_LOCATION + "/var" + ETC_LOCATION = ONE_LOCATION + "/etc" end + CONF_FILE = ETC_LOCATION + "/quota.conf" + CONF = { :db => "sqlite://#{VAR_LOCATION}/onequota.db", :defaults => { @@ -81,9 +88,11 @@ class Quota ########################################################################### # DB handling ########################################################################### - def initialize(conf={}) - # TBD merge with the conf file - @conf=CONF + def initialize + conf = YAML.load_file(CONF_FILE) + @conf=CONF.merge(conf) {|key,h1,h2| + h1.merge(h2) if h1.instance_of?(Hash) && h2.instance_of?(Hash) + } @client = OpenNebula::Client.new @@ -123,12 +132,7 @@ class Quota # Gets user limits def get(table, uid=nil) if uid - limit=@db[table].filter(:uid => uid).first - if limit - limit - else - @conf[:defaults].merge!(:uid => uid) - end + @db[table].filter(:uid => uid).first else @db[table].all end @@ -151,7 +155,8 @@ class Quota end def get_quota(uid=nil) - get(QUOTA_TABLE, uid) + limit = get(QUOTA_TABLE, uid) + limit ? limit : @conf[:defaults].merge!(:uid => uid) end def delete_quota(uid) @@ -181,7 +186,7 @@ class Quota def check_quotas(user_id, obj, template) info = get_resources(obj, template) - total = get_usage(obj, user_id) + total = get_usage(user_id, obj, true) quota = get_quota(user_id) msg = "" @@ -238,7 +243,7 @@ class Quota usage = get(USAGE_TABLE, user_id) end - usage + usage ? usage : Hash.new end # Retrieve the useful information of the template for the specified From 08fad75a940354b01288a90759887238757023bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Thu, 1 Sep 2011 18:57:53 +0200 Subject: [PATCH 11/24] Feature #788: Change oneuser --time description from hours to seconds --- src/cli/oneuser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/oneuser b/src/cli/oneuser index a740411d0c..43ffc13268 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -104,7 +104,7 @@ cmd=CommandParser::CmdParser.new(ARGV) do :name => "time", :large => "--time x", :format => Integer, - :description => "Token duration in hours, (default 1)" + :description => "Token duration in seconds, defaults to 3600 (1 h)" } create_options = [READ_FILE, PLAIN, SSH, X509, KEY, CERT] From d145cd8303419ebc8a03c873f147f12a15a2279c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Fri, 2 Sep 2011 06:40:35 -0700 Subject: [PATCH 12/24] Feature #788: Add new 'oneuser key' command, similar to the 2.2 'oneauth key' --- src/cli/one_helper/oneuser_helper.rb | 16 +++++++++++----- src/cli/oneuser | 27 ++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 8 deletions(-) diff --git a/src/cli/one_helper/oneuser_helper.rb b/src/cli/one_helper/oneuser_helper.rb index b481f9291b..a9acaf3d46 100644 --- a/src/cli/one_helper/oneuser_helper.rb +++ b/src/cli/one_helper/oneuser_helper.rb @@ -37,7 +37,7 @@ class OneUserHelper < OpenNebulaHelper::OneHelper return -1, "Can not read file: #{arg}" end else - if options[:plain] + if options[:plain] || options[:ssh] password = arg.gsub(/\s/, '') else password = Digest::SHA1.hexdigest(arg) @@ -49,9 +49,11 @@ class OneUserHelper < OpenNebulaHelper::OneHelper def password(options) if options[:ssh] - require 'ssh_auth' + if !options[:key] + return -1, "You have to specify the --key option" + end - options[:key] ||= ENV['HOME']+'/.ssh/id_rsa' + require 'ssh_auth' begin sshauth = SshAuth.new(:private_key=>options[:key]) @@ -61,10 +63,14 @@ class OneUserHelper < OpenNebulaHelper::OneHelper return 0, sshauth.public_key elsif options[:x509] - require 'x509_auth' - options[:cert] ||= ENV['X509_USER_CERT'] + if !options[:cert] + return -1, "You have to specify the --cert option" + end + + require 'x509_auth' + begin cert = [File.read(options[:cert])] x509auth = X509Auth.new(:certs_pem=>cert) diff --git a/src/cli/oneuser b/src/cli/oneuser index 43ffc13268..531db2b15e 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -139,20 +139,21 @@ cmd=CommandParser::CmdParser.new(ARGV) do oneuser create my_user my_password oneuser create my_user /tmp/mypass -r oneuser create my_user --ssh --key /tmp/id_rsa + oneuser create my_user --ssh -r /tmp/public_key oneuser create my_user --x509 --cert /tmp/my_cert.pem EOT command :create, create_desc, :username, [:password, nil], :options=>create_options do - if options[:ssh] or options[:x509] + if args[1] + pass = args[1] + else rc = helper.password(options) if rc.first == 0 pass = rc[1] else exit_with_code *rc end - else - pass = args[1] end helper.create_resource(options) do |user| @@ -175,6 +176,26 @@ cmd=CommandParser::CmdParser.new(ARGV) do helper.login(args[0], options) end + key_desc = <<-EOT.unindent + Generates a public key from a private SSH key + EOT + + command :key, key_desc, :options=>[KEY] do + require 'ssh_auth' + + options[:key] ||= ENV['HOME']+'/.ssh/id_rsa' + + begin + sshauth = SshAuth.new(:private_key=>options[:key]) + rescue Exception => e + exit_with_code -1, e.message + end + + puts sshauth.public_key + exit_with_code 0 + end + + delete_desc = <<-EOT.unindent Deletes the given User EOT From 7213800115c2e07f8c20bd90078f0d4b0fe5b84e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Fri, 2 Sep 2011 16:07:45 +0200 Subject: [PATCH 13/24] Feature #788: Change 'oneuser passwd' to accept a public ssh key --- src/cli/oneuser | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cli/oneuser b/src/cli/oneuser index 531db2b15e..49559db58e 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -212,15 +212,15 @@ cmd=CommandParser::CmdParser.new(ARGV) do command :passwd, passwd_desc, :userid, :password, :options=>create_options do - if options[:ssh] or options[:x509] + if args[1] + pass = args[1] + else rc = helper.password(options) if rc.first == 0 pass = rc[1] else exit_with_code *rc end - else - pass = args[1] end helper.perform_action(args[0],options,"Password changed") do |user| From c8d45c9985cf60109db9c76e4df1693f74c12824 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Fri, 2 Sep 2011 16:14:14 +0200 Subject: [PATCH 14/24] feature #788: Changed description of oneuser key command. Do not hash x509 passwords --- src/cli/one_helper/oneuser_helper.rb | 2 +- src/cli/oneuser | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cli/one_helper/oneuser_helper.rb b/src/cli/one_helper/oneuser_helper.rb index a9acaf3d46..bca7a4c27c 100644 --- a/src/cli/one_helper/oneuser_helper.rb +++ b/src/cli/one_helper/oneuser_helper.rb @@ -37,7 +37,7 @@ class OneUserHelper < OpenNebulaHelper::OneHelper return -1, "Can not read file: #{arg}" end else - if options[:plain] || options[:ssh] + if options[:plain] || options[:ssh] || options[:x509] password = arg.gsub(/\s/, '') else password = Digest::SHA1.hexdigest(arg) diff --git a/src/cli/oneuser b/src/cli/oneuser index 531db2b15e..bab3cf787c 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -177,7 +177,7 @@ cmd=CommandParser::CmdParser.new(ARGV) do end key_desc = <<-EOT.unindent - Generates a public key from a private SSH key + Shows a public key from a private SSH key. Use it as password for the SSH authentication mechanism. EOT command :key, key_desc, :options=>[KEY] do From 6fd97be3029459a11e4fac505a978b6e307bc916 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Fri, 2 Sep 2011 16:17:26 +0200 Subject: [PATCH 15/24] feature #788: better formatting for oneuser command help --- src/cli/oneuser | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/cli/oneuser b/src/cli/oneuser index 778e340dce..0f35fd6c0d 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -177,7 +177,8 @@ cmd=CommandParser::CmdParser.new(ARGV) do end key_desc = <<-EOT.unindent - Shows a public key from a private SSH key. Use it as password for the SSH authentication mechanism. + Shows a public key from a private SSH key. Use it as password + for the SSH authentication mechanism. EOT command :key, key_desc, :options=>[KEY] do From 2709d144a8dd02b8d19794a06c034dfcc7a1e402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Fri, 2 Sep 2011 16:20:14 +0200 Subject: [PATCH 16/24] Feature #788: Make 'oneuser passwd' second parameter optional, so --key can be used --- src/cli/oneuser | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/oneuser b/src/cli/oneuser index 0f35fd6c0d..e46b8a757b 100755 --- a/src/cli/oneuser +++ b/src/cli/oneuser @@ -211,7 +211,7 @@ cmd=CommandParser::CmdParser.new(ARGV) do Changes the given User's password EOT - command :passwd, passwd_desc, :userid, :password, + command :passwd, passwd_desc, :userid, [:password, nil], :options=>create_options do if args[1] pass = args[1] From 860846cb948004fa5164024e44036b883d5fdc6d Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Fri, 2 Sep 2011 16:46:03 +0200 Subject: [PATCH 17/24] feature #788: Disable quotas by default --- src/authm_mad/remotes/quota/quota.conf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/authm_mad/remotes/quota/quota.conf b/src/authm_mad/remotes/quota/quota.conf index 04cbc0f8a3..5ada4f7f9b 100644 --- a/src/authm_mad/remotes/quota/quota.conf +++ b/src/authm_mad/remotes/quota/quota.conf @@ -21,7 +21,7 @@ # Default quotas #------------------------------------------------------------------------------- :defaults: - :cpu: nil, - :memory: nil, - :num_vms: nil, - :storage: nil + :cpu: + :memory: + :num_vms: + :storage: From e8ea488ad2f030f14fb9a6bafe7f533a438fafb6 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Fri, 2 Sep 2011 16:46:50 +0200 Subject: [PATCH 18/24] feature #788: Check the quota type --- src/authm_mad/remotes/quota/quota.rb | 43 +++++++++++++++++++--------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/authm_mad/remotes/quota/quota.rb b/src/authm_mad/remotes/quota/quota.rb index 64a8d3ae8e..19525804db 100644 --- a/src/authm_mad/remotes/quota/quota.rb +++ b/src/authm_mad/remotes/quota/quota.rb @@ -190,13 +190,21 @@ class Quota quota = get_quota(user_id) msg = "" - info.each { |quota_name, quota_requested| - spent = total[quota_name].to_i + quota_requested.to_i - if quota[quota_name] && spent > quota[quota_name].to_i - msg << " #{quota_name.to_s.upcase} quota exceeded " - msg << "(Quota: #{quota[quota_name].to_i}, " - msg << "Used: #{spent.to_i}, " - msg << "Asked: #{quota_requested.to_i})." + info.each { |qname, quota_requested| + unless quota[qname] + next + end + + used = send(DB_QUOTA_SCHEMA[qname].name.to_sym, total[qname]) + request = send(DB_QUOTA_SCHEMA[qname].name.to_sym, quota_requested) + limit = send(DB_QUOTA_SCHEMA[qname].name.to_sym, quota[qname]) + spent = used + request + + if spent > limit + msg << " #{qname.to_s.upcase} quota exceeded " + msg << "(Quota: #{limit}, " + msg << "Used: #{used}, " + msg << "Asked: #{request})." end } @@ -217,8 +225,6 @@ class Quota # Usage ########################################################################### def get_usage(user_id, resource=nil, force=false) - usage = Hash.new - if force if RESOURCES.include?(resource) resources = [resource] @@ -226,24 +232,33 @@ class Quota resources = RESOURCES end + usage = Hash.new + resources.each{ |res| pool = get_pool(res, user_id) - base_xpath = "/#{res}_POOL/#{resource}" Quota.const_get("#{res}_USAGE".to_sym).each { |key, params| - usage[key] ||= 0 pool.each_xpath("#{base_xpath}/#{params[:xpath]}") { |elem| - usage[key] += params[:count] ? 1 : elem.to_i + if elem + usage[key] ||= 0 + if params[:count] + usage[key] += 1 + else + usage[key] += send(DB_QUOTA_SCHEMA[key].name.to_sym, elem) + end + end } } - set(USAGE_TABLE, user_id, usage) + set(USAGE_TABLE, user_id, usage) unless usage.empty? + usage.merge!(:uid => user_id) } else usage = get(USAGE_TABLE, user_id) + usage ||= {:uid => user_id} end - usage ? usage : Hash.new + usage end # Retrieve the useful information of the template for the specified From a3fa3818522cdf8ed09f27d5fde80155ff46c8dc Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Fri, 2 Sep 2011 16:47:23 +0200 Subject: [PATCH 19/24] Use the same format for the CLI options help --- src/cli/command_parser.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli/command_parser.rb b/src/cli/command_parser.rb index 3878d5630e..3fb18ae7bd 100755 --- a/src/cli/command_parser.rb +++ b/src/cli/command_parser.rb @@ -223,7 +223,7 @@ EOT @opts.each{ |o| str = "" - str << o[:short] if o[:short] + str << o[:short].split(' ').first << ', ' if o[:short] str << o[:large] printf opt_format, str, o[:description] From 6b2c903ea28ff964d14d0ffc870b0e18bbec4438 Mon Sep 17 00:00:00 2001 From: Daniel Molina Date: Fri, 2 Sep 2011 16:48:50 +0200 Subject: [PATCH 20/24] feature #788: Add quota tests --- .../remotes/quota/test/fixtures/imagepool.xml | 26 ++ .../remotes/quota/test/fixtures/vm.xml | 46 +++ .../remotes/quota/test/fixtures/vmpool.xml | 49 +++ .../remotes/quota/test/helper/mock_client.rb | 53 +++ .../remotes/quota/test/helper/test_helper.rb | 31 ++ .../remotes/quota/test/quota_spec.rb | 341 ++++++++++++++++++ 6 files changed, 546 insertions(+) create mode 100644 src/authm_mad/remotes/quota/test/fixtures/imagepool.xml create mode 100644 src/authm_mad/remotes/quota/test/fixtures/vm.xml create mode 100644 src/authm_mad/remotes/quota/test/fixtures/vmpool.xml create mode 100644 src/authm_mad/remotes/quota/test/helper/mock_client.rb create mode 100644 src/authm_mad/remotes/quota/test/helper/test_helper.rb create mode 100644 src/authm_mad/remotes/quota/test/quota_spec.rb diff --git a/src/authm_mad/remotes/quota/test/fixtures/imagepool.xml b/src/authm_mad/remotes/quota/test/fixtures/imagepool.xml new file mode 100644 index 0000000000..f71c914557 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/fixtures/imagepool.xml @@ -0,0 +1,26 @@ + + <% images.each do |id,image| %> + + <%= id %> + <%= image[:uid] ? image[:uid] : 0 %> + <%= image[:gid] ? image[:gid] : 0 %> + <%= image[:uname] ? image[:uname] : 'oneadmin' %> + <%= image[:gname] ? image[:gname] : 'oneadmin' %> + <%= image[:name] ? image[:name] : 'ttylinux' %> + <%= image[:type] ? image[:type] : 0 %> + <%= image[:pub] ? image[:pub] : 0 %> + <%= image[:persistent] ? image[:persistent] : 0 %> + 1314875019 + <%= image[:source] ? image[:source] : '/etc/hosts' %> + <%= image[:state] ? image[:state] : 1 %> + <%= image[:size] ? image[:size] : 100 %> + 0 + + + <% end %> + \ No newline at end of file diff --git a/src/authm_mad/remotes/quota/test/fixtures/vm.xml b/src/authm_mad/remotes/quota/test/fixtures/vm.xml new file mode 100644 index 0000000000..1560a16029 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/fixtures/vm.xml @@ -0,0 +1,46 @@ + + <%= id %> + <%= vm[:uid] ? vm[:uid] : 0 %> + <%= vm[:gid] ? vm[:gid] : 0 %> + <%= vm[:name] ? vm[:uid] : 'pepe' %> + <%= vm[:last_poll] ? vm[:last_poll] : '1309275256' %> + <%= vm[:state] ? vm[:state] : 3 %> + 3 + 1309275252 + 0 + dummy + <%= vm[:memory] ? vm[:memory] : 128 %> + <%= vm[:cpu] ? vm[:cpu] : 1 %> + <%= vm[:net_tx] ? vm[:net_tx] : 0 %> + <%= vm[:net_rx] ? vm[:net_rx] : 0 %> + + <% if history = vm[:history] %> + + <% history.each do |h| %> + + <%= h[:seq] ? h[:seq] : 0 %> + <%= h[:hostname] ? h[:hostname] : "kvxen" %> + /Users/dmolina/trabajo/acctmoni/install/var/ + <%= h[:hid] ? h[:hid] : 0 %> + 1309275256 + 0 + vmm_dummy + tm_dummy + <%= h[:pstime] ? h[:pstime] : 0 %> + <%= h[:petime] ? h[:petime] : 0 %> + <%= h[:rstime] ? h[:rstime] : 0 %> + <%= h[:retime] ? h[:retime] : 0 %> + <%= h[:estime] ? h[:estime] : 0 %> + <%= h[:eetime] ? h[:eetime] : 0 %> + <%= h[:reason] ? h[:reason] : 0 %> + + <% end %> + + <% end %> + \ No newline at end of file diff --git a/src/authm_mad/remotes/quota/test/fixtures/vmpool.xml b/src/authm_mad/remotes/quota/test/fixtures/vmpool.xml new file mode 100644 index 0000000000..1ab1fa3112 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/fixtures/vmpool.xml @@ -0,0 +1,49 @@ + + <% vms.each do |id,vm| %> + + <%= id %> + <%= vm[:uid] ? vm[:uid] : 0 %> + <%= vm[:gid] ? vm[:gid] : 0 %> + <%= vm[:name] ? vm[:uid] : 'pepe' %> + <%= vm[:last_poll] ? vm[:last_poll] : '1309275256' %> + <%= vm[:state] ? vm[:state] : 3 %> + 3 + 1309275252 + 0 + dummy + <%= vm[:memory] ? vm[:memory] : 128 %> + <%= vm[:cpu] ? vm[:cpu] : 1 %> + <%= vm[:net_tx] ? vm[:net_tx] : 0 %> + <%= vm[:net_rx] ? vm[:net_rx] : 0 %> + + <% if history = vm[:history] %> + + <% h = history.last %> + + <%= h[:seq] ? h[:seq] : 0 %> + <%= h[:hostname] ? h[:hostname] : "kvxen" %> + /Users/dmolina/trabajo/acctmoni/install/var/ + <%= h[:hid] ? h[:hid] : 0 %> + 1309275256 + 0 + vmm_dummy + tm_dummy + <%= h[:pstime] ? h[:pstime] : 0 %> + <%= h[:petime] ? h[:petime] : 0 %> + <%= h[:rstime] ? h[:rstime] : 0 %> + <%= h[:retime] ? h[:retime] : 0 %> + <%= h[:estime] ? h[:estime] : 0 %> + <%= h[:eetime] ? h[:eetime] : 0 %> + <%= h[:reason] ? h[:reason] : 0 %> + + + <% end %> + + <% end %> + diff --git a/src/authm_mad/remotes/quota/test/helper/mock_client.rb b/src/authm_mad/remotes/quota/test/helper/mock_client.rb new file mode 100644 index 0000000000..e92264dea2 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/helper/mock_client.rb @@ -0,0 +1,53 @@ +require 'erb' + +FPATH = "./fixtures/" + +class MockClient + def initialize + @vms = Hash.new + @done_vms = Hash.new + + @images = Hash.new + end + + + def call(action, *args) + xmlrpc_action = "one."+action + + case xmlrpc_action + when "one.vm.info" + id = args[0] + vm = @vms[id] + return ERB.new(File.read(FPATH+'vm.xml')).result(binding) + when "one.vmpool.info" + case args[3] + when -1 + vms = @vms + return ERB.new(File.read(FPATH+'vmpool.xml')).result(binding) + when 6 then + vms = @done_vms + return ERB.new(File.read(FPATH+'vmpool.xml')).result(binding) + end + when "one.imagepool.info" + images = @images + return ERB.new(File.read(FPATH+'imagepool.xml')).result(binding) + end + end + + def add_vm(id, values) + if values[:state] == 6 + @done_vms[id] = values.clone + else + @vms[id] = values.clone + end + end + + def delete_vm(id) + @vms.delete(id) + @vms_done.delete(id) + end + + def add_image(id, values) + @images[id] = values + end +end \ No newline at end of file diff --git a/src/authm_mad/remotes/quota/test/helper/test_helper.rb b/src/authm_mad/remotes/quota/test/helper/test_helper.rb new file mode 100644 index 0000000000..0735253de4 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/helper/test_helper.rb @@ -0,0 +1,31 @@ + +ONE_LOCATION = ENV['ONE_LOCATION'] +if !ONE_LOCATION + RUBY_LIB_LOCATION="/usr/lib/one/ruby" +else + RUBY_LIB_LOCATION=ONE_LOCATION+"/lib/ruby" +end + +$: << RUBY_LIB_LOCATION + +$: << './helper' +$: << '.' +$: << '..' + +require 'mock_client' +require 'opennebula' +require 'quota' + +class Quota + def set_client(client) + @client = client + end + + def rm_and_set_testdb + `rm -f /tmp/onequota_test.db` + @db=Sequel.connect("sqlite:///tmp/onequota_test.db") + + create_table(QUOTA_TABLE) + create_table(USAGE_TABLE) + end +end diff --git a/src/authm_mad/remotes/quota/test/quota_spec.rb b/src/authm_mad/remotes/quota/test/quota_spec.rb new file mode 100644 index 0000000000..893ac26d34 --- /dev/null +++ b/src/authm_mad/remotes/quota/test/quota_spec.rb @@ -0,0 +1,341 @@ +require 'helper/test_helper' + +describe "Quota testing" do + before(:all) do + @mock_client = MockClient.new + + @quota = Quota.new + @quota.set_client(@mock_client) + @quota.rm_and_set_testdb + + @uid1 = 0 + @quota1 = { + :cpu => 2.4, + :memory => 1024, + :num_vms => 4, + :storage => 10000 + } + + @uid2 = 1 + @quota2 = { + :cpu => 1.2, + :memory => 512, + :num_vms => 2, + :storage => 5000 + } + + # Generate VM ACL request + vm_template = <<-EOT +