From 955c1025191402f77babd9e73d36f190730e6513 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Thu, 16 Jan 2014 17:23:14 +0100 Subject: [PATCH] feature #2371: add pagination to ruby oca, cli and sunstone There is a new pool method called get_hash that uses pagination when possible. The default page size can be changed with the environment variable ONE_POOL_PAGE_SIZE. Any value > 2 will set a page size, a non numeric value disables pagination. Paginated pools are parsed using sax instead of dom so this makes the parse a bit faster. It will also detect Ox gem (https://github.com/ohler55/ox) and will use it as it is much faster than Nokogiri. Screen cleaning in top command is done after retrieving the pool reducing the flickering. --- src/cli/cli_helper.rb | 12 ++-- src/cli/one_helper.rb | 17 +++-- src/oca/ruby/opennebula/client.rb | 44 +++++++++++- src/oca/ruby/opennebula/pool.rb | 53 ++++++++++++++ src/oca/ruby/opennebula/xml_pool.rb | 1 + src/oca/ruby/opennebula/xml_utils.rb | 99 +++++++++++++++++++++++++++ src/sunstone/models/SunstoneServer.rb | 5 +- 7 files changed, 215 insertions(+), 16 deletions(-) diff --git a/src/cli/cli_helper.rb b/src/cli/cli_helper.rb index 5ffbe579a7..5034b8b3a5 100644 --- a/src/cli/cli_helper.rb +++ b/src/cli/cli_helper.rb @@ -176,11 +176,11 @@ module CLIHelper begin while true + data = block.call + CLIHelper.scr_cls CLIHelper.scr_move(0,0) - data = block.call - show(data, options) sleep delay end @@ -216,8 +216,8 @@ module CLIHelper end begin - print res_data.collect{|l| - (0..ncolumns-1).collect{ |i| + res_data.each{|l| + puts (0..ncolumns-1).collect{ |i| dat=l[i] col=@default_columns[i] @@ -225,8 +225,8 @@ module CLIHelper str=CLIHelper.color_state(str) if i==stat_column str - }.join(' ') - }.join("\n").gsub(/ *$/, '') + }.join(' ').rstrip + } rescue Errno::EPIPE end diff --git a/src/cli/one_helper.rb b/src/cli/one_helper.rb index 2f925ae904..16cb97d67c 100644 --- a/src/cli/one_helper.rb +++ b/src/cli/one_helper.rb @@ -357,21 +357,26 @@ EOT pool = factory_pool(filter_flag) - rc = pool.info - return -1, rc.message if OpenNebula.is_error?(rc) - if options[:xml] + # TODO: use paginated functions + rc=pool.info + return -1, rc.message if OpenNebula.is_error?(rc) return 0, pool.to_xml(true) else table = format_pool(options) if top table.top(options) { - pool.info - pool_to_array(pool) + array=pool_to_array(pool.get_hash) + if OpenNebula.is_error?(array) + STDERR.puts array.mesage + exit(-1) + end + array } else - array=pool_to_array(pool) + array=pool_to_array(pool.get_hash) + return -1, array.message if OpenNebula.is_error?(array) if options[:ids] array=array.select do |element| diff --git a/src/oca/ruby/opennebula/client.rb b/src/oca/ruby/opennebula/client.rb index e89e831643..16a9ca8bbb 100644 --- a/src/oca/ruby/opennebula/client.rb +++ b/src/oca/ruby/opennebula/client.rb @@ -15,9 +15,34 @@ #--------------------------------------------------------------------------- # require 'xmlrpc/client' +require 'bigdecimal' +require 'stringio' + module OpenNebula - if OpenNebula::NOKOGIRI + attr_accessor :pool_page_size + + if OpenNebula::OX + class OxStreamParser < XMLRPC::XMLParser::AbstractStreamParser + def initialize + @parser_class = OxParser + end + + class OxParser < Ox::Sax + include XMLRPC::XMLParser::StreamParserMixin + + alias :text :character + alias :end_element :endElement + alias :start_element :startElement + + def parse(str) + Ox.sax_parse(self, StringIO.new(str), + :symbolize => false, + :convert_special => true) + end + end + end + elsif OpenNebula::NOKOGIRI class NokogiriStreamParser < XMLRPC::XMLParser::AbstractStreamParser def initialize @parser_class = NokogiriParser @@ -39,6 +64,19 @@ module OpenNebula end end + DEFAULT_POOL_PAGE_SIZE = 2000 + + if size=ENV['ONE_POOL_PAGE_SIZE'] + if size.strip.match(/^\d+$/) && size.to_i >= 2 + @pool_page_size = size.to_i + else + @pool_page_size = nil + end + else + @pool_page_size = DEFAULT_POOL_PAGE_SIZE + end + + # The client class, represents the connection with the core and handles the # xml-rpc calls. class Client @@ -91,7 +129,9 @@ module OpenNebula @server = XMLRPC::Client.new2(@one_endpoint, nil, timeout) - if OpenNebula::NOKOGIRI + if defined?(OxStreamParser) + @server.set_parser(OxStreamParser.new) + elsif OpenNebula::NOKOGIRI @server.set_parser(NokogiriStreamParser.new) elsif XMLPARSER @server.set_parser(XMLRPC::XMLParser::XMLStreamParser.new) diff --git a/src/oca/ruby/opennebula/pool.rb b/src/oca/ruby/opennebula/pool.rb index 95f4c9b7c6..340dbee00c 100644 --- a/src/oca/ruby/opennebula/pool.rb +++ b/src/oca/ruby/opennebula/pool.rb @@ -24,6 +24,9 @@ module OpenNebula include Enumerable alias_method :each_with_xpath, :each + PAGINATED_POOLS=%w{VM_POOL IMAGE_POOL TEMPLATE_POOL VN_POOL + DOCUMENT_POOL} + protected #pool:: _String_ XML name of the root element #element:: _String_ XML name of the Pool elements @@ -156,5 +159,55 @@ module OpenNebula return str end + + # Gets a hash from a pool + # + # size:: nil => default page size + # < 2 => not paginated + # >=2 => page size + # + # The default page size can be changed with the environment variable + # ONE_POOL_PAGE_SIZE. Any value > 2 will set a page size, a non + # numeric value disables pagination. + def get_hash(size=nil) + allow_paginated = PAGINATED_POOLS.include?(@pool_name) + + if OpenNebula.pool_page_size && allow_paginated && + ( ( size && size >= 2 ) || !size ) + size = OpenNebula.pool_page_size if !size + { @pool_name => { @element_name => info_paginated(size) } } + else + rc=info + return rc if OpenNebula.is_error?(rc) + to_hash + end + end + + # Gets a pool in hash form using pagination + # + # size:: _Integer_ size of each page + def info_paginated(size) + array=Array.new + current=0 + + parser=ParsePoolSax.new(@pool_name, @element_name) + + while true + a=@client.call("#{@pool_name.delete('_').downcase}.info", + -2, current, -size, -1) + return a if OpenNebula.is_error?(a) + + a_array=parser.parse(a) + + array += a_array + current += size + break if !a || a_array.length false, + :convert_special => true) + end + + class PoolSax < Ox::Sax + include ParsePoolBase + + alias :text :characters + end + end + elsif NOKOGIRI + class ParsePoolSax < ParsePoolSaxBase + def initialize(pool_name, elem_name) + super(pool_name, elem_name) + @parser = Nokogiri::XML::SAX::Parser.new(@pool_sax) + end + + def sax_parse(str) + @parser.parse(str) + end + + class PoolSax < Nokogiri::XML::SAX::Document + include ParsePoolBase + + alias :cdata_block :characters + end + end + end end diff --git a/src/sunstone/models/SunstoneServer.rb b/src/sunstone/models/SunstoneServer.rb index 7efe602abe..54858faf89 100644 --- a/src/sunstone/models/SunstoneServer.rb +++ b/src/sunstone/models/SunstoneServer.rb @@ -65,12 +65,13 @@ class SunstoneServer < CloudServer return [404, error.to_json] end - rc = pool.info + rc = pool.get_hash if OpenNebula.is_error?(rc) return [500, rc.to_json] else - return [200, pool.to_json] + STDERR.puts rc.inspect + return [200, rc.to_json] end end