feature #2289: monitor all VMs in EC2 IM

Get information from all EC2 instances in IM driver and provide
this information to OpenNebula so a VM poll is not executed per VM.
The monitoring will be much faster as VM is very expensive and the time
it takes to get info about one VM is very similar to getting information
about all VMs.

To keep track of the OpenNebula ID in EC2 machines a new tag is added
to EC2 instances (ONE_ID). This is used when monitoring VMs. VMs without
this tag wont be correctly reported to OpenNebula and will be monitored
with the old method (VM poll).

This version is kind of a PoC. The driver should be refactored so it
is similar to other drivers, that is, one script per action. The library
that holds the functionality will be held in a library file in VMM
driver as VMware drivers. Special care should be taken read
EC2 parameters as now is needed by both VMM and IM.

src/im_mad/ec2/one_im_ec2.rb

@@ -20,8 +20,10 @@ ONE_LOCATION=ENV["ONE_LOCATION"]
 
 if !ONE_LOCATION
     RUBY_LIB_LOCATION="/usr/lib/one/ruby"
+    MAD_LOCATION="/usr/lib/one/mads"
 else
     RUBY_LIB_LOCATION=ONE_LOCATION+"/lib/ruby"
+    MAD_LOCATION=ONE_LOCATION+"/lib/mads"
 end
 
 $: << RUBY_LIB_LOCATION
@@ -66,9 +68,20 @@ class EC2InformationManagerDriver < OpenNebulaDriver
     # The monitor action, just print the capacity info and hostname
     def action_monitor(num, host, not_used)
         info   = "HOSTNAME=\"#{host}\"\n#{@info}"
+        info << get_vm_info
         info64 = Base64::encode64(info).strip.delete("\n")
         send_message("MONITOR", RESULT[:success], num, info64)
     end
+
+    def get_vm_info
+        exe=LocalCommand.run(MAD_LOCATION+'/one_vmm_ec2 --poll')
+
+        if exe.code==0
+            exe.stdout
+        else
+            ''
+        end
+    end
 end
 
 # The EC2 Information Driver main program

src/vmm_mad/ec2/one_vmm_ec2.rb

@@ -212,6 +212,10 @@ class EC2Driver < VirtualMachineDriver
             exec_and_log_ec2(:authorize, ec2_info, 'default', id)
         end
 
+        LocalCommand.run(
+            "#{EC2_LOCATION}/bin/ec2-create-tags #{deploy_id} -t ONE_ID=#{id}",
+            log_method(id))
+
         if ec2_value(ec2_info, 'TAGS')
             exec_and_log_ec2(:tags, ec2_info, deploy_id, id)
         end
@@ -254,19 +258,24 @@ class EC2Driver < VirtualMachineDriver
 
         deploy_id = msg.elements["DEPLOY_ID"].text
 
-        info =  "#{POLL_ATTRIBUTE[:usedmemory]}=0 " \
-                "#{POLL_ATTRIBUTE[:usedcpu]}=0 " \
-                "#{POLL_ATTRIBUTE[:nettx]}=0 " \
-                "#{POLL_ATTRIBUTE[:netrx]}=0"
-
-
         exe = exec_and_log_ec2(:describe, nil, deploy_id, id)
         if exe.code != 0
             send_message(ACTION[:poll], RESULT[:failure], id, exe.stderr)
             return
         end
 
-        exe.stdout.match(Regexp.new("INSTANCE\\s+#{deploy_id}\\s+(.+)"))
+        info = EC2Driver.parse_poll(exe.stdout, deploy_id)
+
+        send_message(ACTION[:poll], RESULT[:success], id, info)
+    end
+
+    def self.parse_poll(text, deploy_id)
+        text.match(Regexp.new("INSTANCE\\s+#{deploy_id}\\s+(.+)"))
+
+        info =  "#{POLL_ATTRIBUTE[:usedmemory]}=0 " \
+                "#{POLL_ATTRIBUTE[:usedcpu]}=0 " \
+                "#{POLL_ATTRIBUTE[:nettx]}=0 " \
+                "#{POLL_ATTRIBUTE[:netrx]}=0"
 
         if !$1
             info << " #{POLL_ATTRIBUTE[:state]}=#{VM_STATE[:deleted]}"
@@ -284,7 +293,7 @@ class EC2Driver < VirtualMachineDriver
             end
         end
 
-        send_message(ACTION[:poll], RESULT[:success], id, info)
+        info
     end
 
 private
@@ -396,8 +405,43 @@ private
     end
 end
 
+def monitor_all_vms
+    exe = LocalCommand.run(
+            "#{EC2_LOCATION}/bin/ec2-describe-instances",
+            lambda { |str| STDERR.puts str })
+
+    if exe.code != 0
+        exit -1
+    end
+
+    puts "VM_POLL=YES"
+
+    exe.stdout.split(/^RESERVATION\s.*?$/).each do |vm|
+        m=vm.match(/^INSTANCE\s+(\S+)/)
+        next if !m
+
+        deploy_id = m[1]
+
+        one_id='-1'
+
+        vm.scan(/^TAG.*ONE_ID\s+(\d+)/) {|i| one_id = i.first }
+
+        poll_data=EC2Driver.parse_poll(vm, deploy_id)
+
+        puts "VM=["
+        puts "  ID=#{one_id},"
+        puts "  DEPLOY_ID=#{deploy_id},"
+        puts "  POLL=\"#{poll_data}\" ]"
+    end
+end
+
 # EC2Driver Main program
 
+if ARGV[0]=='--poll'
+    monitor_all_vms
+    exit(0)
+end
+
 ec2_conf = ARGV.last
 
 if ec2_conf