Merge branch 'master' into feature-2816

src/cli/cli_helper.rb

@@ -303,7 +303,7 @@ module CLIHelper
 
             begin
                 if options[:csv]
-                    puts CSV.generate_line(@default_columns)
+                    puts CSV.generate_line(@default_columns) if !options[:noheader]
                     res_data.each {|l| puts CSV.generate_line(l) }
                 else
                     res_data.each{|l|

src/cli/one_helper.rb

@@ -516,11 +516,13 @@ EOT
         #  List pool functions
         #-----------------------------------------------------------------------
         def start_pager
-            pager = ENV['ONE_PAGER'] || 'less'
+            pager = ENV['ONE_PAGER'] || 'more'
 
             # Start pager, defaults to less
             p_r, p_w = IO.pipe
 
+            Signal.trap('PIPE', 'SIG_IGN')
+
             lpid = fork do
                 $stdin.reopen(p_r)
 
@@ -531,7 +533,7 @@ EOT
 
                 exec([pager, pager])
             end
-            
+
             # Send listing to pager pipe
             $stdout.close
             $stdout = p_w.dup
@@ -547,6 +549,9 @@ EOT
 
             begin
                 Process.wait(lpid)
+            rescue Interrupt
+                Process.kill("TERM", lpid)
+                Process.wait(lpid)
             rescue Errno::ECHILD
             end
         end
@@ -558,12 +563,11 @@ EOT
                 elements = 0
                 page     = ""
 
-                pool.each {|e| 
-                    elements += 1 
+                pool.each {|e|
+                    elements += 1
                     page << e.to_xml(true) << "\n"
                 }
             else
-            
                 pname = pool.pool_name
                 ename = pool.element_name
 
@@ -585,8 +589,8 @@ EOT
         # output
         #-----------------------------------------------------------------------
         def list_pool_table(table, pool, options, filter_flag)
-            if $stdout.isatty and (!options.key?:no_pager) 
-                size = $stdout.winsize[0] - 1 
+            if $stdout.isatty and (!options.key?:no_pager)
+                size = $stdout.winsize[0] - 1
 
                 # ----------- First page, check if pager is needed -------------
                 rc = pool.get_page(size, 0)
@@ -662,8 +666,8 @@ EOT
         # List pool in XML format, pagination is used in interactive output
         #-----------------------------------------------------------------------
         def list_pool_xml(pool, options, filter_flag)
-            if $stdout.isatty 
-                size = $stdout.winsize[0] - 1 
+            if $stdout.isatty
+                size = $stdout.winsize[0] - 1
 
                 # ----------- First page, check if pager is needed -------------
                 rc = pool.get_page(size, 0)

src/dm/DispatchManagerStates.cc

@@ -314,6 +314,8 @@ void  DispatchManager::resubmit_action(int vid)
 
         vm->set_state(VirtualMachine::PENDING);
 
+        vm->set_deploy_id(""); //reset the deploy-id
+
         vmpool->update(vm);
 
         vm->unlock();

src/im_mad/remotes/common.d/collectd-client.rb

@@ -88,13 +88,12 @@ class CollectdClient
 
             # Collect the Data
             ts = Time.now
-            data = run_probes
+
+            # Send signal to itself to run probes and send the data
+            Process.kill('HUP', $$)
 
             run_probes_time = (Time.now - ts).to_i
 
-            # Send the Data
-            send data
-
             # Sleep during the Cycle
             sleep_time = @monitor_push_period - run_probes_time
             sleep_time = 0 if sleep_time < 0
@@ -130,4 +129,16 @@ sleep rand monitor_push_period
 # Start push monitorization
 client = CollectdClient.new(hypervisor, number, host, port, probes_args,
                             monitor_push_period)
+
+Signal.trap('HUP') do
+    # ignore another HUP until we handle this one
+    this_handler = Signal.trap('HUP', 'IGNORE')
+
+    data = client.run_probes
+    client.send data
+
+    # set the handler back
+    Signal.trap('HUP', this_handler)
+end
+
 client.monitor

src/im_mad/remotes/kvm-probes.d/machines-models.rb

@@ -30,14 +30,9 @@ begin
     machines = []
     models   = []
 
-    Open3.popen3("virsh -r -c qemu:///system capabilities") {|i, o, e, t|
-        if t.value.exitstatus != 0
-            exit -1
-        end
-
-        capabilities = o.read
-    }
-
+    cmd = 'virsh -r -c qemu:///system capabilities'
+    capabilities, e, s = Open3.capture3(cmd)
+    exit(-1) unless s.success?
 
     cap_xml = REXML::Document.new(capabilities)
     cap_xml = cap_xml.root
@@ -94,12 +89,9 @@ begin
             end
         }
 
-        cpu_models = ""
-        Open3.popen3("virsh -r -c qemu:///system cpu-models #{a}") {|i, o, e, t|
-            break if t.value.exitstatus != 0
-
-            cpu_models = o.read
-        }
+        cmd = "virsh -r -c qemu:///system cpu-models #{a}"
+        cpu_models, e, s = Open3.capture3(cmd)
+        break unless s.success?
 
         cpu_models.each_line { |l|
             l.chomp!

src/mad/sh/scripts_common.sh

@@ -994,3 +994,9 @@ function get_nic_information {
     OUTBOUND_PEAK_KB="${XPATH_ELEMENTS[j++]}"
     ORDER="${XPATH_ELEMENTS[j++]}"
 }
+
+function hup_collectd
+{
+    SEND_HUP='kill -HUP `cat /tmp/one-collectd-client.pid` || true'
+    ssh_exec_and_log_no_error $1 "$SEND_HUP"
+}

src/market_mad/remotes/linuxcontainers/monitor

@@ -31,7 +31,7 @@ class LinuxContainersMarket
     #---------------------------------------------------------------------------
     DEFAULTS = {
         :url    => 'https://images.linuxcontainers.org',
-        :sizemb => 5120,
+        :sizemb => 2560,
         :fs     => 'ext4',
         :format => 'raw',
         :agent  => 'OpenNebula'

src/tm_mad/common/delete

@@ -67,4 +67,4 @@ EOF
 )
 
 ssh_exec_and_log $DST_HOST "$delete_file" "Error deleting $DST_PATH"
-
+hup_collectd $DST_HOST

src/tm_mad/common/mkimage

@@ -70,5 +70,6 @@ EOF
 log "Making filesystem of ${SIZE}M and type $FSTYPE at $DST"
 
 ssh_exec_and_log $DST_HOST "$MKSCRIPT" "Could not create image $DST_PATH"
+hup_collectd $DST_HOST
 
 exit 0

src/tm_mad/common/resize

@@ -43,3 +43,4 @@ fi
 
 ssh_exec_and_log "${SRC_HOST}" "qemu-img resize ${SRC_PATH} ${SIZE}M" \
                  "Error resizing image ${SRC_PATH}"
+hup_collectd $SRC_HOST

src/tm_mad/fs_lvm/clone

@@ -142,4 +142,5 @@ EOF
 
 ssh_exec_and_log "$DST_HOST" "$CLONE_CMD" \
         "Error cloning $SRC_PATH to $LV_NAME"
+hup_collectd $DST_HOST
 exit 0

src/tm_mad/fs_lvm/cpds

@@ -66,5 +66,6 @@ log "Dumping $SRC to $DST"
 
 ssh_exec_and_log "$SRC_HOST" "$DUMP_CMD" \
     "Error dumping $SRC to $DST"
+hup_collectd $SRC_HOST
 
 exit 0

src/tm_mad/fs_lvm/delete

@@ -84,3 +84,5 @@ else
     ssh_exec_and_log "$DST_HOST" "$DELETE_CMD" \
         "Error deleting $DST_PATH"
 fi
+
+hup_collectd $DST_HOST

src/tm_mad/fs_lvm/mv

@@ -154,3 +154,6 @@ fi
 
 ssh_exec_and_log "$DST_HOST" "mv $SRC_PATH $DST_PATH" \
     "Error moving VM files to another System DS: $SRC_PATH to $DST_PATH in $DST_HOST"
+
+hup_collectd $DST_HOST
+hup_collectd $SRC_HOST

src/tm_mad/fs_lvm/mvds

@@ -75,5 +75,6 @@ ssh_exec_and_log "$SRC_HOST" "$DUMP_CMD" \
 LOCK="tm-fs_lvm-${DS_SYS_ID}.lock"
 exclusive "${LOCK}" 120 ssh_exec_and_log "$SRC_HOST" "$DELETE_CMD" \
     "Error dumping $SRC to $DST"
+hup_collectd $SRC_HOST
 
 exit 0

src/tm_mad/ssh/clone

@@ -93,3 +93,4 @@ if [ -n "$ORIGINAL_SIZE" -a "$SIZE" -gt "$ORIGINAL_SIZE" ]; then
        "Error resizing image $DST"
 fi
 
+hup_collectd $DST_HOST

src/tm_mad/ssh/cpds

@@ -120,5 +120,6 @@ fi
 
 log "Moving $SRC to datastore as $DST"
 exec_and_log "$SCP -r $SRC $DST" "Error copying $SRC to $DST"
+hup_collectd $SRC_HOST
 
 exit 0

src/tm_mad/ssh/mv

@@ -85,5 +85,7 @@ EOF
 )
 
 ssh_exec_and_log "$SRC_HOST" "$TAR_SSH" "Error copying disk directory to target host"
+hup_collectd $DST_HOST
+hup_collectd $SRC_HOST
 
 exit 0

src/tm_mad/ssh/mvds

@@ -53,6 +53,7 @@ exec_and_log "$SCP -r $SRC $DST" "Error copying $SRC to $DST"
 
 if $SSH $SRC_HOST ls ${SRC_PATH_SNAP} >/dev/null 2>&1; then
     exec_and_log "rsync -r --delete ${SRC_HOST}:${SRC_PATH_SNAP}/ ${DST_SNAP}"
+    hup_collectd $SRC_HOST
 fi
 
 exit 0

src/tm_mad/ssh/snap_create

@@ -82,4 +82,5 @@ EOT
 
 ssh_exec_and_log "${SRC_HOST}" "${CMD}" \
                  "Error creating snapshot ${SNAP_PATH}"
+hup_collectd $SRC_HOST
 

src/tm_mad/ssh/snap_delete

@@ -70,4 +70,5 @@ CURRENT_PATH=${DISK_PATH}
 
 ssh_exec_and_log "${SRC_HOST}" "rm ${SNAP_PATH}" \
                  "Error deleting snapshot ${SNAP_PATH}"
+hup_collectd $SRC_HOST
 

src/tm_mad/ssh/snap_revert

@@ -77,4 +77,5 @@ EOF
 
 ssh_exec_and_log "${SRC_HOST}" "${CMD}" \
                  "Error reverting snapshot to ${SNAP_PATH}"
+hup_collectd $SRC_HOST
 

src/vmm/VirtualMachineManager.cc

@@ -2420,6 +2420,19 @@ void VirtualMachineManager::detach_nic_action(
         return;
     }
 
+    int uid = vm->get_created_by_uid();
+    int owner_id = vm->get_uid();
+    vm->unlock();
+
+    password = Nebula::instance().get_upool()->get_token_password(uid, owner_id);
+
+    vm = vmpool->get(vid);
+
+    if (vm == 0)
+    {
+        return;
+    }
+
     if (!vm->hasHistory())
     {
         goto error_history;

src/vmm_mad/remotes/lib/lxd/container.rb

@@ -169,9 +169,8 @@ class Container
         err = 'cannot create user data directory:'
         rc, o, e = Command.execute("sudo #{cmd}", true) if e.include?(err)
 
-        return [rc, o, e] unless rc != 0
-
-        OpenNebula.log_error("#{__method__}: Failed to run command #{cmd}: #{e}")
+        log = "Failed to run command #{cmd}: #{e}"
+        OpenNebula.log_error("#{__method__}: #{log}") unless rc.zero?
 
         [rc, o, e]
     end

src/vmm_mad/remotes/lib/lxd/mapper/qcow2.rb

@@ -20,7 +20,7 @@ $LOAD_PATH.unshift File.dirname(__FILE__)
 
 require 'mapper'
 
-class Qcow2Mapper <  Mapper
+class Qcow2Mapper < Mapper
 
     # Max number of block devices. This should be set to the parameter used
     # to load the nbd kernel module (default in kernel is 16)
@@ -32,18 +32,20 @@ class Qcow2Mapper <  Mapper
         return if device.empty?
 
         dsrc = one_vm.disk_source(disk)
-        cmd  = "#{COMMANDS[:nbd]} -c #{device} #{dsrc}"
+        File.chmod(0o664, dsrc) if File.symlink?(one_vm.sysds_path)
 
-        File.chmod(0664, dsrc) if File.symlink?(one_vm.sysds_path)
+        map = "#{COMMANDS[:nbd]} -c #{device} #{dsrc}"
+        rc, _out, err = Command.execute(map, false)
 
-        rc, _out, err = Command.execute(cmd, true)
-
-        if rc != 0
+        unless rc.zero?
             OpenNebula.log_error("#{__method__}: #{err}")
             return
         end
 
-        sleep 0.5 # TODO: improve settledown, lsblk -f fails
+        sleep 5 # wait for parts to come out
+
+        partitions = lsblk(device)
+        show_parts(device) unless partitions[0]['type'] == 'part'
 
         device
     end
@@ -51,7 +53,7 @@ class Qcow2Mapper <  Mapper
     def do_unmap(device, _one_vm, _disk, _directory)
         cmd = "#{COMMANDS[:nbd]} -d #{device}"
 
-        rc, _out, err = Command.execute(cmd, true)
+        rc, _out, err = Command.execute(cmd, false)
 
         return true if rc.zero?
 
@@ -61,21 +63,32 @@ class Qcow2Mapper <  Mapper
 
     private
 
-    def nbd_device()
+    def show_parts(device)
+        get_parts = "#{COMMANDS[:kpartx]} -s -av #{device}"
+
+        rc, _out, err = Command.execute(get_parts, false)
+
+        unless rc.zero?
+            OpenNebula.log_error("#{__method__}: #{err}")
+            return
+        end
+    end
+
+    def nbd_device
         sys_parts = lsblk('')
         device_id = -1
         nbds      = []
 
-        sys_parts.each { |p|
+        sys_parts.each do |p|
             m = p['name'].match(/nbd(\d+)/)
-            next if !m
+            next unless m
 
             nbds << m[1].to_i
-        }
+        end
 
-        NBDS_MAX.times { |i|
+        NBDS_MAX.times do |i|
             return "/dev/nbd#{i}" unless nbds.include?(i)
-        }
+        end
 
         OpenNebula.log_error("#{__method__}: Cannot find free nbd device")
 

src/vmm_mad/remotes/lib/lxd/mapper/raw.rb

@@ -44,7 +44,7 @@ class FSRawMapper < Mapper
 
         return true if rc.zero?
 
-        OpenNebula.log_error("#{__method__}: #{err}") if rc != 0
+        OpenNebula.log_error("#{__method__}: #{err}")
         nil
     end
 

src/vmm_mad/remotes/lxd/poll

@@ -130,9 +130,16 @@ module LXD
             state
         end
 
+        def lxc_path(vm_name)
+            path = 'lxc/' + vm_name
+            path = "#{ENV['LXC_CGROUP_PREFIX']}#{path}" if ENV['LXC_CGROUP_PREFIX']
+        end
+
         def get_memory(vm_name)
-            stat = File.read('/sys/fs/cgroup/memory/lxc/' + vm_name + '/memory.usage_in_bytes').to_i
+            stat = File.read('/sys/fs/cgroup/memory/' + lxc_path(vm_name) + '/memory.usage_in_bytes').to_i
             stat / 1024
+        rescue StandardError
+            0
         end
 
         def get_net_statistics(vmd)
@@ -167,7 +174,7 @@ module LXD
             cpu_jiffies = get_cpu_jiffies - start_cpu_jiffies
 
             vm_names.each do |vm_name|
-                cpu_used[vm_name] = (get_process_jiffies(vm_name).to_f - 
+                cpu_used[vm_name] = (get_process_jiffies(vm_name).to_f -
                         cpu_used[vm_name]) / cpu_jiffies
 
                 cpu_used[vm_name] = (cpu_used[vm_name] * multiplier).round(2)
@@ -196,7 +203,8 @@ module LXD
         def get_process_jiffies(vm_name)
             begin
                 jiffies = 0
-                stat = File.read('/sys/fs/cgroup/cpu,cpuacct/lxc/' + vm_name + '/cpuacct.stat')
+
+                stat = File.read('/sys/fs/cgroup/cpu,cpuacct/' + lxc_path(vm_name) + '/cpuacct.stat')
                 stat.lines.each {|line| jiffies += line.split(' ')[1] }
             rescue StandardError
                 return 0
@@ -222,20 +230,19 @@ module LXD
             arch     = container.architecture
             capacity = container.expanded_config
 
-            cpu = ""
-            vcpu= ""
-            mem = ""
+            cpu = ''
+            vcpu = ''
+            mem = ''
 
             if capacity
                 cpu  = capacity['limits.cpu.allowance']
                 vcpu = capacity['limits.cpu']
-                mem  = capacity['limits.memory'] 
+                mem  = capacity['limits.memory']
             end
 
-            cpu  = "50%"  if !cpu || cpu.empty?
-            vcpu = "1"    if !vcpu || vcpu.empty?
-            mem  = "512M" if !mem || mem.empty?
- 
+            cpu  = '50%'  if !cpu || cpu.empty?
+            vcpu = '1'    if !vcpu || vcpu.empty?
+            mem  = '512MB' if !mem || mem.empty?
             cpu = cpu.chomp('%').to_f / 100
             mem = parse_memory(mem)
 

src/vnm_mad/remotes/lib/nic.rb

@@ -110,7 +110,12 @@ module VNMMAD
                 end
 
                 if deploy_id && vm.vm_info[:dumpxml].nil?
-                    vm.vm_info[:dumpxml] = YAML.safe_load(`lxc config show #{deploy_id} 2>/dev/null`)
+                    cmd = "lxc config show #{deploy_id} 2>/dev/null"
+
+                    config = YAML.safe_load(`#{cmd}`)
+                    config = YAML.safe_load(`sudo #{cmd}`) if config.nil?
+
+                    vm.vm_info[:dumpxml] = config
 
                     vm.vm_info.each_key do |k|
                         vm.vm_info[k] = nil if vm.vm_info[k].to_s.strip.empty?