* F OpenNebula/one#5506 (#5599,#5506): LXC cgroup improvements

src/mad/ruby/scripts_common.rb

@@ -28,6 +28,11 @@ module OpenNebula
         log_function("INFO", message)
     end
 
+    # Logs an info message
+    def self.log_warning(message)
+        log_function('WARNING', message)
+    end
+
     # Logs an error message
     def self.log_error(message)
         log_function("ERROR", message)

src/vmm_mad/remotes/lib/lxc/client.rb

@@ -24,17 +24,22 @@ class LXCClient
 
     # LXC CLI Commands
     COMMANDS = {
-        :attach     => 'sudo lxc-attach',
-        :config     => 'sudo lxc-config',
-        :console    => 'sudo lxc-console',
-        :create     => 'sudo lxc-create',
-        :destroy    => 'sudo lxc-destroy',
-        :info       => 'sudo lxc-info',
-        :ls         => 'sudo lxc-ls',
-        :start      => 'sudo lxc-start',
-        :stop       => 'sudo lxc-stop'
+        :attach     => 'lxc-attach',
+        :config     => 'lxc-config',
+        :console    => 'lxc-console',
+        :create     => 'lxc-create',
+        :destroy    => 'lxc-destroy',
+        :info       => 'lxc-info',
+        :ls         => 'lxc-ls',
+        :start      => 'lxc-start',
+        :stop       => 'lxc-stop'
     }
 
+    COMMANDS.each_value do |value|
+        value.prepend 'sudo '
+        value << ' --logpriority ERROR'
+    end
+
     # Returns LXC version
     def version
         _, rc, = Command.execute_log("#{COMMANDS[:ls]} --version")
@@ -47,24 +52,24 @@ class LXCClient
 
     def start(name, options = {})
         cmd = append_options("#{COMMANDS[:start]} -n '#{name}'", options)
-        Command.execute_rc_log(cmd)
+        Command.container_cmd(name, cmd)
     end
 
     def stop(name, options = {})
         cmd = append_options("#{COMMANDS[:stop]} -n '#{name}'", options)
-        Command.execute_rc_log(cmd)
+        Command.container_cmd(name, cmd)
     end
 
     def create(name, options = {})
         options[:template] ||= 'none' # Template is mandatory
 
         cmd = append_options("#{COMMANDS[:create]} -n '#{name}'", options)
-        Command.execute_rc_log(cmd)
+        Command.container_cmd(name, cmd)
     end
 
     def destroy(name, options = {})
         cmd = append_options("#{COMMANDS[:destroy]} -n '#{name}'", options)
-        Command.execute_rc_log(cmd)
+        Command.container_cmd(name, cmd)
     end
 
     #-----------------------------------------------------------------------

src/vmm_mad/remotes/lib/lxc/command.rb

@@ -24,4 +24,11 @@ module Command
 
     LOCK_FILE = '/tmp/onelxc-lock'
 
+    def self.container_cmd(name, cmd)
+        return true if execute_rc_log(cmd)
+
+        STDERR.puts "Check container logs at /var/log/lxc/#{name}"
+        false
+    end
+
 end

src/vmm_mad/remotes/lib/lxc/opennebula_vm.rb

@@ -117,11 +117,47 @@ class LXCVM < OpenNebulaVM
             lxc["lxc.net.#{i}.veth.pair"] = "one-#{@vm_id}-#{nic['NIC_ID']}"
         end
 
-        # Add cgroup limitations
         # rubocop:disable Layout/LineLength
-        lxc['lxc.cgroup.cpu.shares']            = cpu_shares
-        lxc['lxc.cgroup.memory.limit_in_bytes'] = memmory_limit_in_bytes
-        lxc['lxc.cgroup.memory.oom_control']    = 1 # Avoid OOM to kill the process when limit is reached
+
+        # Add cgroup limitations
+
+        # rubocop:disable Style/ConditionalAssignment
+        cg_set = if cgroup_ver == 2
+                     CGROUP_NAMES.keys[1]
+                 else
+                     CGROUP_NAMES.keys[0]
+                 end
+        # rubocop:enable Style/ConditionalAssignment
+
+        pre= "lxc.#{cg_set}."
+
+        lxc["#{pre}cpu.#{CGROUP_NAMES[cg_set][:cpu]}"] = cpu_shares
+
+        numa_nodes = get_numa_nodes
+
+        if !numa_nodes.empty?
+            nodes = []
+            cores = []
+
+            numa_nodes.each do |node|
+                nodes << node['MEMORY_NODE_ID']
+                cores << node['CPUS']
+            end
+
+            lxc["#{pre}cpuset.#{CGROUP_NAMES[cg_set][:cores]}"] = cores.join(',')
+            lxc["#{pre}cpuset.#{CGROUP_NAMES[cg_set][:nodes]}"] = nodes.join(',')
+        end
+
+        memory = limits_memory
+
+        lxc["#{pre}memory.#{CGROUP_NAMES[cg_set][:memory_max]}"] = memory
+        lxc["#{pre}memory.#{CGROUP_NAMES[cg_set][:memory_low]}"] = (memory.chomp.to_f*0.9).ceil
+
+        lxc["#{pre}memory.#{CGROUP_NAMES[cg_set][:swap]}"] = limits_memory_swap('LXC_SWAP') if swap_limitable?
+
+        # Avoid OOM to kill the process when limit is reached
+        lxc["#{pre}memory.#{CGROUP_NAMES[cg_set][:oom]}"] = 1
+
         # rubocop:enable Layout/LineLength
 
         # User mapping
@@ -312,7 +348,6 @@ class Disk
             path = 'context'
             opts = 'none rbind,ro,create=dir,optional 0 0'
         else
-            # TODO: Adjustable guest mountpoint
             path = "media/one-disk.#{@id}"
             opts = 'none rbind,create=dir,optional 0 0'
         end

src/vmm_mad/remotes/lib/opennebula_vm.rb

@@ -24,6 +24,28 @@ class OpenNebulaVM
 
     CGROUP_DEFAULT_SHARES = 1024
 
+    CGROUP_NAMES ={
+        'cgroup' => {
+            :cpu        => 'shares',
+            :cores      => 'cpus',
+            :nodes      => 'mems',
+            :memory_max => 'limit_in_bytes',
+            :memory_low => 'soft_limit_in_bytes',
+            :swap       => 'memsw.limit_in_bytes',
+            :oom        => 'oom_control'
+        },
+        'cgroup2' => {
+            :cpu        => 'weight',
+            :cores      => 'cpus',
+            :nodes      => 'mems',
+            :memory_max => 'max',
+            :memory_low => 'low',
+            :swap       => 'swap.max',
+            :oom        => 'oom.group'
+
+        }
+    }
+
     #---------------------------------------------------------------------------
     # Class Constructor
     #---------------------------------------------------------------------------
@@ -60,6 +82,13 @@ class OpenNebulaVM
         !@xml['//TEMPLATE/CONTEXT/DISK_ID'].empty?
     end
 
+    # Returns cgroup version
+    def cgroup_ver
+        return 2 unless `mount | grep 'type cgroup2'`.empty?
+
+        1
+    end
+
     def wild?
         @vm_name && !@vm_name.include?('one-')
     end
@@ -81,6 +110,18 @@ class OpenNebulaVM
         @xml.elements('//TEMPLATE/DISK')
     end
 
+    def get_numa_nodes
+        @xml.elements('//TEMPLATE/NUMA_NODE')
+    end
+
+    def swap_limitable?
+        if File.exist?('/sys/fs/cgroup/memory/memory.memsw.limit_in_bytes')
+            return true
+        end
+
+        OpenNebula.log_warning('swap limiting via cgroups not supported')
+    end
+
     def location
         "#{sysds_path}/#{vm_id}"
     end
@@ -149,7 +190,11 @@ class OpenNebulaVM
         shares_val
     end
 
-    # Return the value for memmory.limit_in_bytes cgroup based on the value of
+    def get_memory
+        @xml['//TEMPLATE/MEMORY']
+    end
+
+    # Return the value for memory.limit_in_bytes cgroup based on the value of
     # MEMORY.
     #
     # memory.limit_in_bytes
@@ -158,10 +203,15 @@ class OpenNebulaVM
     # is possible to use suffixes to represent larger units - k or K for
     # kilobytes, m or M for megabytes, and g or G for gigabytes.
     # (https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-memory)
-    def memmory_limit_in_bytes
-        default_units = 'M' # MEMORY units are in MB
+    def limits_memory
+        "#{get_memory}M"
+    end
 
-        "#{@xml['//TEMPLATE/MEMORY']}#{default_units}"
+    def limits_memory_swap(hypervisor_attr)
+        memory = get_memory.to_i
+        swap = @xml["/VM/USER_TEMPLATE/#{hypervisor_attr}"].to_i
+
+        "#{swap + memory}M"
     end
 
     private