1
0
mirror of https://github.com/OpenNebula/one.git synced 2025-02-02 09:47:00 +03:00

M #: generalize Domain, Domains & ProcessList

co-authored-by: Christian González <cgonzalez@opennebula.io>
This commit is contained in:
Ruben S. Montero 2020-05-18 13:48:42 +02:00
parent 441861c4cc
commit 7e43c4632a
No known key found for this signature in database
GPG Key ID: A0CEA6FA880A1D87
5 changed files with 300 additions and 433 deletions

View File

@ -1229,7 +1229,9 @@ IM_PROBES_LIB_FILES="\
src/im_mad/remotes/lib/numa_common.rb \
src/im_mad/remotes/lib/probe_db.rb \
src/im_mad/remotes/lib/vcenter.rb \
src/im_mad/remotes/lib/nsx.rb"
src/im_mad/remotes/lib/nsx.rb \
src/im_mad/remotes/lib/domain.rb \
src/im_mad/remotes/lib/process_list.rb"
# KVM PROBES
IM_PROBES_KVM_FILES="\

View File

@ -0,0 +1,154 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
#-------------------------------------------------------------------------------
# This class represents a base domain, information includes:
#-------------------------------------------------------------------------------
class BaseDomain
attr_reader :vm, :name
def initialize(name)
@name = name
@vm = {}
end
# Get domain attribute by name.
def [](name)
@vm[name]
end
def []=(name, value)
@vm[name] = value
end
# Merge hash value into the domain attributes
def merge!(map)
@vm.merge!(map)
end
# Builds an OpenNebula Template with the monitoring keys. E.g.
# CPU=125.2
# MEMORY=1024
# NETTX=224324
# NETRX=213132
# ...
#
# Keys are defined in MONITOR_KEYS constant
#
# @return [String] OpenNebula template encoded in base64
def to_monitor
mon_s = ''
MONITOR_KEYS.each do |k|
next unless @vm[k.to_sym]
mon_s << "#{k.upcase}=\"#{@vm[k.to_sym]}\"\n"
end
Base64.strict_encode64(mon_s)
end
MONITOR_KEYS = %w[cpu memory netrx nettx diskrdbytes diskwrbytes diskrdiops
diskwriops]
end
#-------------------------------------------------------------------------------
# This class represents a base domains
#-------------------------------------------------------------------------------
class BaseDomains
include ProcessList
def initialize
@vms = {}
end
# Get the list of VMs (known to OpenNebula) and their monitor info
# including process usage
#
# @return [Hash] with KVM Domain classes indexed by their uuid
def info
info_each(true) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Get the list of VMs and their info
# not including process usage.
#
# @return [Hash] with KVM Domain classes indexed by their uuid
def state_info
info_each(false) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Return a message string with VM monitor information
def to_monitor
mon_s = ''
@vms.each do |_uuid, vm|
mon_s << "VM = [ ID=\"#{vm[:id]}\", UUID=\"#{vm[:uuid]}\","
mon_s << " MONITOR=\"#{vm.to_monitor}\"]\n"
end
mon_s
end
private
# Generic build method for the info list. It filters and builds the
# domain list based on the given block
# @param[Boolean] do_process, to get process information
def info_each(do_process)
return unless block_given?
vm_ps = ProcessList.process_list if do_process
names = ProcessList.retrieve_names
return @vms if names.empty?
names.each do |name|
vm = yield(name)
@vms[vm[:uuid]] = vm if vm
end
return @vms unless do_process
vm_ps.each do |uuid, ps|
next unless @vms[uuid]
@vms[uuid].merge!(ps)
end
@vms
end
end

View File

@ -22,6 +22,9 @@ require 'json'
require 'base64'
require 'client'
require_relative 'process_list'
require_relative 'domain'
#-------------------------------------------------------------------------------
# Firecracker Monitor Module. This module provides basic functionality to
# retrieve Firecracker instances information
@ -114,49 +117,15 @@ module Firecracker
end
#-------------------------------------------------------------------------------
# This module gets the pid, memory and cpu usage of a set of process that
# includes a -uuid argument (qemu-kvm vms).
#
# Usage is computed based on the fraction of jiffies used by the process
# relative to the system during AVERAGE_SECS (1s)
# Extends ProcessList module defined at process_list.rb
#-------------------------------------------------------------------------------
module ProcessList
# Number of seconds to average process usage
AVERAGE_SECS = 1
# list of process indexed by uuid, each entry:
# :pid
# :memory
# :cpu
def self.process_list
pids = []
procs = {}
ps = `ps auxwww`
ps.each_line do |l|
m = l.match(/firecracker.+(one-\d+)/)
next unless m
l = l.split(/\s+/)
swap = `cat /proc/#{l[1]}/status 2>/dev/null | grep VmSwap`
swap = swap.split[1] || 0
procs[m[1]] = {
:pid => l[1],
:memory => l[5].to_i + swap.to_i
}
pids << l[1]
end
cpu = cpu_info(pids)
procs.each {|_i, p| p[:cpu] = cpu[p[:pid]] || 0 }
procs
end
# Regex used to retrieve microVMs process info
PS_REGEX = /firecracker.+(one-\d+)/
def self.retrieve_names
ps = `ps auxwww`
@ -172,62 +141,6 @@ module ProcessList
domains
end
# Get cpu usage in 100% for a set of PIDs
# param[Array] pids of the arrys to compute the CPU usage
# result[Array] array of cpu usage
def self.cpu_info(pids)
multiplier = Integer(`grep -c processor /proc/cpuinfo`) * 100
cpu_ini = {}
j_ini = jiffies
pids.each do |pid|
cpu_ini[pid] = proc_jiffies(pid).to_f
end
sleep AVERAGE_SECS
cpu_j = jiffies - j_ini
cpu = {}
pids.each do |pid|
cpu[pid] = (proc_jiffies(pid) - cpu_ini[pid]) / cpu_j
cpu[pid] = (cpu[pid] * multiplier).round(2)
end
cpu
end
# CPU tics used in the system
def self.jiffies
stat = File.open('/proc/stat', 'r') {|f| f.readline }
j = 0
stat.split(' ')[1..-3].each {|num| j += num.to_i }
j
rescue StandardError
0
end
# CPU tics used by a process
def self.proc_jiffies(pid)
stat = File.read("/proc/#{pid}/stat")
j = 0
data = stat.lines.first.split(' ')
[13, 14, 15, 16].each {|col| j += data[col].to_i }
j
rescue StandardError
0
end
end
#-------------------------------------------------------------------------------
@ -245,19 +158,12 @@ end
# @vm[:diskrdiops]
# @vm[:diskwriops]
#
# This class uses the KVM and ProcessList interface
# This class uses the Firecracker and ProcessList interface
#-------------------------------------------------------------------------------
class Domain
attr_reader :vm, :name
def initialize(name)
@name = name
@vm = {}
end
class Domain < BaseDomain
# Gets the information of the domain, fills the @vm hash using ProcessList
# and virsh dominfo
# and ps command
def info
# Flush the microVM metrics
hash = Firecracker.retrieve_info(@name)
@ -286,42 +192,6 @@ class Domain
io_stats
end
# Get domain attribute by name.
def [](name)
@vm[name]
end
def []=(name, value)
@vm[name] = value
end
# Merge hash value into the domain attributes
def merge!(map)
@vm.merge!(map)
end
# Builds an OpenNebula Template with the monitoring keys. E.g.
# CPU=125.2
# MEMORY=1024
# NETTX=224324
# NETRX=213132
# ...
#
# Keys are defined in MONITOR_KEYS constant
#
# @return [String] OpenNebula template encoded in base64
def to_monitor
mon_s = ''
MONITOR_KEYS.each do |k|
next unless @vm[k.to_sym]
mon_s << "#{k.upcase}=\"#{@vm[k.to_sym]}\"\n"
end
Base64.strict_encode64(mon_s)
end
private
# --------------------------------------------------------------------------
@ -337,9 +207,6 @@ class Domain
'Running' => 'RUNNING'
}
MONITOR_KEYS = %w[cpu memory netrx nettx diskrdbytes diskwrbytes diskrdiops
diskwriops]
# Get the I/O stats of the domain as provided by Libvirt command domstats
# The metrics are aggregated for all DIKS and NIC
def io_stats
@ -394,87 +261,11 @@ module DomainList
############################################################################
# This is the implementation class for the module logic
############################################################################
class FirecrackerDomains
class FirecrackerDomains < BaseDomains
include Firecracker
include ProcessList
def initialize
@vms = {}
end
# Get the list of VMs (known to OpenNebula) and their monitor info
# including process usage
#
# @return [Hash] with KVM Domain classes indexed by their uuid
def info
info_each(true) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Get the list of VMs and their info
# not including process usage.
#
# @return [Hash] with KVM Domain classes indexed by their uuid
def state_info
info_each(false) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Return a message string with VM monitor information
def to_monitor
mon_s = ''
@vms.each do |_uuid, vm|
mon_s << "VM = [ ID=\"#{vm[:id]}\", "
mon_s << "DEPLOY_ID=\"#{vm[:deploy_id]}\","
mon_s << " MONITOR=\"#{vm.to_monitor}\"]\n"
end
mon_s
end
private
# Generic build method for the info list. It filters and builds the
# domain list based on the given block
# @param[Boolean] do_process, to get process information
def info_each(do_process)
return unless block_given?
vm_ps = ProcessList.process_list if do_process
names = ProcessList.retrieve_names
return @vms if names.empty?
names.each do |name|
vm = yield(name)
@vms[vm[:uuid]] = vm if vm
end
return @vms unless do_process
vm_ps.each do |uuid, ps|
next unless @vms[uuid]
@vms[uuid].merge!(ps)
end
@vms
end
end
end

View File

@ -20,6 +20,9 @@ require 'open3'
require 'base64'
require 'rexml/document'
require_relative 'process_list'
require_relative 'domain'
ENV['LANG'] = 'C'
ENV['LC_ALL'] = 'C'
@ -74,104 +77,29 @@ module KVM
end
#-------------------------------------------------------------------------------
# This module gets the pid, memory and cpu usage of a set of process that
# includes a -uuid argument (qemu-kvm vms).
#
# Usage is computed based on the fraction of jiffies used by the process
# relative to the system during AVERAGE_SECS (1s)
# Extends ProcessList module defined at process_list.rb
#-------------------------------------------------------------------------------
module ProcessList
# Number of seconds to average process usage
AVERAGE_SECS = 1
# list of process indexed by uuid, each entry:
# :pid
# :memory
# :cpu
def self.process_list
pids = []
procs = {}
ps = `ps auxwww`
# Regex used to retrieve VMs process info
PS_REGEX = /-uuid ([a-z0-9\-]+) /
ps.each_line do |l|
m = l.match(/-uuid ([a-z0-9\-]+) /)
next unless m
def self.retrieve_names
text, _e, s = KVM.virsh(:list, '')
names = []
l = l.split(/\s+/)
return names if s.exitstatus != 0
swap = `cat /proc/#{l[1]}/status 2>/dev/null | grep VmSwap`
swap = swap.split[1] || 0
lines = text.split(/\n/)[2..-1]
procs[m[1]] = {
:pid => l[1],
:memory => l[5].to_i + swap.to_i
}
pids << l[1]
names = lines.map do |line|
line.split(/\s+/).delete_if {|d| d.empty? }[1]
end
cpu = cpu_info(pids)
procs.each {|_i, p| p[:cpu] = cpu[p[:pid]] || 0 }
procs
end
# Get cpu usage in 100% for a set of PIDs
# param[Array] pids of the arrys to compute the CPU usage
# result[Array] array of cpu usage
def self.cpu_info(pids)
multiplier = Integer(`grep -c processor /proc/cpuinfo`) * 100
cpu_ini = {}
j_ini = jiffies
pids.each do |pid|
cpu_ini[pid] = proc_jiffies(pid).to_f
end
sleep AVERAGE_SECS
cpu_j = jiffies - j_ini
cpu = {}
pids.each do |pid|
cpu[pid] = (proc_jiffies(pid) - cpu_ini[pid]) / cpu_j
cpu[pid] = (cpu[pid] * multiplier).round(2)
end
cpu
end
# CPU tics used in the system
def self.jiffies
stat = File.open('/proc/stat', 'r') {|f| f.readline }
j = 0
stat.split(' ')[1..-3].each {|num| j += num.to_i }
j
rescue StandardError
0
end
# CPU tics used by a process
def self.proc_jiffies(pid)
stat = File.read("/proc/#{pid}/stat")
j = 0
data = stat.lines.first.split(' ')
[13, 14, 15, 16].each {|col| j += data[col].to_i }
j
rescue StandardError
0
names
end
end
@ -194,14 +122,7 @@ end
#
# This class uses the KVM and ProcessList interface
#-------------------------------------------------------------------------------
class Domain
attr_reader :vm, :name
def initialize(name)
@name = name
@vm = {}
end
class Domain < BaseDomain
# Gets the information of the domain, fills the @vm hash using ProcessList
# and virsh dominfo
@ -257,20 +178,6 @@ class Domain
io_stats
end
# Get domain attribute by name.
def [](name)
@vm[name]
end
def []=(name, value)
@vm[name] = value
end
# Merge hash value into the domain attributes
def merge!(map)
@vm.merge!(map)
end
# Convert the output of dumpxml for this domain to an OpenNebula template
# that can be imported. This method is for wild VMs.
#
@ -331,28 +238,6 @@ class Domain
''
end
# Builds an OpenNebula Template with the monitoring keys. E.g.
# CPU=125.2
# MEMORY=1024
# NETTX=224324
# NETRX=213132
# ...
#
# Keys are defined in MONITOR_KEYS constant
#
# @return [String] OpenNebula template encoded in base64
def to_monitor
mon_s = ''
MONITOR_KEYS.each do |k|
next unless @vm[k.to_sym]
mon_s << "#{k.upcase}=\"#{@vm[k.to_sym]}\"\n"
end
Base64.strict_encode64(mon_s)
end
private
# --------------------------------------------------------------------------
@ -385,9 +270,6 @@ class Domain
}
}
MONITOR_KEYS = %w[cpu memory netrx nettx diskrdbytes diskwrbytes diskrdiops
diskwriops]
# Get the I/O stats of the domain as provided by Libvirt command domstats
# The metrics are aggregated for all DIKS and NIC
def io_stats
@ -461,29 +343,11 @@ module DomainList
############################################################################
# This is the implementation class for the module logic
############################################################################
class KVMDomains
class KVMDomains < BaseDomains
include KVM
include ProcessList
def initialize
@vms = {}
end
# Get the list of VMs (known to OpenNebula) and their monitor info
# including process usage
#
# @return [Hash] with KVM Domain classes indexed by their uuid
def info
info_each(true) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Get the list of wild VMs (not known to OpenNebula) and their monitor
# information including process usage
#
@ -504,28 +368,6 @@ module DomainList
end
end
def state_info
info_each(false) do |name|
vm = Domain.new name
next if vm.info == -1
vm
end
end
# Return a message string with VM monitor information
def to_monitor
mon_s = ''
@vms.each do |_uuid, vm|
mon_s << "VM = [ ID=\"#{vm[:id]}\", DEPLOY_ID=\"#{vm[:deploy_id]}\","
mon_s << " MONITOR=\"#{vm.to_monitor}\"]\n"
end
mon_s
end
# Return a message string with wild VM information
def wilds_to_monitor
mon_s = ''
@ -541,45 +383,6 @@ module DomainList
mon_s
end
private
# Generic build method for the info list. It filters and builds the
# domain list based on the given block
# @param[Boolean] do_process, to get process information
def info_each(do_process)
return unless block_given?
vm_ps = ProcessList.process_list if do_process
text, _e, s = KVM.virsh(:list, '')
return {} if s.exitstatus != 0
lines = text.split(/\n/)[2..-1]
return @vms if lines.nil?
names = lines.map do |line|
line.split(/\s+/).delete_if {|d| d.empty? }[1]
end
names.each do |name|
vm = yield(name)
@vms[vm[:uuid]] = vm if vm
end
return @vms unless do_process
vm_ps.each do |uuid, ps|
next unless @vms[uuid]
@vms[uuid].merge!(ps)
end
@vms
end
end
end

View File

@ -0,0 +1,117 @@
#!/usr/bin/env ruby
# -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
# not use this file except in compliance with the License. You may obtain #
# a copy of the License at #
# #
# http://www.apache.org/licenses/LICENSE-2.0 #
# #
# Unless required by applicable law or agreed to in writing, software #
# distributed under the License is distributed on an "AS IS" BASIS, #
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
# See the License for the specific language governing permissions and #
# limitations under the License. #
#--------------------------------------------------------------------------- #
#-------------------------------------------------------------------------------
# This module gets the pid, memory and cpu usage of a set of process that
# includes a -uuid argument (qemu-kvm vms).
#
# Usage is computed based on the fraction of jiffies used by the process
# relative to the system during AVERAGE_SECS (1s)
#-------------------------------------------------------------------------------
module ProcessList
# list of process indexed by uuid, each entry:
# :pid
# :memory
# :cpu
def self.process_list
pids = []
procs = {}
ps = `ps auxwww`
ps.each_line do |l|
m = l.match(PS_REGEX)
next unless m
l = l.split(/\s+/)
swap = `cat /proc/#{l[1]}/status 2>/dev/null | grep VmSwap`
swap = swap.split[1] || 0
procs[m[1]] = {
:pid => l[1],
:memory => l[5].to_i + swap.to_i
}
pids << l[1]
end
cpu = cpu_info(pids)
procs.each {|_i, p| p[:cpu] = cpu[p[:pid]] || 0 }
procs
end
# Get cpu usage in 100% for a set of PIDs
# param[Array] pids of the arrys to compute the CPU usage
# result[Array] array of cpu usage
def self.cpu_info(pids)
multiplier = Integer(`grep -c processor /proc/cpuinfo`) * 100
cpu_ini = {}
j_ini = jiffies
pids.each do |pid|
cpu_ini[pid] = proc_jiffies(pid).to_f
end
sleep AVERAGE_SECS
cpu_j = jiffies - j_ini
cpu = {}
pids.each do |pid|
cpu[pid] = (proc_jiffies(pid) - cpu_ini[pid]) / cpu_j
cpu[pid] = (cpu[pid] * multiplier).round(2)
end
cpu
end
# CPU tics used in the system
def self.jiffies
stat = File.open('/proc/stat', 'r') {|f| f.readline }
j = 0
stat.split(' ')[1..-3].each {|num| j += num.to_i }
j
rescue StandardError
0
end
# CPU tics used by a process
def self.proc_jiffies(pid)
stat = File.read("/proc/#{pid}/stat")
j = 0
data = stat.lines.first.split(' ')
[13, 14, 15, 16].each {|col| j += data[col].to_i }
j
rescue StandardError
0
end
end