F #6668: Add Prometheus support for OpenNebula

install.sh

@@ -121,6 +121,14 @@ if [ -z "$ROOT" ] ; then
     DOCS_LOCATION="/usr/share/doc/one"
     SUNSTONE_MAIN_JS_LOCATION="$VAR_LOCATION/sunstone"
 
+    ONEPROMETHEUS_SYSTEMD_LOCATION="/lib/systemd/system"
+    ONEPROMETHEUS_VAR_ALERTMANAGER_LOCATION="/var/lib/alertmanager"
+    ONEPROMETHEUS_VAR_PROMETHEUS_LOCATION="/var/lib/prometheus"
+
+    ONEPROMETHEUS_DIRS="$ONEPROMETHEUS_SYSTEMD_LOCATION \
+                        $ONEPROMETHEUS_VAR_ALERTMANAGER_LOCATION \
+                        $ONEPROMETHEUS_VAR_PROMETHEUS_LOCATION"
+
     if [ "$CLIENT" = "yes" ]; then
         MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $ETC_LOCATION"
 
@@ -161,7 +169,7 @@ if [ -z "$ROOT" ] ; then
                    $LOG_LOCATION $RUN_LOCATION $LOCK_LOCATION \
                    $SYSTEM_DS_LOCATION $DEFAULT_DS_LOCATION $MAN_LOCATION \
                    $VM_LOCATION $ONEGATE_LOCATION $ONEFLOW_LOCATION \
-                   $SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION"
+                   $SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION $ONEPROMETHEUS_DIRS"
 
         DELETE_DIRS="$LIB_LOCATION $ETC_LOCATION $LOG_LOCATION $VAR_LOCATION \
                      $RUN_LOCATION $SHARE_DIRS"
@@ -191,6 +199,14 @@ else
     DOCS_LOCATION="$ROOT/share/doc"
     SUNSTONE_MAIN_JS_LOCATION="$VAR_LOCATION/sunstone"
 
+    ONEPROMETHEUS_SYSTEMD_LOCATION="$LIB_LOCATION/systemd"
+    ONEPROMETHEUS_VAR_ALERTMANAGER_LOCATION="$ROOT/var/alertmanager"
+    ONEPROMETHEUS_VAR_PROMETHEUS_LOCATION="$ROOT/var/prometheus"
+
+    ONEPROMETHEUS_DIRS="$ONEPROMETHEUS_SYSTEMD_LOCATION \
+                        $ONEPROMETHEUS_VAR_ALERTMANAGER_LOCATION \
+                        $ONEPROMETHEUS_VAR_PROMETHEUS_LOCATION"
+
     if [ "$CLIENT" = "yes" ]; then
         MAKE_DIRS="$BIN_LOCATION $LIB_LOCATION $ETC_LOCATION"
 
@@ -220,7 +236,8 @@ else
                    $INCLUDE_LOCATION $SHARE_LOCATION $SYSTEM_DS_LOCATION \
                    $DEFAULT_DS_LOCATION $MAN_LOCATION $DOCS_LOCATION \
                    $VM_LOCATION $ONEGATE_LOCATION $ONEFLOW_LOCATION \
-                   $SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION $LOCK_LOCATION $RUN_LOCATION"
+                   $SUNSTONE_MAIN_JS_LOCATION $ONEHEM_LOCATION $LOCK_LOCATION $RUN_LOCATION \
+                   $ONEPROMETHEUS_DIRS"
 
         DELETE_DIRS="$MAKE_DIRS"
 
@@ -244,8 +261,10 @@ SHARE_DIRS="$SHARE_LOCATION/examples \
             $SHARE_LOCATION/start-scripts \
             $SHARE_LOCATION/conf \
             $SHARE_LOCATION/context \
-            $SHARE_LOCATION/onecfg
-            $SHARE_LOCATION/onecfg/etc"
+            $SHARE_LOCATION/onecfg \
+            $SHARE_LOCATION/onecfg/etc \
+            $SHARE_LOCATION/grafana \
+            $SHARE_LOCATION/prometheus"
 
 ETC_DIRS="$ETC_LOCATION/vmm_exec \
           $ETC_LOCATION/hm \
@@ -264,7 +283,9 @@ ETC_DIRS="$ETC_LOCATION/vmm_exec \
           $ETC_LOCATION/fireedge/sunstone/admin \
           $ETC_LOCATION/fireedge/sunstone/user \
           $ETC_LOCATION/fireedge/sunstone/groupadmin \
-          $ETC_LOCATION/fireedge/sunstone/cloud"
+          $ETC_LOCATION/fireedge/sunstone/cloud \
+          $ETC_LOCATION/alertmanager \
+          $ETC_LOCATION/prometheus"
 
 LIB_DIRS="$LIB_LOCATION/ruby \
           $LIB_LOCATION/ruby/opennebula \
@@ -308,7 +329,12 @@ LIB_DIRS="$LIB_LOCATION/ruby \
           $LIB_LOCATION/onecfg/lib/config/type \
           $LIB_LOCATION/onecfg/lib/config/type/augeas \
           $LIB_LOCATION/onecfg/lib/config/type/yaml \
-          $LIB_LOCATION/onecfg/lib/patch"
+          $LIB_LOCATION/onecfg/lib/patch \
+          $LIB_LOCATION/alertmanager \
+          $LIB_LOCATION/libvirt_exporter \
+          $LIB_LOCATION/node_exporter \
+          $LIB_LOCATION/opennebula_exporter \
+          $LIB_LOCATION/prometheus"
 
 VAR_DIRS="$VAR_LOCATION/remotes \
           $VAR_LOCATION/remotes/etc \
@@ -714,6 +740,29 @@ INSTALL_FILES=(
     SSH_SH_OVERRIDE_LIB_FILES:$LIB_LOCATION/sh/override
     SSH_SHARE_FILES:$SHARE_LOCATION/ssh
     CONTEXT_SHARE:$SHARE_LOCATION/context
+
+    ONEPROMETHEUS_ALERTMANAGER_BIN_FILES:$BIN_LOCATION
+    ONEPROMETHEUS_ALERTMANAGER_CONFIG_FILES:$ETC_LOCATION/alertmanager
+    ONEPROMETHEUS_ALERTMANAGER_FILES:$LIB_LOCATION/alertmanager
+    ONEPROMETHEUS_ALERTMANAGER_SYSTEMD_FILES:$ONEPROMETHEUS_SYSTEMD_LOCATION
+
+    ONEPROMETHEUS_GRAFANA_FILES:$SHARE_LOCATION/grafana
+
+    ONEPROMETHEUS_LIBVIRT_EXPORTER_FILES:$LIB_LOCATION/libvirt_exporter
+    ONEPROMETHEUS_LIBVIRT_EXPORTER_SYSTEMD_FILES:$ONEPROMETHEUS_SYSTEMD_LOCATION
+
+    ONEPROMETHEUS_NODE_EXPORTER_BIN_FILES:$BIN_LOCATION
+    ONEPROMETHEUS_NODE_EXPORTER_FILES:$LIB_LOCATION/node_exporter
+    ONEPROMETHEUS_NODE_EXPORTER_SYSTEMD_FILES:$ONEPROMETHEUS_SYSTEMD_LOCATION
+
+    ONEPROMETHEUS_OPENNEBULA_EXPORTER_FILES:$LIB_LOCATION/opennebula_exporter
+    ONEPROMETHEUS_OPENNEBULA_EXPORTER_SYSTEMD_FILES:$ONEPROMETHEUS_SYSTEMD_LOCATION
+
+    ONEPROMETHEUS_PROMETHEUS_BIN_FILES:$BIN_LOCATION
+    ONEPROMETHEUS_PROMETHEUS_CONFIG_FILES:$ETC_LOCATION/prometheus
+    ONEPROMETHEUS_PROMETHEUS_FILES:$LIB_LOCATION/prometheus
+    ONEPROMETHEUS_PROMETHEUS_SHARE_FILES:$SHARE_LOCATION/prometheus
+    ONEPROMETHEUS_PROMETHEUS_SYSTEMD_FILES:$ONEPROMETHEUS_SYSTEMD_LOCATION
 )
 
 INSTALL_CLIENT_FILES=(
@@ -2995,6 +3044,53 @@ XSD_FILES="share/doc/xsd/acct.xsd \
 
 CONTEXT_SHARE=$(find share/context/ -type f \( ! -iname "*.sh" ! -iname "SConstruct" \))
 
+#-------------------------------------------------------------------------------
+# PROMETHEUS
+#-------------------------------------------------------------------------------
+
+# ALERTMANAGER
+ONEPROMETHEUS_ALERTMANAGER_BIN_FILES="src/oneprometheus/vendor/alertmanager/alertmanager \
+                                      src/oneprometheus/vendor/alertmanager/amtool"
+ONEPROMETHEUS_ALERTMANAGER_CONFIG_FILES="src/oneprometheus/alertmanager/etc/alertmanager.yml"
+ONEPROMETHEUS_ALERTMANAGER_FILES="src/oneprometheus/vendor/alertmanager/LICENSE \
+                                  src/oneprometheus/vendor/alertmanager/NOTICE"
+ONEPROMETHEUS_ALERTMANAGER_SYSTEMD_FILES="src/oneprometheus/alertmanager/systemd/opennebula-alertmanager.service"
+
+# GRAFANA
+ONEPROMETHEUS_GRAFANA_FILES="src/oneprometheus/grafana/share/dashboards/"
+
+# LIBVIRT-EXPORTER
+ONEPROMETHEUS_LIBVIRT_EXPORTER_FILES="src/oneprometheus/opennebula-libvirt-exporter/src/libvirt_collector.rb \
+                                      src/oneprometheus/opennebula-libvirt-exporter/src/libvirt_exporter.rb"
+ONEPROMETHEUS_LIBVIRT_EXPORTER_SYSTEMD_FILES="src/oneprometheus/opennebula-libvirt-exporter/systemd/opennebula-libvirt-exporter.service"
+
+# NODE-EXPORTER
+ONEPROMETHEUS_NODE_EXPORTER_BIN_FILES="src/oneprometheus/vendor/node_exporter/node_exporter"
+ONEPROMETHEUS_NODE_EXPORTER_FILES="src/oneprometheus/vendor/node_exporter/LICENSE \
+                                   src/oneprometheus/vendor/node_exporter/NOTICE"
+ONEPROMETHEUS_NODE_EXPORTER_SYSTEMD_FILES="src/oneprometheus/node_exporter/systemd/opennebula-node-exporter.service"
+
+# OPENNEBULA-EXPORTER
+ONEPROMETHEUS_OPENNEBULA_EXPORTER_FILES="src/oneprometheus/opennebula-exporter/src/opennebula_collector.rb \
+                                         src/oneprometheus/opennebula-exporter/src/opennebula_datastore_collector.rb \
+                                         src/oneprometheus/opennebula-exporter/src/opennebula_exporter.rb \
+                                         src/oneprometheus/opennebula-exporter/src/opennebula_host_collector.rb \
+                                         src/oneprometheus/opennebula-exporter/src/opennebula_server_collector.rb \
+                                         src/oneprometheus/opennebula-exporter/src/opennebula_vm_collector.rb"
+ONEPROMETHEUS_OPENNEBULA_EXPORTER_SYSTEMD_FILES="src/oneprometheus/opennebula-exporter/systemd/opennebula-exporter.service"
+
+# PROMETHEUS
+ONEPROMETHEUS_PROMETHEUS_BIN_FILES="src/oneprometheus/vendor/prometheus/prometheus \
+                                    src/oneprometheus/vendor/prometheus/promtool"
+ONEPROMETHEUS_PROMETHEUS_CONFIG_FILES="src/oneprometheus/prometheus/etc/prometheus.yml \
+                                       src/oneprometheus/prometheus/etc/rules.yml"
+ONEPROMETHEUS_PROMETHEUS_FILES="src/oneprometheus/vendor/prometheus/console_libraries/ \
+                                src/oneprometheus/vendor/prometheus/consoles/ \
+                                src/oneprometheus/vendor/prometheus/LICENSE \
+                                src/oneprometheus/vendor/prometheus/NOTICE"
+ONEPROMETHEUS_PROMETHEUS_SHARE_FILES="src/oneprometheus/prometheus/share/patch_datasources.rb"
+ONEPROMETHEUS_PROMETHEUS_SYSTEMD_FILES="src/oneprometheus/prometheus/systemd/opennebula-prometheus.service"
+
 #-----------------------------------------------------------------------------
 #-----------------------------------------------------------------------------
 # INSTALL.SH SCRIPT

src/oneprometheus/alertmanager/etc/alertmanager.yml

@@ -0,0 +1,16 @@
+route:
+  group_by: ['alertname']
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 1h
+  receiver: 'web.hook'
+receivers:
+  - name: 'web.hook'
+    webhook_configs:
+      - url: 'http://127.0.0.1:5001/'
+inhibit_rules:
+  - source_match:
+      severity: 'critical'
+    target_match:
+      severity: 'warning'
+    equal: ['alertname', 'dev', 'instance']

src/oneprometheus/alertmanager/systemd/opennebula-alertmanager.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=OpenNebula Alertmanager
+
+[Service]
+User=oneadmin
+Group=oneadmin
+Type=simple
+
+ExecStart=/usr/bin/alertmanager \
+          --config.file=/etc/one/alertmanager/alertmanager.yml \
+          --storage.path=/var/lib/alertmanager/data/
+
+[Install]
+WantedBy=multi-user.target

src/oneprometheus/grafana/share/patch_dashboards.rb

@@ -0,0 +1,62 @@
+#!/usr/bin/env ruby
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+SELF = File.dirname File.realpath(__FILE__)
+
+def patch_dashboard(document)
+    templating_list = document.dig 'templating', 'list'
+    return document if templating_list.nil?
+
+    datasource = {
+        'hide'    => 0,
+        'label'   => 'datasource',
+        'name'    => 'DS_PROMETHEUS',
+        'options' => [],
+        'query'   => 'prometheus',
+        'refresh' => 1,
+        'regex'   => '',
+        'type'    => 'datasource'
+    }
+
+    # Make sure patching is idempotent.
+    return document if templating_list.find do
+        |item| item == datasource
+    end
+
+    templating_list.prepend datasource
+
+    return document
+end
+
+if caller.empty?
+    dirs = ARGV.empty? ? ["#{SELF}/dashboards/"] : ARGV
+    dirs.each do |dir|
+        Dir["#{dir}/*.json"].each do |path|
+            document = JSON.load File.read(path)
+            document = patch_dashboard document
+
+            FileUtils.cp path, "#{path}.#{Time.now.utc.to_i}.bak"
+
+            File.write path, JSON.pretty_generate(document)
+        end
+    end
+end

src/oneprometheus/node_exporter/systemd/opennebula-node-exporter.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=OpenNebula Prometheus Node Exporter
+
+[Service]
+User=oneadmin
+Group=oneadmin
+Type=simple
+
+ExecStart=/usr/bin/node_exporter
+
+[Install]
+WantedBy=multi-user.target

src/oneprometheus/opennebula-exporter/src/opennebula_collector.rb

@@ -0,0 +1,116 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'opennebula'
+require 'prometheus/client'
+
+require_relative 'opennebula_server_collector'
+require_relative 'opennebula_host_collector'
+require_relative 'opennebula_datastore_collector'
+require_relative 'opennebula_vm_collector'
+
+
+module Prometheus
+
+    # Patch base classes to clear internal stores each scrape
+    # this allows for some metrics to disappear between scrapes
+    # Concurrent scrapes are not supported
+    module Client
+        class Registry
+            def clear
+                @mutex.synchronize do
+                    @metrics.each_value { |m| m.clear if m }
+                end
+            end
+        end
+
+        class Metric
+            def clear
+                @store.clear
+            end
+        end
+
+        class DataStores::Synchronized
+            private
+
+            class MetricStore
+                def clear
+                    synchronize { @internal_store.clear }
+                end
+            end
+        end
+    end
+
+    module Middleware
+        # OpenNebulaCollector Rack middlware
+
+        # By default metrics are registered on the global registry. Set the
+        # `:registry` option to use a custom registry.
+
+        # By default metrics all have the namespace "opennebula". Set
+        # `:namespace` to something else if you like.
+        class OpenNebulaCollector
+            attr_reader :app, :registry
+
+            NAMESPACE = 'opennebula'
+
+            def initialize(app, options = {})
+                @app      = app
+                @registry = Client.registry
+                @client   = OpenNebula::Client.new
+                @co_mutex = Mutex.new
+
+                @collectors = []
+
+                @collectors << OpenNebulaServerCollector.new(
+                                      @registry, @client, NAMESPACE)
+                @collectors << OpenNebulaHostCollector.new(
+                                      @registry, @client, NAMESPACE)
+                @collectors << OpenNebulaDatastoreCollector.new(
+                                      @registry, @client, NAMESPACE)
+                @collectors << OpenNebulaVMCollector.new(
+                                      @registry, @client, NAMESPACE)
+            end
+
+            def call(env)
+                collect(env) { @app.call(env) }
+            end
+
+            protected
+
+            def collect(env)
+                response = yield
+
+                @co_mutex.synchronize do
+                    @registry.clear
+
+                    @collectors.each do |c|
+                        begin
+                            c.collect
+                        rescue StandardError
+                            nil
+                        end
+                    end
+                end
+
+                response
+            rescue StandardError
+                nil
+            end
+
+        end
+    end
+end

src/oneprometheus/opennebula-exporter/src/opennebula_datastore_collector.rb

@@ -0,0 +1,110 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'opennebula'
+
+class OpenNebulaDatastoreCollector
+
+    LABELS = %i[one_datastore_id]
+
+    # --------------------------------------------------------------------------
+    # Datastore metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_datastore_total
+    #   - opennebula_datastore_total_bytes
+    #   - opennebula_datastore_used_bytes
+    #   - opennebula_datastore_free_bytes
+    #   - opennebula_datastore_images
+    # --------------------------------------------------------------------------
+    DATASTORE_METRICS = {
+        'datastore_total' => {
+            :type   => :gauge,
+            :docstr => 'Total number of datastores defined in OpenNebula',
+            :labels => {}
+        },
+        'datastore_total_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Total capacity of the datastore',
+            :value  => ->(v) { Integer(v['TOTAL_MB']) * 1024 * 1024 },
+            :labels => LABELS
+        },
+        'datastore_used_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Capacity being used in the dastore',
+            :value  => ->(v) { Integer(v['USED_MB']) * 1024 * 1024 },
+            :labels => LABELS
+        },
+        'datastore_free_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Available capacity in the datastore',
+            :value  => ->(v) { Integer(v['FREE_MB']) * 1024 * 1024},
+            :labels => LABELS
+        },
+        'datastore_images' => {
+            :type   => :gauge,
+            :docstr => 'Number of images stored in the datastore',
+            :value  => ->(v) {
+                ids = v.retrieve_elements('IMAGES/ID')
+
+                if ids
+                    ids.size
+                else
+                    0
+                end
+            },
+            :labels => LABELS
+        }
+    }
+
+    def initialize(registry, client, namespace)
+        @client  = client
+        @metrics = {}
+
+        DATASTORE_METRICS.each do |name, conf|
+            @metrics[name] = registry.method(conf[:type]).call(
+                      "#{namespace}_#{name}".to_sym,
+                      :docstring => conf[:docstr],
+                      :labels    => conf[:labels])
+        end
+    end
+
+    def collect
+        ds_pool = OpenNebula::DatastorePool.new(@client)
+        rc      = ds_pool.info_all
+
+        raise rc.message if OpenNebula.is_error?(rc)
+
+        dss = ds_pool.retrieve_xmlelements('/DATASTORE_POOL/DATASTORE')
+
+        @metrics['datastore_total'].set(dss.length)
+
+        dss.each do |ds|
+            labels = { :one_datastore_id => Integer(ds['ID']) }
+
+            DATASTORE_METRICS.each do |name, conf|
+                next unless conf[:value]
+
+                metric = @metrics[name]
+                value  = conf[:value].call(ds)
+
+                next unless metric
+
+                metric.set(value, :labels => labels)
+            end
+        end
+    end
+end
+

src/oneprometheus/opennebula-exporter/src/opennebula_exporter.rb

@@ -0,0 +1,91 @@
+#!/usr/bin/ruby
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+ONE_LOCATION = ENV['ONE_LOCATION']
+
+if !ONE_LOCATION
+    LOG_LOCATION      = '/var/log/one'
+    VAR_LOCATION      = '/var/lib/one'
+    ETC_LOCATION      = '/etc/one'
+    SHARE_LOCATION    = '/usr/share/one'
+    RUBY_LIB_LOCATION = '/usr/lib/one/ruby'
+    GEMS_LOCATION     = '/usr/share/one/gems'
+else
+    VAR_LOCATION      = ONE_LOCATION + '/var'
+    LOG_LOCATION      = ONE_LOCATION + '/var'
+    ETC_LOCATION      = ONE_LOCATION + '/etc'
+    SHARE_LOCATION    = ONE_LOCATION + '/share'
+    RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby'
+    GEMS_LOCATION     = ONE_LOCATION + '/share/gems'
+end
+
+
+if File.directory?(GEMS_LOCATION)
+    $LOAD_PATH.reject! {|l| l =~ /vendor_ruby/ }
+    require 'rubygems'
+    Gem.use_paths(File.realpath(GEMS_LOCATION))
+
+    # for some platforms, we redistribute newer base Ruby gems which
+    # should be loaded instead of default ones in the distributions
+    %w[openssl json].each do |name|
+        begin
+            gem name
+        rescue LoadError
+            # ignore
+        end
+    end
+end
+
+$LOAD_PATH << RUBY_LIB_LOCATION
+$:.unshift File.dirname(__FILE__)
+
+
+##############################################################################
+# Required libraries
+##############################################################################
+require 'rubygems'
+require 'sinatra'
+
+require 'prometheus/middleware/exporter'
+require 'prometheus/middleware/collector'
+require 'opennebula_collector'
+
+
+use Rack::Deflater
+use Prometheus::Middleware::OpenNebulaCollector
+use Prometheus::Middleware::Exporter
+
+
+get '/' do
+    body = '<html>'\
+           '<head><title>OpenNebula Exporter</title></head>'\
+           '<body>'\
+           '<h1>OpenNebula Exporter</h1>'\
+           '<p><a href="/metrics">Metrics</a></p>'\
+           '</body>'\
+           '</html>'
+    [200, {'Content-Type' => 'text/html'}, body]
+end
+
+
+# Default Options
+set :port, 9925
+set :bind, '0.0.0.0'
+
+# Run the Sinatra application
+set :run, false
+Sinatra::Application.run!

src/oneprometheus/opennebula-exporter/src/opennebula_host_collector.rb

@@ -0,0 +1,139 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'opennebula'
+
+class OpenNebulaHostCollector
+
+    LABELS = %i[one_host_id]
+
+    # --------------------------------------------------------------------------
+    # Host metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_host_total
+    #   - opennebula_host_state
+    #   - opennebula_host_mem_total_bytes
+    #   - opennebula_host_mem_maximum_bytes
+    #   - opennebula_host_cpu_total_ratio
+    #   - opennebula_host_cpu_maximum_ratio
+    #   - opennebula_host_cpu_usage_ratio
+    #   - opennebula_host_vms
+    # --------------------------------------------------------------------------
+    HOST_METRICS = {
+        'host_total' => {
+            :type   => :gauge,
+            :docstr => 'Total number of hosts defined in OpenNebula',
+            :labels => {}
+        },
+        'host_state' => {
+            :type   => :gauge,
+            :docstr => 'Host state 0:init 2:monitored 3:error 4:disabled ' \
+                       '8:offline',
+            :value  => ->(v) { Integer(v['STATE']) },
+            :labels => LABELS
+        },
+        'host_mem_total_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Total memory capacity',
+            :value  => ->(v) { Integer(v['HOST_SHARE/TOTAL_MEM']) * 1024 },
+            :labels => LABELS
+        },
+        'host_mem_maximum_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Total memory capacity considering overcommitment',
+            :value  => ->(v) { Integer(v['HOST_SHARE/MAX_MEM']) * 1024 },
+            :labels => LABELS
+        },
+        'host_mem_usage_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Total memory capacity allocated to VMs',
+            :value  => ->(v) { Integer(v['HOST_SHARE/MEM_USAGE']) * 1024 },
+            :labels => LABELS
+        },
+        'host_cpu_total_ratio' => {
+            :type   => :gauge,
+            :docstr => 'Total CPU capacity',
+            :xpath  => 'HOST_SHARE/TOTAL_CPU',
+            :value  => ->(v) { Integer(v['HOST_SHARE/TOTAL_CPU']) },
+            :labels => LABELS
+        },
+        'host_cpu_maximum_ratio' => {
+            :type   => :gauge,
+            :docstr => 'Total CPU capacity considering overcommitment',
+            :value  => ->(v) { Integer(v['HOST_SHARE/MAX_CPU']) },
+            :labels => LABELS
+        },
+        'host_cpu_usage_ratio' => {
+            :type   => :gauge,
+            :docstr => 'Total CPU capacity allocated to VMs',
+            :value  => ->(v) { Integer(v['HOST_SHARE/CPU_USAGE']) },
+            :labels => LABELS
+        },
+        'host_vms' => {
+            :type   => :gauge,
+            :docstr => 'Number of VMs allocated to the host',
+            :value  => ->(v) {
+                ids = v.retrieve_elements('VMS/ID')
+
+                if ids
+                    ids.size
+                else
+                    0
+                end
+            },
+            :labels => LABELS
+        }
+    }
+
+    def initialize(registry, client, namespace)
+        @client  = client
+        @metrics = {}
+
+        HOST_METRICS.each do |name, conf|
+            @metrics[name] = registry.method(conf[:type]).call(
+                      "#{namespace}_#{name}".to_sym,
+                      :docstring => conf[:docstr],
+                      :labels    => conf[:labels])
+        end
+    end
+
+    def collect
+        host_pool = OpenNebula::HostPool.new(@client)
+        rc        = host_pool.info_all!
+
+        raise rc.message if OpenNebula.is_error?(rc)
+
+        hosts = host_pool.retrieve_xmlelements('/HOST_POOL/HOST')
+
+        @metrics['host_total'].set(hosts.length)
+
+        hosts.each do |host|
+            labels = { :one_host_id => Integer(host['ID']) }
+
+            HOST_METRICS.each do |name, conf|
+                next unless conf[:value]
+
+                metric = @metrics[name]
+                value  = conf[:value].call(host)
+
+                next unless metric
+
+                metric.set(value, :labels => labels)
+            end
+        end
+    end
+end
+

src/oneprometheus/opennebula-exporter/src/opennebula_server_collector.rb

@@ -0,0 +1,95 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'socket'
+
+class OpenNebulaServerCollector
+
+    FQDN = Addrinfo.getaddrinfo(Socket.gethostname, nil).first.getnameinfo.first
+
+    LABELS = %i[one_server_fqdn]
+
+    # --------------------------------------------------------------------------
+    # Server metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_server_state
+    # --------------------------------------------------------------------------
+    SERVER_METRICS = {
+        'oned_state' => {
+            :type   => :gauge,
+            :docstr => 'OpenNebula oned service state 0:down 1:up',
+            :labels => LABELS,
+            :systemd => 'opennebula.service'
+        },
+        'scheduler_state' => {
+            :type   => :gauge,
+            :docstr => 'OpenNebula scheduler service state 0:down 1:up',
+            :labels => LABELS,
+            :systemd => 'opennebula-scheduler.service'
+        },
+        'flow_state' => {
+            :type   => :gauge,
+            :docstr => 'OpenNebula Flow service state 0:down 1:up',
+            :labels => LABELS,
+            :systemd => 'opennebula-flow.service'
+        },
+        'hem_state' => {
+            :type   => :gauge,
+            :docstr => 'OpenNebula hook manager service state 0:down 1:up',
+            :labels => LABELS,
+            :systemd => 'opennebula-hem.service'
+        },
+        'gate_state' => {
+            :type   => :gauge,
+            :docstr => 'OpenNebula Gate service state 0:down 1:up',
+            :labels => LABELS,
+            :systemd => 'opennebula-gate.service'
+        }
+    }
+
+    def initialize(registry, client, namespace)
+        @client  = client
+        @metrics = {}
+
+        SERVER_METRICS.each do |name, conf|
+            @metrics[name] = registry.method(conf[:type]).call(
+                      "#{namespace}_#{name}".to_sym,
+                      :docstring => conf[:docstr],
+                      :labels    => conf[:labels])
+        end
+    end
+
+    def collect
+        SERVER_METRICS.each do |name, conf|
+            @metrics[name].set(
+                is_active(conf[:systemd]),
+                :labels => { :one_server_fqdn => FQDN }
+            )
+        end
+    end
+
+    private
+
+    def is_active(service)
+        if `systemctl show --value -p ActiveState #{service}`.strip == 'active'
+            1
+        else
+            0
+        end
+    rescue
+        0
+    end
+end

src/oneprometheus/opennebula-exporter/src/opennebula_vm_collector.rb

@@ -0,0 +1,191 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'opennebula'
+
+class OpenNebulaVMCollector
+
+    LABELS = %i[one_vm_id]
+
+    # --------------------------------------------------------------------------
+    # VM metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_vm_total
+    #   - opennebula_vm_state
+    #   - opennebula_vm_lcm_state
+    #   - opennebula_vm_mem_total_bytes
+    #   - opennebula_vm_cpu_ratio
+    #   - opennebula_vm_cpu_vcpus
+    #   - opennebula_vm_disks
+    #   - opennebula_vm_disk_size_bytes
+    #   - opennebula_vm_nics
+    # --------------------------------------------------------------------------
+    VM_METRICS = {
+        'vm_total' => {
+            :type   => :gauge,
+            :docstr => 'Total number of VMs defined in OpenNebula',
+            :labels => {}
+        },
+        'vm_host_id' => {
+            :type   => :gauge,
+            :docstr => 'Host ID where the VM is allocated',
+            :value  => ->(v) {
+                hid = v['HISTORY_RECORDS/HISTORY[last()]/HID']
+                
+                if !hid || hid.empty?
+                    -1
+                else
+                    Integer(hid)
+                end
+            },
+            :labels => LABELS
+        },
+        'vm_state' => {
+            :type   => :gauge,
+            :docstr => 'VM state 0:init 1:pending 2:hold 3:active ' \
+                       '4:stopped 5:suspended 6:done 8:poweroff ' \
+                       '9:undeployed 10:clonning',
+            :value  => ->(v) { Integer(v['STATE']) },
+            :labels => LABELS
+        },
+        'vm_lcm_state' => {
+            :type   => :gauge,
+            :docstr => 'VM LCM state, only relevant for state 3 (active)',
+            :value  => ->(v) { Integer(v['LCM_STATE']) },
+            :labels => LABELS
+        },
+        'vm_mem_total_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Total memory capacity',
+            :value  => ->(v) { Integer(v['TEMPLATE/MEMORY']) * 1024 },
+            :labels => LABELS
+        },
+        'vm_cpu_ratio' => {
+            :type   => :gauge,
+            :docstr => 'Total CPU capacity requested by the VM',
+            :value  => ->(v) { Float(v['TEMPLATE/CPU']) },
+            :labels => LABELS
+        },
+        'vm_cpu_vcpus' => {
+            :type   => :gauge,
+            :docstr => 'Total number of virtual CPUs',
+            :value  => ->(v) {
+                vcpus = v['TEMPLATE/VCPU']
+
+                if !vcpus || vcpus.empty?
+                    1
+                else
+                    Integer(vcpus)
+                end
+            },
+            :labels => LABELS
+        }
+    }
+
+    DISK_METRICS = {
+        'vm_disks' => {
+            :type   => :gauge,
+            :docstr => 'Total number of disks',
+            :labels => LABELS
+        },
+        'vm_disk_size_bytes' => {
+            :type   => :gauge,
+            :docstr => 'Size of the VM disk',
+            :value  => ->(i, v) { Integer(v["TEMPLATE/DISK [ DISK_ID = #{i} ]/SIZE"]) * 1024 * 1024 },
+            :labels => LABELS + %i[ disk_id ]
+        }
+    }
+
+    NIC_METRICS = {
+        'vm_nics' => {
+            :type   => :gauge,
+            :docstr => 'Total number of network interfaces',
+            :labels => LABELS
+        }
+    }
+
+    def initialize(registry, client, namespace)
+        @client  = client
+        @metrics = {}
+
+        [VM_METRICS, DISK_METRICS, NIC_METRICS].each do |m|
+            m.each do |name, conf|
+                @metrics[name] = registry.method(conf[:type]).call(
+                          "#{namespace}_#{name}".to_sym,
+                          :docstring => conf[:docstr],
+                          :labels    => conf[:labels])
+            end
+        end
+    end
+
+    def collect
+        vm_pool = OpenNebula::VirtualMachinePool.new(@client)
+        rc      = vm_pool.info_all
+
+        raise rc.message if OpenNebula.is_error?(rc)
+
+        vms = vm_pool.retrieve_xmlelements('/VM_POOL/VM')
+
+        @metrics['vm_total'].set(vms.length)
+
+        vms.each do |vm|
+            labels = { :one_vm_id => Integer(vm['ID']) }
+
+            VM_METRICS.each do |name, conf|
+                next unless conf[:value]
+
+                metric = @metrics[name]
+                value  = conf[:value].call(vm)
+
+                next unless metric
+
+                metric.set(value, :labels => labels)
+            end
+
+            disks = vm.retrieve_elements('TEMPLATE/DISK/DISK_ID')
+
+            @metrics['vm_disks'].set(disks.length, :labels => labels)
+
+            labels_disk = labels.clone
+
+            disks.each do |id|
+                labels_disk[:disk_id] = id
+
+                DISK_METRICS.each do |name, conf|
+                    next unless conf[:value]
+
+                    metric = @metrics[name]
+                    value  = conf[:value].call(id, vm)
+
+                    next unless metric
+
+                    metric.set(value, :labels => labels_disk)
+                end
+            end
+
+            nics  = vm.retrieve_elements('TEMPLATE/NIC/NIC_ID')
+            pnics = vm.retrieve_elements('TEMPLATE/PCI/NIC_ID')
+
+            tnics = 0
+
+            tnics = tnics + nics.length if nics
+            tnics = tnics + pnics.length if pnics
+
+            @metrics['vm_nics'].set(tnics, :labels => labels)
+        end
+    end
+end
+

src/oneprometheus/opennebula-exporter/systemd/opennebula-exporter.service

@@ -0,0 +1,12 @@
+[Unit]
+Description=OpenNebula Prometheus Exporter
+
+[Service]
+User=oneadmin
+Group=oneadmin
+Type=simple
+
+ExecStart=/usr/bin/ruby /usr/lib/one/opennebula_exporter/opennebula_exporter.rb
+
+[Install]
+WantedBy=multi-user.target

src/oneprometheus/opennebula-libvirt-exporter/src/libvirt_collector.rb

@@ -0,0 +1,663 @@
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+require 'open3'
+require 'prometheus/client'
+require 'rexml/document'
+
+ENV['LANG']   = 'C'
+ENV['LC_ALL'] = 'C'
+
+#-------------------------------------------------------------------------------
+#  Libvirt Module. This module provides basic functionality to execute
+#  virsh commands
+#-------------------------------------------------------------------------------
+module Libvirt
+
+    # Default URI to connect to libvirt daemon
+    LIBVIRT_URI = 'qemu:///system'
+
+    # Constants for virsh commands
+    COMMANDS = {
+        :dominfo  => "virsh --connect #{LIBVIRT_URI} --readonly dominfo",
+        :domstate => "virsh --connect #{LIBVIRT_URI} --readonly domstate",
+        :list     => "virsh --connect #{LIBVIRT_URI} --readonly list",
+        :dumpxml  => "virsh --connect #{LIBVIRT_URI} --readonly dumpxml",
+        :domstats => "virsh --connect #{LIBVIRT_URI} --readonly domstats"
+    }
+
+    # @param command [Symbol] as defined in the module CONF constant
+    def self.virsh(command, arguments)
+        Open3.capture3("#{COMMANDS[command]} #{arguments}")
+    end
+
+end
+
+#-------------------------------------------------------------------------------
+#  Libvirt Module. This module provides basic functionality to execute
+#  virsh commands
+#
+#  [1] https://libvirt.org/html/libvirt-libvirt-domain.html#virConnectGetAllDomainStats
+#-------------------------------------------------------------------------------
+class LibvirtDomain
+
+    attr_reader :domain
+
+    # --------------------------------------------------------------------------
+    # Instance variables that store domain statitics for different areas. Each
+    # stat may refer to single components (e.g. cpu):
+    #  @cpu = {
+    #    "time"   => "99196706981502",
+    #    "user"   => "85358250000000",
+    #    "system" => "12549140000000"
+    #  }
+    #
+    # or multiple devices (e.g. disk block devices) indexed by ID
+    #  @block = {
+    #    "count" => "2",
+    #    "0" => {
+    #      "name"      => "vda",
+    #      "rd_reqs"   => "207979",
+    #      "rd_bytes"  => "8762597888",
+    #       ...
+    #       "physical" => "4921245696"
+    #    },
+    #    "1" => {
+    #       "name"     => "hda",
+    #       "rd_reqs"  => "317",
+    #       "rd_bytes" => "1145692",
+    #       ...
+    #
+    # Metadata for OpenNebula VMs are availbale through @one
+    #  @one = {
+    #    :name             => "ubuntu2204-func-6-4-2-072c-0.test",
+    #    :system_datastore => "/var/lib/one//datastores/0/161",
+    #    :uname =>"oneadmin",
+    #    :uid   =>"0",
+    #    :gname =>"oneadmin",
+    #    :gid   =>"0",
+    #    :opennebula_version =>"6.4.1",
+    #    :stime           =>"1664446859",
+    #    :deployment_time =>"1665072011"
+    #  }
+    # --------------------------------------------------------------------------
+    STATS = [:state, :cpu, :balloon, :vcpu, :net, :block]
+
+    def initialize(domain, stats_s)
+        @labels = {
+            :domain    => domain,
+            :one_vm_id => domain.split('-')[1]
+        }
+
+        stats_s.each_line do |l|
+            next if l.empty?
+
+            name, value = l.split('=')
+
+            next if name.nil? || name.empty? || value.nil? || value.empty?
+
+            parts = name.split('.')
+            sname = "@#{parts[0].strip}"
+
+            instance_variable_set(sname, {}) unless instance_variable_defined?(sname)
+
+            stat = instance_variable_get(sname)
+
+            if parts.size == 2 || (parts.size > 2 && !parts[1].match(/[0-9]+/))
+                index = 1
+            else
+                index = 2
+                stat[parts[1]] ||= {}
+
+                stat = stat[parts[1]]
+            end
+
+            metric = parts[index..-1].join('_').gsub('-', '_')
+
+            stat[metric] = value.chop
+        end
+
+        @one = {}
+
+        return unless ENV['ONE_PROMETHEUS_VM_NAMES']
+
+        # Add Domain metadata
+        out, _err, rc = Libvirt.virsh(:dumpxml, "#{domain}")
+
+        begin
+            if rc.success?
+                doc = REXML::Document.new(out)
+
+                @one[:name] = doc.elements['domain/name'].text
+
+                doc.elements.each('domain/metadata/one:vm/*') do |elem|
+                    @one[elem.name.to_sym] = elem.text
+                end
+            end
+        rescue StandardError
+        end
+
+        # metadata labels
+        @labels[:one_vm_name] = ''
+        @labels[:one_vm_name] = @one[:name] if @one[:name]
+    end
+
+    # --------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
+    def set_metrics(metrics)
+        STATS.each do |stat_class|
+            next unless instance_variable_defined?("@#{stat_class}")
+
+            stat     = instance_variable_get("@#{stat_class}")
+            lmetrics = LibvirtCollector.const_get("#{stat_class.upcase}_METRICS")
+
+            stat.each do |stat_name, value|
+                if stat_name.match(/[0-9]+/)
+                    value.each do |sstat_name, svalue|
+                        mname  = "#{stat_class}.#{sstat_name}"
+                        metric = metrics[mname]
+                        conf   = lmetrics[mname]
+
+                        gauge_set(metric, conf, stat_name, svalue, value)
+                    end
+                else
+                    mname  = "#{stat_class}.#{stat_name}"
+                    metric = metrics[mname]
+                    conf   = lmetrics[mname]
+
+                    gauge_set(metric, conf, stat_name, value, stat)
+                end
+            end
+        end
+    end
+
+    private
+
+    # --------------------------------------------------------------------------
+    # --------------------------------------------------------------------------
+    def gauge_set(metric, conf, name, value, stat)
+        return if conf.nil? || metric.nil?
+
+        if conf[:render]
+            mvalue = conf[:render].call(value)
+        else
+            mvalue = Integer(value)
+        end
+
+        if conf[:render_labels]
+            mlabels = @labels.merge(conf[:render_labels].call(name, stat))
+        else
+            mlabels = @labels
+        end
+
+        metric.set(mvalue, :labels => mlabels)
+    end
+
+end
+
+# ------------------------------------------------------------------------------
+# TBD
+#
+#  Gauge is used as metric type as it can go back to 0 when the domain resets
+# ------------------------------------------------------------------------------
+module Prometheus
+    # Patch base classes to clear internal stores each scrape
+    # this allows for some metrics to disappear between scrapes
+    # Concurrent scrapes are not supported
+    module Client
+        class Registry
+            def clear
+                @mutex.synchronize do
+                    @metrics.each_value { |m| m.clear if m }
+                end
+            end
+        end
+
+        class Metric
+            def clear
+                @store.clear
+            end
+        end
+
+        class DataStores::Synchronized
+            private
+
+            class MetricStore
+                def clear
+                    synchronize { @internal_store.clear }
+                end
+            end
+        end
+    end
+end
+
+class LibvirtCollector < Prometheus::Middleware::Collector
+
+    include Libvirt
+
+    # Labels that be added to every metric
+    #   - domain: the libvirt domain ID
+    #   - one_vm_id: the opennebula associated VM ID.
+    #   - one_vm_name: the opennebula NAME attribute of the VM
+    LIBVIRT_LABELS = [:domain, :one_vm_id]
+    LIBVIRT_LABELS << :one_vm_name if ENV['ONE_PROMETHEUS_VM_NAMES']
+
+    # Metrics are named within this namesapce
+    NAMESPACE = 'opennebula_libvirt'
+
+    # --------------------------------------------------------------------------
+    # STATE metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_state
+    STATE_METRICS = {
+        'state.state' => {
+            :name   => 'state',
+            :type   => :gauge,
+            :docstr => 'State of the domain 0:no_state, 1:running, 2:blocked, ' \
+                       '3:paused, 4:shutdown, 5:shutoff, 6:chrased, 7:suspended (PM)',
+            :labels => LIBVIRT_LABELS + [:reason],
+            :render_labels => ->(_i, stat) { { :reason => stat['reason'] } }
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    # CPU metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_cpu_seconds_total
+    #   - opennebula_libvirt_cpu_system_seconds_total
+    #   - opennebula_libvirt_cpu_user_seconds_total
+    # --------------------------------------------------------------------------
+    CPU_METRICS = {
+        'cpu.time' => {
+            :name   => 'cpu_seconds_total',
+            :type   => :gauge,
+            :docstr => 'Total CPU time used by the domain',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS
+        },
+        'cpu.system' => {
+            :name   => 'cpu_system_seconds_total',
+            :type   => :gauge,
+            :docstr => 'System CPU time used by the domain',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS
+        },
+        'cpu.user' => {
+            :name   => 'cpu_user_seconds_total',
+            :type   => :gauge,
+            :docstr => 'User CPU time used by the domain',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    # Memory metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_memory_total_bytes
+    #   - opennebula_libvirt_memory_maximum_bytes
+    #   - opennebula_libvirt_memory_swapin_bytes_total
+    #   - opennebula_libvirt_memory_swapout_bytes_total
+    #   - opennebula_libvirt_memory_unsed_bytes
+    #   - opennebula_libvirt_memory_available_bytes
+    #   - opennebula_libvirt_memory_rss_bytes
+    # --------------------------------------------------------------------------
+    BALLOON_METRICS = {
+        'balloon.current' => {
+            :name   => 'memory_total_bytes',
+            :type   => :gauge,
+            :docstr => 'Total memory currently used by the domain',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.maximum' => {
+            :name   => 'memory_maximum_bytes',
+            :type   => :gauge,
+            :docstr => 'Total memory currently used by the domain',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.swap_in' => {
+            :name   => 'memory_swapin_bytes_total',
+            :type   => :gauge,
+            :docstr => 'Amount of data read from swap space',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.swap_out' => {
+            :name   => 'memory_swapout_bytes_total',
+            :type   => :gauge,
+            :docstr => 'Amount of data written out to swap space',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.unused' => {
+            :name   => 'memory_unsed_bytes',
+            :type   => :gauge,
+            :docstr => 'Amount of memory left unused by the system',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.available' => {
+            :name   => 'memory_available_bytes',
+            :type   => :gauge,
+            :docstr => 'Amount of usable memory as seen by the domain',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        },
+        'balloon.rss' => {
+            :name   => 'memory_rss_bytes',
+            :type   => :gauge,
+            :docstr => 'Resident Set Size of running domain\'s process',
+            :render => ->(v) { Integer(v) * 1024 },
+            :labels => LIBVIRT_LABELS
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    # Virtual CPU metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_vcpu_online
+    #   - opennebula_libvirt_vcpu_maximum
+    #   - opennebula_libvirt_vcpu_state
+    #   - opennebula_libvirt_vcpu_time_seconds_total
+    #   - opennebula_libvirt_vcpu_wait_seconds_total
+    # --------------------------------------------------------------------------
+    VCPU_METRICS = {
+        'vcpu.current' => {
+            :name   => 'vcpu_online',
+            :type   => :gauge,
+            :docstr => 'Current number of online virtual CPUs',
+            :labels => LIBVIRT_LABELS
+        },
+        'vcpu.maximum' => {
+            :name   => 'vcpu_maximum',
+            :type   => :gauge,
+            :docstr => 'Maximum number of online virtual CPUs',
+            :labels => LIBVIRT_LABELS
+        },
+        'vcpu.state' => {
+            :name   => 'vcpu_state',
+            :type   => :gauge,
+            :docstr => 'State of the virtual CPU 0:offline, 1:running, 2:blocked',
+            :labels => LIBVIRT_LABELS + [:vcpu],
+            :render_labels => ->(i, _stat) { { :vcpu => i } }
+        },
+        'vcpu.time' => {
+            :name   => 'vcpu_time_seconds_total',
+            :type   => :gauge,
+            :docstr => 'vitual cpu time spent by virtual CPU',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS + [:vcpu],
+            :render_labels => ->(i, _stat) { { :vcpu => i } }
+        },
+        'vcpu.wait' => {
+            :name   => 'vcpu_wait_seconds_total',
+            :type   => :gauge,
+            :docstr => 'Time the vCPU wants to run, but the host scheduler' \
+                       ' has something else running ahead of it',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS + [:vcpu],
+            :render_labels => ->(i, _stat) { { :vcpu => i } }
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    # Network interface metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_net_vnics
+    #   - opennebula_libvirt_net_rx_total_bytes
+    #   - opennebula_libvirt_net_rx_packets
+    #   - opennebula_libvirt_net_rx_errors
+    #   - opennebula_libvirt_net_rx_drops
+    #   - opennebula_libvirt_net_tx_total_bytes
+    #   - opennebula_libvirt_net_tx_packets
+    #   - opennebula_libvirt_net_tx_errors
+    #   - opennebula_libvirt_net_tx_drops
+    # --------------------------------------------------------------------------
+    NET_METRICS = {
+        'net.count' => {
+            :name   => 'net_devices',
+            :type   => :gauge,
+            :docstr => 'Total number of network interfaces on this domain',
+            :labels => LIBVIRT_LABELS,
+        },
+        'net.rx_bytes' => {
+            :name   => 'net_rx_total_bytes',
+            :type   => :gauge,
+            :docstr => 'Total bytes received by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.rx_pkts' => {
+            :name   => 'net_rx_packets',
+            :type   => :gauge,
+            :docstr => 'Total number of packets received by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.rx_errs' => {
+            :name   => 'net_rx_errors',
+            :type   => :gauge,
+            :docstr => 'Total number of receive errors',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.rx_drop' => {
+            :name   => 'net_rx_drops',
+            :type   => :gauge,
+            :docstr => 'Total number of receive packets dropped by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.tx_bytes' => {
+            :name   => 'net_tx_total_bytes',
+            :type   => :gauge,
+            :docstr => 'Total bytes transmitted by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.tx_pkts' => {
+            :name   => 'net_tx_packets',
+            :type   => :gauge,
+            :docstr => 'Total number of packets transmitted by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.tx_errs' => {
+            :name   => 'net_tx_errors',
+            :type   => :gauge,
+            :docstr => 'Total number of transmission errors',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        },
+        'net.tx_drop' => {
+            :name   => 'net_tx_drops',
+            :type   => :gauge,
+            :docstr => 'Total number of transmit packets dropped by the vNIC',
+            :labels => LIBVIRT_LABELS + [:vnic, :device],
+            :render_labels => ->(i, stat) { { :vnic => i, :device => stat['name'] } }
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    # Block device metrics
+    # --------------------------------------------------------------------------
+    #   - opennebula_libvirt_block_devices
+    #   - opennebula_libvirt_block_rd_requests
+    #   - opennebula_libvirt_block_rd_bytes
+    #   - opennebula_libvirt_block_rd_time_seconds
+    #   - opennebula_libvirt_block_wr_requests
+    #   - opennebula_libvirt_block_wr_bytes
+    #   - opennebula_libvirt_block_wr_time_seconds
+    #   - opennebula_libvirt_block_virtual_bytes
+    #   - opennebula_libvirt_block_physical_bytes
+    # --------------------------------------------------------------------------
+    BLOCK_METRICS = {
+        'block.count' => {
+            :name   => 'block_devices',
+            :type   => :gauge,
+            :docstr => 'Total number of block devices on this domain',
+            :labels => LIBVIRT_LABELS,
+        },
+        'block.rd_reqs' => {
+            :name   => 'block_rd_requests',
+            :type   => :gauge,
+            :docstr => 'Total number of read requests',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.rd_bytes' => {
+            :name   => 'block_rd_bytes',
+            :type   => :gauge,
+            :docstr => 'Total number of read bytes',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.rd_times' => {
+            :name   => 'block_rd_time_seconds',
+            :type   => :gauge,
+            :docstr => 'Total time spent on reads',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.wr_reqs' => {
+            :name   => 'block_wr_requests',
+            :type   => :gauge,
+            :docstr => 'Total number of write requests',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.wr_bytes' => {
+            :name   => 'block_wr_bytes',
+            :type   => :gauge,
+            :docstr => 'Total number of written bytes',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.wr_times' => {
+            :name   => 'block_wr_time_seconds',
+            :type   => :gauge,
+            :docstr => 'Total time spent on writes',
+            :render => ->(v) { Integer(v) / 10**9 },
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.capacity' => {
+            :name   => 'block_virtual_bytes',
+            :type   => :gauge,
+            :docstr => 'Virtual size of the device',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        },
+        'block.physical' => {
+            :name   => 'block_physical_bytes',
+            :type   => :gauge,
+            :docstr => 'Physical size of the container of the backing image',
+            :labels => LIBVIRT_LABELS + [:disk, :device],
+            :render_labels => ->(i, stat) { { :disk => i, :device => stat['name'] } }
+        }
+    }
+
+    # --------------------------------------------------------------------------
+    ALL_METRICS = [
+        STATE_METRICS,
+        CPU_METRICS,
+        BALLOON_METRICS,
+        VCPU_METRICS,
+        NET_METRICS,
+        BLOCK_METRICS
+    ]
+    # --------------------------------------------------------------------------
+
+    def initialize(app)
+        # Initialize registry and metrics
+        super(app, :metrics_prefix => NAMESPACE)
+
+        @libvirt_metrics = {}
+        @rec_mutex       = Mutex.new
+
+        # Define all metrics in the Promethues client registry
+        ALL_METRICS.each do |m|
+            m.each do |name, conf|
+                @libvirt_metrics[name] = @registry.method(conf[:type]).call(
+                    "#{NAMESPACE}_#{conf[:name]}".to_sym,
+                    :docstring => conf[:docstr],
+                    :labels    => conf[:labels]
+                )
+            end
+        end
+
+        # Libvirt globael state
+        @libvirt_state_metric = @registry.gauge(
+            "#{NAMESPACE}_daemon_up".to_sym,
+            :docstring => 'State of the libvirt daemon 0:down 1:up'
+        )
+    end
+
+    def record(env, code, duration)
+        @rec_mutex.synchronize { single_record(env, code, duration) }
+    end
+
+    private
+
+    def single_record(env, code, duration)
+        @registry.clear
+
+        #super(env, code, duration)
+        Prometheus::Middleware::Collector.instance_method(:record).bind(self).call(env, code, duration)
+
+        out, _err, rc = Libvirt.virsh(:domstats, '')
+
+        if !rc.success?
+            @libvirt_state_metric.set(0)
+            return
+        end
+
+        @libvirt_state_metric.set(1)
+
+        domain = { :id => '', :txt => '' }
+
+        set_metrics = lambda do
+            ld = LibvirtDomain.new(domain[:id], domain[:txt])
+            ld.set_metrics(@libvirt_metrics)
+        end
+
+        out.each_line do |line|
+            m = line.match(/Domain: '([^']*)'/)
+
+            if m
+                if domain[:txt].empty?
+                    domain[:id] = m[1]
+                else
+                    set_metrics.call
+
+                    domain[:txt] = ''
+                    domain[:id]  = m[1]
+                end
+            else
+                domain[:txt] << line
+            end
+        end
+
+        # Make sure to handle the last domain!
+        set_metrics.call
+    rescue StandardError
+        nil
+    end
+
+end

src/oneprometheus/opennebula-libvirt-exporter/src/libvirt_exporter.rb

@@ -0,0 +1,79 @@
+#!/usr/bin/env ruby
+
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+#--------------------------------------------------------------------------- #
+
+ONE_LOCATION = ENV['ONE_LOCATION']
+
+if ONE_LOCATION.nil?
+    RUBY_LIB_LOCATION = '/usr/lib/one/ruby/'
+    GEMS_LOCATION     = '/usr/share/one/gems/'
+else
+    RUBY_LIB_LOCATION = ONE_LOCATION + '/lib/ruby/'
+    GEMS_LOCATION     = ONE_LOCATION + '/share/gems/'
+end
+
+# %%RUBYGEMS_SETUP_BEGIN%%
+if File.directory?(GEMS_LOCATION)
+    real_gems_path = File.realpath(GEMS_LOCATION)
+    if !defined?(Gem) || Gem.path != [real_gems_path]
+        $LOAD_PATH.reject! {|p| p =~ /vendor_ruby/ }
+
+        # Suppress warnings from Rubygems
+        # https://github.com/OpenNebula/one/issues/5379
+        begin
+            verb = $VERBOSE
+            $VERBOSE = nil
+            require 'rubygems'
+            Gem.use_paths(real_gems_path)
+        ensure
+            $VERBOSE = verb
+        end
+    end
+end
+# %%RUBYGEMS_SETUP_END%%
+
+$LOAD_PATH << RUBY_LIB_LOCATION
+
+require 'sinatra'
+
+require 'prometheus/middleware/exporter'
+require 'prometheus/middleware/collector'
+
+require_relative './libvirt_collector'
+
+use Rack::Deflater
+use LibvirtCollector
+use Prometheus::Middleware::Exporter
+
+get '/' do
+    body = '<html>'\
+           '<head><title>OpenNebula Libvirt Exporter</title></head>'\
+           '<body>'\
+           '<h1>OpenNebula Libvirt Exporter</h1>'\
+           '<p><a href="/metrics">Metrics</a></p>'\
+           '</body>'\
+           '</html>'
+    [200, { 'Content-Type' => 'text/html' }, body]
+end
+
+# Default Options
+set :bind, '0.0.0.0'
+set :port, 9926
+
+# Run the Sinatra application
+set :run, false
+Sinatra::Application.run!

src/oneprometheus/opennebula-libvirt-exporter/systemd/opennebula-libvirt-exporter.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=OpenNebula Prometheus Libvirt Exporter
+
+[Service]
+User=oneadmin
+Group=oneadmin
+Type=simple
+
+# Comment out the following line if you don't need VM name labels
+Environment=ONE_PROMETHEUS_VM_NAMES=
+
+ExecStart=/usr/bin/ruby /usr/lib/one/libvirt_exporter/libvirt_exporter.rb
+
+[Install]
+WantedBy=multi-user.target

src/oneprometheus/prometheus/etc/prometheus.yml

@@ -0,0 +1,25 @@
+global:
+  scrape_interval:     15s # Set the scrape interval to every 15 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to the global default (10s).
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+  - static_configs:
+    - targets: ['localhost:9093']
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files: ['rules.yml']
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: prometheus
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+    - targets: ['localhost:9090']

src/oneprometheus/prometheus/etc/rules.yml

@@ -0,0 +1,102 @@
+groups:
+- name: AllInstances
+  rules:
+  - alert: InstanceDown
+    expr: up == 0
+    for: 30s
+    annotations:
+      title: 'Instance {{ $labels.instance }} down'
+      description: '{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 30 seconds'
+    labels: { severity: critical }
+
+  - alert: DiskFree
+    expr: (node_filesystem_avail_bytes{mountpoint="/",fstype!="rootfs"} / node_filesystem_size_bytes{mountpoint="/",fstype!="rootfs"} * 100) <= 10
+    for: 30s
+    annotations:
+      title: 'Instance {{ $labels.instance }} has less than 10% of free space in rootfs'
+    labels: { severity: warning }
+
+  - alert: FreeMemory10
+    expr: ((node_memory_MemFree_bytes * 100) / node_memory_MemTotal_bytes) <= 10
+    for: 30s
+    annotations:
+      title: 'Instance {{ $labels.instance }} has less than 10% of free memory'
+    labels: { severity: warning }
+
+  - alert: LoadAverage15
+    expr: node_load15 > count without(cpu, mode) (node_cpu_seconds_total{mode="idle"}) * 0.8
+    for: 30s
+    annotations:
+      title: 'Instance {{ $labels.instance }} has higher load average then 80% CPU count in last 15 minutes'
+    labels: { severity: warning }
+
+  - alert: RebootInLast5Minutes
+    expr: rate(node_boot_time_seconds[5m]) > 0
+    annotations:
+      title: 'Instance {{ $labels.instance }} has has been rebooted in last 5 minutes'
+    labels: { severity: warning }
+
+- name: OpenNebulaHosts
+  rules:
+  - alert: HostDown
+    expr: opennebula_host_state != 2
+    for: 30s
+    annotations:
+      title: "OpenNebula host {{ $labels.one_host_id }} down"
+      description: "OpenNebula host {{ $labels.one_host_id }} of job {{ $labels.job }} has been down for more than 30 seconds"
+    labels: { severity: critical }
+
+  - alert: LibvirtDown
+    expr: opennebula_libvirt_daemon_up == 0
+    for: 30s
+    annotations:
+      title: "Libvirt daemon on host {{ $labels.one_host_id }} down"
+      description: "Libvirt daemon on host {{ $labels.one_host_id }} of job {{ $labels.job }} has been down for more than 30 seconds"
+    labels: { severity: critical }
+
+
+- name: OpenNebulaVirtualMachines
+  rules:
+  - alert: VMFailed
+    expr: >
+      count((opennebula_vm_lcm_state >= 36 and opennebula_vm_lcm_state <=42) or
+        opennebula_vm_lcm_state == 44 or
+        (opennebula_vm_lcm_state >= 46 and opennebula_vm_lcm_state <=50) or
+        opennebula_vm_lcm_state == 61) by (instance, name) > 0
+    for: 30s
+    annotations:
+      title: "OpenNebula VMs in failed state"
+      description: "OpenNebula VMs of job {{ $labels.job }} are in failed state for more than 30 seconds"
+    labels: { severity: critical }
+
+  - alert: VMPending
+    expr: count(opennebula_vm_state == 1) by (instance, name) > 0
+    for: 300s
+    annotations:
+      title: "OpenNebula VMs in pending"
+      description: "OpenNebula VMs of job {{ $labels.job }} are in pending state for more than 300 seconds"
+    labels: { severity: critical }
+
+- name: OpenNebulaServices
+  rules:
+  - alert: OnedDown
+    expr: opennebula_oned_state == 0
+    for: 30s
+    annotations:
+      title: "OpenNebula oned {{ $labels.one_server_fqdn }} down"
+      description: "OpenNebula oned {{ $labels.one_server_fqdn }} of job {{ $labels.job }} has been down for more than 30 seconds"
+    labels: { severity: critical }
+  - alert: SchedulerDown
+    expr: opennebula_scheduler_state == 0
+    for: 30s
+    annotations:
+      title: "OpenNebula scheduler {{ $labels.one_server_fqdn }} down"
+      description: "OpenNebula scheduler {{ $labels.one_server_fqdn }} of job {{ $labels.job }} has been down for more than 30 seconds"
+    labels: { severity: critical }
+  - alert: HookManagerDown
+    expr: opennebula_hem_state == 0
+    for: 30s
+    annotations:
+      title: "OpenNebula hook manager {{ $labels.one_server_fqdn }} down"
+      description: "OpenNebula hook manager {{ $labels.one_server_fqdn }} of job {{ $labels.job }} has been down for more than 30 seconds"
+    labels: { severity: critical }

src/oneprometheus/prometheus/share/metrics_to_md.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+
+metrics = {}
+
+STDIN.each_line do |l|
+  h = l.match(/# HELP ([^\s]*) (.*)$/)
+
+  if h
+      metrics[h[1]] ||= {}
+      metrics[h[1]][:help] = h[2]
+      next
+  end
+
+  t = l.match(/# TYPE ([^\s]*) (.*)$/)
+
+  if t
+      metrics[t[1]] ||= {}
+      metrics[t[1]][:type] = t[2]
+  end
+end
+
+puts " Name | Description | Type "
+puts "----- | ----------- | -----"
+
+metrics.each do |k,v|
+   puts "#{k} | #{v[:help]} | #{v[:type]}"
+end

src/oneprometheus/prometheus/share/patch_datasources.rb

@@ -0,0 +1,221 @@
+#!/usr/bin/env ruby
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+# frozen_string_literal: true
+
+ONE_LOCATION = ENV['ONE_LOCATION']
+
+if ONE_LOCATION.nil?
+    ONEPROMETHEUS_ETC_LOCATION = '/etc/one/'
+else
+    ONEPROMETHEUS_ETC_LOCATION = ONE_LOCATION + '/etc/'
+end
+
+require 'optparse'
+require 'fileutils'
+require 'socket'
+require 'uri'
+require 'yaml'
+require 'resolv'
+require 'ipaddr'
+
+LOCAL_IPS = Socket.ip_address_list.map { |ip| ip.ip_address }
+
+def list_to_dict(list, key: 'job_name')
+    list.each_with_object({}) do |item, dict|
+        dict[item[key]] = item
+    end
+end
+
+def dict_to_list(dict)
+    dict.each_with_object([]) do |item, list|
+        list << item[1]
+    end
+end
+
+def file(path, content, mode: 'u=rw,go=r',
+         overwrite: false,
+         backup: false)
+    return if !overwrite && File.exist?(path)
+
+    if content.nil?
+        FileUtils.mkdir_p path
+    else
+        if overwrite && backup && File.exist?(path)
+            FileUtils.cp path, "#{path}.#{Time.now.utc.to_i}.bak"
+        end
+
+        FileUtils.mkdir_p File.dirname path
+        File.write path, content
+    end
+
+    begin
+        FileUtils.chmod mode, path
+    rescue StandardError
+        nil
+    end
+end
+
+def onezone_show(zone_name_or_id = 'OpenNebula')
+    output = `onezone show '#{zone_name_or_id}' --yaml`
+    result = $?.to_i
+
+    exit(-1) if result != 0
+
+    YAML.safe_load output
+end
+
+def detect_servers(zone_name_or_id = 'OpenNebula')
+    servers = onezone_show(zone_name_or_id)&.dig('ZONE', 'SERVER_POOL', 'SERVER')
+
+    if servers.is_a?(Hash)
+        servers = [servers]
+    end
+
+    addresses = servers&.map do |server|
+        hostname = URI(server['ENDPOINT']).host
+        Addrinfo.ip(hostname).ip_address
+    end
+
+    interface_addresses = Socket.ip_address_list.map do |address|
+        address.ip_address
+    end
+
+    address = addresses&.find do |addr|
+        interface_addresses.include? addr
+    end
+
+    index = addresses&.index address
+    addresses&.delete_at index
+
+    [addresses || [], address || Socket.gethostname]
+end
+
+def onehost_list
+    hosts = YAML.safe_load `onehost list --yaml`
+    hosts = hosts.dig('HOST_POOL', 'HOST') || []
+    hosts = [hosts] if hosts.is_a? Hash
+
+    hosts.select { |h| h['TEMPLATE']['HOSTNAME'] }
+end
+
+def is_local?(srv)
+    ip_regex = /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/
+
+    if srv.match(ip_regex)
+        return LOCAL_IPS.include?(srv) || IPAddr.new('127.0.0.0/8').include?(srv)
+    else
+        begin
+            ip = Resolv.getaddress(srv)
+        rescue
+            return Socket.gethostname == srv
+        end
+
+        return LOCAL_IPS.include?(ip) || IPAddr.new('127.0.0.0/8').include?(ip)
+    end
+end
+
+def patch_datasources(document, zone_name_or_id = 'OpenNebula')
+    hosts = onehost_list
+
+    servers, myself = detect_servers zone_name_or_id
+
+    # Alertmanager
+    document['alerting']['alertmanagers'] = [{
+        'static_configs' => [{
+            'targets' => (servers + [myself]).map do |server|
+                "#{server}:9093"
+            end
+        }]
+    }]
+
+    scrape_configs = []
+
+    # OpenNebula exporter
+    scrape_configs << {
+        'job_name' => 'opennebula_exporter',
+        'static_configs' => [{
+            'targets' => ["#{myself}:9925"]
+        }]
+    }
+
+    # Node exporter
+    node_exporters = []
+
+    node_exporters += [{
+        'targets' => servers.map { |server| "#{server}:9100" }
+    }] unless servers.empty?
+
+    node_exporters += hosts.map do |host|
+        { 'targets' => ["#{host['TEMPLATE']['HOSTNAME']}:9100"],
+          'labels' => { 'one_host_id' => host['ID'] } }
+    end unless hosts.empty?
+
+    # if localhost is not included in hosts already
+    node_exporters += [{ 'targets' => ["#{myself}:9100"] }] \
+        unless hosts.map { |h| h['TEMPLATE']['HOSTNAME'] }.any? { |h| is_local?(h) }
+
+    scrape_configs << {
+        'job_name' => 'node_exporter',
+        'static_configs' => node_exporters
+    }
+
+    # Libvirt exporter
+    scrape_configs << {
+        'job_name' => 'libvirt_exporter',
+        'static_configs' => hosts.map do |host|
+            { 'targets' => ["#{host['TEMPLATE']['HOSTNAME']}:9926"],
+              'labels' => { 'one_host_id' => host['ID'] } }
+        end
+    } unless hosts.empty?
+
+    document['scrape_configs'] = dict_to_list(
+        list_to_dict(document['scrape_configs']).merge(list_to_dict(scrape_configs))
+    )
+
+    document
+end
+
+if caller.empty?
+    options = { :zone_name_or_id => 'OpenNebula' }
+
+    OptionParser.new do |opts|
+        opts.banner = "Usage: #{$PROGRAM_NAME} [options]"
+
+        opts.on '-z ZONE_NAME_OR_ID',
+                '--zone-name-or-id ZONE_NAME_OR_ID',
+                'OpenNebula Zone name or ID' do |zone_name_or_id|
+            options[:zone_name_or_id] = zone_name_or_id
+        end
+
+        opts.on_tail '-h', '--help', 'Show this message' do
+          puts opts
+          exit
+        end
+    end.parse!
+
+    prometheus_yml_path = "#{ONEPROMETHEUS_ETC_LOCATION}/prometheus/prometheus.yml"
+
+    prometheus_yml = patch_datasources YAML.load_file(prometheus_yml_path),
+                                       options[:zone_name_or_id]
+
+    file prometheus_yml_path,
+         YAML.dump(prometheus_yml),
+         :mode => 'ug=rw,o=',
+         :overwrite => true,
+         :backup => true
+end

src/oneprometheus/prometheus/share/patch_datasources_spec.rb

@@ -0,0 +1,448 @@
+# frozen_string_literal: true
+
+require 'rspec'
+require 'socket'
+require 'uri'
+require 'yaml'
+
+require_relative 'patch_datasources'
+
+RSpec.describe 'list_to_dict / dict_to_list' do
+    it 'should convert list to dict' do
+        provided = [ { 'key' => 'a', 'field' => 1 },
+                     { 'key' => 'b', 'field' => 2 } ]
+
+        expected = { 'a' => { 'key' => 'a', 'field' => 1 },
+                     'b' => { 'key' => 'b', 'field' => 2 } }
+
+        expect(list_to_dict(provided, key: 'key')).to eq expected
+    end
+    it 'should convert dict to list' do
+        provided = { 'a' => { 'key' => 'a', 'field' => 1 },
+                     'b' => { 'key' => 'b', 'field' => 2 } }
+
+        expected = [ { 'key' => 'a', 'field' => 1 },
+                     { 'key' => 'b', 'field' => 2 } ]
+
+        expect(dict_to_list(provided)).to eq expected
+    end
+end
+
+RSpec.describe 'detect_servers' do
+    it 'should detect 0 peers' do
+        allow(self).to receive(:onezone_show).and_return YAML.safe_load(<<~DOCUMENT)
+        ---
+        ZONE:
+          ID: '0'
+          NAME: OpenNebula
+          STATE: '0'
+          TEMPLATE:
+            ENDPOINT: http://localhost:2633/RPC2
+          SERVER_POOL: {}
+        DOCUMENT
+
+        allow(Socket).to receive(:ip_address_list).and_return [
+            Addrinfo.ip('127.0.0.1'),
+            Addrinfo.ip('192.168.150.1')
+        ]
+
+        expect(detect_servers).to eq [
+            [], Socket.gethostname
+        ]
+    end
+    it 'should detect 2 peers' do
+        allow(self).to receive(:onezone_show).and_return YAML.safe_load(<<~DOCUMENT)
+        ---
+        ZONE:
+          ID: '0'
+          NAME: OpenNebula
+          STATE: '0'
+          TEMPLATE:
+            ENDPOINT: http://localhost:2633/RPC2
+          SERVER_POOL:
+            SERVER:
+            - ENDPOINT: http://192.168.150.1:2633/RPC2
+              ID: '0'
+              NAME: Node-1
+              STATE: '2'
+              TERM: '22'
+              VOTEDFOR: '1'
+              COMMIT: '97912'
+              LOG_INDEX: '97912'
+              FEDLOG_INDEX: "-1"
+            - ENDPOINT: http://192.168.150.2:2633/RPC2
+              ID: '1'
+              NAME: Node-2
+              STATE: '3'
+              TERM: '22'
+              VOTEDFOR: '1'
+              COMMIT: '97912'
+              LOG_INDEX: '97912'
+              FEDLOG_INDEX: "-1"
+            - ENDPOINT: http://192.168.150.3:2633/RPC2
+              ID: '2'
+              NAME: Node-3
+              STATE: '2'
+              TERM: '22'
+              VOTEDFOR: "-1"
+              COMMIT: '97912'
+              LOG_INDEX: '97912'
+              FEDLOG_INDEX: "-1"
+        DOCUMENT
+
+        allow(Socket).to receive(:ip_address_list).and_return [
+            Addrinfo.ip('127.0.0.1'),
+            Addrinfo.ip('192.168.150.2')
+        ]
+
+        expect(detect_servers).to eq [
+            ['192.168.150.1', '192.168.150.3'], '192.168.150.2'
+        ]
+    end
+end
+
+RSpec.describe 'patch_datasources' do
+    before(:all) do
+        @onehost_list = YAML.safe_load(<<~DOCUMENT)
+        ---
+          - ID: '1'
+            NAME: omicron
+            STATE: '2'
+            PREV_STATE: '2'
+            IM_MAD: kvm
+            VM_MAD: kvm
+            CLUSTER_ID: '0'
+            CLUSTER: default
+            HOST_SHARE:
+              MEM_USAGE: '0'
+              CPU_USAGE: '0'
+              TOTAL_MEM: '65219716'
+              TOTAL_CPU: '1200'
+              MAX_MEM: '65219716'
+              MAX_CPU: '1200'
+              RUNNING_VMS: '0'
+              VMS_THREAD: '1'
+              DATASTORES:
+                DISK_USAGE: '0'
+                FREE_DISK: '501528'
+                MAX_DISK: '781195'
+                USED_DISK: '279668'
+              PCI_DEVICES: {}
+              NUMA_NODES:
+                NODE:
+                  CORE:
+                  - CPUS: 10:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '5'
+                  - CPUS: 8:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '4'
+                  - CPUS: 6:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '3'
+                  - CPUS: 4:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '2'
+                  - CPUS: 2:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '1'
+                  - CPUS: 0:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '0'
+                  HUGEPAGE:
+                  - FREE: '0'
+                    PAGES: '0'
+                    SIZE: '2048'
+                    USAGE: '0'
+                  - FREE: '0'
+                    PAGES: '0'
+                    SIZE: '1048576'
+                    USAGE: '0'
+                  MEMORY:
+                    DISTANCE: '0'
+                    FREE: '0'
+                    TOTAL: '65219716'
+                    USAGE: '0'
+                    USED: '0'
+                  NODE_ID: '0'
+            VMS: {}
+            TEMPLATE:
+              ARCH: x86_64
+              CGROUPS_VERSION: '1'
+              CPUSPEED: '3349'
+              HOSTNAME: omicron
+              HYPERVISOR: kvm
+              IM_MAD: kvm
+              KVM_CPU_MODEL: EPYC-Rome
+              KVM_CPU_MODELS: 486 pentium pentium2 pentium3 pentiumpro coreduo n270 core2duo
+                qemu32 kvm32 cpu64-rhel5 cpu64-rhel6 qemu64 kvm64 Conroe Penryn Nehalem Nehalem-IBRS
+                Westmere Westmere-IBRS SandyBridge SandyBridge-IBRS IvyBridge IvyBridge-IBRS
+                Haswell-noTSX Haswell-noTSX-IBRS Haswell Haswell-IBRS Broadwell-noTSX Broadwell-noTSX-IBRS
+                Broadwell Broadwell-IBRS Skylake-Client Skylake-Client-IBRS Skylake-Client-noTSX-IBRS
+                Skylake-Server Skylake-Server-IBRS Skylake-Server-noTSX-IBRS Cascadelake-Server
+                Cascadelake-Server-noTSX Icelake-Client Icelake-Client-noTSX Icelake-Server
+                Icelake-Server-noTSX athlon phenom Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4
+                Opteron_G5 EPYC EPYC-IBPB EPYC-Rome EPYC-Milan Dhyana
+              KVM_MACHINES: pc-i440fx-focal ubuntu pc-0.15 pc-i440fx-2.12 pc-i440fx-2.0 pc-i440fx-xenial
+                pc-q35-4.2 q35 pc-i440fx-2.5 pc-i440fx-4.2 pc pc-q35-xenial pc-i440fx-1.5
+                pc-0.12 pc-q35-2.7 pc-q35-eoan-hpb pc-i440fx-disco-hpb pc-i440fx-zesty pc-q35-artful
+                pc-i440fx-trusty pc-i440fx-2.2 pc-i440fx-eoan-hpb pc-q35-focal-hpb pc-1.1
+                pc-q35-bionic-hpb pc-i440fx-artful pc-i440fx-2.7 pc-i440fx-yakkety pc-q35-2.4
+                pc-q35-cosmic-hpb pc-q35-2.10 pc-i440fx-1.7 pc-0.14 pc-q35-2.9 pc-i440fx-2.11
+                pc-q35-3.1 pc-q35-4.1 pc-i440fx-2.4 pc-1.3 pc-i440fx-4.1 pc-q35-eoan pc-i440fx-2.9
+                pc-i440fx-bionic-hpb isapc pc-i440fx-1.4 pc-q35-cosmic pc-q35-2.6 pc-i440fx-3.1
+                pc-q35-bionic pc-q35-disco-hpb pc-i440fx-cosmic pc-q35-2.12 pc-i440fx-bionic
+                pc-q35-disco pc-i440fx-cosmic-hpb pc-i440fx-2.1 pc-1.0 pc-i440fx-wily pc-i440fx-2.6
+                pc-q35-4.0.1 pc-i440fx-1.6 pc-0.13 pc-q35-2.8 pc-i440fx-2.10 pc-q35-3.0 pc-q35-zesty
+                pc-q35-4.0 microvm pc-i440fx-2.3 pc-q35-focal ubuntu-q35 pc-i440fx-disco pc-1.2
+                pc-i440fx-4.0 pc-i440fx-focal-hpb pc-i440fx-2.8 pc-i440fx-eoan pc-q35-2.5
+                pc-i440fx-3.0 pc-q35-yakkety pc-q35-2.11
+              MODELNAME: AMD Ryzen 5 PRO 5650U with Radeon Graphics
+              RESERVED_CPU: ''
+              RESERVED_MEM: ''
+              VERSION: 6.4.1
+              VM_MAD: kvm
+            MONITORING: {}
+          - ID: '0'
+            NAME: epsilon
+            STATE: '2'
+            PREV_STATE: '2'
+            IM_MAD: kvm
+            VM_MAD: kvm
+            CLUSTER_ID: '0'
+            CLUSTER: default
+            HOST_SHARE:
+              MEM_USAGE: '0'
+              CPU_USAGE: '0'
+              TOTAL_MEM: '65219716'
+              TOTAL_CPU: '1200'
+              MAX_MEM: '65219716'
+              MAX_CPU: '1200'
+              RUNNING_VMS: '0'
+              VMS_THREAD: '1'
+              DATASTORES:
+                DISK_USAGE: '0'
+                FREE_DISK: '501528'
+                MAX_DISK: '781195'
+                USED_DISK: '279668'
+              PCI_DEVICES: {}
+              NUMA_NODES:
+                NODE:
+                  CORE:
+                  - CPUS: 10:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '5'
+                  - CPUS: 8:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '4'
+                  - CPUS: 6:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '3'
+                  - CPUS: 4:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '2'
+                  - CPUS: 2:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '1'
+                  - CPUS: 0:-1
+                    DEDICATED: 'NO'
+                    FREE: '1'
+                    ID: '0'
+                  HUGEPAGE:
+                  - FREE: '0'
+                    PAGES: '0'
+                    SIZE: '2048'
+                    USAGE: '0'
+                  - FREE: '0'
+                    PAGES: '0'
+                    SIZE: '1048576'
+                    USAGE: '0'
+                  MEMORY:
+                    DISTANCE: '0'
+                    FREE: '0'
+                    TOTAL: '65219716'
+                    USAGE: '0'
+                    USED: '0'
+                  NODE_ID: '0'
+            VMS: {}
+            TEMPLATE:
+              ARCH: x86_64
+              CGROUPS_VERSION: '1'
+              CPUSPEED: '2715'
+              HOSTNAME: epsilon
+              HYPERVISOR: kvm
+              IM_MAD: kvm
+              KVM_CPU_MODEL: EPYC-Rome
+              KVM_CPU_MODELS: 486 pentium pentium2 pentium3 pentiumpro coreduo n270 core2duo
+                qemu32 kvm32 cpu64-rhel5 cpu64-rhel6 qemu64 kvm64 Conroe Penryn Nehalem Nehalem-IBRS
+                Westmere Westmere-IBRS SandyBridge SandyBridge-IBRS IvyBridge IvyBridge-IBRS
+                Haswell-noTSX Haswell-noTSX-IBRS Haswell Haswell-IBRS Broadwell-noTSX Broadwell-noTSX-IBRS
+                Broadwell Broadwell-IBRS Skylake-Client Skylake-Client-IBRS Skylake-Client-noTSX-IBRS
+                Skylake-Server Skylake-Server-IBRS Skylake-Server-noTSX-IBRS Cascadelake-Server
+                Cascadelake-Server-noTSX Icelake-Client Icelake-Client-noTSX Icelake-Server
+                Icelake-Server-noTSX athlon phenom Opteron_G1 Opteron_G2 Opteron_G3 Opteron_G4
+                Opteron_G5 EPYC EPYC-IBPB EPYC-Rome EPYC-Milan Dhyana
+              KVM_MACHINES: pc-i440fx-focal ubuntu pc-0.15 pc-i440fx-2.12 pc-i440fx-2.0 pc-i440fx-xenial
+                pc-q35-4.2 q35 pc-i440fx-2.5 pc-i440fx-4.2 pc pc-q35-xenial pc-i440fx-1.5
+                pc-0.12 pc-q35-2.7 pc-q35-eoan-hpb pc-i440fx-disco-hpb pc-i440fx-zesty pc-q35-artful
+                pc-i440fx-trusty pc-i440fx-2.2 pc-i440fx-eoan-hpb pc-q35-focal-hpb pc-1.1
+                pc-q35-bionic-hpb pc-i440fx-artful pc-i440fx-2.7 pc-i440fx-yakkety pc-q35-2.4
+                pc-q35-cosmic-hpb pc-q35-2.10 pc-i440fx-1.7 pc-0.14 pc-q35-2.9 pc-i440fx-2.11
+                pc-q35-3.1 pc-q35-4.1 pc-i440fx-2.4 pc-1.3 pc-i440fx-4.1 pc-q35-eoan pc-i440fx-2.9
+                pc-i440fx-bionic-hpb isapc pc-i440fx-1.4 pc-q35-cosmic pc-q35-2.6 pc-i440fx-3.1
+                pc-q35-bionic pc-q35-disco-hpb pc-i440fx-cosmic pc-q35-2.12 pc-i440fx-bionic
+                pc-q35-disco pc-i440fx-cosmic-hpb pc-i440fx-2.1 pc-1.0 pc-i440fx-wily pc-i440fx-2.6
+                pc-q35-4.0.1 pc-i440fx-1.6 pc-0.13 pc-q35-2.8 pc-i440fx-2.10 pc-q35-3.0 pc-q35-zesty
+                pc-q35-4.0 microvm pc-i440fx-2.3 pc-q35-focal ubuntu-q35 pc-i440fx-disco pc-1.2
+                pc-i440fx-4.0 pc-i440fx-focal-hpb pc-i440fx-2.8 pc-i440fx-eoan pc-q35-2.5
+                pc-i440fx-3.0 pc-q35-yakkety pc-q35-2.11
+              MODELNAME: AMD Ryzen 5 PRO 5650U with Radeon Graphics
+              RESERVED_CPU: ''
+              RESERVED_MEM: ''
+              VERSION: 6.4.1
+              VM_MAD: kvm
+            MONITORING: {}
+        DOCUMENT
+
+        @provided = YAML.safe_load(<<~DOCUMENT)
+        global:
+          scrape_interval:     15s
+          evaluation_interval: 15s
+        alerting:
+          alertmanagers:
+          - static_configs:
+            - targets: ['localhost:9093']
+        rule_files: ['rules.yml']
+        scrape_configs:
+        - job_name: prometheus
+          static_configs:
+          - targets: ['localhost:9090']
+        DOCUMENT
+    end
+
+    it 'should patch prometheus datasources for 1 peer' do
+        allow(self).to receive(:onehost_list).and_return @onehost_list
+
+        allow(self).to receive(:detect_servers).and_return [
+            [], '127.0.0.1'
+        ]
+
+        expected = YAML.safe_load(<<~DOCUMENT)
+        ---
+        global:
+          scrape_interval: 15s
+          evaluation_interval: 15s
+        alerting:
+          alertmanagers:
+          - static_configs:
+            - targets:
+              - 127.0.0.1:9093
+        rule_files:
+        - rules.yml
+        scrape_configs:
+        - job_name: prometheus
+          static_configs:
+          - targets:
+            - localhost:9090
+        - job_name: opennebula_exporter
+          static_configs:
+          - targets:
+            - 127.0.0.1:9925
+        - job_name: node_exporter
+          static_configs:
+          - targets:
+            - omicron:9100
+            labels:
+              one_host_id: '1'
+          - targets:
+            - epsilon:9100
+            labels:
+              one_host_id: '0'
+          - targets:
+            - 127.0.0.1:9100
+        - job_name: libvirt_exporter
+          static_configs:
+          - targets:
+            - omicron:9926
+            labels:
+              one_host_id: '1'
+          - targets:
+            - epsilon:9926
+            labels:
+              one_host_id: '0'
+        DOCUMENT
+
+        expect(patch_datasources(@provided)).to eq expected
+    end
+
+    it 'should patch prometheus datasources for 3 peers' do
+        allow(self).to receive(:onehost_list).and_return @onehost_list
+
+        allow(self).to receive(:detect_servers).and_return [
+            ['192.168.150.1', '192.168.150.3'], '192.168.150.2'
+        ]
+
+        expected = YAML.safe_load(<<~DOCUMENT)
+        ---
+        global:
+          scrape_interval: 15s
+          evaluation_interval: 15s
+        alerting:
+          alertmanagers:
+          - static_configs:
+            - targets:
+              - 192.168.150.1:9093
+              - 192.168.150.3:9093
+              - 192.168.150.2:9093
+        rule_files:
+        - rules.yml
+        scrape_configs:
+        - job_name: prometheus
+          static_configs:
+          - targets:
+            - localhost:9090
+        - job_name: opennebula_exporter
+          static_configs:
+          - targets:
+            - 192.168.150.2:9925
+        - job_name: node_exporter
+          static_configs:
+          - targets:
+            - 192.168.150.1:9100
+            - 192.168.150.3:9100
+          - targets:
+            - omicron:9100
+            labels:
+              one_host_id: '1'
+          - targets:
+            - epsilon:9100
+            labels:
+              one_host_id: '0'
+          - targets:
+            - 192.168.150.2:9100
+        - job_name: libvirt_exporter
+          static_configs:
+          - targets:
+            - omicron:9926
+            labels:
+              one_host_id: '1'
+          - targets:
+            - epsilon:9926
+            labels:
+              one_host_id: '0'
+        DOCUMENT
+
+        expect(patch_datasources(@provided)).to eq expected
+    end
+end

src/oneprometheus/prometheus/systemd/opennebula-prometheus.service

@@ -0,0 +1,14 @@
+[Unit]
+Description=OpenNebula Prometheus Server
+
+[Service]
+User=oneadmin
+Group=oneadmin
+Type=simple
+
+ExecStart=/usr/bin/prometheus \
+          --config.file=/etc/one/prometheus/prometheus.yml \
+          --storage.tsdb.path=/var/lib/prometheus/data/
+
+[Install]
+WantedBy=multi-user.target

src/oneprometheus/vendor/download

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# -------------------------------------------------------------------------- #
+# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems                #
+#                                                                            #
+# Licensed under the Apache License, Version 2.0 (the "License"); you may    #
+# not use this file except in compliance with the License. You may obtain    #
+# a copy of the License at                                                   #
+#                                                                            #
+# http://www.apache.org/licenses/LICENSE-2.0                                 #
+#                                                                            #
+# Unless required by applicable law or agreed to in writing, software        #
+# distributed under the License is distributed on an "AS IS" BASIS,          #
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   #
+# See the License for the specific language governing permissions and        #
+# limitations under the License.                                             #
+# -------------------------------------------------------------------------- #
+
+: "${ALERTMANAGER_VERSION:=0.24.0}"
+
+: "${NODE_EXPORTER_VERSION:=1.4.0}"
+
+: "${PROMETHEUS_VERSION:=2.37.1}"
+
+set -o errexit -o nounset -o pipefail
+
+which curl dirname gzip realpath tar xargs 1>/dev/null
+
+readonly SELF=$(realpath "$0" | xargs dirname) && cd "$SELF/"
+
+readonly ALERTMANAGER_URL="https://github.com/prometheus/alertmanager/releases/download/v$ALERTMANAGER_VERSION/alertmanager-$ALERTMANAGER_VERSION.linux-amd64.tar.gz"
+
+readonly NODE_EXPORTER_URL="https://github.com/prometheus/node_exporter/releases/download/v$NODE_EXPORTER_VERSION/node_exporter-$NODE_EXPORTER_VERSION.linux-amd64.tar.gz"
+
+readonly PROMETHEUS_URL="https://github.com/prometheus/prometheus/releases/download/v$PROMETHEUS_VERSION/prometheus-$PROMETHEUS_VERSION.linux-amd64.tar.gz"
+
+install -d "$SELF/alertmanager/"
+curl -fsSL "$ALERTMANAGER_URL" | tar -xzf- --strip-components=1 -C "$SELF/alertmanager/"
+
+install -d "$SELF/node_exporter/"
+curl -fsSL "$NODE_EXPORTER_URL" | tar -xzf- --strip-components=1 -C "$SELF/node_exporter/"
+
+install -d "$SELF/prometheus/"
+curl -fsSL "$PROMETHEUS_URL" | tar -xzf- --strip-components=1 -C "$SELF/prometheus/"