F #3859: Monitoring & Scheduler

* Merge monitor information into Host XML document
  * Add new param to vmpool.monitoring call
  * Add new param to hostpool.monitoring call
  * Update OCA - Ruby

co-authored-by: Christian González <cgonzalez@opennebula.io>

include/ObjectXML.h

@@ -247,6 +247,15 @@ public:
      */
     int add_node(const char * xpath_expr, xmlNodePtr node, const char * new_name);
 
+    /**
+     *  Removes nodes from the object by xPath
+     *
+     *  @param xpath_expr Path of the parent node
+     *
+     *  @return number of elements removed
+     */
+    int remove_nodes(const char * xpath_expr);
+
     /**
      *  Frees a vector of XMLNodes, as returned by the get_nodes function
      *    @param content the vector of xmlNodePtr

src/host/HostPool.cc

@@ -175,7 +175,7 @@ int HostPool::dump_monitoring(
 
     if ( !where.empty() )
     {
-        cmd << " AND " << where;
+        cmd << " WHERE " << where;
     }
 
     cmd << " ORDER BY hid, " << one_db::host_monitor_table << ".last_mon_time;";

src/oca/ruby/opennebula/host_pool.rb

@@ -99,9 +99,16 @@ module OpenNebula
 
         # Retrieves the monitoring data for all the Hosts in the pool, in XML
         #
+        # @param [Integer] num Optional number of monitoring records to be
+        #   retrieved. If nill all records are retrieved
+        #
         # @return [String] VM monitoring data, in XML
-        def monitoring_xml()
-            return @client.call(HOST_POOL_METHODS[:monitoring])
+        def monitoring_xml(num = nil)
+            return @client.call(HOST_POOL_METHODS[:monitoring]) if num.nil?
+
+            @client.call(HOST_POOL_METHODS[:monitoring], num.to_i)
         end
+
     end
+
 end

src/oca/ruby/opennebula/virtual_machine_pool.rb

@@ -199,10 +199,13 @@ module OpenNebula
         #
         # @param [Integer] filter_flag Optional filter flag to retrieve all or
         #   part of the Pool. Possible values: INFO_ALL, INFO_GROUP, INFO_MINE.
-        #
+        # @param [Integer] num Optional number of monitoring records to be
+        #   retrieved. If nill all records are retrieved
         # @return [String] VM monitoring data, in XML
-        def monitoring_xml(filter_flag=INFO_ALL)
-            return @client.call(VM_POOL_METHODS[:monitoring], filter_flag)
+        def monitoring_xml(filter_flag=INFO_ALL, num=nil)
+            return @client.call(VM_POOL_METHODS[:monitoring], filter_flag) if num.nil?
+
+            @client.call(VM_POOL_METHODS[:monitoring], filter_flag, num.to_i)
         end
 
         # Processes all the history records, and stores the monthly cost for

src/rm/RequestManagerPoolInfoFilter.cc

@@ -35,6 +35,7 @@
 #include "VMTemplatePool.h"
 #include "VNTemplatePool.h"
 #include "ZonePool.h"
+#include "OneDB.h"
 
 using namespace std;
 
@@ -497,8 +498,9 @@ void VirtualMachinePoolMonitoring::request_execute(
     int filter_flag = xmlrpc_c::value_int(paramList.getInt(1));
 
     string oss;
-    string        where;
-    int           rc;
+    string where;
+    string and_clause = "";
+    int rc;
 
     if ( filter_flag < GROUP )
     {
@@ -507,7 +509,25 @@ void VirtualMachinePoolMonitoring::request_execute(
         return;
     }
 
-    where_filter(att, filter_flag, -1, -1, "", "", false, false, false, where);
+    if (paramList.size() > 2)
+    {
+        ostringstream oss;
+        int num_rows = xmlrpc_c::value_int(paramList.getInt(2));
+
+        oss << one_db::vm_monitor_table << ".last_poll in "
+            << "(select last_poll from " << one_db::vm_monitor_table << " as t "
+            << "where t.vmid = " << one_db::vm_monitor_table << ".vmid "
+            << "ORDER by last_poll DESC";
+
+        if (num_rows != -1)
+        {
+            oss << " LIMIT " << num_rows << ")";
+        }
+
+        and_clause = oss.str();
+    }
+
+    where_filter(att, filter_flag, -1, -1, and_clause, "", false, false, false, where);
 
     rc = (static_cast<VirtualMachinePool *>(pool))->dump_monitoring(oss, where);
 
@@ -691,10 +711,30 @@ void HostPoolMonitoring::request_execute(
 {
     string oss;
     string where;
+    string and_clause = "";
 
     int rc;
 
-    where_filter(att, ALL, -1, -1, "", "", false, false, false, where);
+    if (paramList.size() > 1)
+    {
+        ostringstream oss;
+        int num_rows = xmlrpc_c::value_int(paramList.getInt(1));
+
+        oss << one_db::host_monitor_table << ".last_mon_time in "
+            << "(select last_mon_time "
+            << "from " << one_db::host_monitor_table << " as t "
+            << "where t.hid = " << one_db::host_monitor_table << ".hid "
+            << "ORDER by last_mon_time DESC";
+
+        if (num_rows != -1)
+        {
+            oss << " LIMIT " << num_rows << ")";
+        }
+
+        and_clause = oss.str();
+    }
+
+    where_filter(att, ALL, -1, -1, and_clause, "", false, false, false, where);
 
     rc = (static_cast<HostPool *>(pool))->dump_monitoring(oss, where);
 

src/scheduler/include/HostPoolXML.h

@@ -20,7 +20,9 @@
 
 #include "PoolXML.h"
 #include "HostXML.h"
-#include "ClusterPoolXML.h"
+
+class ClusterPoolXML;
+class MonitorPoolXML;
 
 using namespace std;
 
@@ -53,6 +55,13 @@ public:
      */
     void merge_clusters(ClusterPoolXML * clpool);
 
+    /**
+     * Add the last MONITORING information to each Host
+     *
+     * @param mpool Monitoring Pool with last data
+     */
+    void merge_monitoring(MonitorPoolXML * mpool);
+
 protected:
 
     int get_suitable_nodes(vector<xmlNodePtr>& content)

src/scheduler/include/MonitorXML.h

@@ -0,0 +1,113 @@
+/* -------------------------------------------------------------------------- */
+/* Copyright 2002-2020, OpenNebula Project, OpenNebula Systems                */
+/*                                                                            */
+/* Licensed under the Apache License, Version 2.0 (the "License"); you may    */
+/* not use this file except in compliance with the License. You may obtain    */
+/* a copy of the License at                                                   */
+/*                                                                            */
+/* http://www.apache.org/licenses/LICENSE-2.0                                 */
+/*                                                                            */
+/* Unless required by applicable law or agreed to in writing, software        */
+/* distributed under the License is distributed on an "AS IS" BASIS,          */
+/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.   */
+/* See the License for the specific language governing permissions and        */
+/* limitations under the License.                                             */
+/* -------------------------------------------------------------------------- */
+
+#ifndef MONITOR_XML_H_
+#define MONITOR_XML_H_
+
+#include <vector>
+#include "PoolXML.h"
+
+class MonitorXML : public ObjectXML
+{
+public:
+    MonitorXML(const std::string &xml_doc):ObjectXML(xml_doc)
+    {
+        init_attributes();
+    };
+
+    MonitorXML(const xmlNodePtr node):ObjectXML(node)
+    {
+        init_attributes();
+    };
+
+    int get_oid() const
+    {
+        return oid;
+    };
+
+private:
+    int oid;
+
+    void init_attributes()
+    {
+        xpath(oid, "/MONITORING/ID", -1);
+    };
+};
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+class MonitorPoolXML : public PoolXML
+{
+public:
+
+    MonitorPoolXML (Client* client):PoolXML(client){};
+
+    ~MonitorPoolXML() = default;
+
+    /**
+     *  Gets an object from the pool
+     *   @param oid the object unique identifier
+     *
+     *   @return a pointer to the object, 0 in case of failure
+     */
+    MonitorXML * get(int oid) const
+    {
+        return static_cast<MonitorXML *>(PoolXML::get(oid));
+    };
+
+protected:
+
+    int get_suitable_nodes(std::vector<xmlNodePtr>& content)
+    {
+        return get_nodes("/MONITORING_DATA/MONITORING", content);
+    };
+
+    void add_object(xmlNodePtr node)
+    {
+        if ( node == 0 || node->children == 0 )
+        {
+            NebulaLog::log("MONITOR",Log::ERROR,
+                           "XML Node is not a valid MONITORING record");
+            return;
+        }
+
+        MonitorXML* monitor = new MonitorXML(node);
+
+        objects.insert(pair<int,ObjectXML*>(monitor->get_oid(), monitor));
+    }
+
+    int load_info(xmlrpc_c::value &result)
+    {
+        try
+        {
+            client->call("one.hostpool.monitoring", "i", &result, 1);
+
+            return 0;
+        }
+        catch (exception const& e)
+        {
+            ostringstream   oss;
+            oss << "Exception raised: " << e.what();
+
+            NebulaLog::log("MONITOR", Log::ERROR, oss);
+
+            return -1;
+        }
+    }
+};
+
+#endif /* MONITOR_XML_H_ */

src/scheduler/include/RankPolicy.h

@@ -73,30 +73,44 @@ private:
             return;
         }
 
+        NebulaLog::log("RANK", Log::DDEBUG, "Rank evaluation for expression : "
+                + srank);
+
         for (unsigned int i=0; i<resources.size(); rank=0, i++)
         {
             resource = pool->get(resources[i]->oid);
 
-            if ( resource != 0 )
+            if ( resource == nullptr )
             {
-                rc = resource->eval_arith(srank, rank, &errmsg);
+                continue;
+            }
 
-                if (rc != 0)
+            rc = resource->eval_arith(srank, rank, &errmsg);
+
+            if (rc != 0)
+            {
+                ostringstream oss;
+
+                oss << "Computing rank, expression: " << srank;
+
+                if (errmsg != 0)
                 {
-                    ostringstream oss;
+                    oss << ", error: " << errmsg;
+                    errmsg = 0;
 
-                    oss << "Computing rank, expression: " << srank;
-
-                    if (errmsg != 0)
-                    {
-                        oss << ", error: " << errmsg;
-                        errmsg = 0;
-
-                        free(errmsg);
-                    }
-
-                    NebulaLog::log("RANK",Log::ERROR,oss);
+                    free(errmsg);
                 }
+
+                NebulaLog::log("RANK",Log::ERROR,oss);
+            }
+
+            if (NebulaLog::log_level() >= Log::DDEBUG)
+            {
+                ostringstream oss;
+
+                oss << "ID: " << resources[i]->oid << " Rank: " << rank;
+
+                NebulaLog::log("RANK", Log::DDEBUG, oss);
             }
 
             priority.push_back(rank);

src/scheduler/include/Scheduler.h

@@ -28,6 +28,7 @@
 #include "SchedulerPolicy.h"
 #include "ActionManager.h"
 #include "AclXML.h"
+#include "MonitorXML.h"
 
 using namespace std;
 
@@ -78,6 +79,7 @@ protected:
         vnetpool(0),
         vmgpool(0),
         vmapool(0),
+        hmonpool(0),
         timer(0),
         one_xmlrpc(""),
         machines_limit(0),
@@ -93,6 +95,7 @@ protected:
     {
         delete hpool;
         delete clpool;
+        delete hmonpool;
 
         delete vmpool;
         delete vm_roles_pool;
@@ -129,6 +132,8 @@ protected:
 
     VirtualMachineActionsPoolXML* vmapool;
 
+    MonitorPoolXML * hmonpool;
+
     // ---------------------------------------------------------------
     // Scheduler Policies
     // ---------------------------------------------------------------

src/scheduler/src/pool/HostPoolXML.cc

@@ -15,6 +15,8 @@
 /* -------------------------------------------------------------------------- */
 
 #include "HostPoolXML.h"
+#include "MonitorXML.h"
+#include "ClusterPoolXML.h"
 
 int HostPoolXML::set_up()
 {
@@ -128,3 +130,39 @@ void HostPoolXML::merge_clusters(ClusterPoolXML * clpool)
 
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
+
+void HostPoolXML::merge_monitoring(MonitorPoolXML * mpool)
+{
+    map<int,ObjectXML*>::iterator it;
+
+    vector<xmlNodePtr> nodes;
+
+    for (it=objects.begin(); it!=objects.end(); it++)
+    {
+        HostXML* host = static_cast<HostXML*>(it->second);
+
+        MonitorXML* monitor = mpool->get(host->get_hid());
+
+        if ( monitor == nullptr )
+        {
+            continue;
+        }
+
+        nodes.clear();
+
+        monitor->get_nodes("/MONITORING", nodes);
+
+        if (!nodes.empty())
+        {
+            host->remove_nodes("/HOST/MONITORING");
+
+            host->add_node("/HOST", nodes[0], "MONITORING");
+        }
+
+        monitor->free_nodes(nodes);
+    }
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+

src/scheduler/src/pool/HostXML.cc

@@ -25,11 +25,13 @@
 
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
-int HostXML::host_num_paths =  4;
+int HostXML::host_num_paths =  6;
 
 const char *HostXML::host_paths[] = {
     "/HOST/TEMPLATE/",
     "/HOST/HOST_SHARE/",
+    "/HOST/MONITORING/CAPACITY/",
+    "/HOST/MONITORING/SYSTEM/",
     "/HOST/",
     "/HOST/CLUSTER_TEMPLATE/"};
 

src/scheduler/src/sched/Scheduler.cc

@@ -318,8 +318,9 @@ void Scheduler::start()
     acls  = new AclXML(client, zone_id);
     upool = new UserPoolXML(client);
 
-    hpool  = new HostPoolXML(client);
-    clpool = new ClusterPoolXML(client);
+    hpool    = new HostPoolXML(client);
+    clpool   = new ClusterPoolXML(client);
+    hmonpool = new MonitorPoolXML(client);
 
     dspool     = new SystemDatastorePoolXML(client);
     img_dspool = new ImageDatastorePoolXML(client);
@@ -493,6 +494,15 @@ int Scheduler::set_up_pools()
         return rc;
     }
 
+    rc = hmonpool->set_up();
+
+    if ( rc != 0 )
+    {
+        return rc;
+    }
+
+    hpool->merge_monitoring(hmonpool);
+
     return 0;
 };
 

src/xml/ObjectXML.cc

@@ -463,6 +463,44 @@ int ObjectXML::rename_nodes(const char * xpath_expr, const char * new_name)
     return renamed;
 }
 
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+int ObjectXML::remove_nodes(const char * xpath_expr)
+{
+    xmlXPathObjectPtr obj = xmlXPathEvalExpression(
+        reinterpret_cast<const xmlChar *>(xpath_expr), ctx);
+
+    if (obj == 0 || obj->nodesetval == 0)
+    {
+        return 0;
+    }
+
+    xmlNodeSetPtr ns = obj->nodesetval;
+
+    int size    = ns->nodeNr;
+    int removed = size;
+
+    for(int i = 0; i < size; ++i)
+    {
+        xmlNodePtr cur = ns->nodeTab[i];
+
+        if ( cur == 0 || cur->type != XML_ELEMENT_NODE )
+        {
+            removed--;
+            continue;
+        }
+
+        xmlUnlinkNode(cur);
+
+        xmlFreeNode(cur);
+    }
+
+    xmlXPathFreeObject(obj);
+
+    return removed;
+}
+
 /* ************************************************************************ */
 /* Host :: Parse functions to compute rank and evaluate requirements        */
 /* ************************************************************************ */