feature #1685: Clean network in CLEANUP. Adds cleanup_action to VMM. Make use of new action in LCM

include/TransferManager.h

@@ -54,6 +54,7 @@ public:
         EPILOG_DELETE,
         EPILOG_DELETE_PREVIOUS,
         EPILOG_DELETE_STOP,
+        EPILOG_DELETE_BOTH,
         CHECKPOINT,
         DRIVER_CANCEL,
         FINALIZE
@@ -138,6 +139,20 @@ public:
         VirtualMachine *        vm,
         ostream&                xfr);
 
+    /**
+     *  This function generates the the epilog_delete sequence for current,
+     *  front-end and previous hosts.
+     *    @param vm pointer to VM, locked
+     *    @param xfr stream to write the commands
+     *    @param local true to delete the front-end
+     *    @param previous true to delete the previous host
+     *
+     *    @return 0 on success
+     */
+    int epilog_delete_commands(VirtualMachine *vm,
+                               ostream&        xfr,
+                               bool            local,
+                               bool            previous);
 private:
     /**
      *  Thread id for the Transfer Manager
@@ -247,7 +262,8 @@ private:
     void epilog_stop_action(int vid);
 
     /**
-     *  This function starts the epilog_delete sequence
+     *  This function starts the epilog_delete sequence in the current host
+     *    @param vid the Virtual Machine ID
      */
     void epilog_delete_action(int vid)
     {
@@ -256,6 +272,8 @@ private:
 
     /**
      *  This function starts the epilog_delete_stop sequence on the local host
+     *  i.e. the front-end (the VM is not running)
+     *    @param vid the Virtual Machine ID
      */
     void epilog_delete_stop_action(int vid)
     {
@@ -263,21 +281,22 @@ private:
     }
 
     /**
-     *  This function starts the epilog_delete sequence
+     *  This function starts the epilog_delete sequence on the previous host
+     *    @param vid the Virtual Machine ID
      */
-    void epilog_delete_action(bool local, int vid);
+    void epilog_delete_previous_action(int vid);
+
+    /**
+     *  This function starts the epilog_delete sequence on the current and
+     *  previous hosts
+     *    @param vid the Virtual Machine ID
+     */
+    void epilog_delete_both_action(int vid);
 
     /**
      *  This function starts the epilog_delete sequence
      */
-    int epilog_delete_commands(VirtualMachine *vm,
+    void epilog_delete_action(bool local, int vid);
-                               ostream&        xfr,
-                               bool            local,
-                               bool            previous);
-    /**
-     *  This function starts the epilog_delete sequence on the previous host
-     */
-    void epilog_delete_previous_action(int vid);
 
     /**
      *  This function starts the epilog sequence

include/VirtualMachineManager.h

@@ -49,6 +49,8 @@ public:
         SHUTDOWN,
         CANCEL,
         CANCEL_PREVIOUS,
+        CLEANUP,
+        CLEANUP_BOTH,
         MIGRATE,
         RESTORE,
         REBOOT,
@@ -257,6 +259,13 @@ private:
     void cancel_previous_action(
         int vid);
 
+    /**
+     *  Cleanups a host (cancel VM + delete disk images).
+     *    @param vid the id of the VM.
+     */
+    void cleanup_action(
+        int vid, bool cancel_previous);
+
     /**
      *  Function to migrate (live) a VM (MIGRATE action).
      *    @param vid the id of the VM.

include/VirtualMachineManagerDriver.h

@@ -30,7 +30,7 @@ using namespace std;
  *  VirtualMachineManagerDriver provides a base class to implement VM Manager
  *  Drivers. This class implements the protocol and recover functions
  *  from the Mad interface. Classes derived from the VirtualMachineManagerDriver
- *  must implement the deployment function to generate specific VM  
+ *  must implement the deployment function to generate specific VM
  *  deployment information for the unerlying MAD.
  */
 class VirtualMachineManagerDriver : public Mad
@@ -53,7 +53,7 @@ public:
         string&     message);
 
     /**
-     *  TODO: What do we need here? just poll the active VMs to recover 
+     *  TODO: What do we need here? just poll the active VMs to recover
      *  connections? Or an specific recover action from the MAD?
      */
     void recover();
@@ -67,25 +67,25 @@ public:
     virtual int deployment_description(
         const VirtualMachine *  vm,
         const string&           file_name) const = 0;
-    
+
-protected:	
+protected:
     /**
-     *  Gets a configuration attr from driver configuration file (single 
+     *  Gets a configuration attr from driver configuration file (single
      *  version)
      *    @param name of config attribute
      *    @param value of the attribute
      */
     void get_default(
-    	const char *  name, 
+    	const char *  name,
         string&       value) const
     {
     	string sn = name;
-    	
+
     	driver_conf.get(sn,value);
-    }    
+    }
 
     /**
-     *  Gets a configuration attr from driver configuration file (vector 
+     *  Gets a configuration attr from driver configuration file (vector
      *  version)
      *    @param name of config vector attribute for the domain
      *    @param vname of the attribute
@@ -95,20 +95,20 @@ protected:
     	const char *  name,
     	const char *  vname,
         string&       value) const;
-    
+
-private:	
+private:
-	/**	
+	/**
 	 *  Configuration file for the driver
 	 */
 	Template	driver_conf;
-	
+
     /**
      *  Pointer to the Virtual Machine Pool, to access VMs
      */
     VirtualMachinePool * vmpool;
 
     friend class VirtualMachineManager;
-      
+
     /**
      *  Sends a deploy request to the MAD: "DEPLOY ID XML_DRV_MSG"
      *    @param oid the virtual machine id.
@@ -169,6 +169,18 @@ private:
         write_drv("CANCEL", oid, drv_msg);
     }
 
+    /**
+     *  Sends a cleanup request to the MAD: "CLEANUP ID XML_DRV_MSG"
+     *    @param oid the virtual machine id.
+     *    @param drv_msg xml data for the mad operation
+     */
+    void cleanup (
+        const int     oid,
+        const string& drv_msg) const
+    {
+        write_drv("CLEANUP", oid, drv_msg);
+    }
+
     /**
      *  Sends a checkpoint request to the MAD: "CHECKPOINT ID XML_DRV_MSG"
      *    @param oid the virtual machine id.
@@ -263,7 +275,7 @@ private:
         ostringstream os;
 
         os << aname << " " << oid << " " << msg << endl;
-    
+
         write(os);
     }
 };

src/lcm/LifeCycleActions.cc

@@ -683,9 +683,7 @@ void  LifeCycleManager::clean_up_vm(VirtualMachine * vm)
             vmpool->update_history(vm);
 
             vmm->trigger(VirtualMachineManager::DRIVER_CANCEL,vid);
-            vmm->trigger(VirtualMachineManager::CANCEL,vid);
+            vmm->trigger(VirtualMachineManager::CLEANUP,vid);
-
-            tm->trigger(TransferManager::EPILOG_DELETE,vid);
         break;
 
         case VirtualMachine::MIGRATE:
@@ -701,11 +699,7 @@ void  LifeCycleManager::clean_up_vm(VirtualMachine * vm)
             hpool->del_capacity(vm->get_previous_hid(), vm->get_oid(), cpu, mem, disk);
 
             vmm->trigger(VirtualMachineManager::DRIVER_CANCEL,vid);
-
+            vmm->trigger(VirtualMachineManager::CLEANUP_BOTH,vid);
-            vmm->trigger(VirtualMachineManager::CANCEL,vid);
-            vmm->trigger(VirtualMachineManager::CANCEL_PREVIOUS,vid);
-
-            tm->trigger(TransferManager::EPILOG_DELETE,vid);
         break;
 
         case VirtualMachine::SAVE_STOP:
@@ -714,9 +708,7 @@ void  LifeCycleManager::clean_up_vm(VirtualMachine * vm)
             vmpool->update_history(vm);
 
             vmm->trigger(VirtualMachineManager::DRIVER_CANCEL,vid);
-            vmm->trigger(VirtualMachineManager::CANCEL,vid);
+            vmm->trigger(VirtualMachineManager::CLEANUP,vid);
-
-            tm->trigger(TransferManager::EPILOG_DELETE,vid);
         break;
 
         case VirtualMachine::SAVE_MIGRATE:
@@ -742,8 +734,7 @@ void  LifeCycleManager::clean_up_vm(VirtualMachine * vm)
             vmpool->update_history(vm);
 
             tm->trigger(TransferManager::DRIVER_CANCEL,vid);
-            tm->trigger(TransferManager::EPILOG_DELETE,vid);
+            tm->trigger(TransferManager::EPILOG_DELETE_BOTH,vid);
-            tm->trigger(TransferManager::EPILOG_DELETE_PREVIOUS,vid);
         break;
 
         case VirtualMachine::EPILOG_STOP:

src/tm/TransferManager.cc

@@ -115,6 +115,10 @@ void TransferManager::trigger(Actions action, int _vid)
         aname = "EPILOG_DELETE_PREVIOUS";
         break;
 
+    case EPILOG_DELETE_BOTH:
+        aname = "EPILOG_DELETE_BOTH";
+        break;
+
     case CHECKPOINT:
         aname = "CHECKPOINT";
         break;
@@ -183,6 +187,10 @@ void TransferManager::do_action(const string &action, void * arg)
     {
         epilog_delete_previous_action(vid);
     }
+    else if (action == "EPILOG_DELETE_BOTH")
+    {
+        epilog_delete_both_action(vid);
+    }
     else if (action == "CHECKPOINT")
     {
         checkpoint_action(vid);
@@ -1394,8 +1402,7 @@ error_driver:
 
 error_file:
     os << "epilog_delete, could not open file: " << xfr_name;
-    os << ". You may need to manually clean " << vm->get_hostname()
+    os << ". You may need to manually clean the host (current)";
-       << ":" << vm->get_remote_system_dir();
     goto error_common;
 
 error_common:
@@ -1468,8 +1475,81 @@ error_driver:
 
 error_file:
     os << "epilog_delete_previous, could not open file: " << xfr_name;
-    os << ". You may need to manually clean " << vm->get_previous_hostname()
+    os << ". You may need to manually clean the host (previous)";
-       << ":" << vm->get_remote_system_dir();
+    goto error_common;
+
+error_common:
+    vm->log("TM", Log::ERROR, os);
+    (nd.get_lcm())->trigger(LifeCycleManager::EPILOG_FAILURE, vid);
+
+    vm->unlock();
+    return;
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
+void TransferManager::epilog_delete_both_action(int vid)
+{
+    ostringstream os;
+
+    ofstream xfr;
+    string   xfr_name;
+
+    VirtualMachine * vm;
+    Nebula&          nd = Nebula::instance();
+
+    const TransferManagerDriver * tm_md;
+
+    int rc;
+
+    // ------------------------------------------------------------------------
+    // Setup & Transfer script
+    // ------------------------------------------------------------------------
+    vm = vmpool->get(vid,true);
+
+    if (vm == 0)
+    {
+        return;
+    }
+
+    tm_md = get();
+
+    if (tm_md == 0)
+    {
+        goto error_driver;
+    }
+
+    xfr_name = vm->get_transfer_file() + ".delete_both";
+    xfr.open(xfr_name.c_str(),ios::out | ios::trunc);
+
+    if (xfr.fail() == true)
+    {
+        goto error_file;
+    }
+
+    rc = epilog_delete_commands(vm, xfr, false, false); //current
+    rc = epilog_delete_commands(vm, xfr, false, true);  //previous
+
+    if ( rc != 0 )
+    {
+        goto error_common;
+    }
+
+    xfr.close();
+
+    tm_md->transfer(vid, xfr_name);
+
+    vm->unlock();
+    return;
+
+error_driver:
+    os << "epilog_delete_both, error getting TM driver.";
+    goto error_common;
+
+error_file:
+    os << "epilog_delete_both, could not open file: " << xfr_name;
+    os << ". You may need to manually clean hosts (previous & current)";
     goto error_common;
 
 error_common:

src/vmm/VirtualMachineManager.cc

@@ -146,6 +146,14 @@ void VirtualMachineManager::trigger(Actions action, int _vid)
         aname = "CANCEL_PREVIOUS";
         break;
 
+    case CLEANUP:
+        aname = "CLEANUP";
+        break;
+
+    case CLEANUP_BOTH:
+        aname = "CLEANUP_BOTH";
+        break;
+
     case MIGRATE:
         aname = "MIGRATE";
         break;
@@ -234,6 +242,14 @@ void VirtualMachineManager::do_action(const string &action, void * arg)
     {
         cancel_previous_action(vid);
     }
+    else if (action == "CLEANUP")
+    {
+        cleanup_action(vid, false);
+    }
+    else if (action == "CLEANUP_BOTH")
+    {
+        cleanup_action(vid, true);
+    }
     else if (action == "MIGRATE")
     {
         migrate_action(vid);
@@ -925,6 +941,106 @@ error_common:
 /* -------------------------------------------------------------------------- */
 /* -------------------------------------------------------------------------- */
 
+void VirtualMachineManager::cleanup_action(
+    int vid, bool cancel_previous)
+{
+    int rc;
+
+    VirtualMachine * vm;
+    ostringstream    os;
+
+    string   vm_tmpl;
+    string * drv_msg;
+
+    string m_hostname = "";
+    string m_net_drv  = "";
+
+    const VirtualMachineManagerDriver *   vmd;
+
+    Nebula& nd = Nebula::instance();
+
+    // Get the VM from the pool
+    vm = vmpool->get(vid,true);
+
+    if (vm == 0)
+    {
+        return;
+    }
+
+    if (!vm->hasHistory())
+    {
+        goto error_history;
+    }
+
+    // Get the driver for this VM
+    vmd = get(vm->get_vmm_mad());
+
+    if ( vmd == 0 )
+    {
+        goto error_driver;
+    }
+
+    if ( cancel_previous && vm->hasPreviousHistory() )
+    {
+        m_hostname = vm->get_previous_hostname();
+        m_net_drv  = vm->get_previous_vnm_mad();
+    }
+
+    rc = nd.get_tm()->epilog_delete_commands(vm, os, false, false);
+
+    if ( rc != 0 )
+    {
+        goto error_common;
+    }
+
+    // Invoke driver method
+    drv_msg = format_message(
+        vm->get_hostname(),
+        vm->get_vnm_mad(),
+        m_hostname,
+        m_net_drv,
+        vm->get_deploy_id(),
+        "",
+        "",
+        "",
+        os.str(),
+        "",
+        vm->to_xml(vm_tmpl));
+
+    vmd->cleanup(vid, *drv_msg);
+
+    delete drv_msg;
+
+    vm->unlock();
+
+    return;
+
+error_history:
+    os.str("");
+    os << "cleanup_action, VM has no history";
+    goto error_common;
+
+error_driver:
+    os.str("");
+    os << "cleanup_action, error getting driver " << vm->get_vmm_mad();
+
+error_common:
+    if ( vm->get_lcm_state() == VirtualMachine::CLEANUP)
+    {
+        Nebula              &ne = Nebula::instance();
+        LifeCycleManager *  lcm = ne.get_lcm();
+
+        //TODO
+        //lcm->trigger(LifeCycleManager::CLEANUP_FAILURE, vid);
+    }
+
+    vm->unlock();
+    return;
+}
+
+/* -------------------------------------------------------------------------- */
+/* -------------------------------------------------------------------------- */
+
 void VirtualMachineManager::migrate_action(
     int vid)
 {
@@ -1366,7 +1482,7 @@ void VirtualMachineManager::attach_action(
     vm_tm_mad = vm->get_tm_mad();
     opennebula_hostname = nd.get_nebula_hostname();
 
-    rc = Nebula::instance().get_tm()->prolog_transfer_command(
+    rc = nd.get_tm()->prolog_transfer_command(
             vm,
             disk,
             vm_tm_mad,

src/vmm_mad/exec/one_vmm_exec.rb

@@ -627,18 +627,33 @@ class ExecDriver < VirtualMachineDriver
                 :action     => :cancel,
                 :parameters => [:deploy_id, :host],
                 :no_fail    => true
+            },
+            {
+                :driver  => :vnm,
+                :action  => :clean,
+                :no_fail => true
             }
         ]
 
         # Cancel the VM at the previous host (in case of migration)
-        steps <<
+        if !mhost.empty?
+            steps <<
             {
                 :driver      => :vmm,
                 :action      => :cancel,
                 :parameters  => [:deploy_id, :dest_host],
                 :destination => true,
                 :no_fail     => true
-            } if !mhost.empty?
+            }
+
+            steps <<
+            {
+                :driver  => :vnm,
+                :action  => :clean,
+                :destination => true,
+                :no_fail => true
+            }
+        end
 
         # Cleans VM disk images and directory
         tm_command.each_line { |tc|