From dac9db95d208a19a73e9a2aa238098acf5f3ff82 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Tue, 16 Jun 2015 23:40:09 +0200 Subject: [PATCH 01/16] feature #3782: Size for snapshots (Virtual size). Renamed is* functions to is_* in VirtualMachine. --- include/Snapshots.h | 8 ++++- include/VirtualMachine.h | 11 ++++-- src/dm/DispatchManagerStates.cc | 2 +- src/rm/RequestManagerVirtualMachine.cc | 8 ++--- src/tm/TransferManager.cc | 10 +++--- src/vm/Snapshots.cc | 25 +++++++++++-- src/vm/VirtualMachine.cc | 50 +++++++++++++++++--------- 7 files changed, 81 insertions(+), 33 deletions(-) diff --git a/include/Snapshots.h b/include/Snapshots.h index adbfea9a8c..43a6719392 100644 --- a/include/Snapshots.h +++ b/include/Snapshots.h @@ -76,9 +76,10 @@ public: /** * Creates a new (empty) snapshot of the active disk * @param tag description of this snapshot (optional) + * @param size_mb of the snapshot (virtual size) * @return id of the new snapshot */ - int create_snapshot(const string& tag); + int create_snapshot(const string& tag, unsigned int size_mb); /** * Check if an snapshot can be deleted (no children, no active) @@ -168,6 +169,11 @@ public: return (snap != 0); } + /** + * @return total snapshot size (virtual) in mb + */ + unsigned int get_total_size() const; + /** * Get Attribute from the given snapshot * @param id of the snapshot diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index 2ca85bfa57..f79433abbf 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -1233,17 +1233,22 @@ public: /** * Check if the given disk is volatile */ - static bool isVolatile(const VectorAttribute * disk); + static bool is_volatile(const VectorAttribute * disk); /** * Check if the template contains a volatile disk */ - static bool isVolatile(const Template * tmpl); + static bool is_volatile(const Template * tmpl); + + /** + * Check if the disk is persistent + */ + static bool is_persistent(const VectorAttribute * disk); /** * Check if the themplate is for an imported VM */ - bool isImported() const; + bool is_imported() const; /** * Return the total SIZE of volatile disks diff --git a/src/dm/DispatchManagerStates.cc b/src/dm/DispatchManagerStates.cc index a7728b2490..fde85534a5 100644 --- a/src/dm/DispatchManagerStates.cc +++ b/src/dm/DispatchManagerStates.cc @@ -236,7 +236,7 @@ void DispatchManager::done_action(int vid) gid = vm->get_gid(); tmpl = vm->clone_template(); - if (vm->isImported()) + if (vm->is_imported()) { deploy_id = vm->get_deploy_id(); } diff --git a/src/rm/RequestManagerVirtualMachine.cc b/src/rm/RequestManagerVirtualMachine.cc index b29f0f13f5..0527305886 100644 --- a/src/rm/RequestManagerVirtualMachine.cc +++ b/src/rm/RequestManagerVirtualMachine.cc @@ -519,7 +519,7 @@ void VirtualMachineAction::request_execute(xmlrpc_c::paramList const& paramList, return; } - if (vm->isImported() && ( + if (vm->is_imported() && ( action == History::DELETE_RECREATE_ACTION || action == History::UNDEPLOY_ACTION || action == History::UNDEPLOY_HARD_ACTION || @@ -848,7 +848,7 @@ void VirtualMachineDeploy::request_execute(xmlrpc_c::paramList const& paramList, return; } - if (vm->isImported()) + if (vm->is_imported()) { dm->import(vm); } @@ -992,7 +992,7 @@ void VirtualMachineMigrate::request_execute(xmlrpc_c::paramList const& paramList return; } - if (vm->isImported()) + if (vm->is_imported()) { failure_response(ACTION, request_error("Migration is not supported for imported VMs",""), @@ -1561,7 +1561,7 @@ void VirtualMachineAttach::request_execute(xmlrpc_c::paramList const& paramList, RequestAttributes att_quota(vm_perms.uid, vm_perms.gid, att); - volatile_disk = VirtualMachine::isVolatile(tmpl); + volatile_disk = VirtualMachine::is_volatile(tmpl); if ( volatile_disk ) { diff --git a/src/tm/TransferManager.cc b/src/tm/TransferManager.cc index c4d95f34fd..cdc7735a84 100644 --- a/src/tm/TransferManager.cc +++ b/src/tm/TransferManager.cc @@ -907,7 +907,7 @@ void TransferManager::prolog_migr_action(int vid) disk->vector_value_str("DISK_ID", disk_id); - if ( VirtualMachine::isVolatile(disk) == true ) + if ( VirtualMachine::is_volatile(disk) == true ) { tm_mad = vm_tm_mad; ds_id = vm->get_ds_id(); @@ -1045,7 +1045,7 @@ void TransferManager::prolog_resume_action(int vid) disk->vector_value_str("DISK_ID", disk_id); - if ( VirtualMachine::isVolatile(disk) == true ) + if ( VirtualMachine::is_volatile(disk) == true ) { tm_mad = vm_tm_mad; ds_id = vm->get_ds_id(); @@ -1277,7 +1277,7 @@ void TransferManager::epilog_transfer_command( int ds_id_i; int vv_rc = 0; - if ( VirtualMachine::isVolatile(disk) == true ) + if ( VirtualMachine::is_volatile(disk) == true ) { tm_mad = vm->get_tm_mad(); ds_id_i= vm->get_ds_id(); @@ -1479,7 +1479,7 @@ void TransferManager::epilog_stop_action(int vid) disk->vector_value_str("DISK_ID", disk_id); - if ( VirtualMachine::isVolatile(disk) == true ) + if ( VirtualMachine::is_volatile(disk) == true ) { tm_mad = vm_tm_mad; ds_id = vm->get_ds_id(); @@ -1630,7 +1630,7 @@ int TransferManager::epilog_delete_commands(VirtualMachine *vm, disk->vector_value_str("DISK_ID", disk_id); - if ( VirtualMachine::isVolatile(disk) == true ) + if ( VirtualMachine::is_volatile(disk) == true ) { tm_mad = vm_tm_mad; ds_id = vm_ds_id; diff --git a/src/vm/Snapshots.cc b/src/vm/Snapshots.cc index 2cee81882b..30276ded36 100644 --- a/src/vm/Snapshots.cc +++ b/src/vm/Snapshots.cc @@ -122,15 +122,16 @@ void Snapshots::init() /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -int Snapshots::create_snapshot(const string& tag) +int Snapshots::create_snapshot(const string& tag, unsigned int size_mb) { VectorAttribute * snapshot = new VectorAttribute("SNAPSHOT"); if (!tag.empty()) { - snapshot->replace("TAG",tag); + snapshot->replace("TAG", tag); } + snapshot->replace("SIZE", size_mb); snapshot->replace("ID", next_snapshot); snapshot->replace("DATE", static_cast(time(0))); snapshot->replace("PARENT", active); @@ -317,3 +318,23 @@ bool Snapshots::test_delete(unsigned int id, string& error) const return true; } + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +unsigned int Snapshots::get_total_size() const +{ + map::const_iterator it; + unsigned int size_mb, total_mb = 0; + + for ( it = snapshot_pool.begin(); it != snapshot_pool.end(); it++) + { + if (it->second->vector_value("SIZE", size_mb) == 0) + { + total_mb += size_mb; + } + } + + return total_mb; +} + diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index a87af5db8f..19985f91e5 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -2227,7 +2227,7 @@ VectorAttribute * VirtualMachine::delete_attach_disk(Snapshots **snap) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -bool VirtualMachine::isVolatile(const VectorAttribute * disk) +bool VirtualMachine::is_volatile(const VectorAttribute * disk) { string type = disk->vector_value("TYPE"); @@ -2239,21 +2239,30 @@ bool VirtualMachine::isVolatile(const VectorAttribute * disk) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -bool VirtualMachine::isVolatile(const Template * tmpl) +bool VirtualMachine::is_persistent(const VectorAttribute * disk) +{ + bool pers_disk; + + disk->vector_value("PERSISTENT", pers_disk); + + return pers_disk; +} + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +bool VirtualMachine::is_volatile(const Template * tmpl) { vector disks; + const VectorAttribute * disk; + int num_disks = tmpl->get("DISK", disks); for (int i = 0 ; i < num_disks ; i++) { - const VectorAttribute * disk = dynamic_cast(disks[i]); + disk =static_cast(disks[i]); - if (disk == 0) - { - continue; - } - - if (VirtualMachine::isVolatile(disk)) + if (is_volatile(disk)) { return true; } @@ -2265,13 +2274,13 @@ bool VirtualMachine::isVolatile(const Template * tmpl) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -bool VirtualMachine::isImported() const +bool VirtualMachine::is_imported() const { - bool is_imported = false; + bool imported = false; - get_template_attribute("IMPORTED", is_imported); + get_template_attribute("IMPORTED", imported); - return is_imported; + return imported; } /* -------------------------------------------------------------------------- */ @@ -2299,7 +2308,7 @@ long long VirtualMachine::get_volatile_disk_size(Template * tmpl) continue; } - if (!VirtualMachine::isVolatile(disk)) + if (!VirtualMachine::is_volatile(disk)) { continue; } @@ -4168,6 +4177,7 @@ int VirtualMachine::get_snapshot_disk(string& ds_id, string& tm_mad, int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) { map::iterator it; + unsigned int size_mb; int snap_id; VectorAttribute * disk; @@ -4180,19 +4190,25 @@ int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) return -1; } - if (isVolatile(disk)) + if (is_volatile(disk)) { error = "Cannot make snapshots on volatile disks"; return -1; } + if (disk->vector_value("SIZE", size_mb) != 0 ) + { + error = "Wrong size in disk"; + return -1; + } + it = snapshots.find(did); if ( it == snapshots.end() ) { Snapshots * snap = new Snapshots(did); - snap_id = snap->create_snapshot(tag); + snap_id = snap->create_snapshot(tag, size_mb); if (snap_id != -1) { @@ -4205,7 +4221,7 @@ int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) } else { - snap_id = it->second->create_snapshot(tag); + snap_id = it->second->create_snapshot(tag, size_mb); } if (snap_id != -1) From 9542d396b81031f4eb7690f72c31a00b4fd09217 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 17 Jun 2015 10:10:28 +0200 Subject: [PATCH 02/16] feature #3782: Update quotas on non-persistent snapshots --- include/VirtualMachine.h | 6 +- src/rm/RequestManagerVirtualMachine.cc | 92 +++++++++++++++++++++++++- src/vm/VirtualMachine.cc | 44 +++++++----- 3 files changed, 121 insertions(+), 21 deletions(-) diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index f79433abbf..132fcf323e 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -1261,6 +1261,11 @@ public: */ void get_security_groups(set& sgs) const; + /** + * + */ + const VectorAttribute* get_disk(int disk_id) const; + // ------------------------------------------------------------------------ // Context related functions // ------------------------------------------------------------------------ @@ -1997,7 +2002,6 @@ private: static_cast(*this).get_disk(disk_id)); }; - const VectorAttribute* get_disk(int disk_id) const; protected: diff --git a/src/rm/RequestManagerVirtualMachine.cc b/src/rm/RequestManagerVirtualMachine.cc index 0527305886..4aa5a7ca51 100644 --- a/src/rm/RequestManagerVirtualMachine.cc +++ b/src/rm/RequestManagerVirtualMachine.cc @@ -2343,6 +2343,17 @@ void VirtualMachineDiskSnapshotCreate::request_execute( Nebula& nd = Nebula::instance(); DispatchManager * dm = nd.get_dm(); + VirtualMachine * vm; + + PoolObjectAuth vm_perms; + + const VectorAttribute * disk; + + VectorAttribute * delta_disk = 0; + VirtualMachineTemplate * deltas = 0; + + Template ds_deltas; + int rc; int snap_id; string error_str; @@ -2356,19 +2367,94 @@ void VirtualMachineDiskSnapshotCreate::request_execute( return; } + // ------------------------------------------------------------------------ + // Check quotas for the new snapshot + // ------------------------------------------------------------------------ + if ((vm = get_vm(id, att)) == 0) + { + return; + } + + disk = (const_cast(vm))->get_disk(did); + + if (disk == 0) + { + failure_response(ACTION, request_error("VM disk does not exists", ""), att); + + vm->unlock(); + + return; + } + + string disk_size = disk->vector_value("SIZE"); + string ds_id = disk->vector_value("DATASTORE_ID"); + bool persistent = VirtualMachine::is_persistent(disk); + + vm->get_permissions(vm_perms); + + vm->unlock(); + + RequestAttributes att_quota(vm_perms.uid, vm_perms.gid, att); + + if (VirtualMachine::is_volatile(disk)) + { + failure_response(ACTION, request_error("Cannot make snapshots on " + "volatile disks",""), att); + return; + } + else if (persistent) + { + ds_deltas.add("DATASTORE", ds_id); + ds_deltas.add("SIZE", disk_size); + + if (!quota_authorization(&ds_deltas, Quotas::DATASTORE, att_quota)) + { + return; + } + } + else + { + deltas = new VirtualMachineTemplate(); + + delta_disk = new VectorAttribute("DISK"); + delta_disk->replace("TYPE", "FS"); + delta_disk->replace("SIZE", disk_size); + + deltas->add("VMS", 0); + deltas->set(delta_disk); + + if (!quota_resize_authorization(id, deltas, att_quota)) + { + delete deltas; + return; + } + } + + // ------------------------------------------------------------------------ + // Do the snapshot + // ------------------------------------------------------------------------ rc = dm->disk_snapshot_create(id, did, tag, snap_id, error_str); if ( rc != 0 ) { - failure_response(ACTION, - request_error(error_str, ""), - att); + if (persistent) + { + quota_rollback(&ds_deltas, Quotas::DATASTORE, att_quota); + } + else + { + quota_rollback(deltas, Quotas::VM, att_quota); + } + + failure_response(ACTION, request_error(error_str, ""), att); } else { success_response(snap_id, att); } + delete deltas; + return; } diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 19985f91e5..723e8376b7 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -2288,35 +2288,37 @@ bool VirtualMachine::is_imported() const long long VirtualMachine::get_volatile_disk_size(Template * tmpl) { - long long size = 0; + long long disk_size, size = 0; vector disks; - int num_disks = tmpl->get("DISK", disks); + const VectorAttribute * disk; - if (num_disks == 0) - { - return size; - } + int num_disks = tmpl->get("DISK", disks); for (int i = 0 ; i < num_disks ; i++) { - long long disk_size; - const VectorAttribute * disk = dynamic_cast(disks[i]); + disk = dynamic_cast(disks[i]); if (disk == 0) { continue; } - if (!VirtualMachine::is_volatile(disk)) + if (is_volatile(disk)) { - continue; + if (disk->vector_value("SIZE", disk_size) == 0) + { + size += disk_size; + } + } + else if (!is_persistent(disk)) + { + if (disk->vector_value("DISK_SNAPSHOT_TOTAL_SIZE", disk_size) == 0) + { + size += disk_size; + } } - if (disk->vector_value("SIZE", disk_size) == 0) - { - size += disk_size; - } } return size; @@ -4177,7 +4179,7 @@ int VirtualMachine::get_snapshot_disk(string& ds_id, string& tm_mad, int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) { map::iterator it; - unsigned int size_mb; + unsigned int size_mb, snap_size; int snap_id; VectorAttribute * disk; @@ -4208,7 +4210,8 @@ int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) { Snapshots * snap = new Snapshots(did); - snap_id = snap->create_snapshot(tag, size_mb); + snap_id = snap->create_snapshot(tag, size_mb); + snap_size = size_mb; if (snap_id != -1) { @@ -4221,13 +4224,15 @@ int VirtualMachine::new_disk_snapshot(int did, const string& tag, string& error) } else { - snap_id = it->second->create_snapshot(tag, size_mb); + snap_id = it->second->create_snapshot(tag, size_mb); + snap_size = it->second->get_total_size(); } if (snap_id != -1) { disk->replace("DISK_SNAPSHOT_ACTIVE", "YES"); disk->replace("DISK_SNAPSHOT_ID", snap_id); + disk->replace("DISK_SNAPSHOT_TOTAL_SIZE", snap_size); } return snap_id; @@ -4288,6 +4293,7 @@ void VirtualMachine::delete_disk_snapshot(int did, int snap_id) { map::iterator it; VectorAttribute * disk = get_disk(did); + unsigned int snap_size; if ( disk == 0 ) { @@ -4303,6 +4309,10 @@ void VirtualMachine::delete_disk_snapshot(int did, int snap_id) it->second->delete_snapshot(snap_id); + snap_size = it->second->get_total_size(); + + disk->replace("DISK_SNAPSHOT_TOTAL_SIZE", snap_size); + if (it->second->size() == 0) { Snapshots * tmp = it->second; From b2101281ecaed6a7f81b85f7557d9c4648ad6fe9 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 17 Jun 2015 16:02:12 +0200 Subject: [PATCH 03/16] feature #3782: Update quotas for snapshots on non-persistent disks on disk detach and vm done --- src/rm/RequestManagerVirtualMachine.cc | 25 +++++++++---------------- src/vm/VirtualMachinePool.cc | 23 +++++++++++++---------- 2 files changed, 22 insertions(+), 26 deletions(-) diff --git a/src/rm/RequestManagerVirtualMachine.cc b/src/rm/RequestManagerVirtualMachine.cc index 4aa5a7ca51..106e4a9d09 100644 --- a/src/rm/RequestManagerVirtualMachine.cc +++ b/src/rm/RequestManagerVirtualMachine.cc @@ -2348,11 +2348,9 @@ void VirtualMachineDiskSnapshotCreate::request_execute( PoolObjectAuth vm_perms; const VectorAttribute * disk; - VectorAttribute * delta_disk = 0; - VirtualMachineTemplate * deltas = 0; - Template ds_deltas; + Template deltas; int rc; int snap_id; @@ -2404,28 +2402,25 @@ void VirtualMachineDiskSnapshotCreate::request_execute( } else if (persistent) { - ds_deltas.add("DATASTORE", ds_id); - ds_deltas.add("SIZE", disk_size); + deltas.add("DATASTORE", ds_id); + deltas.add("SIZE", disk_size); - if (!quota_authorization(&ds_deltas, Quotas::DATASTORE, att_quota)) + if (!quota_authorization(&deltas, Quotas::DATASTORE, att_quota)) { return; } } else { - deltas = new VirtualMachineTemplate(); - delta_disk = new VectorAttribute("DISK"); delta_disk->replace("TYPE", "FS"); delta_disk->replace("SIZE", disk_size); - deltas->add("VMS", 0); - deltas->set(delta_disk); + deltas.add("VMS", 0); + deltas.set(delta_disk); - if (!quota_resize_authorization(id, deltas, att_quota)) + if (!quota_resize_authorization(id, &deltas, att_quota)) { - delete deltas; return; } } @@ -2439,11 +2434,11 @@ void VirtualMachineDiskSnapshotCreate::request_execute( { if (persistent) { - quota_rollback(&ds_deltas, Quotas::DATASTORE, att_quota); + quota_rollback(&deltas, Quotas::DATASTORE, att_quota); } else { - quota_rollback(deltas, Quotas::VM, att_quota); + quota_rollback(&deltas, Quotas::VM, att_quota); } failure_response(ACTION, request_error(error_str, ""), att); @@ -2453,8 +2448,6 @@ void VirtualMachineDiskSnapshotCreate::request_execute( success_response(snap_id, att); } - delete deltas; - return; } diff --git a/src/vm/VirtualMachinePool.cc b/src/vm/VirtualMachinePool.cc index 43f83fd109..a722922d30 100644 --- a/src/vm/VirtualMachinePool.cc +++ b/src/vm/VirtualMachinePool.cc @@ -1063,12 +1063,23 @@ void VirtualMachinePool::delete_attach_disk(int vid) int image_id; tmpl.set(disk); + tmpl.add("VMS", 0); - if ( disk->vector_value("IMAGE_ID", image_id) == 0 ) + if (VirtualMachine::is_volatile(disk)) { - // Disk using an Image + Quotas::quota_del(Quotas::VM, uid, gid, &tmpl); + } + else + { + disk->vector_value("IMAGE_ID", image_id); + Quotas::quota_del(Quotas::IMAGE, uid, gid, &tmpl); + if (!VirtualMachine::is_persistent(disk)) + { + Quotas::quota_del(Quotas::VM, uid, gid, &tmpl); + } + if (snap != 0) { imagem->set_image_snapshots(image_id, *snap, false); @@ -1077,14 +1088,6 @@ void VirtualMachinePool::delete_attach_disk(int vid) imagem->release_image(oid, image_id, false); } - else // Volatile disk - { - // It is an update of the volatile counter without - // shutting destroying a VM - tmpl.add("VMS", 0); - - Quotas::quota_del(Quotas::VM, uid, gid, &tmpl); - } } } From 9964cb27e25f577bbe20708c62c49b687db67354 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 17 Jun 2015 17:45:37 +0200 Subject: [PATCH 04/16] feature #3782: Quotas for single snapshot deletion --- include/Snapshots.h | 9 ++++++++- include/VirtualMachine.h | 6 +++++- src/lcm/LifeCycleStates.cc | 30 ++++++++++++++++++++++++++++-- src/vm/Snapshots.cc | 20 ++++++++++++++++++-- src/vm/VirtualMachine.cc | 34 ++++++++++++++++++++++++++++++++-- 5 files changed, 91 insertions(+), 8 deletions(-) diff --git a/include/Snapshots.h b/include/Snapshots.h index 43a6719392..75722e56c6 100644 --- a/include/Snapshots.h +++ b/include/Snapshots.h @@ -174,6 +174,13 @@ public: */ unsigned int get_total_size() const; + /** + * Get the size (virtual) in mb of the given snapshot + * @param id of the snapshot + * @return size or 0 if not found + */ + unsigned int get_snapshot_size(unsigned int id) const; + /** * Get Attribute from the given snapshot * @param id of the snapshot @@ -181,7 +188,7 @@ public: * * @return value or empty if not found */ - string get_snapshot_attribute(unsigned int id, const char* name); + string get_snapshot_attribute(unsigned int id, const char* name) const; private: diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index 132fcf323e..d3f485337c 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -24,6 +24,7 @@ #include "Log.h" #include "NebulaLog.h" #include "NebulaUtil.h" +#include "Quotas.h" #include #include @@ -1534,8 +1535,11 @@ public: * called before actually deleting the snapshot. * @param disk_id of the disk * @param snap_id of the snapshot + * @param type of quota used by this snapshot + * @param quotas template with snapshot usage */ - void delete_disk_snapshot(int disk_id, int snap_id); + void delete_disk_snapshot(int disk_id, int snap_id, Quotas::QuotaType& type, + Template **quotas); /** * Get information about the disk to take the snapshot from diff --git a/src/lcm/LifeCycleStates.cc b/src/lcm/LifeCycleStates.cc index 76954ab173..18c739ea26 100644 --- a/src/lcm/LifeCycleStates.cc +++ b/src/lcm/LifeCycleStates.cc @@ -1722,6 +1722,9 @@ void LifeCycleManager::disk_snapshot_success(int vid) { string disk_id, tm_mad, ds_id, snap_id; + Quotas::QuotaType qt; + Template *quotas = 0; + VirtualMachine * vm = vmpool->get(vid,true); if ( vm == 0 ) @@ -1743,6 +1746,9 @@ void LifeCycleManager::disk_snapshot_success(int vid) int isnap_id = strtol(snap_id.c_str(),NULL,0); int idisk_id = strtol(disk_id.c_str(),NULL,0); + int uid = vm->get_uid(); + int gid = vm->get_gid(); + switch (vm->get_lcm_state()) { case VirtualMachine::DISK_SNAPSHOT_POWEROFF: @@ -1753,7 +1759,7 @@ void LifeCycleManager::disk_snapshot_success(int vid) case VirtualMachine::DISK_SNAPSHOT_DELETE_POWEROFF: vm->log("LCM", Log::INFO, "VM disk snapshot deleted."); - vm->delete_disk_snapshot(idisk_id, isnap_id); + vm->delete_disk_snapshot(idisk_id, isnap_id, qt, "as); break; default: @@ -1770,6 +1776,13 @@ void LifeCycleManager::disk_snapshot_success(int vid) vm->unlock(); + if ( quotas != 0 ) + { + Quotas::quota_del(qt, uid, gid, quotas); + + delete quotas; + } + return; } @@ -1780,6 +1793,9 @@ void LifeCycleManager::disk_snapshot_failure(int vid) { string disk_id, tm_mad, ds_id, snap_id; + Quotas::QuotaType qt; + Template *quotas = 0; + VirtualMachine * vm = vmpool->get(vid,true); if ( vm == 0 ) @@ -1801,11 +1817,14 @@ void LifeCycleManager::disk_snapshot_failure(int vid) int isnap_id = strtol(snap_id.c_str(),NULL,0); int idisk_id = strtol(disk_id.c_str(),NULL,0); + int uid = vm->get_uid(); + int gid = vm->get_gid(); + switch (vm->get_lcm_state()) { case VirtualMachine::DISK_SNAPSHOT_POWEROFF: vm->log("LCM", Log::ERROR, "Could not take disk snapshot."); - vm->delete_disk_snapshot(idisk_id, isnap_id); + vm->delete_disk_snapshot(idisk_id, isnap_id, qt, "as); break; case VirtualMachine::DISK_SNAPSHOT_DELETE_POWEROFF: @@ -1827,6 +1846,13 @@ void LifeCycleManager::disk_snapshot_failure(int vid) vm->unlock(); + if ( quotas != 0 ) + { + Quotas::quota_del(qt, uid, gid, quotas); + + delete quotas; + } + return; } diff --git a/src/vm/Snapshots.cc b/src/vm/Snapshots.cc index 30276ded36..fbc6235b4c 100644 --- a/src/vm/Snapshots.cc +++ b/src/vm/Snapshots.cc @@ -271,9 +271,9 @@ const VectorAttribute * Snapshots::get_snapshot(unsigned int id) const /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -string Snapshots::get_snapshot_attribute(unsigned int id, const char * name) +string Snapshots::get_snapshot_attribute(unsigned int id, const char * name) const { - VectorAttribute * snapshot = get_snapshot(id); + const VectorAttribute * snapshot = get_snapshot(id); if (snapshot == 0) { @@ -283,6 +283,22 @@ string Snapshots::get_snapshot_attribute(unsigned int id, const char * name) return snapshot->vector_value(name); } +/* -------------------------------------------------------------------------- */ + +unsigned int Snapshots::get_snapshot_size(unsigned int id) const +{ + unsigned int snap_size = 0; + + const VectorAttribute * snapshot = get_snapshot(id); + + if (snapshot != 0) + { + snapshot->vector_value("SIZE", snap_size); + } + + return snap_size; +} + /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 723e8376b7..0747336fca 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -4289,12 +4289,17 @@ int VirtualMachine::revert_disk_snapshot(int did, int snap_id) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -void VirtualMachine::delete_disk_snapshot(int did, int snap_id) +void VirtualMachine::delete_disk_snapshot(int did, int snap_id, + Quotas::QuotaType& type, Template **quotas) { map::iterator it; - VectorAttribute * disk = get_disk(did); unsigned int snap_size; + VectorAttribute * delta_disk; + VectorAttribute * disk = get_disk(did); + + *quotas = 0; + if ( disk == 0 ) { return; @@ -4307,6 +4312,8 @@ void VirtualMachine::delete_disk_snapshot(int did, int snap_id) return; } + unsigned int ssize = it->second->get_snapshot_size(snap_id); + it->second->delete_snapshot(snap_id); snap_size = it->second->get_total_size(); @@ -4321,5 +4328,28 @@ void VirtualMachine::delete_disk_snapshot(int did, int snap_id) delete tmp; } + + if ( is_persistent(disk) ) + { + *quotas = new Template(); + + (*quotas)->add("DATASTORE", disk->vector_value("DATASTORE_ID")); + (*quotas)->add("SIZE", (long long) ssize); + + type = Quotas::DATASTORE; + } + else + { + *quotas = new Template(); + + delta_disk = new VectorAttribute("DISK"); + delta_disk->replace("TYPE", "FS"); + delta_disk->replace("SIZE", ssize); + + (*quotas)->add("VMS", 0); + (*quotas)->set(delta_disk); + + type = Quotas::VM; + } } From a2e49d42647f71535947f2d4d79836931054d5df Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Thu, 18 Jun 2015 12:30:25 +0200 Subject: [PATCH 05/16] feature #3782: Snapshots do not increment IMAGE counter in datastore for persistent images --- src/rm/RequestManagerVirtualMachine.cc | 1 + src/um/QuotaDatastore.cc | 16 ++++++++++++++-- src/vm/VirtualMachine.cc | 1 + 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/rm/RequestManagerVirtualMachine.cc b/src/rm/RequestManagerVirtualMachine.cc index 106e4a9d09..1ec4480232 100644 --- a/src/rm/RequestManagerVirtualMachine.cc +++ b/src/rm/RequestManagerVirtualMachine.cc @@ -2404,6 +2404,7 @@ void VirtualMachineDiskSnapshotCreate::request_execute( { deltas.add("DATASTORE", ds_id); deltas.add("SIZE", disk_size); + deltas.add("IMAGES", 0); if (!quota_authorization(&deltas, Quotas::DATASTORE, att_quota)) { diff --git a/src/um/QuotaDatastore.cc b/src/um/QuotaDatastore.cc index 762ece997c..4fe303af02 100644 --- a/src/um/QuotaDatastore.cc +++ b/src/um/QuotaDatastore.cc @@ -33,6 +33,7 @@ bool QuotaDatastore::check(Template * tmpl, Quotas& default_quotas, string& erro string ds_id; int size; + int images, images_req = 1; tmpl->get("DATASTORE", ds_id); @@ -48,7 +49,12 @@ bool QuotaDatastore::check(Template * tmpl, Quotas& default_quotas, string& erro return false; } - ds_request.insert(make_pair("IMAGES",1)); + if (tmpl->get("IMAGES", images) && images >= 0) + { + images_req = images; + } + + ds_request.insert(make_pair("IMAGES", images_req)); ds_request.insert(make_pair("SIZE", size)); return check_quota(ds_id, ds_request, default_quotas, error); @@ -63,6 +69,7 @@ void QuotaDatastore::del(Template * tmpl) string ds_id; int size; + int images, images_req = 1; tmpl->get("DATASTORE", ds_id); @@ -76,7 +83,12 @@ void QuotaDatastore::del(Template * tmpl) return; } - ds_request.insert(make_pair("IMAGES",1)); + if (tmpl->get("IMAGES", images) && images >= 0) + { + images_req = images; + } + + ds_request.insert(make_pair("IMAGES", images_req)); ds_request.insert(make_pair("SIZE", size)); del_quota(ds_id, ds_request); diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 0747336fca..9227accf47 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -4335,6 +4335,7 @@ void VirtualMachine::delete_disk_snapshot(int did, int snap_id, (*quotas)->add("DATASTORE", disk->vector_value("DATASTORE_ID")); (*quotas)->add("SIZE", (long long) ssize); + (*quotas)->add("IMAGES",0 ); type = Quotas::DATASTORE; } From f9bec6a019cf9519a36ab3e759baf10888001a68 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Fri, 19 Jun 2015 12:49:45 +0200 Subject: [PATCH 06/16] feature #3782: Return disk and snapshot information for Ceph disks --- src/vmm_mad/remotes/poll_xen_kvm.rb | 66 ++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/src/vmm_mad/remotes/poll_xen_kvm.rb b/src/vmm_mad/remotes/poll_xen_kvm.rb index 13d699f9f7..aeb14ef0bd 100755 --- a/src/vmm_mad/remotes/poll_xen_kvm.rb +++ b/src/vmm_mad/remotes/poll_xen_kvm.rb @@ -313,27 +313,64 @@ module KVM def self.get_disk_usage(xml) return {} if !JSON_LOADED - doc=REXML::Document.new(xml) + doc = REXML::Document.new(xml) size = 0 data = { - :disk_actual_size => 0.0, + :disk_actual_size => 0.0, :disk_virtual_size => 0.0 } doc.elements.each('domain/devices/disk/source') do |ele| - next if !ele.attributes['file'] + # read the disk path (for regular disks) + file = ele.attributes['file'] rescue nil - text = `qemu-img info --output=json #{ele.attributes['file']}` - next if !$? || !$?.success? + # get protocol and name (for ceph) + protocol = ele.attributes['protocol'] rescue nil + name = ele.attributes['name'] rescue nil - json = JSON.parse(text) + if protocol == "rbd" + # Ceph + auth = ele.parent.elements["auth"].attributes["username"] rescue nil + auth = "--id #{auth}" if !auth.nil? - data[:disk_actual_size] += json['actual-size'].to_f/1024/1024 - data[:disk_virtual_size] += json['virtual-size'].to_f/1024/1024 + pool, image = name.split('/') + disk_id = image.split('-')[-1].to_i + + images_list = rbd_pool(pool, auth) + images_doc = REXML::Document.new(images_list) + + xpath = "images/image[image='#{image}']/size" + image_size = images_doc.elements[xpath].text.to_f/1024/1024 + + data[:disk_actual_size] += image_size + data[:disk_virtual_size] += image_size + + images_doc.elements.each("images/snapshot") do |snap| + next unless snap.elements["image"].text.start_with?(image) + + snap_id = snap.elements["snapshot"].text.to_i + + snap_size = snap.elements["size"].text.to_f/1024/1024 + + data["snap_size_#{disk_id}_#{snap_id}".to_sym] = snap_size + + data[:disk_actual_size] += snap_size + data[:disk_virtual_size] += snap_size + end + else file + # Regular Disk + text = `qemu-img info --output=json #{file}` + next if !$? || !$?.success? + + json = JSON.parse(text) + + data[:disk_actual_size] += json['actual-size'].to_f/1024/1024 + data[:disk_virtual_size] += json['virtual-size'].to_f/1024/1024 + end end - data[:disk_actual_size] = data[:disk_actual_size].round + data[:disk_actual_size] = data[:disk_actual_size].round data[:disk_virtual_size] = data[:disk_virtual_size].round data @@ -456,9 +493,18 @@ OS=[ARCH="#{arch}"] #{vnc_txt} EOT - return uuid, template end + + def self.rbd_pool(pool, auth = nil) + @@rbd_pool ||= {} + + if @@rbd_pool[pool].nil? + @@rbd_pool[pool] = `rbd #{auth} ls -l -p #{pool} --format xml` + end + + @@rbd_pool[pool] + end end ################################################################################ From dfdbc516d1aefb77d15c69f397db427a906ecf6f Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Fri, 19 Jun 2015 13:47:17 +0200 Subject: [PATCH 07/16] feature #3782: Quotas for one.image snapshots actions --- src/image/ImageManagerActions.cc | 4 +++- src/image/ImageManagerDriver.cc | 39 ++++++++++++++++++++++++++++++++ src/rm/RequestManagerChown.cc | 2 +- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/image/ImageManagerActions.cc b/src/image/ImageManagerActions.cc index 6846353ac8..320c1596ad 100644 --- a/src/image/ImageManagerActions.cc +++ b/src/image/ImageManagerActions.cc @@ -498,6 +498,8 @@ int ImageManager::delete_image(int iid, string& error_str) ipool->update(img); } + unsigned int snap_size = (img->get_snapshots()).get_total_size(); + img->unlock(); delete drv_msg; @@ -507,7 +509,7 @@ int ImageManager::delete_image(int iid, string& error_str) Template img_usage; img_usage.add("DATASTORE", ds_id); - img_usage.add("SIZE", size); + img_usage.add("SIZE", size + snap_size); Quotas::ds_del(uid, gid, &img_usage); diff --git a/src/image/ImageManagerDriver.cc b/src/image/ImageManagerDriver.cc index 6bb6fa1aac..41bec760e8 100644 --- a/src/image/ImageManagerDriver.cc +++ b/src/image/ImageManagerDriver.cc @@ -18,6 +18,7 @@ #include "ImagePool.h" #include "NebulaLog.h" +#include "Quotas.h" #include "Nebula.h" #include @@ -692,6 +693,9 @@ static void snap_delete_action(istringstream& is, ostringstream oss; string info; + unsigned int snap_size; + int ds_id, uid, gid; + Image * image = ipool->get(id, true); if ( image == 0 ) @@ -715,6 +719,11 @@ static void snap_delete_action(istringstream& is, if ( result == "SUCCESS") { + ds_id = image->get_ds_id(); + uid = image->get_uid(); + gid = image->get_gid(); + snap_size = (image->get_snapshots()).get_snapshot_size(snap_id); + image->delete_snapshot(snap_id); } else @@ -738,6 +747,17 @@ static void snap_delete_action(istringstream& is, ipool->update(image); image->unlock(); + + if (result == "SUCCESS") + { + Template quotas; + + quotas.add("DATASTORE", ds_id); + quotas.add("SIZE", (long long) snap_size); + quotas.add("IMAGES",0 ); + + Quotas::ds_del(uid, gid, "as); + } } /* -------------------------------------------------------------------------- */ @@ -808,6 +828,9 @@ static void snap_flatten_action(istringstream& is, ostringstream oss; string info; + unsigned int snap_size; + int ds_id, uid, gid; + Image * image = ipool->get(id, true); if ( image == 0 ) @@ -817,6 +840,11 @@ static void snap_flatten_action(istringstream& is, if ( result == "SUCCESS") { + ds_id = image->get_ds_id(); + uid = image->get_uid(); + gid = image->get_gid(); + snap_size = (image->get_snapshots()).get_total_size(); + image->clear_snapshots(); } else @@ -842,6 +870,17 @@ static void snap_flatten_action(istringstream& is, ipool->update(image); image->unlock(); + + if (result == "SUCCESS") + { + Template quotas; + + quotas.add("DATASTORE", ds_id); + quotas.add("SIZE", (long long) snap_size); + quotas.add("IMAGES",0 ); + + Quotas::ds_del(uid, gid, "as); + } } /* -------------------------------------------------------------------------- */ diff --git a/src/rm/RequestManagerChown.cc b/src/rm/RequestManagerChown.cc index 6c73c1a5b1..d7b1fcc52e 100644 --- a/src/rm/RequestManagerChown.cc +++ b/src/rm/RequestManagerChown.cc @@ -60,7 +60,7 @@ PoolObjectSQL * RequestManagerChown::get_and_quota( tmpl = new Template; tmpl->add("DATASTORE", img->get_ds_id()); - tmpl->add("SIZE", img->get_size()); + tmpl->add("SIZE",img->get_size()+img->get_snapshots().get_total_size()); qtype = Quotas::DATASTORE; } From 552e6a8f96cca917a191d1eab6aa872b9472e19a Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Fri, 19 Jun 2015 17:01:42 +0200 Subject: [PATCH 08/16] feature #3781: Parse disk_*_size from monitor driver --- include/VirtualMachineManagerDriver.h | 4 ++++ src/vmm/VirtualMachineManagerDriver.cc | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/include/VirtualMachineManagerDriver.h b/include/VirtualMachineManagerDriver.h index a0da01e156..eab4e93529 100644 --- a/include/VirtualMachineManagerDriver.h +++ b/include/VirtualMachineManagerDriver.h @@ -367,6 +367,8 @@ private: * @param cpu used by the VM (rate) * @param net_tx transmitted bytes (total) * @param net_rx received bytes (total) + * @param disk_actual actual disk usage for VM (total mb) + * @param disk_virtual virtual disk usage for VM (total mb) * @param state of the vm * @param custom monitor information */ @@ -376,6 +378,8 @@ private: int &memory, long long &net_tx, long long &net_rx, + long long &disk_actual, + long long &disk_virtual, char &state, map &custom); diff --git a/src/vmm/VirtualMachineManagerDriver.cc b/src/vmm/VirtualMachineManagerDriver.cc index c64efbe287..9bdb94d886 100644 --- a/src/vmm/VirtualMachineManagerDriver.cc +++ b/src/vmm/VirtualMachineManagerDriver.cc @@ -602,6 +602,8 @@ void VirtualMachineManagerDriver::process_poll( int memory; long long net_tx; long long net_rx; + long long dactual; + long long dvirtual; char state; map custom; @@ -616,7 +618,8 @@ void VirtualMachineManagerDriver::process_poll( /* Parse VM info */ /* ---------------------------------------------------------------------- */ - rc = parse_vm_info(monitor_str, cpu, memory, net_tx, net_rx, state, custom); + rc = parse_vm_info(monitor_str, cpu, memory, net_tx, net_rx, dactual, + dvirtual, state, custom); if (rc == -1) //Parse error, ignore this monitor data { @@ -724,6 +727,8 @@ int VirtualMachineManagerDriver::parse_vm_info( int &memory, long long &net_tx, long long &net_rx, + long long &disk_actual, + long long &disk_virtual, char &state, map &custom) { @@ -744,6 +749,9 @@ int VirtualMachineManagerDriver::parse_vm_info( net_rx = -1; state = '-'; + disk_actual = -1; + disk_virtual = -1; + custom.clear(); is.str(monitor_str); @@ -794,6 +802,14 @@ int VirtualMachineManagerDriver::parse_vm_info( { tiss >> state; } + else if (var == "DISK_ACTUAL_SIZE") + { + tiss >> disk_actual; + } + else if (var == "DISK_VIRTUAL_SIZE") + { + tiss >> disk_virtual; + } else if (!var.empty()) { string val; From 3e6159f70fe7b33363c69058fe26ac6dac590960 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Sat, 20 Jun 2015 00:42:20 +0200 Subject: [PATCH 09/16] feature #3781: DISK_VIRTUAL/ACTUAL_SIZE first class element in VM template --- include/VirtualMachine.h | 22 +++++++++++++++++----- share/doc/xsd/acct.xsd | 7 +++++++ share/doc/xsd/vm.xsd | 7 +++++++ src/cli/etc/oneacct.yaml | 12 +++++++++++- src/cli/one_helper/oneacct_helper.rb | 11 ++++++++++- src/cli/one_helper/onevm_helper.rb | 6 +++++- src/mad/ruby/VirtualMachineDriver.rb | 12 +++++++----- src/vm/VirtualMachine.cc | 26 ++++++++++++++++++++++---- src/vmm/VirtualMachineManagerDriver.cc | 2 +- src/vmm_mad/dummy/one_vmm_dummy.rb | 4 +++- 10 files changed, 90 insertions(+), 19 deletions(-) diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index d3f485337c..af50620bc6 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -400,10 +400,12 @@ public: * @param _net_rx received bytes (total) */ void update_info( - const int _memory, - const int _cpu, - const long long _net_tx, - const long long _net_rx, + int _memory, + int _cpu, + long long _net_tx, + long long _net_rx, + long long _disk_actual, + long long _disk_virtual, const map &custom); /** @@ -414,7 +416,7 @@ public: { map empty; - update_info(0, 0, -1, -1, empty); + update_info(0, 0, -1, -1, -1, -1, empty); } /** @@ -1702,6 +1704,16 @@ private: */ long long net_rx; + /** + * Network usage, received bytes + */ + long long disk_actual; + + /** + * Network usage, received bytes + */ + long long disk_virtual; + /** * History record, for the current host */ diff --git a/share/doc/xsd/acct.xsd b/share/doc/xsd/acct.xsd index 9b1b9d1870..73765a2837 100644 --- a/share/doc/xsd/acct.xsd +++ b/share/doc/xsd/acct.xsd @@ -187,6 +187,13 @@ + + + + + + + diff --git a/share/doc/xsd/vm.xsd b/share/doc/xsd/vm.xsd index fb37186bf9..8debc2c7c1 100644 --- a/share/doc/xsd/vm.xsd +++ b/share/doc/xsd/vm.xsd @@ -120,6 +120,13 @@ + + + + + + + diff --git a/src/cli/etc/oneacct.yaml b/src/cli/etc/oneacct.yaml index d7e487aa83..23ded42030 100644 --- a/src/cli/etc/oneacct.yaml +++ b/src/cli/etc/oneacct.yaml @@ -46,6 +46,14 @@ :desc: Data sent to the network :size: 6 +:DISK_ACTUAL: + :desc: Total disk size + :size: 6 + +:DISK_VIRTUAL: + :desc: Total virtual disk size + :size: 6 + :default: - :VID - :HOSTNAME @@ -55,4 +63,6 @@ - :MEMORY - :CPU - :NET_RX -- :NET_TX \ No newline at end of file +- :NET_TX +- :DISK_ACTUAL +- :DISK_VIRTUAL \ No newline at end of file diff --git a/src/cli/one_helper/oneacct_helper.rb b/src/cli/one_helper/oneacct_helper.rb index 830466c8bc..8e18c4ab4a 100644 --- a/src/cli/one_helper/oneacct_helper.rb +++ b/src/cli/one_helper/oneacct_helper.rb @@ -165,7 +165,16 @@ class AcctHelper < OpenNebulaHelper::OneHelper OpenNebulaHelper.unit_to_str(d["VM"]["NET_TX"].to_i / 1024.0, {}) end - default :VID, :HOSTNAME, :ACTION, :REASON, :START_TIME, :END_TIME, :MEMORY, :CPU, :NET_RX, :NET_TX + column :DISK_ACTUAL, "Total disk size used", :size=>6 do |d| + # DISK size is measured in mb, unit_to_str expects KBytes + OpenNebulaHelper.unit_to_str(d["VM"]["DISK_ACTUAL_SIZE"].to_i * 1024.0, {}) + end + + column :DISK_VIRTUAL, "Total disk virtual size used", :size=>6 do |d| + # DISK size is measured in mb, unit_to_str expects KBytes + OpenNebulaHelper.unit_to_str(d["VM"]["DISK_VIRTUAL_SIZE"].to_i * 1024.0, {}) + end + default :VID, :HOSTNAME, :ACTION, :REASON, :START_TIME, :END_TIME, :MEMORY, :CPU, :NET_RX, :NET_TX, :DISK_ACTUAL, :DISK_VIRTUAL end SHOWBACK_TABLE = CLIHelper::ShowTable.new("oneshowback.yaml", nil) do diff --git a/src/cli/one_helper/onevm_helper.rb b/src/cli/one_helper/onevm_helper.rb index 22e4d6e3da..d80e8bd606 100644 --- a/src/cli/one_helper/onevm_helper.rb +++ b/src/cli/one_helper/onevm_helper.rb @@ -506,7 +506,9 @@ class OneVMHelper < OpenNebulaHelper::OneHelper "USED MEMORY" => "MEMORY", "USED CPU" => "CPU", "NET_TX" => "NET_TX", - "NET_RX" => "NET_RX" + "NET_RX" => "NET_RX", + "DISK SIZE (ACTUAL)" => "DISK_ACTUAL_SIZE", + "DISK SIZE (VIRTUAL)" => "DISK_VIRTUAL_SIZE" } poll_attrs.each { |k,v| @@ -514,6 +516,8 @@ class OneVMHelper < OpenNebulaHelper::OneHelper puts str % [k,vm[v]] elsif k == "USED MEMORY" puts str % [k, OpenNebulaHelper.unit_to_str(vm[v].to_i, {})] + elsif k =~ /DISK/ + puts str % [k, OpenNebulaHelper.unit_to_str(vm[v].to_i*1024, {})] else puts str % [k, OpenNebulaHelper.unit_to_str(vm[v].to_i/1024, {})] end diff --git a/src/mad/ruby/VirtualMachineDriver.rb b/src/mad/ruby/VirtualMachineDriver.rb index 5414e2c1c6..8cc46daa9d 100644 --- a/src/mad/ruby/VirtualMachineDriver.rb +++ b/src/mad/ruby/VirtualMachineDriver.rb @@ -51,11 +51,13 @@ class VirtualMachineDriver < OpenNebulaDriver } POLL_ATTRIBUTE = { - :usedmemory => "USEDMEMORY", - :usedcpu => "USEDCPU", - :nettx => "NETTX", - :netrx => "NETRX", - :state => "STATE" + :usedmemory => "USEDMEMORY", + :usedcpu => "USEDCPU", + :nettx => "NETTX", + :netrx => "NETRX", + :state => "STATE", + :disk_actual => "DISK_ACTUAL_SIZE", + :disk_virtual=> "DISK_VIRTUAL_SIZE" } VM_STATE = { diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 9227accf47..aa7e1472d4 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -62,6 +62,8 @@ VirtualMachine::VirtualMachine(int id, cpu(0), net_tx(0), net_rx(0), + disk_actual(0), + disk_virtual(0), history(0), previous_history(0), _log(0) @@ -3674,6 +3676,8 @@ string& VirtualMachine::to_xml_extended(string& xml, int n_history) const << "" << cpu << "" << "" << net_tx << "" << "" << net_rx << "" + << "" << disk_actual << "" + << ""<< disk_virtual<< "" << obj_template->to_xml(template_xml) << user_obj_template->to_xml(user_template_xml); @@ -3748,6 +3752,8 @@ int VirtualMachine::from_xml(const string &xml_str) rc += xpath(cpu, "/VM/CPU", 0); rc += xpath(net_tx, "/VM/NET_TX", 0); rc += xpath(net_rx, "/VM/NET_RX", 0); + rc += xpath(disk_actual, "/VM/DISK_ACTUAL_SIZE", 0); + rc += xpath(disk_virtual,"/VM/DISK_VIRTUAL_SIZE", 0); // Permissions rc += perms_from_xml(); @@ -3856,10 +3862,12 @@ string VirtualMachine::get_system_dir() const /* -------------------------------------------------------------------------- */ void VirtualMachine::update_info( - const int _memory, - const int _cpu, - const long long _net_tx, - const long long _net_rx, + int _memory, + int _cpu, + long long _net_tx, + long long _net_rx, + long long _disk_actual, + long long _disk_virtual, const map &custom) { map::const_iterator it; @@ -3886,6 +3894,16 @@ void VirtualMachine::update_info( net_rx = _net_rx; } + if (_disk_actual != -1) + { + disk_actual = _disk_actual; + } + + if (_disk_virtual != -1) + { + disk_virtual = _disk_virtual; + } + for (it = custom.begin(); it != custom.end(); it++) { replace_template_attribute(it->first, it->second); diff --git a/src/vmm/VirtualMachineManagerDriver.cc b/src/vmm/VirtualMachineManagerDriver.cc index 9bdb94d886..1d0c937e19 100644 --- a/src/vmm/VirtualMachineManagerDriver.cc +++ b/src/vmm/VirtualMachineManagerDriver.cc @@ -647,7 +647,7 @@ void VirtualMachineManagerDriver::process_poll( if (vm->get_state() == VirtualMachine::ACTIVE) { - vm->update_info(memory, cpu, net_tx, net_rx, custom); + vm->update_info(memory, cpu, net_tx, net_rx, dactual, dvirtual, custom); vmpool->update(vm); diff --git a/src/vmm_mad/dummy/one_vmm_dummy.rb b/src/vmm_mad/dummy/one_vmm_dummy.rb index c85864ddca..0513ee1343 100755 --- a/src/vmm_mad/dummy/one_vmm_dummy.rb +++ b/src/vmm_mad/dummy/one_vmm_dummy.rb @@ -195,7 +195,9 @@ class DummyDriver < VirtualMachineDriver "#{POLL_ATTRIBUTE[:nettx]}=#{prev_nettx+(50*rand(3))} " \ "#{POLL_ATTRIBUTE[:netrx]}=#{prev_netrx+(100*rand(4))} " \ "#{POLL_ATTRIBUTE[:usedmemory]}=#{max_memory * (rand(80)+20)/100} " \ - "#{POLL_ATTRIBUTE[:usedcpu]}=#{max_cpu * (rand(95)+5)/100}" + "#{POLL_ATTRIBUTE[:usedcpu]}=#{max_cpu * (rand(95)+5)/100} " \ + "#{POLL_ATTRIBUTE[:disk_actual]}=#{rand(1024)} " \ + "#{POLL_ATTRIBUTE[:disk_virtual]}=#{1024}" send_message(ACTION[:poll],result,id,monitor_info) end From 872ef3af738a3343bd8b3bd3060b72bb2995d678 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Sat, 20 Jun 2015 02:29:47 +0200 Subject: [PATCH 10/16] feature #3718: Add disk_cost, use disk_actual_size (needs to revisit) --- include/VirtualMachinePool.h | 4 +++- share/etc/oned.conf | 3 ++- src/nebula/Nebula.cc | 13 +++++++++- src/nebula/NebulaTemplate.cc | 1 + src/vm/VirtualMachine.cc | 15 ++++++++++++ src/vm/VirtualMachinePool.cc | 46 ++++++++++++++++++++++++------------ 6 files changed, 64 insertions(+), 18 deletions(-) diff --git a/include/VirtualMachinePool.h b/include/VirtualMachinePool.h index a819020292..d5eb5d163c 100644 --- a/include/VirtualMachinePool.h +++ b/include/VirtualMachinePool.h @@ -40,7 +40,8 @@ public: time_t expire_time, bool on_hold, float default_cpu_cost, - float default_mem_cost); + float default_mem_cost, + float default_disk_cost); ~VirtualMachinePool(){}; @@ -385,6 +386,7 @@ private: */ static float _default_cpu_cost; static float _default_mem_cost; + static float _default_disk_cost; /** * Callback used to get an int in the DB it is used by VM Pool in: diff --git a/share/etc/oned.conf b/share/etc/oned.conf index b942dc390d..78285ce04a 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -118,7 +118,8 @@ FEDERATION = [ DEFAULT_COST = [ CPU_COST = 0, - MEMORY_COST = 0 + MEMORY_COST = 0, + DISK_COST = 0 ] #******************************************************************************* diff --git a/src/nebula/Nebula.cc b/src/nebula/Nebula.cc index 41001a6eb8..9eb6f280fc 100644 --- a/src/nebula/Nebula.cc +++ b/src/nebula/Nebula.cc @@ -468,6 +468,7 @@ void Nebula::start(bool bootstrap_only) bool vm_submit_on_hold; float cpu_cost; float mem_cost; + float disk_cost; vector vm_hooks; vector host_hooks; @@ -515,6 +516,7 @@ void Nebula::start(bool bootstrap_only) cpu_cost = 0; mem_cost = 0; + disk_cost= 0; if (rc != 0) { @@ -534,6 +536,14 @@ void Nebula::start(bool bootstrap_only) { mem_cost = 0; } + + + rc = vatt->vector_value("DISK_COST", disk_cost); + + if (rc != 0) + { + disk_cost = 0; + } } vmpool = new VirtualMachinePool(db, @@ -544,7 +554,8 @@ void Nebula::start(bool bootstrap_only) vm_expiration, vm_submit_on_hold, cpu_cost, - mem_cost); + mem_cost, + disk_cost); hpool = new HostPool(db, host_hooks, diff --git a/src/nebula/NebulaTemplate.cc b/src/nebula/NebulaTemplate.cc index 876310daf0..30f195389f 100644 --- a/src/nebula/NebulaTemplate.cc +++ b/src/nebula/NebulaTemplate.cc @@ -224,6 +224,7 @@ void OpenNebulaTemplate::set_conf_default() vvalue.clear(); vvalue.insert(make_pair("CPU_COST","0")); vvalue.insert(make_pair("MEMORY_COST","0")); + vvalue.insert(make_pair("DISK_COST","0")); vattribute = new VectorAttribute("DEFAULT_COST",vvalue); conf_default.insert(make_pair(vattribute->name(),vattribute)); diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index aa7e1472d4..049ee40070 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -391,6 +391,17 @@ int VirtualMachine::insert(SqlDB * db, string& error_str) obj_template->add("MEMORY_COST", fvalue); } + if ( user_obj_template->get("DISK_COST", fvalue) == true ) + { + if ( fvalue < 0 ) + { + goto error_disk_cost; + } + + user_obj_template->erase("DISK_COST"); + obj_template->add("DISK_COST", fvalue); + } + // ------------------------------------------------------------------------ // Check the OS attribute // ------------------------------------------------------------------------ @@ -545,6 +556,10 @@ error_memory_cost: error_str = "MEMORY_COST attribute must be a positive float or integer value."; goto error_common; +error_disk_cost: + error_str = "DISK_COST attribute must be a positive float or integer value."; + goto error_common; + error_one_vms: error_str = "Trying to import an OpenNebula VM: 'one-*'."; goto error_common; diff --git a/src/vm/VirtualMachinePool.cc b/src/vm/VirtualMachinePool.cc index a722922d30..d27417b3a1 100644 --- a/src/vm/VirtualMachinePool.cc +++ b/src/vm/VirtualMachinePool.cc @@ -29,6 +29,7 @@ time_t VirtualMachinePool::_monitor_expiration; bool VirtualMachinePool::_submit_on_hold; float VirtualMachinePool::_default_cpu_cost; float VirtualMachinePool::_default_mem_cost; +float VirtualMachinePool::_default_disk_cost; const char * VirtualMachinePool::import_table = "vm_import"; @@ -52,7 +53,8 @@ VirtualMachinePool::VirtualMachinePool( time_t expire_time, bool on_hold, float default_cpu_cost, - float default_mem_cost) + float default_mem_cost, + float default_disk_cost) : PoolSQL(db, VirtualMachine::table, true, false) { const VectorAttribute * vattr; @@ -67,6 +69,7 @@ VirtualMachinePool::VirtualMachinePool( _submit_on_hold = on_hold; _default_cpu_cost = default_cpu_cost; _default_mem_cost = default_mem_cost; + _default_disk_cost= default_disk_cost; if ( _monitor_expiration == 0 ) { @@ -591,20 +594,24 @@ static string put_time(time_t t) */ struct SBRecord { - SBRecord(float c, float m, float h): cpu_cost(c), mem_cost(m), hours(h){}; - SBRecord(): cpu_cost(0), mem_cost(0), hours(0){}; + SBRecord(float c, float m, float d, float h): cpu_cost(c), mem_cost(m), + disk_cost(d), hours(h){}; + + SBRecord(): cpu_cost(0), mem_cost(0), disk_cost(0), hours(0){}; ostringstream& to_xml(ostringstream &oss) { string cpuc_s = one_util::float_to_str(cpu_cost); string memc_s = one_util::float_to_str(mem_cost); + string diskc_s= one_util::float_to_str(disk_cost); string hour_s = one_util::float_to_str(hours); - string cost_s = one_util::float_to_str(cpu_cost + mem_cost); + string cost_s = one_util::float_to_str(cpu_cost + mem_cost + disk_cost); - oss << "" << cpuc_s << "" + oss << "" << cpuc_s << "" << ""<< memc_s << "" + << "" << diskc_s<< "" << "" << cost_s << "" - << "" << hour_s << ""; + << "" << hour_s << ""; return oss; }; @@ -613,11 +620,13 @@ struct SBRecord { { cpu_cost = 0; mem_cost = 0; + disk_cost= 0; hours = 0; }; float cpu_cost; float mem_cost; + float disk_cost; float hours; }; @@ -650,14 +659,16 @@ int VirtualMachinePool::calculate_showback( string sql_cmd_separator; string sql_cmd_end; - tm tmp_tm; - int vid; - int h_stime; - int h_etime; - float cpu_cost; - float mem_cost; - float cpu; - int mem; + tm tmp_tm; + int vid; + int h_stime; + int h_etime; + float cpu_cost; + float mem_cost; + float disk_cost; + float cpu; + float disk; + int mem; #ifdef SBDEBUG ostringstream debug; @@ -791,9 +802,11 @@ int VirtualMachinePool::calculate_showback( history.xpath(cpu, "/HISTORY/VM/TEMPLATE/CPU", 0); history.xpath(mem, "/HISTORY/VM/TEMPLATE/MEMORY", 0); + history.xpath(disk, "/HISTORY/VM/DISK_ACTUAL_SIZE", 0); history.xpath(cpu_cost, "/HISTORY/VM/TEMPLATE/CPU_COST", _default_cpu_cost); history.xpath(mem_cost, "/HISTORY/VM/TEMPLATE/MEMORY_COST", _default_mem_cost); + history.xpath(disk_cost,"/HISTORY/VM/TEMPLATE/DISK_COST", _default_disk_cost); #ifdef SBDDEBUG int seq; @@ -805,8 +818,10 @@ int VirtualMachinePool::calculate_showback( << "h_etime " << h_etime << endl << "cpu_cost " << cpu_cost << endl << "mem_cost " << mem_cost << endl + << "disk_cost " << disk_cost << endl << "cpu " << cpu << endl - << "mem " << mem; + << "mem " << mem << endl + << "disk " << disk; NebulaLog::log("SHOWBACK", Log::DEBUG, debug); #endif @@ -841,6 +856,7 @@ int VirtualMachinePool::calculate_showback( totals[t].cpu_cost += cpu_cost * cpu * n_hours; totals[t].mem_cost += mem_cost * mem * n_hours; + totals[t].disk_cost+= disk_cost* disk* n_hours; totals[t].hours += n_hours; } } From d310b31016e811415d01ea2f56603944365268a5 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Mon, 22 Jun 2015 13:17:11 -0400 Subject: [PATCH 11/16] feature #3782: snap_create for persistent qcow2 --- src/tm_mad/qcow2/snap_create | 41 +++++++++++++++++++++++++++++++----- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/src/tm_mad/qcow2/snap_create b/src/tm_mad/qcow2/snap_create index f91356d7d1..22efdd8091 100755 --- a/src/tm_mad/qcow2/snap_create +++ b/src/tm_mad/qcow2/snap_create @@ -31,17 +31,48 @@ else DATASTORES=$ONE_LOCATION/var/datastores fi +DRIVER_PATH=$(dirname $0) + . $TMCOMMON SRC_PATH=`arg_path $SRC` -SYSTEM_DS_PATH=$(dirname ${SRC_PATH}) +#------------------------------------------------------------------------------- +# Get Image information +#------------------------------------------------------------------------------- DISK_ID=$(basename ${SRC} | cut -d. -f2) -DISK_PATH="${SYSTEM_DS_PATH}/disk.${DISK_ID}" -SNAP_DIR="${DISK_PATH}.snap" -SNAP_PATH="${SNAP_DIR}/${SNAP_ID}" -SNAP_PATH_RELATIVE=$(basename ${SNAP_PATH}) + +XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" + +unset i j XPATH_ELEMENTS + +while IFS= read -r -d '' element; do + XPATH_ELEMENTS[i++]="$element" +done < <(onevm show -x $VMID| $XPATH \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE) + +DISK_SRC="${XPATH_ELEMENTS[j++]}" +CLONE="${XPATH_ELEMENTS[j++]}" + + +SYSTEM_DS_PATH=$(dirname ${SRC_PATH}) +IMAGE_DS_PATH=$(dirname ${DISK_SRC}) + + +if [ "$CLONE" = "YES" ]; then + DISK_PATH="${SYSTEM_DS_PATH}/disk.${DISK_ID}" + SNAP_DIR="${DISK_PATH}.snap" + SNAP_PATH="${SNAP_DIR}/${SNAP_ID}" + SNAP_PATH_RELATIVE=$(basename ${SNAP_PATH}) +else + DISK_PATH=$DISK_SRC + SNAP_DIR="${DISK_SRC}.snap" + SNAP_PATH="${SNAP_DIR}/${SNAP_ID}" + SNAP_PATH_RELATIVE=$(basename ${SNAP_PATH}) +fi + CURRENT_PATH=${DISK_PATH} if [ ! -d ${SNAP_PATH} ]; then From 96bea2e6b2148dd97ad1ca19def2582d832cb582 Mon Sep 17 00:00:00 2001 From: Javi Fontan Date: Tue, 23 Jun 2015 09:20:16 -0400 Subject: [PATCH 12/16] feature #3782: cpds support for qcow2 snapshots --- src/tm_mad/qcow2/cpds | 38 ++++++++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/src/tm_mad/qcow2/cpds b/src/tm_mad/qcow2/cpds index ac9ec3f823..a1accce276 100755 --- a/src/tm_mad/qcow2/cpds +++ b/src/tm_mad/qcow2/cpds @@ -16,19 +16,20 @@ # limitations under the License. # #--------------------------------------------------------------------------- # -# mvds host:remote_system_ds/disk.i fe:SOURCE vmid dsid +# mvds host:remote_system_ds/disk.i fe:SOURCE snapid vmid dsid # - fe is the front-end hostname # - SOURCE is the path of the disk image in the form DS_BASE_PATH/disk # - host is the target host to deploy the VM # - remote_system_ds is the path for the system datastore in the host # - vmid is the id of the VM # - dsid is the target datastore (0 is the system datastore) +# - snapid is the snapshot id. "-1" for none SRC=$1 DST=$2 - -VMID=$3 -DSID=$4 +SNAP_ID=$3 +VMID=$4 +DSID=$5 if [ -z "${ONE_LOCATION}" ]; then TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh @@ -36,8 +37,29 @@ else TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh fi +DRIVER_PATH=$(dirname $0) + . $TMCOMMON +#------------------------------------------------------------------------------- +# Get Image information +#------------------------------------------------------------------------------- + +DISK_ID=$(basename ${SRC} | cut -d. -f2) + +XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" + +unset i j XPATH_ELEMENTS + +while IFS= read -r -d '' element; do + XPATH_ELEMENTS[i++]="$element" +done < <(onevm show -x $VMID| $XPATH \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE) + +DISK_SRC="${XPATH_ELEMENTS[j++]}" +CLONE="${XPATH_ELEMENTS[j++]}" + #------------------------------------------------------------------------------- # Set dst path and dir #------------------------------------------------------------------------------- @@ -52,6 +74,14 @@ DST_DS_PATH="$(dirname $(dirname $DST_ARG_PATH))" DST_PATH="${SRC_DS_PATH}${DST_ARG_PATH##$DST_DS_PATH}" +if [ "$CLONE" != "YES" ]; then + SRC_PATH="$DISK_SRC" +fi + +if [ "$SNAP_ID" != "-1" ]; then + SRC_PATH="$SRC_PATH.snap/$SNAP_ID" +fi + #------------------------------------------------------------------------------- # Move the image back to the datastore #------------------------------------------------------------------------------- From 5f1d630fc695cfb38857d3778ad7fae5fa369106 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Tue, 23 Jun 2015 17:22:46 +0200 Subject: [PATCH 13/16] feature #3718: Rename update_info method to set_deploy_id --- include/VirtualMachine.h | 3 +-- src/lcm/LifeCycleActions.cc | 2 +- src/vmm/VirtualMachineManagerDriver.cc | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index af50620bc6..0b22542840 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -385,8 +385,7 @@ public: * Updates VM dynamic information (id). * @param _deploy_id the VMM driver specific id */ - void update_info( - const string& _deploy_id) + void set_deploy_id(const string& _deploy_id) { deploy_id = _deploy_id; }; diff --git a/src/lcm/LifeCycleActions.cc b/src/lcm/LifeCycleActions.cc index 81a3ec4e08..f5cf8fae8a 100644 --- a/src/lcm/LifeCycleActions.cc +++ b/src/lcm/LifeCycleActions.cc @@ -1180,7 +1180,7 @@ void LifeCycleManager::recover(VirtualMachine * vm, bool success) oss << "one-" << vm->get_oid(); - vm->update_info(oss.str()); + vm->set_deploy_id(oss.str()); } lcm_action = LifeCycleManager::DEPLOY_SUCCESS; diff --git a/src/vmm/VirtualMachineManagerDriver.cc b/src/vmm/VirtualMachineManagerDriver.cc index 1d0c937e19..01a1fb838a 100644 --- a/src/vmm/VirtualMachineManagerDriver.cc +++ b/src/vmm/VirtualMachineManagerDriver.cc @@ -256,7 +256,7 @@ void VirtualMachineManagerDriver::protocol(const string& message) const is >> deploy_id; - vm->update_info(deploy_id); + vm->set_deploy_id(deploy_id); vmpool->update(vm); From 685a46365dc34905118458816e60e4b7ea1c83ef Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Tue, 23 Jun 2015 21:52:10 +0200 Subject: [PATCH 14/16] feature #3718: New template for MONITORING data, it can accomodate any data, and support the general OpenNebula Template format. --- include/VirtualMachine.h | 48 +++++----- src/cli/one_helper/oneacct_helper.rb | 8 +- src/cli/one_helper/onevm_helper.rb | 12 +-- src/oca/ruby/opennebula/virtual_machine.rb | 18 ++-- .../ruby/opennebula/virtual_machine_pool.rb | 31 +++---- src/vm/VirtualMachine.cc | 90 ++++++++----------- src/vmm/VirtualMachineManagerDriver.cc | 70 ++++++--------- src/vmm_mad/dummy/one_vmm_dummy.rb | 8 +- 8 files changed, 124 insertions(+), 161 deletions(-) diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index 0b22542840..82178cd4a9 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -18,6 +18,7 @@ #define VIRTUAL_MACHINE_H_ #include "VirtualMachineTemplate.h" +#include "VirtualMachineMonitorInfo.h" #include "PoolSQL.h" #include "History.h" #include "Image.h" @@ -385,7 +386,7 @@ public: * Updates VM dynamic information (id). * @param _deploy_id the VMM driver specific id */ - void set_deploy_id(const string& _deploy_id) + void set_deploy_id (const string& _deploy_id) { deploy_id = _deploy_id; }; @@ -393,19 +394,8 @@ public: /** * Updates VM dynamic information (usage counters), and updates last_poll, * and copies it to history record for acct. - * @param _memory Kilobytes used by the VM (total) - * @param _cpu used by the VM (rate) - * @param _net_tx transmitted bytes (total) - * @param _net_rx received bytes (total) */ - void update_info( - int _memory, - int _cpu, - long long _net_tx, - long long _net_rx, - long long _disk_actual, - long long _disk_virtual, - const map &custom); + int update_info(const string& monitor_data); /** * Clears the VM monitor information: usage counters, last_poll, @@ -413,9 +403,18 @@ public: */ void reset_info() { - map empty; + last_poll = time(0); - update_info(0, 0, -1, -1, -1, -1, empty); + monitoring.clear(); + + set_vm_info(); + + clear_template_monitor_error(); + } + + const VirtualMachineMonitorInfo& get_info() const + { + return monitoring; } /** @@ -1733,24 +1732,25 @@ private: */ map snapshots; - // ------------------------------------------------------------------------- - // Logging & Dirs - // ------------------------------------------------------------------------- + /** + * User template to store custom metadata. This template can be updated + */ + VirtualMachineTemplate * user_obj_template; + + /** + * Monitoring information for the VM + */ + VirtualMachineMonitorInfo monitoring; /** * Log class for the virtual machine, it writes log messages in * $ONE_LOCATION/var/$VID/vm.log * or, in case that OpenNebula is installed in root * /var/log/one/$VM_ID.log - * For the syslog... TODO + * For the syslog it will use the predefined /var/log/ locations */ Log * _log; - /** - * User template to store custom metadata. This template can be updated - * - */ - VirtualMachineTemplate * user_obj_template; // ************************************************************************* // DataBase implementation (Private) diff --git a/src/cli/one_helper/oneacct_helper.rb b/src/cli/one_helper/oneacct_helper.rb index 8e18c4ab4a..bdae3293e3 100644 --- a/src/cli/one_helper/oneacct_helper.rb +++ b/src/cli/one_helper/oneacct_helper.rb @@ -157,22 +157,22 @@ class AcctHelper < OpenNebulaHelper::OneHelper column :NET_RX, "Data received from the network", :size=>6 do |d| # NET is measured in bytes, unit_to_str expects KBytes - OpenNebulaHelper.unit_to_str(d["VM"]["NET_RX"].to_i / 1024.0, {}) + OpenNebulaHelper.unit_to_str(d["VM"]["MONITORING/NETRX"].to_i / 1024.0, {}) end column :NET_TX, "Data sent to the network", :size=>6 do |d| # NET is measured in bytes, unit_to_str expects KBytes - OpenNebulaHelper.unit_to_str(d["VM"]["NET_TX"].to_i / 1024.0, {}) + OpenNebulaHelper.unit_to_str(d["VM"]["MONITORING/NETTX"].to_i / 1024.0, {}) end column :DISK_ACTUAL, "Total disk size used", :size=>6 do |d| # DISK size is measured in mb, unit_to_str expects KBytes - OpenNebulaHelper.unit_to_str(d["VM"]["DISK_ACTUAL_SIZE"].to_i * 1024.0, {}) + OpenNebulaHelper.unit_to_str(d["VM"]["MONITORING/DISK_ACTUAL_SIZE"].to_i * 1024.0, {}) end column :DISK_VIRTUAL, "Total disk virtual size used", :size=>6 do |d| # DISK size is measured in mb, unit_to_str expects KBytes - OpenNebulaHelper.unit_to_str(d["VM"]["DISK_VIRTUAL_SIZE"].to_i * 1024.0, {}) + OpenNebulaHelper.unit_to_str(d["VM"]["MONITORING/DISK_VIRTUAL_SIZE"].to_i * 1024.0, {}) end default :VID, :HOSTNAME, :ACTION, :REASON, :START_TIME, :END_TIME, :MEMORY, :CPU, :NET_RX, :NET_TX, :DISK_ACTUAL, :DISK_VIRTUAL end diff --git a/src/cli/one_helper/onevm_helper.rb b/src/cli/one_helper/onevm_helper.rb index d80e8bd606..fb2aa12050 100644 --- a/src/cli/one_helper/onevm_helper.rb +++ b/src/cli/one_helper/onevm_helper.rb @@ -503,12 +503,12 @@ class OneVMHelper < OpenNebulaHelper::OneHelper CLIHelper.print_header(str_h1 % "VIRTUAL MACHINE MONITORING",false) poll_attrs = { - "USED MEMORY" => "MEMORY", - "USED CPU" => "CPU", - "NET_TX" => "NET_TX", - "NET_RX" => "NET_RX", - "DISK SIZE (ACTUAL)" => "DISK_ACTUAL_SIZE", - "DISK SIZE (VIRTUAL)" => "DISK_VIRTUAL_SIZE" + "USED MEMORY" => "MONITORING/USEDMEMORY", + "USED CPU" => "MONITORING/USEDCPU", + "NET_TX" => "MONITORING/NETTX", + "NET_RX" => "MONITORING/NETRX", + "DISK SIZE (ACTUAL)" => "MONITORING/DISK_ACTUAL_SIZE", + "DISK SIZE (VIRTUAL)" => "MONITORING/DISK_VIRTUAL_SIZE" } poll_attrs.each { |k,v| diff --git a/src/oca/ruby/opennebula/virtual_machine.rb b/src/oca/ruby/opennebula/virtual_machine.rb index cb686f1f4e..4db5f0ae9b 100644 --- a/src/oca/ruby/opennebula/virtual_machine.rb +++ b/src/oca/ruby/opennebula/virtual_machine.rb @@ -535,19 +535,15 @@ module OpenNebula # the requested xpath expressions, and an Array of 'timestamp, value'. # # @example - # vm.monitoring( ['CPU', 'NET_TX', 'TEMPLATE/CUSTOM_PROBE'] ) + # vm.monitoring( ['MONITORING/USEDCPU', 'MONITORING/NETTX'] ) # - # { "NET_TX" => - # [["1337264510", "210"], - # ["1337264553", "220"], - # ["1337264584", "230"]], - # "TEMPLATE/CUSTOM_PROBE" => - # [], - # "CPU" => - # [["1337264510", "0"], - # ["1337264553", "0"], - # ["1337264584", "0"]] + # { + # "MONITORING/USEDCPU"=>[["1435085098", "47"], ["1435085253", "5"], + # ["1435085410", "48"], ["1435085566", "3"], ["1435088136", "2"]], + # "MONITORING/NETTX"=>[["1435085098", "0"], ["1435085253", "50"], + # ["1435085410", "50"], ["1435085566", "50"], ["1435085723", "50"]] # } + # def monitoring(xpath_expressions) return super(VM_METHODS[:monitoring], 'VM', 'LAST_POLL', xpath_expressions) diff --git a/src/oca/ruby/opennebula/virtual_machine_pool.rb b/src/oca/ruby/opennebula/virtual_machine_pool.rb index 4cf721432b..708d91a643 100644 --- a/src/oca/ruby/opennebula/virtual_machine_pool.rb +++ b/src/oca/ruby/opennebula/virtual_machine_pool.rb @@ -132,23 +132,24 @@ module OpenNebula # and an Array of 'timestamp, value'. # # @example - # vm_pool.monitoring( ['CPU', 'NET_TX', 'TEMPLATE/CUSTOM_PROBE'] ) + # vm_pool.monitoring( ['MONITORING/USEDCPU', 'MONITORING/NETTX'] ) # - # {"1"=> - # {"CPU"=> - # [["1337608271", "0"], ["1337608301", "0"], ["1337608331", "0"]], - # "NET_TX"=> - # [["1337608271", "510"], ["1337608301", "510"], ["1337608331", "520"]], - # "TEMPLATE/CUSTOM_PROBE"=> - # []}, + # {"3"=> + # { + # "MONITORING/USEDCPU"=>[["1435085098", "47"], ["1435085253", "5"], + # ["1435085410", "48"], ["1435085566", "3"], ["1435088136", "2"]], + # "MONITORING/NETTX"=>[["1435085098", "0"], ["1435085253", "50"], + # ["1435085410", "50"], ["1435085566", "50"], ["1435085723", "50"]] + # }, + # "43" => + # { + # "MONITORING/USEDCPU"=>[["1435085098", "47"], ["1435085253", "5"], + # ["1435085410", "48"], ["1435085566", "3"], ["1435088136", "2"]], + # "MONITORING/NETTX"=>[["1435085098", "0"], ["1435085253", "50"], + # ["1435085410", "50"], ["1435085566", "50"], ["1435085723", "50"]] + # } + # } # - # "0"=> - # {"CPU"=> - # [["1337608271", "0"], ["1337608301", "0"], ["1337608331", "0"]], - # "NET_TX"=> - # [["1337608271", "510"], ["1337608301", "510"], ["1337608331", "520"]], - # "TEMPLATE/CUSTOM_PROBE"=> - # []}} def monitoring(xpath_expressions, filter_flag=INFO_ALL) return super(VM_POOL_METHODS[:monitoring], 'VM', 'LAST_POLL', xpath_expressions, filter_flag) diff --git a/src/vm/VirtualMachine.cc b/src/vm/VirtualMachine.cc index 049ee40070..693afa9b7a 100644 --- a/src/vm/VirtualMachine.cc +++ b/src/vm/VirtualMachine.cc @@ -58,12 +58,6 @@ VirtualMachine::VirtualMachine(int id, stime(time(0)), etime(0), deploy_id(""), - memory(0), - cpu(0), - net_tx(0), - net_rx(0), - disk_actual(0), - disk_virtual(0), history(0), previous_history(0), _log(0) @@ -3664,6 +3658,7 @@ string& VirtualMachine::to_xml_extended(string& xml, int n_history) const { string template_xml; string user_template_xml; + string monitoring_xml; string history_xml; string perm_xml; string snap_xml; @@ -3687,12 +3682,7 @@ string& VirtualMachine::to_xml_extended(string& xml, int n_history) const << "" << stime << "" << "" << etime << "" << "" << deploy_id << "" - << "" << memory << "" - << "" << cpu << "" - << "" << net_tx << "" - << "" << net_rx << "" - << "" << disk_actual << "" - << ""<< disk_virtual<< "" + << monitoring.to_xml(monitoring_xml) << obj_template->to_xml(template_xml) << user_obj_template->to_xml(user_template_xml); @@ -3763,13 +3753,6 @@ int VirtualMachine::from_xml(const string &xml_str) rc += xpath(etime, "/VM/ETIME", 0); rc += xpath(deploy_id, "/VM/DEPLOY_ID",""); - rc += xpath(memory, "/VM/MEMORY", 0); - rc += xpath(cpu, "/VM/CPU", 0); - rc += xpath(net_tx, "/VM/NET_TX", 0); - rc += xpath(net_rx, "/VM/NET_RX", 0); - rc += xpath(disk_actual, "/VM/DISK_ACTUAL_SIZE", 0); - rc += xpath(disk_virtual,"/VM/DISK_VIRTUAL_SIZE", 0); - // Permissions rc += perms_from_xml(); @@ -3798,6 +3781,20 @@ int VirtualMachine::from_xml(const string &xml_str) ObjectXML::free_nodes(content); content.clear(); + // Virtual Machine Monitoring + + ObjectXML::get_nodes("/VM/MONITORING", content); + + if (content.empty()) + { + return -1; + } + + rc += monitoring.from_xml_node(content[0]); + + ObjectXML::free_nodes(content); + content.clear(); + // Virtual Machine user template ObjectXML::get_nodes("/VM/USER_TEMPLATE", content); @@ -3876,57 +3873,40 @@ string VirtualMachine::get_system_dir() const /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ -void VirtualMachine::update_info( - int _memory, - int _cpu, - long long _net_tx, - long long _net_rx, - long long _disk_actual, - long long _disk_virtual, - const map &custom) +int VirtualMachine::update_info(const string& monitor_data) { - map::const_iterator it; + int rc; + string error; + + ostringstream oss; last_poll = time(0); - if (_memory != -1) - { - memory = _memory; - } + rc = monitoring.update(monitor_data, error); - if (_cpu != -1) + if ( rc != 0) { - cpu = _cpu; - } + oss << "Ignoring monitoring information, error:" << error + << ". Monitor information was: " << monitor_data; - if (_net_tx != -1) - { - net_tx = _net_tx; - } + NebulaLog::log("VMM", Log::ERROR, oss); - if (_net_rx != -1) - { - net_rx = _net_rx; - } + set_template_error_message(oss.str()); - if (_disk_actual != -1) - { - disk_actual = _disk_actual; - } + log("VMM", Log::ERROR, oss); - if (_disk_virtual != -1) - { - disk_virtual = _disk_virtual; - } - - for (it = custom.begin(); it != custom.end(); it++) - { - replace_template_attribute(it->first, it->second); + return -1; } set_vm_info(); clear_template_monitor_error(); + + oss << "VM " << oid << " successfully monitored: " << monitor_data; + + NebulaLog::log("VMM", Log::DEBUG, oss); + + return 0; }; /* -------------------------------------------------------------------------- */ diff --git a/src/vmm/VirtualMachineManagerDriver.cc b/src/vmm/VirtualMachineManagerDriver.cc index 01a1fb838a..fb8ec46a24 100644 --- a/src/vmm/VirtualMachineManagerDriver.cc +++ b/src/vmm/VirtualMachineManagerDriver.cc @@ -596,64 +596,50 @@ void VirtualMachineManagerDriver::process_poll( VirtualMachine* vm, const string& monitor_str) { - int rc; - - int cpu; - int memory; - long long net_tx; - long long net_rx; - long long dactual; - long long dvirtual; - char state; - - map custom; - ostringstream oss; + char state; Nebula &ne = Nebula::instance(); LifeCycleManager* lcm = ne.get_lcm(); VirtualMachinePool* vmpool = ne.get_vmpool(); - /* ---------------------------------------------------------------------- */ - /* Parse VM info */ - /* ---------------------------------------------------------------------- */ - - rc = parse_vm_info(monitor_str, cpu, memory, net_tx, net_rx, dactual, - dvirtual, state, custom); - - if (rc == -1) //Parse error, ignore this monitor data - { - oss << "Ignoring monitoring information, parse error." - << " Monitor information was: " - << monitor_str; - - NebulaLog::log("VMM", Log::ERROR, oss); - - vm->set_template_error_message(oss.str()); - - vm->log("VMM", Log::ERROR, oss); - - vmpool->update(vm); - - return; - } - - oss << "VM " << vm->get_oid() << " successfully monitored: " << monitor_str; - NebulaLog::log("VMM", Log::DEBUG, oss); - /* ---------------------------------------------------------------------- */ /* Update VM info only for VMs in ACTIVE */ /* ---------------------------------------------------------------------- */ if (vm->get_state() == VirtualMachine::ACTIVE) { - vm->update_info(memory, cpu, net_tx, net_rx, dactual, dvirtual, custom); + if (vm->update_info(monitor_str) == 0) + { + vmpool->update_history(vm); + + vmpool->update_monitoring(vm); + } vmpool->update(vm); - vmpool->update_history(vm); + const VirtualMachineMonitorInfo &minfo = vm->get_info(); - vmpool->update_monitoring(vm); + state = minfo.get_state(); + } + else + { + VirtualMachineMonitorInfo minfo; + string error; + + if (minfo.update(monitor_str, error) != 0) + { + ostringstream oss; + + oss << "Ignoring monitoring information, error:" << error + << ". Monitor information was: " << monitor_str; + + NebulaLog::log("VMM", Log::ERROR, oss); + + return; + }; + + state = minfo.get_state(); } /* ---------------------------------------------------------------------- */ diff --git a/src/vmm_mad/dummy/one_vmm_dummy.rb b/src/vmm_mad/dummy/one_vmm_dummy.rb index 0513ee1343..f8b902cee6 100755 --- a/src/vmm_mad/dummy/one_vmm_dummy.rb +++ b/src/vmm_mad/dummy/one_vmm_dummy.rb @@ -180,13 +180,13 @@ class DummyDriver < VirtualMachineDriver end prev_nettx = 0 - if msg.elements["VM/NET_TX"] - prev_nettx = msg.elements["VM/NET_TX"].text.to_i + if msg.elements["VM/MONITORING/NET_TX"] + prev_nettx = msg.elements["VM/MONITORING/NET_TX"].text.to_i end prev_netrx = 0 - if msg.elements["VM/NET_RX"] - prev_netrx = msg.elements["VM/NET_RX"].text.to_i + if msg.elements["VM/MONITORING/NET_RX"] + prev_netrx = msg.elements["VM/MONITORING/NET_RX"].text.to_i end # monitor_info: string in the form "VAR=VAL VAR=VAL ... VAR=VAL" From c6d7c6046ff6add4a12d2356d5124416852b8580 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 24 Jun 2015 14:19:03 +0200 Subject: [PATCH 15/16] feature #3782: Make IMAGE attribute mandatory --- src/image/Image.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/image/Image.cc b/src/image/Image.cc index 2ff499e3b1..d5712ed961 100644 --- a/src/image/Image.cc +++ b/src/image/Image.cc @@ -411,7 +411,7 @@ int Image::from_xml(const string& xml) rc += xpath(cloning_ops, "/IMAGE/CLONING_OPS", -1); rc += xpath(cloning_id, "/IMAGE/CLONING_ID", -1); - xpath(target_snapshot, "/IMAGE/TARGET_SNAPSHOT", -1); + rc += xpath(target_snapshot, "/IMAGE/TARGET_SNAPSHOT", -1); rc += xpath(ds_id, "/IMAGE/DATASTORE_ID", -1); rc += xpath(ds_name, "/IMAGE/DATASTORE", "not_found"); From 83a4165fffc1b18765d67623f9b067bf80d37ef8 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Wed, 24 Jun 2015 14:19:40 +0200 Subject: [PATCH 16/16] feature #3718: Added missing include file --- include/VirtualMachineMonitorInfo.h | 76 +++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 include/VirtualMachineMonitorInfo.h diff --git a/include/VirtualMachineMonitorInfo.h b/include/VirtualMachineMonitorInfo.h new file mode 100644 index 0000000000..464f99da04 --- /dev/null +++ b/include/VirtualMachineMonitorInfo.h @@ -0,0 +1,76 @@ +/* -------------------------------------------------------------------------- */ +/* Copyright 2002-2015, OpenNebula Project (OpenNebula.org), C12G Labs */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); you may */ +/* not use this file except in compliance with the License. You may obtain */ +/* a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ +/* See the License for the specific language governing permissions and */ +/* limitations under the License. */ +/* -------------------------------------------------------------------------- */ + +#ifndef VIRTUAL_MACHINE_MONITOR_INFO_H_ +#define VIRTUAL_MACHINE_MONITOR_INFO_H_ + +#include "Template.h" + +#include + +using namespace std; + +/** + * Virtual Machine Monitor class, stores the monitor data for the VM + */ +class VirtualMachineMonitorInfo : public Template +{ +public: + VirtualMachineMonitorInfo():Template(false,'=',"MONITORING"){}; + + ~VirtualMachineMonitorInfo(){}; + + /** + * Update the monitoring information with data from the probes + * @param monitor_data of the VM + * @param error description if any + * @return 0 on success + */ + int update(const string& monitor_data, string& error) + { + char * error_c = 0; + + clear(); + + int rc = parse(monitor_data, &error_c); + + if (rc != 0) + { + error = error_c; + + free(error_c); + } + + return rc; + }; + + char get_state() const + { + string state_str; + + get("STATE", state_str); + + if (state_str.empty()) + { + return '-'; + } + + return state_str[0]; + }; +}; + +#endif +