shaba/one
1
0
Fork 0
mirror of https://github.com/OpenNebula/one.git synced 2025-03-28 14:50:08 +03:00
Code Releases Wiki Activity

F #3503: Support for DPDK. Added ISOLCPUS to isolate cpus from the NUMA scheduler

Browse Source
This commit is contained in:
Ruben S. Montero 2019-07-25 16:01:17 +02:00
parent 7e24e6e5d7
commit 3b4b0b2723
No known key found for this signature in database
GPG Key ID: A0CEA6FA880A1D87
17 changed files with 225 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
include/HostShare.h
View File

@ -292,6 +292,12 @@ public:
mem_usage -= memory;
}
/**
* Reserve CPU IDs
* @param rcpus list of reserved cpu ids (comma separated)
*/
void reserve_cpus(const std::string& rcpus);
/**
* Prints the NUMA node to an output stream.
*/
@ -537,6 +543,19 @@ public:
*/
void del(HostShareCapacity &sr);
/**
*
*/
void reserve_cpus(const std::string &cpu_ids)
{
for (auto it = nodes.begin(); it != nodes.end(); ++it)
{
it->second->reserve_cpus(cpu_ids);
it->second->update_cores();
}
};
/**
* Prints the NUMA nodes to an output stream.
*/
@ -756,6 +775,15 @@ public:
*/
void update_capacity(Host *host);
/**
* Reserve CPUs in the numa nodes
*/
void reserve_cpus(const std::string &cpu_ids)
{
numa.reserve_cpus(cpu_ids);
}
/**
* Return the number of running VMs in this host
*/

11
include/VirtualNetwork.h
View File

@ -68,8 +68,9 @@ public:
UNDEFINED = 0,
LINUX = 1,
OPENVSWITCH = 2,
VCENTER_PORT_GROUPS = 3,
BRNONE = 4
OPENVSWITCH_DPDK = 3,
VCENTER_PORT_GROUPS = 4,
BRNONE = 5
};
static string driver_to_str(VirtualNetworkDriver ob)
@ -142,6 +143,8 @@ public:
return "linux";
case OPENVSWITCH:
return "openvswitch";
case OPENVSWITCH_DPDK:
return "openvswitch_dpdk";
case VCENTER_PORT_GROUPS:
return "vcenter_port_groups";
case BRNONE:
@ -160,6 +163,10 @@ public:
{
return OPENVSWITCH;
}
else if ( ob == "openvswitch_dpdk" )
{
return OPENVSWITCH_DPDK;
}
else if ( ob == "vcenter_port_groups" )
{
return VCENTER_PORT_GROUPS;

1
share/etc/oned.conf
View File

@ -1529,6 +1529,7 @@ VN_MAD_CONF = [
VN_MAD_CONF = [
NAME = "ovswitch",
BRIDGE_TYPE = "openvswitch"
#openvswitch or openvswitch_dpdk
]
VN_MAD_CONF = [

3
src/cli/one_helper/onehost_helper.rb
View File

@ -770,6 +770,9 @@ class OneHostHelper < OpenNebulaHelper::OneHelper
if c[1] == '-1'
ret += '-'
free += 1
elsif c[1] == '-2'
ret += 'I'
used += 1
elsif c[1] != '-1' && info['FREE'] == '0'
ret += 'X'
used += 1

5
src/host/Host.cc
View File

@ -742,6 +742,7 @@ int Host::post_update_template(string& error)
{
string new_im_mad;
string new_vm_mad;
string cpu_ids;
map<std::string, unsigned int>::const_iterator it;
@ -774,7 +775,11 @@ int Host::post_update_template(string& error)
replace_template_attribute("IM_MAD", im_mad_name);
replace_template_attribute("VM_MAD", vmm_mad_name);
get_template_attribute("ISOLCPUS", cpu_ids);
host_share.update_capacity(this);
host_share.reserve_cpus(cpu_ids);
return 0;
};

84
src/host/HostShare.cc
View File

@ -498,7 +498,7 @@ HostShareNode::Core::Core(unsigned int _i, const std::string& _c, int fc):id(_i)
continue;
}
if (!(thread_s >> vm_id) || vm_id < -1)
if (!(thread_s >> vm_id) || vm_id < -2)
{
vm_id = -1;
}
@ -556,6 +556,86 @@ VectorAttribute * HostShareNode::HugePage::to_attribute()
return vpage;
}
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
void HostShareNode::reserve_cpus(const std::string& cpu_ids)
{
std::vector<unsigned int> ids;
std::set<unsigned int> core_ids;
one_util::split(cpu_ids, ',', ids);
//--------------------------------------------------------------------------
// Reserve / free CPUS based on the cpu list
//--------------------------------------------------------------------------
for (auto it = cores.begin(); it != cores.end(); ++it)
{
struct Core &c = it->second;
for (auto jt = c.cpus.begin(); jt != c.cpus.end(); ++jt)
{
bool update = false;
for (auto kt = ids.begin() ; kt != ids.end(); ++kt)
{
if ( *kt == jt->first )
{
update = true;
break;
}
}
if ( update )
{
if ( jt->second == -1 ) //reserve
{
jt->second = -2;
core_ids.insert(it->first);
}
}
else if ( jt->second == -2 ) //free
{
jt->second = -1;
core_ids.insert(it->first);
}
}
}
//--------------------------------------------------------------------------
// Recompute free cpus for the cores that have been updated
//--------------------------------------------------------------------------
for (auto it = core_ids.begin(); it != core_ids.end(); ++it)
{
auto jt = cores.find(*it);
if ( jt == cores.end() )
{
continue;
}
struct Core &c = jt->second;
unsigned int fcpus = 0;
for (auto kt = c.cpus.begin(); kt != c.cpus.end(); ++kt)
{
if ( kt->second == -1 )
{
fcpus++;
}
}
if ( c.free_cpus == 0 && fcpus != 0) //CORE policy
{
continue;
}
c.free_cpus = fcpus;
}
}
// -----------------------------------------------------------------------------
// -----------------------------------------------------------------------------
@ -1032,7 +1112,7 @@ void HostShareNUMA::set_monitorization(Template &ht)
HostShareNode& node = get_node(node_id);
node.set_hugepage(size, free, pages, 0, true);
node.set_hugepage(size, pages, free, 0, true);
}
std::vector<VectorAttribute *> memory;

13
src/sunstone/public/app/tabs/hosts-tab/panels/numa.js
View File

@ -76,6 +76,7 @@ define(function(require) {
Sunstone.runAction(RESOURCE + ".update_template", that.element.ID, template_str);
}
});
$("#"+ISOLCPUSINPUT).click(function(e){
if(that.element && that.element.ID && that.element.TEMPLATE){
var template = $.extend({}, that.element.TEMPLATE);
@ -128,7 +129,7 @@ define(function(require) {
)
.add(
$("<div/>",{"class":'columns small-2 text-center'}).append(
$("<button/>",{"class": "button", "id": ISOLCPUSINPUT }).text("Send")
$("<button/>",{"class": "button", "id": ISOLCPUSINPUT }).text("Update")
)
)
);
@ -136,6 +137,7 @@ define(function(require) {
$("#placeNumaInfo").append(selectTable);
$("#placeNumaInfo").append(isolcpusTable);
numaNodes.map(function(node,i){
var displaySubtitle = true;
var title = $("<h4/>");
@ -166,7 +168,8 @@ define(function(require) {
placeBody = tBody.append($("<tr/>")).find("tr:last");
}
placeBody.append(
$("<td/>",{"colspan":2,"class":"text-center"}).append(
$("<td/>",{"colspan":2,"class":"text-center"}).append(
$("<h6/>").text(core.ID? "Core "+core.ID : "")
)
);
@ -175,13 +178,16 @@ define(function(require) {
if(cpus instanceof Array){
cpus.map(function(cpu){
var cpuInfo = cpu.split(":");
var state = (cpuInfo && cpuInfo[1] && Number(cpuInfo[1])>=0? "busy" : (Number(cpuInfo[1]) === -1? "free" : "isolated"));
placeBody.find("td:last").append(
$("<div/>",{"class":"small-6 columns cpu "+state}).append(
$("<div/>",{"class":""}).text("CPU #"+cpuInfo[0]).add(
cpuInfo && cpuInfo[1] && cpuInfo[1] >= 0?
$("<a/>",{"class":"","href":"/#vms-tab/"+cpuInfo[1]}).text("VM #"+cpuInfo[1])
:
$("<div/>",{"class":"no-vm"}).text(Number(cpuInfo[1]) === -1?"FREE":"ISOLATED")
)
)
@ -215,9 +221,11 @@ define(function(require) {
//HUGEPAGE
if(infoNode.HUGEPAGE){
var infoHugepages = infoNode.HUGEPAGE;
if (!(infoHugepages instanceof Array)) {
infoHugepages = [infoHugepages];
}
hugepage.append($("<h6/>").text("Hugepage"));
var hugepageTable = $("<table/>");
hugepageTable.append(
@ -250,6 +258,7 @@ define(function(require) {
}
}
$("#placeNumaInfo").append(title.add(subtitle).add(coreTable).add($("<div/>",{"class":"row"}).append(memory.add(hugepage))));
Tips.setup();
});
}

1
src/sunstone/public/scss/_settings.scss
View File

@ -67,6 +67,7 @@ $dark-black: #000;
//$black: #333;
$white: #fefefe;
$isolated: #7d7b7b;
$free: #cacaca;
$busy: #4DBBD3;
$deploying-color: #C7731F;

3
src/sunstone/public/scss/_table.scss
View File

@ -59,6 +59,7 @@ table {
color: $white;
border: 1px solid $white;
}
.isolated{
background-color: $isolated;
font-weight: bold;
@ -123,4 +124,4 @@ span .fa-times:hover {
button.search-dropdown {
box-shadow: none !important;
border-style: none !important;
}
}

40
src/vmm/LibVirtDriverKVM.cc
View File

@ -326,14 +326,7 @@ static void vtopol(ofstream& file, const VectorAttribute * topology,
mboss << "\t<memoryBacking>\n";
mboss << "\t\t<hugepages>\n";
mboss << "\t\t\t<page size=" << one_util::escape_xml_attr(hpsz_kb);
if (!mnodes.str().empty())
{
mboss << " nodeset='" << mnodes.str() << "'";
}
mboss << "/>\n";
mboss << "\t\t\t<page size=" << one_util::escape_xml_attr(hpsz_kb) << "/>\n";
mboss << "\t\t</hugepages>\n";
mboss << "\t</memoryBacking>\n";
@ -460,6 +453,7 @@ int LibVirtDriver::deployment_description_kvm(
string filter = "";
string virtio_queues = "";
string bridge_type = "";
string nic_id = "";
string i_avg_bw = "";
string i_peak_bw = "";
@ -1256,6 +1250,7 @@ int LibVirtDriver::deployment_description_kvm(
for(int i=0; i<num; i++)
{
nic_id = nic[i]->vector_value("NIC_ID");
bridge = nic[i]->vector_value("BRIDGE");
vn_mad = nic[i]->vector_value("VN_MAD");
mac = nic[i]->vector_value("MAC");
@ -1283,15 +1278,30 @@ int LibVirtDriver::deployment_description_kvm(
}
else
{
file << "\t\t<interface type='bridge'>" << endl;
if (VirtualNetwork::str_to_bridge_type(bridge_type) == VirtualNetwork::OPENVSWITCH)
switch(VirtualNetwork::str_to_bridge_type(bridge_type))
{
file << "\t\t\t<virtualport type='openvswitch'/>" << endl;
}
case VirtualNetwork::UNDEFINED:
case VirtualNetwork::LINUX:
case VirtualNetwork::VCENTER_PORT_GROUPS:
case VirtualNetwork::BRNONE:
file << "\t\t<interface type='bridge'>\n"
<< "\t\t\t<source bridge="
<< one_util::escape_xml_attr(bridge) << "/>\n";
break;
file << "\t\t\t<source bridge="
<< one_util::escape_xml_attr(bridge) << "/>\n";
case VirtualNetwork::OPENVSWITCH:
file << "\t\t<interface type='bridge'>\n"
<< "\t\t\t<virtualport type='openvswitch'/>\n"
<< "\t\t\t<source bridge="
<< one_util::escape_xml_attr(bridge) << "/>\n";
break;
case VirtualNetwork::OPENVSWITCH_DPDK:
file << "\t\t<interface type='vhostuser'>\n"
<< "\t\t\t<source type='unix' mode='server' path='"
<< vm->get_system_dir() << "/" << target << "' />\n";
break;
}
}
if( !mac.empty() )

1
src/vmm_mad/exec/one_vmm_exec.rb
View File

@ -142,6 +142,7 @@ class VmmAction
ID DEPLOY_ID TEMPLATE/CONTEXT USER_TEMPLATE
TEMPLATE/SECURITY_GROUP_RULE
HISTORY_RECORDS/HISTORY/HOSTNAME
HISTORY_RECORDS/HISTORY/DS_ID
HISTORY_RECORDS/HISTORY/VM_MAD
]

1
src/vnm/VirtualNetwork.cc
View File

@ -1403,6 +1403,7 @@ int VirtualNetwork::parse_bridge_type(const string &vn_mad, string &error_str)
{
goto error;
}
bridge_type = br_type;
}

5
src/vnm_mad/remotes/OpenNebulaNetwork.conf
View File

@ -40,6 +40,10 @@
# Enable ARP Cache Poisoning Prevention Rules
:arp_cache_poisoning: true
# Default path to create DPDK sockets. This needs to be change only if
# DATASTORE_LOCATION in oned.conf has been changed
:datastore_location: /var/lib/one/datastores
################################################################################
# 802.1Q Options
################################################################################
@ -102,6 +106,7 @@
#
# :ovs_bridge_conf:
# :stp_enable: true
# :datapath_type: netdev
# These options will be added to the ip link add command. For example:

2
src/vnm_mad/remotes/lib/nic.rb
View File

@ -80,7 +80,7 @@ module VNMMAD
if dumpxml
dumpxml_root = REXML::Document.new(dumpxml).root
xpath = "devices/interface[@type='bridge']/" \
xpath = "devices/interface[@type='bridge' or @type='vhostuser']/" \
"mac[@address='#{self[:mac]}']/../target"
tap = dumpxml_root.elements[xpath]

9
src/vnm_mad/remotes/lib/vm.rb
View File

@ -84,6 +84,15 @@ module VNMMAD
@vm_root.root.elements[xpath].text
end
def system_dir(ds_location)
ds_id_p = 'HISTORY_RECORDS/HISTORY/DS_ID'
ds_id = @vm_root.root.elements[ds_id_p].text
vm_id = @vm_root.root.elements['ID'].text
"#{ds_location}/#{ds_id}/#{vm_id}"
end
private
# Method to build the associated Hash from a NIC

1
src/vnm_mad/remotes/lib/vnmmad.rb
View File

@ -51,6 +51,7 @@ rescue
:vlan_mtu => "1500",
:ipset_maxelem => "65536",
:keep_empty_bridge => false,
:datastore_location => '/var/lib/one/datastores'
}
end

51
src/vnm_mad/remotes/ovswitch/OpenvSwitch.rb
View File

@ -26,12 +26,6 @@ class OpenvSwitchVLAN < VNMMAD::VNMDriver
xpath_filter ||= XPATH_FILTER
super(vm, xpath_filter, deploy_id)
@vm.nics.each do |nic|
if nic[:bridge_ovs] && !nic[:bridge_ovs].empty?
nic[:bridge] = nic[:bridge_ovs]
end
end
end
def activate
@ -55,12 +49,14 @@ class OpenvSwitchVLAN < VNMMAD::VNMDriver
unless @bridges[@nic[:bridge]].include? @nic[:vlan_dev]
create_vlan_dev
add_bridge_port(@nic[:vlan_dev])
add_bridge_port(@nic[:vlan_dev], nil)
end
elsif @nic[:phydev]
add_bridge_port(@nic[:phydev])
add_bridge_port(@nic[:phydev], nil)
end
add_bridge_port(nic[:target], dpdk_vm(@nic[:target])) if dpdk?
if @nic[:tap].nil?
# In net/pre action, we just need to ensure the bridge is
# created so the libvirt/QEMU can add VM interfaces into that.
@ -136,13 +132,17 @@ class OpenvSwitchVLAN < VNMMAD::VNMDriver
# Remove flows
del_flows
# delete port from bridge in case of dpdk
del_bridge_port(@nic[:target]) if dpdk?
next if @nic[:phydev].nil?
next if @bridges[@nic[:bridge]].nil?
# Get the name of the vlan device.
get_vlan_dev_name
# Return if the bridge doesn't exist because it was already deleted (handles last vm with multiple nics on the same vlan)
# Return if the bridge doesn't exist because it was already deleted
# (handles last vm with multiple nics on the same vlan)
next if !@bridges.include? @nic[:bridge]
# Return if we want to keep the empty bridge
@ -386,6 +386,21 @@ private
nil
end
# Return true when usgin dpdk
def dpdk?
@nic[:bridge_type] == 'openvswitch_dpdk'
end
# Path to the vm port socket /var/lib/one/datastores/0/23/one-23-0
def dpdk_vm(port)
"#{@vm.system_dir(@nic[:conf][:datastore_location])}/#{port}"
end
# Path to bridge folder for non VM links
def dpdk_br
"#{@nic[:conf][:datastore_location]}/ovs-#{@nic[:bridge]}"
end
# Creates an OvS bridge if it does not exists, and brings it up.
# This function IS FINAL, exits if action cannot be completed
def create_bridge
@ -408,14 +423,28 @@ private
end
# Add port into OvS bridge
def add_bridge_port(port)
def add_bridge_port(port, dpdk_path = nil)
return if @bridges[@nic[:bridge]].include? port
OpenNebula.exec_and_log("#{command(:ovs_vsctl)} add-port #{@nic[:bridge]} #{port}")
ovs_cmd = "#{command(:ovs_vsctl)} add-port #{@nic[:bridge]} #{port}"
if dpdk_path && dpdk?
ovs_cmd << " -- set Interface #{port} type=dpdkvhostuserclient"\
" options:vhost-server-path=#{dpdk_path}"
end
OpenNebula.exec_and_log(ovs_cmd)
@bridges[@nic[:bridge]] << port
end
# Delete port from OvS bridge
def del_bridge_port(port)
OpenNebula.exec_and_log("#{command(:ovs_vsctl)} del-port #{@nic[:bridge]} #{port}")
@bridges[@nic[:bridge]].delete(port)
end
# Calls ovs-vsctl set bridge to set options stored in ovs_bridge_conf
def set_bridge_options
@nic[:ovs_bridge_conf].each do |option, value|