controllers: improve bgp-evpn

- add new bgp plugin
- add ebgp support
- add loopback support
- move gateway-nodes option to zone as 'exitnodes'
- move external-peers to bgp plugin

PVE/API2/Network/SDN/Controllers.pm

@@ -11,6 +11,7 @@ use PVE::Network::SDN::Zones;
 use PVE::Network::SDN::Controllers;
 use PVE::Network::SDN::Controllers::Plugin;
 use PVE::Network::SDN::Controllers::EvpnPlugin;
+use PVE::Network::SDN::Controllers::BgpPlugin;
 use PVE::Network::SDN::Controllers::FaucetPlugin;
 
 use Storable qw(dclone);

PVE/Network/SDN/Controllers.pm

@@ -13,9 +13,11 @@ use PVE::Network::SDN::Vnets;
 use PVE::Network::SDN::Zones;
 
 use PVE::Network::SDN::Controllers::EvpnPlugin;
+use PVE::Network::SDN::Controllers::BgpPlugin;
 use PVE::Network::SDN::Controllers::FaucetPlugin;
 use PVE::Network::SDN::Controllers::Plugin;
 PVE::Network::SDN::Controllers::EvpnPlugin->register();
+PVE::Network::SDN::Controllers::BgpPlugin->register();
 PVE::Network::SDN::Controllers::FaucetPlugin->register();
 PVE::Network::SDN::Controllers::Plugin->init();
 
@@ -95,24 +97,24 @@ sub generate_controller_config {
     #generate configuration
     my $config = {};
 
-    foreach my $id (keys %{$controller_cfg->{ids}}) {
+    foreach my $id (sort keys %{$controller_cfg->{ids}}) {
 	my $plugin_config = $controller_cfg->{ids}->{$id};
 	my $plugin = PVE::Network::SDN::Controllers::Plugin->lookup($plugin_config->{type});
-	$plugin->generate_controller_config($plugin_config, $plugin_config, $id, $uplinks, $config);
+	$plugin->generate_controller_config($plugin_config, $controller_cfg, $id, $uplinks, $config);
     }
 
-    foreach my $id (keys %{$zone_cfg->{ids}}) {
+    foreach my $id (sort keys %{$zone_cfg->{ids}}) {
 	my $plugin_config = $zone_cfg->{ids}->{$id};
 	my $controllerid = $plugin_config->{controller};
 	next if !$controllerid;
 	my $controller = $controller_cfg->{ids}->{$controllerid};
 	if ($controller) {
 	    my $controller_plugin = PVE::Network::SDN::Controllers::Plugin->lookup($controller->{type});
-	    $controller_plugin->generate_controller_zone_config($plugin_config, $controller, $id, $uplinks, $config);
+	    $controller_plugin->generate_controller_zone_config($plugin_config, $controller, $controller_cfg, $id, $uplinks, $config);
 	}
     }
 
-    foreach my $id (keys %{$vnet_cfg->{ids}}) {
+    foreach my $id (sort keys %{$vnet_cfg->{ids}}) {
 	my $plugin_config = $vnet_cfg->{ids}->{$id};
 	my $zoneid = $plugin_config->{zone};
 	next if !$zoneid;

PVE/Network/SDN/Controllers/BgpPlugin.pm

@@ -0,0 +1,153 @@
+package PVE::Network::SDN::Controllers::BgpPlugin;
+
+use strict;
+use warnings;
+
+use PVE::INotify;
+use PVE::JSONSchema qw(get_standard_option);
+use PVE::Tools qw(run_command file_set_contents file_get_contents);
+
+use PVE::Network::SDN::Controllers::Plugin;
+use PVE::Network::SDN::Zones::Plugin;
+use Net::IP;
+
+use base('PVE::Network::SDN::Controllers::Plugin');
+
+sub type {
+    return 'bgp';
+}
+
+sub properties {
+    return {
+	ebgp => {
+	    type => 'boolean',
+	    optional => 1,
+	    description => "Enable ebgp. (remote-as external)",
+	},
+	loopback => {
+	    description => "source loopback interface.",
+	    type => 'string'
+	},
+        node => get_standard_option('pve-node'),
+    };
+}
+
+sub options {
+    return {
+	'node' => { optional => 0 },
+	'asn' => { optional => 0 },
+	'peers' => { optional => 0 },
+	'ebgp' => { optional => 1 },
+	'loopback' => { optional => 1 },
+    };
+}
+
+# Plugin implementation
+sub generate_controller_config {
+    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+
+    my @peers;
+    @peers = PVE::Tools::split_list($plugin_config->{'peers'}) if $plugin_config->{'peers'};
+
+    my $asn = $plugin_config->{asn};
+    my $ebgp = $plugin_config->{ebgp};
+    my $loopback = $plugin_config->{loopback};
+    my $local_node = PVE::INotify::nodename();
+
+
+    return if !$asn;
+    return if $local_node ne $plugin_config->{node};
+
+    my $bgp = $config->{frr}->{router}->{"bgp $asn"} //= {};
+
+    my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
+
+    my $remoteas = $ebgp ? "external" : $asn;
+
+    #global options
+    my @controller_config = (
+        "bgp router-id $ifaceip",
+        "no bgp default ipv4-unicast",
+        "coalesce-time 1000",
+        "bgp network import-check"
+    );
+
+    push(@{$bgp->{""}}, @controller_config) if keys %{$bgp} == 0;
+
+    @controller_config = ();
+    if($ebgp) {
+	push @controller_config, "no bgp ebgp-requires-policy";
+	push @controller_config, "bgp disable-ebgp-connected-route-check" if $loopback;
+    }
+
+    #BGP neighbors
+    if(@peers) {
+	push @controller_config, "neighbor BGP peer-group";
+	push @controller_config, "neighbor BGP remote-as $remoteas";
+	push @controller_config, "neighbor BGP bfd";
+    }
+
+    # BGP peers
+    foreach my $address (@peers) {
+	push @controller_config, "neighbor $address peer-group BGP";
+    }
+    push(@{$bgp->{""}}, @controller_config);
+
+    # address-family unicast
+    if (@peers) {
+	my $ipversion = Net::IP::ip_is_ipv6($ifaceip) ? "ipv6" : "ipv4";
+	my $mask = Net::IP::ip_is_ipv6($ifaceip) ? "/128" : "32";
+
+	push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "network $ifaceip/$mask") if $loopback;
+	push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "neighbor BGP activate");
+	push(@{$bgp->{"address-family"}->{"$ipversion unicast"}}, "neighbor BGP soft-reconfiguration inbound");
+    }
+
+    return $config;
+}
+
+sub generate_controller_zone_config {
+    my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
+
+}
+
+sub on_delete_hook {
+    my ($class, $controllerid, $zone_cfg) = @_;
+
+    # verify that zone is associated to this controller
+    foreach my $id (keys %{$zone_cfg->{ids}}) {
+	my $zone = $zone_cfg->{ids}->{$id};
+	die "controller $controllerid is used by $id"
+	    if (defined($zone->{controller}) && $zone->{controller} eq $controllerid);
+    }
+}
+
+sub on_update_hook {
+    my ($class, $controllerid, $controller_cfg) = @_;
+
+    # we can only have 1 bgp controller by node
+    my $local_node = PVE::INotify::nodename();
+    my $controllernb = 0;
+    foreach my $id (keys %{$controller_cfg->{ids}}) {
+        next if $id eq $controllerid;
+        my $controller = $controller_cfg->{ids}->{$id};
+        next if $controller->{type} ne "bgp";
+        next if $controller->{node} ne $local_node;
+        $controllernb++;
+        die "only 1 bgp controller can be defined" if $controllernb > 1;
+    }
+}
+
+sub write_controller_config {
+    my ($class, $plugin_config, $config) = @_;
+    return;
+}
+
+sub reload_controller {
+    my ($class) = @_;
+    return;
+}
+
+1;
+
+

PVE/Network/SDN/Controllers/EvpnPlugin.pm

@@ -9,6 +9,7 @@ use PVE::Tools qw(run_command file_set_contents file_get_contents);
 
 use PVE::Network::SDN::Controllers::Plugin;
 use PVE::Network::SDN::Zones::Plugin;
+use Net::IP;
 
 use base('PVE::Network::SDN::Controllers::Plugin');
 
@@ -26,11 +27,6 @@ sub properties {
 	    description => "peers address list.",
 	    type => 'string', format => 'ip-list'
 	},
-	'gateway-nodes' => get_standard_option('pve-node-list'),
-	'gateway-external-peers' => {
-	    description => "upstream bgp peers address list.",
-	    type => 'string', format => 'ip-list'
-	},
     };
 }
 
@@ -38,80 +34,97 @@ sub options {
     return {
 	'asn' => { optional => 0 },
 	'peers' => { optional => 0 },
-	'gateway-nodes' => { optional => 1 },
-	'gateway-external-peers' => { optional => 1 },
     };
 }
 
 # Plugin implementation
 sub generate_controller_config {
-    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+    my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
 
     my @peers;
     @peers = PVE::Tools::split_list($plugin_config->{'peers'}) if $plugin_config->{'peers'};
 
+    my $local_node = PVE::INotify::nodename();
+
     my $asn = $plugin_config->{asn};
-    my $gatewaynodes = $plugin_config->{'gateway-nodes'};
-    my @gatewaypeers;
-    @gatewaypeers = PVE::Tools::split_list($plugin_config->{'gateway-external-peers'}) if $plugin_config->{'gateway-external-peers'};
+    my $ebgp = undef;
+    my $loopback = undef;
+    my $autortas = undef;
+    my $bgprouter = find_bgp_controller($local_node, $controller_cfg);
+    if($bgprouter) {
+	$ebgp = 1 if $plugin_config->{'asn'} ne $bgprouter->{asn};
+	$loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+	$asn = $bgprouter->{asn} if $bgprouter->{asn};
+	$autortas = $plugin_config->{'asn'} if $ebgp;
+    }
 
     return if !$asn;
 
     my $bgp = $config->{frr}->{router}->{"bgp $asn"} //= {};
 
-    my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers);
+    my ($ifaceip, $interface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
 
-    my $is_gateway = undef;
-    my $local_node = PVE::INotify::nodename();
-
-    foreach my $gatewaynode (PVE::Tools::split_list($gatewaynodes)) {
-	$is_gateway = 1 if $gatewaynode eq $local_node;
-    }
+    my $remoteas = $ebgp ? "external" : $asn;
 
+    #global options
     my @controller_config = (
 	"bgp router-id $ifaceip",
 	"no bgp default ipv4-unicast",
 	"coalesce-time 1000",
     );
 
-    foreach my $address (@peers) {
-	next if $address eq $ifaceip;
-	push @controller_config, "neighbor $address remote-as $asn";
-    }
-
-    if ($is_gateway) {
-	foreach my $address (@gatewaypeers) {
-	    push @controller_config, "neighbor $address remote-as external";
-	}
-    }
-    push(@{$bgp->{""}}, @controller_config);
+    push(@{$bgp->{""}}, @controller_config) if keys %{$bgp} == 0;
 
     @controller_config = ();
+    
+    #VTEP neighbors
+    push @controller_config, "neighbor VTEP peer-group";
+    push @controller_config, "neighbor VTEP remote-as $remoteas";
+    push @controller_config, "neighbor VTEP bfd";
+
+    if($ebgp && $loopback) {
+	push @controller_config, "neighbor VTEP ebgp-multihop 10";
+	push @controller_config, "neighbor VTEP update-source $loopback";
+    }
+
+    # VTEP peers
     foreach my $address (@peers) {
 	next if $address eq $ifaceip;
-	push @controller_config, "neighbor $address activate";
+	push @controller_config, "neighbor $address peer-group VTEP";
     }
+
+    push(@{$bgp->{""}}, @controller_config);
+
+    # address-family l2vpn
+    @controller_config = ();
+    push @controller_config, "neighbor VTEP activate";
     push @controller_config, "advertise-all-vni";
+    push @controller_config, "autort as $autortas" if $autortas;
     push(@{$bgp->{"address-family"}->{"l2vpn evpn"}}, @controller_config);
 
-    if ($is_gateway) {
-	# import /32 routes of evpn network from vrf1 to default vrf (for packet return)
-	@controller_config = map { "neighbor $_ activate" } @gatewaypeers;
-
-	push(@{$bgp->{"address-family"}->{"ipv4 unicast"}}, @controller_config);
-	push(@{$bgp->{"address-family"}->{"ipv6 unicast"}}, @controller_config);
-    }
-
     return $config;
 }
 
 sub generate_controller_zone_config {
-    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+    my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
+
+    my $local_node = PVE::INotify::nodename();
 
     my $vrf = "vrf_$id";
     my $vrfvxlan = $plugin_config->{'vrf-vxlan'};
+    my $exitnodes = $plugin_config->{'exitnodes'};
+
     my $asn = $controller->{asn};
-    my $gatewaynodes = $controller->{'gateway-nodes'};
+    my $ebgp = undef;
+    my $loopback = undef;
+    my $autortas = undef;
+    my $bgprouter = find_bgp_controller($local_node, $controller_cfg);
+    if($bgprouter) {
+        $ebgp = 1 if $controller->{'asn'} ne $bgprouter->{asn};
+	$loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+	$asn = $bgprouter->{asn} if $bgprouter->{asn};
+	$autortas = $controller->{'asn'} if $ebgp;
+    }
 
     return if !$vrf || !$vrfvxlan || !$asn;
 
@@ -120,11 +133,18 @@ sub generate_controller_zone_config {
     push @controller_config, "vni $vrfvxlan";
     push(@{$config->{frr}->{vrf}->{"$vrf"}}, @controller_config);
 
-    push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{""}}, "!");
+    #main vrf router
+    @controller_config = ();
+    push @controller_config, "no bgp ebgp-requires-policy" if $ebgp;
+#    push @controller_config, "!";
+    push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{""}}, @controller_config);
 
-    my $local_node = PVE::INotify::nodename();
+    if ($autortas) {
+	push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target import $autortas:$vrfvxlan");
+	push(@{$config->{frr}->{router}->{"bgp $asn vrf $vrf"}->{"address-family"}->{"l2vpn evpn"}}, "route-target export $autortas:$vrfvxlan");
+    }
 
-    my $is_gateway = grep { $_ eq $local_node } PVE::Tools::split_list($gatewaynodes);
+    my $is_gateway = grep { $_ eq $local_node } PVE::Tools::split_list($exitnodes);
     if ($is_gateway) {
 
 	@controller_config = ();
@@ -165,13 +185,31 @@ sub on_update_hook {
 
     # we can only have 1 evpn controller / 1 asn by server
 
+    my $controllernb = 0;
     foreach my $id (keys %{$controller_cfg->{ids}}) {
 	next if $id eq $controllerid;
 	my $controller = $controller_cfg->{ids}->{$id};
-	die "only 1 evpn controller can be defined" if $controller->{type} eq "evpn";
+	next if $controller->{type} ne "evpn";
+	$controllernb++;
+	die "only 1 global evpn controller can be defined" if $controllernb > 1;
     }
 }
 
+sub find_bgp_controller {
+    my ($nodename, $controller_cfg) = @_;
+
+    my $controller = undef;
+    foreach my $id  (keys %{$controller_cfg->{ids}}) {
+        $controller = $controller_cfg->{ids}->{$id};
+        next if $controller->{type} ne 'bgp';
+        next if $controller->{node} ne $nodename;
+	last;
+    }
+
+    return $controller;
+}
+
+
 sub sort_frr_config {
     my $order = {};
     $order->{''} = 0;

PVE/Network/SDN/Controllers/FaucetPlugin.pm

@@ -22,12 +22,12 @@ sub properties {
 
 # Plugin implementation
 sub generate_controller_config {
-    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+    my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
 
 }
 
 sub generate_controller_zone_config {
-    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+    my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
 
     my $dpid = $plugin_config->{'dp-id'};
     my $dphex = printf("%x",$dpid);

PVE/Network/SDN/Controllers/Makefile

@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm FaucetPlugin.pm EvpnPlugin.pm
+SOURCES=Plugin.pm FaucetPlugin.pm EvpnPlugin.pm BgpPlugin.pm
 
 
 PERL5DIR=${DESTDIR}/usr/share/perl5

PVE/Network/SDN/Controllers/Plugin.pm

@@ -70,7 +70,14 @@ sub generate_sdn_config {
 }
 
 sub generate_controller_config {
-    my ($class, $plugin_config, $controller, $id, $uplinks, $config) = @_;
+    my ($class, $plugin_config, $controller_cfg, $id, $uplinks, $config) = @_;
+
+    die "please implement inside plugin";
+}
+
+
+sub generate_controller_zone_config {
+    my ($class, $plugin_config, $controller, $controller_cfg, $id, $uplinks, $config) = @_;
 
     die "please implement inside plugin";
 }

PVE/Network/SDN/Zones/EvpnPlugin.pm

@@ -4,6 +4,7 @@ use strict;
 use warnings;
 use PVE::Network::SDN::Zones::VxlanPlugin;
 use PVE::Exception qw(raise raise_param_exc);
+use PVE::JSONSchema qw(get_standard_option);
 use PVE::Tools qw($IPV4RE);
 use PVE::INotify;
 use PVE::Cluster;
@@ -27,6 +28,7 @@ sub properties {
 	    type => 'string',
 	    description => "Frr router name",
 	},
+	'exitnodes' => get_standard_option('pve-node-list'),
     };
 }
 
@@ -35,7 +37,8 @@ sub options {
     return {
         nodes => { optional => 1},
         'vrf-vxlan' => { optional => 0 },
-        'controller' => { optional => 0 },
+        controller => { optional => 0 },
+	exitnodes => { optional => 1 },
 	mtu => { optional => 1 },
 	dns => { optional => 1 },
 	reversedns => { optional => 1 },
@@ -59,10 +62,14 @@ sub generate_sdn_config {
     my $local_node = PVE::INotify::nodename();
 
     die "missing vxlan tag" if !$tag;
+    die "missing controller" if !$controller;
     warn "vlan-aware vnet can't be enabled with evpn plugin" if $vnet->{vlanaware};
 
     my @peers = PVE::Tools::split_list($controller->{'peers'});
-    my ($ifaceip, $iface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers);
+#    my $bgprouter = PVE::Network::SDN::Controllers::EvpnController::find_bgp_controller($local_node, $controller_cfg);
+#    my $loopback = $bgprouter->{loopback} if $bgprouter->{loopback};
+    my $loopback = undef;
+    my ($ifaceip, $iface) = PVE::Network::SDN::Zones::Plugin::find_local_ip_interface_peers(\@peers, $loopback);
 
     my $mtu = 1450;
     $mtu = $interfaces_config->{$iface}->{mtu} - 50 if $interfaces_config->{$iface}->{mtu};

PVE/Network/SDN/Zones/Plugin.pm

@@ -274,10 +274,17 @@ sub get_local_route_ip {
 
 
 sub find_local_ip_interface_peers {
-    my ($peers) = @_;
+    my ($peers, $iface) = @_;
 
     my $network_config = PVE::INotify::read_file('interfaces');
     my $ifaces = $network_config->{ifaces};
+    
+    #if iface is defined, return ip if exist (if not,try to find it on other ifaces)
+    if ($iface) {
+	my $ip = $ifaces->{$iface}->{address};
+	return ($ip,$iface) if $ip;
+    }
+
     #is a local ip member of peers list ?
     foreach my $address (@{$peers}) {
 	while (my $interface = each %$ifaces) {