fix: populate routes to BGP neighbors (Equinix Metal)

Fixes #8267

Also refactor the code so that we don't fail hard on mutiple bonds, but
it's not clear still how to attach addresses, as they don't have a
interface name field, so for now attaching to the first bond.

Fixes #8411

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/equinix.go

@@ -9,7 +9,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	stderrors "errors"
 	"fmt"
 	"io"
 	"log"
@@ -71,6 +70,12 @@ type Address struct {
 	Gateway    string `json:"gateway"`
 }
 
+// BGPNeighbor holds BGP neighbor info from the equinixmetal metadata.
+type BGPNeighbor struct {
+	AddressFamily int      `json:"address_family"`
+	PeerIPs       []string `json:"peer_ips"`
+}
+
 const (
 	// EquinixMetalUserDataEndpoint is the local metadata endpoint for Equinix.
 	EquinixMetalUserDataEndpoint = "https://metadata.platformequinix.com/userdata"
@@ -122,27 +127,23 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 	// translate the int returned from bond mode metadata to the type needed by network resources
 	bondMode := nethelpers.BondMode(uint8(equinixMetadata.Network.Bonding.Mode))
 
-	// determine bond name and build list of interfaces enslaved by the bond
-	bondName := ""
-
 	hostInterfaces, err := safe.StateListAll[*network.LinkStatus](ctx, st)
 	if err != nil {
 		return nil, fmt.Errorf("error listing host interfaces: %w", err)
 	}
 
-	slaveIndex := 0
+	bondSlaveIndexes := map[string]int{}
+	firstBond := ""
 
 	for _, iface := range equinixMetadata.Network.Interfaces {
 		if iface.Bond == "" {
 			continue
 		}
 
-		if bondName != "" && iface.Bond != bondName {
+		if firstBond == "" {
-			return nil, stderrors.New("encountered multiple bonds. this is unexpected in the equinix metal platform")
+			firstBond = iface.Bond
 		}
 
-		bondName = iface.Bond
-
 		found := false
 
 		hostInterfaceIter := hostInterfaces.Iterator()
@@ -154,17 +155,20 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 			if hostInterfaceIter.Value().TypedSpec().PermanentAddr.String() == iface.MAC {
 				found = true
 
+				slaveIndex := bondSlaveIndexes[iface.Bond]
+
 				networkConfig.Links = append(networkConfig.Links,
 					network.LinkSpecSpec{
 						Name: hostInterfaceIter.Value().Metadata().ID(),
 						Up:   true,
 						BondSlave: network.BondSlave{
-							MasterName: bondName,
+							MasterName: iface.Bond,
 							SlaveIndex: slaveIndex,
 						},
 						ConfigLayer: network.ConfigPlatform,
 					})
-				slaveIndex++
+
+				bondSlaveIndexes[iface.Bond]++
 
 				break
 			}
@@ -173,40 +177,45 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 		if !found {
 			log.Printf("interface with MAC %q wasn't found on the host, adding with the name from metadata", iface.MAC)
 
+			slaveIndex := bondSlaveIndexes[iface.Bond]
+
 			networkConfig.Links = append(networkConfig.Links,
 				network.LinkSpecSpec{
 					ConfigLayer: network.ConfigPlatform,
 					Name:        iface.Name,
 					Up:          true,
 					BondSlave: network.BondSlave{
-						MasterName: bondName,
+						MasterName: iface.Bond,
 						SlaveIndex: slaveIndex,
 					},
 				})
-			slaveIndex++
+
+			bondSlaveIndexes[iface.Bond]++
 		}
 	}
 
-	bondLink := network.LinkSpecSpec{
+	for bondName := range bondSlaveIndexes {
-		ConfigLayer: network.ConfigPlatform,
+		bondLink := network.LinkSpecSpec{
-		Name:        bondName,
+			ConfigLayer: network.ConfigPlatform,
-		Logical:     true,
+			Name:        bondName,
-		Up:          true,
+			Logical:     true,
-		Kind:        network.LinkKindBond,
+			Up:          true,
-		Type:        nethelpers.LinkEther,
+			Kind:        network.LinkKindBond,
-		BondMaster: network.BondMasterSpec{
+			Type:        nethelpers.LinkEther,
-			Mode:       bondMode,
+			BondMaster: network.BondMasterSpec{
-			DownDelay:  200,
+				Mode:       bondMode,
-			MIIMon:     100,
+				DownDelay:  200,
-			UpDelay:    200,
+				MIIMon:     100,
-			HashPolicy: nethelpers.BondXmitPolicyLayer34,
+				UpDelay:    200,
-		},
+				HashPolicy: nethelpers.BondXmitPolicyLayer34,
+			},
+		}
+
+		networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults()
+
+		networkConfig.Links = append(networkConfig.Links, bondLink)
 	}
 
-	networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults()
-
-	networkConfig.Links = append(networkConfig.Links, bondLink)
-
 	// 2. addresses
 
 	publicIPs := []string{}
@@ -233,7 +242,7 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 		networkConfig.Addresses = append(networkConfig.Addresses,
 			network.AddressSpecSpec{
 				ConfigLayer: network.ConfigPlatform,
-				LinkName:    bondName,
+				LinkName:    firstBond,
 				Address:     ipAddr,
 				Scope:       nethelpers.ScopeGlobal,
 				Flags:       nethelpers.AddressFlags(nethelpers.AddressPermanent),
@@ -249,6 +258,7 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 	}
 
 	// 3. routes
+	var privateGateway netip.Addr
 
 	for _, addr := range equinixMetadata.Network.Addresses {
 		if !(addr.Enabled && addr.Management) {
@@ -275,7 +285,7 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 			route := network.RouteSpecSpec{
 				ConfigLayer: network.ConfigPlatform,
 				Gateway:     gw,
-				OutLinkName: bondName,
+				OutLinkName: firstBond,
 				Table:       nethelpers.TableMain,
 				Protocol:    nethelpers.ProtocolStatic,
 				Type:        nethelpers.TypeUnicast,
@@ -298,6 +308,8 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 					return nil, err
 				}
 
+				privateGateway = gw
+
 				dest, err := netip.ParsePrefix(privSubnet)
 				if err != nil {
 					return nil, err
@@ -307,7 +319,7 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 					ConfigLayer: network.ConfigPlatform,
 					Gateway:     gw,
 					Destination: dest,
-					OutLinkName: bondName,
+					OutLinkName: firstBond,
 					Table:       nethelpers.TableMain,
 					Protocol:    nethelpers.ProtocolStatic,
 					Type:        nethelpers.TypeUnicast,
@@ -347,6 +359,36 @@ func (p *EquinixMetal) ParseMetadata(ctx context.Context, equinixMetadata *Metad
 		ProviderID:   fmt.Sprintf("equinixmetal://%s", equinixMetadata.ID),
 	}
 
+	// 6. BGP neighbors
+
+	for _, bgpNeighbor := range equinixMetadata.BGPNeighbors {
+		if bgpNeighbor.AddressFamily != 4 {
+			continue
+		}
+
+		for _, peerIP := range bgpNeighbor.PeerIPs {
+			peer, err := netip.ParseAddr(peerIP)
+			if err != nil {
+				return nil, err
+			}
+
+			route := network.RouteSpecSpec{
+				ConfigLayer: network.ConfigPlatform,
+				Gateway:     privateGateway,
+				Destination: netip.PrefixFrom(peer, 32),
+				OutLinkName: firstBond,
+				Table:       nethelpers.TableMain,
+				Protocol:    nethelpers.ProtocolStatic,
+				Type:        nethelpers.TypeUnicast,
+				Family:      nethelpers.FamilyInet4,
+			}
+
+			route.Normalize()
+
+			networkConfig.Routes = append(networkConfig.Routes, route)
+		}
+	}
+
 	return networkConfig, nil
 }
 

internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/metadata.go

@@ -6,11 +6,12 @@ package equinixmetal
 
 // MetadataConfig holds equinixmetal metadata info.
 type MetadataConfig struct {
-	ID             string   `json:"id"`
+	ID             string        `json:"id"`
-	Hostname       string   `json:"hostname"`
+	Hostname       string        `json:"hostname"`
-	Plan           string   `json:"plan"`
+	Plan           string        `json:"plan"`
-	Metro          string   `json:"metro"`
+	Metro          string        `json:"metro"`
-	Facility       string   `json:"facility"`
+	Facility       string        `json:"facility"`
-	Network        Network  `json:"network"`
+	Network        Network       `json:"network"`
-	PrivateSubnets []string `json:"private_subnets"`
+	BGPNeighbors   []BGPNeighbor `json:"bgp_neighbors"`
+	PrivateSubnets []string      `json:"private_subnets"`
 }

internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/expected.yaml

@@ -95,6 +95,28 @@ routes:
       flags: ""
       protocol: static
       layer: platform
+    - family: inet4
+      dst: 169.254.255.1/32
+      src: ""
+      gateway: 10.66.142.16
+      outLinkName: bond0
+      table: main
+      scope: global
+      type: unicast
+      flags: ""
+      protocol: static
+      layer: platform
+    - family: inet4
+      dst: 169.254.255.2/32
+      src: ""
+      gateway: 10.66.142.16
+      outLinkName: bond0
+      table: main
+      scope: global
+      type: unicast
+      flags: ""
+      protocol: static
+      layer: platform
 hostnames:
     - hostname: infra-green-ci
       domainname: ""

internal/app/machined/pkg/runtime/v1alpha1/platform/equinixmetal/testdata/metadata.json

@@ -111,6 +111,38 @@
         ],
         "metal_gateways": []
     },
+    "bgp_neighbors": [
+        {
+          "address_family": 4,
+          "customer_as": 65000,
+          "customer_ip": "10.67.50.1",
+          "md5_enabled": false,
+          "md5_password": null,
+          "multihop": true,
+          "peer_as": 65530,
+          "peer_ips": [
+            "169.254.255.1",
+            "169.254.255.2"
+          ],
+          "routes_in": [],
+          "routes_out": []
+        },
+        {
+          "address_family": 6,
+          "customer_as": 65000,
+          "customer_ip": "2604:1380:45e1:5000::1",
+          "md5_enabled": false,
+          "md5_password": null,
+          "multihop": true,
+          "peer_as": 65530,
+          "peer_ips": [
+            "fc00:0000:0000:0000:0000:0000:0000:000e",
+            "fc00:0000:0000:0000:0000:0000:0000:000f"
+          ],
+          "routes_in": [],
+          "routes_out": []
+        }
+    ],
     "api_url": "https://metadata.packet.net",
     "phone_home_url": "http://tinkerbell.ny5.packet.net/phone-home",
     "user_state_url": "http://tinkerbell.ny5.packet.net/events"

website/content/v1.6/_index.md

@@ -4,8 +4,8 @@ no_list: true
 linkTitle: "Documentation"
 cascade:
   type: docs
-lastRelease: v1.6.4
+lastRelease: v1.6.7
-kubernetesRelease: "1.29.1"
+kubernetesRelease: "1.29.3"
 prevKubernetesRelease: "1.28.3"
 nvidiaContainerToolkitRelease: "v1.13.5"
 nvidiaDriverRelease: "535.129.03"

website/content/v1.6/talos-guides/install/bare-metal-platforms/equinix-metal.md

@@ -5,8 +5,7 @@ aliases:
   - ../../../bare-metal-platforms/equinix-metal
 ---
 
-You can create a Talos Linux cluster on Equinix Metal in a variety of ways, such as through the EM web UI, the `metal` command line too, or through PXE booting.
+You can create a Talos Linux cluster on Equinix Metal in a variety of ways, such as through the EM web UI, or the `metal` command line tool.
-Talos Linux is a supported OS install option on Equinix Metal, so it's an easy process.
 
 Regardless of the method, the process is:
 
@@ -14,8 +13,8 @@ Regardless of the method, the process is:
 * Generate the configurations using `talosctl`.
 * Provision your machines on Equinix Metal.
 * Push the configurations to your servers (if not done as part of the machine provisioning).
-* configure your Kubernetes endpoint to point to the newly created control plane nodes
+* Configure your Kubernetes endpoint to point to the newly created control plane nodes.
-* bootstrap the cluster
+* Bootstrap the cluster.
 
 ## Define the Kubernetes Endpoint
 
@@ -65,10 +64,16 @@ The convention we use is `#!talos`.
 
 ## Provision the machines in Equinix Metal
 
+Talos Linux can be PXE-booted on Equinix Metal using [Image Factory]({{< relref "../../../learn-more/image-factory" >}}), using the `equinixMetal` platform: e.g.
+`https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64` (this URL references the default schematic and `amd64` architecture).
+
+Follow the Image Factory guide to create a custom schematic, e.g. with CPU microcode updates.
+The PXE boot URL can be used as the iPXE script URL.
+
 ### Using the Equinix Metal UI
 
 Simply select the location and type of machines in the Equinix Metal web interface.
-Select Talos as the Operating System, then select the number of servers to create, and name them (in lowercase only.)
+Select 'Custom iPXE' as the Operating System and enter the Image Factory PXE URL as the iPXE script URL, then select the number of servers to create, and name them (in lowercase only.)
 Under *optional settings*, you can optionally paste in the contents of `controlplane.yaml` that was generated, above (ensuring you add a first line of `#!talos`).
 
 You can repeat this process to create machines of different types for control plane and worker nodes (although you would pass in `worker.yaml` for the worker nodes, as user data).
@@ -81,8 +86,6 @@ If you did not pass in the machine configuration as User Data, you need to provi
 
 This guide assumes the user has a working API token,and the [Equinix Metal CLI](https://github.com/equinix/metal-cli/) installed.
 
-Because Talos Linux is a supported operating system, Talos Linux machines can be provisioned directly via the CLI, using the `-O talos_v1` parameter (for Operating System).
-
 <!-- textlint-disable one-sentence-per-line -->
 > Note: Ensure you have prepended `#!talos` to the `controlplane.yaml` file.
 <!-- textlint-enable one-sentence-per-line -->
@@ -91,49 +94,17 @@ Because Talos Linux is a supported operating system, Talos Linux machines can be
 metal device create \
   --project-id $PROJECT_ID \
   --facility $FACILITY \
-  --operating-system "talos_v1" \
+  --operating-system "custom_ipxe" \
+  --ipxe-script-url "https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64" \
   --plan $PLAN\
   --hostname $HOSTNAME\
   --userdata-file controlplane.yaml
 ```
 
-e.g. `metal device create -p <projectID> -f da11 -O talos_v1 -P c3.small.x86 -H steve.test.11 --userdata-file ./controlplane.yaml`
+e.g. `metal device create -p <projectID> -f da11 -O custom_ipxe -P c3.small.x86 -H steve.test.11 --userdata-file ./controlplane.yaml --ipxe-script-url "https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64"`
 
 Repeat this to create each control plane node desired: there should usually be 3 for a HA cluster.
 
-### Network Booting via iPXE
-
-Talos Linux can be PXE-booted on Equinix Metal using [Image Factory]({{< relref "../../../learn-more/image-factory" >}}), using the `equinixMetal` platform: e.g.
-`https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64` (this URL references the default schematic and `amd64` architecture).
-
-#### Create the Control Plane Nodes
-
-```bash
-metal device create \
-  --project-id $PROJECT_ID \
-  --facility $FACILITY \
-  --ipxe-script-url $PXE_SERVER \
-  --operating-system "custom_ipxe" \
-  --plan $PLAN\
-  --hostname $HOSTNAME\
-  --userdata-file controlplane.yaml
-```
-
-> Note: Repeat this to create each control plane node desired: there should usually be 3 for a HA cluster.
-
-#### Create the Worker Nodes
-
-```bash
-metal device create \
-  --project-id $PROJECT_ID \
-  --facility $FACILITY \
-  --ipxe-script-url $PXE_SERVER \
-  --operating-system "custom_ipxe" \
-  --plan $PLAN\
-  --hostname $HOSTNAME\
-  --userdata-file worker.yaml
-```
-
 ## Update the Kubernetes endpoint
 
 Now our control plane nodes have been created, and we know their IP addresses, we can associate them with the Kubernetes endpoint.

website/content/v1.7/talos-guides/install/bare-metal-platforms/equinix-metal.md

@@ -5,8 +5,7 @@ aliases:
   - ../../../bare-metal-platforms/equinix-metal
 ---
 
-You can create a Talos Linux cluster on Equinix Metal in a variety of ways, such as through the EM web UI, the `metal` command line too, or through PXE booting.
+You can create a Talos Linux cluster on Equinix Metal in a variety of ways, such as through the EM web UI, or the `metal` command line tool.
-Talos Linux is a supported OS install option on Equinix Metal, so it's an easy process.
 
 Regardless of the method, the process is:
 
@@ -14,8 +13,8 @@ Regardless of the method, the process is:
 * Generate the configurations using `talosctl`.
 * Provision your machines on Equinix Metal.
 * Push the configurations to your servers (if not done as part of the machine provisioning).
-* configure your Kubernetes endpoint to point to the newly created control plane nodes
+* Configure your Kubernetes endpoint to point to the newly created control plane nodes.
-* bootstrap the cluster
+* Bootstrap the cluster.
 
 ## Define the Kubernetes Endpoint
 
@@ -65,10 +64,16 @@ The convention we use is `#!talos`.
 
 ## Provision the machines in Equinix Metal
 
+Talos Linux can be PXE-booted on Equinix Metal using [Image Factory]({{< relref "../../../learn-more/image-factory" >}}), using the `equinixMetal` platform: e.g.
+`https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64` (this URL references the default schematic and `amd64` architecture).
+
+Follow the Image Factory guide to create a custom schematic, e.g. with CPU microcode updates.
+The PXE boot URL can be used as the iPXE script URL.
+
 ### Using the Equinix Metal UI
 
 Simply select the location and type of machines in the Equinix Metal web interface.
-Select Talos as the Operating System, then select the number of servers to create, and name them (in lowercase only.)
+Select 'Custom iPXE' as the Operating System and enter the Image Factory PXE URL as the iPXE script URL, then select the number of servers to create, and name them (in lowercase only.)
 Under *optional settings*, you can optionally paste in the contents of `controlplane.yaml` that was generated, above (ensuring you add a first line of `#!talos`).
 
 You can repeat this process to create machines of different types for control plane and worker nodes (although you would pass in `worker.yaml` for the worker nodes, as user data).
@@ -81,8 +86,6 @@ If you did not pass in the machine configuration as User Data, you need to provi
 
 This guide assumes the user has a working API token,and the [Equinix Metal CLI](https://github.com/equinix/metal-cli/) installed.
 
-Because Talos Linux is a supported operating system, Talos Linux machines can be provisioned directly via the CLI, using the `-O talos_v1` parameter (for Operating System).
-
 <!-- textlint-disable one-sentence-per-line -->
 > Note: Ensure you have prepended `#!talos` to the `controlplane.yaml` file.
 <!-- textlint-enable one-sentence-per-line -->
@@ -91,49 +94,17 @@ Because Talos Linux is a supported operating system, Talos Linux machines can be
 metal device create \
   --project-id $PROJECT_ID \
   --facility $FACILITY \
-  --operating-system "talos_v1" \
+  --operating-system "custom_ipxe" \
+  --ipxe-script-url "https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64" \
   --plan $PLAN\
   --hostname $HOSTNAME\
   --userdata-file controlplane.yaml
 ```
 
-e.g. `metal device create -p <projectID> -f da11 -O talos_v1 -P c3.small.x86 -H steve.test.11 --userdata-file ./controlplane.yaml`
+e.g. `metal device create -p <projectID> -f da11 -O custom_ipxe -P c3.small.x86 -H steve.test.11 --userdata-file ./controlplane.yaml --ipxe-script-url "https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64"`
 
 Repeat this to create each control plane node desired: there should usually be 3 for a HA cluster.
 
-### Network Booting via iPXE
-
-Talos Linux can be PXE-booted on Equinix Metal using [Image Factory]({{< relref "../../../learn-more/image-factory" >}}), using the `equinixMetal` platform: e.g.
-`https://pxe.factory.talos.dev/pxe/376567988ad370138ad8b2698212367b8edcb69b5fd68c80be1f2ec7d603b4ba/{{< release >}}/equinixMetal-amd64` (this URL references the default schematic and `amd64` architecture).
-
-#### Create the Control Plane Nodes
-
-```bash
-metal device create \
-  --project-id $PROJECT_ID \
-  --facility $FACILITY \
-  --ipxe-script-url $PXE_SERVER \
-  --operating-system "custom_ipxe" \
-  --plan $PLAN\
-  --hostname $HOSTNAME\
-  --userdata-file controlplane.yaml
-```
-
-> Note: Repeat this to create each control plane node desired: there should usually be 3 for a HA cluster.
-
-#### Create the Worker Nodes
-
-```bash
-metal device create \
-  --project-id $PROJECT_ID \
-  --facility $FACILITY \
-  --ipxe-script-url $PXE_SERVER \
-  --operating-system "custom_ipxe" \
-  --plan $PLAN\
-  --hostname $HOSTNAME\
-  --userdata-file worker.yaml
-```
-
 ## Update the Kubernetes endpoint
 
 Now our control plane nodes have been created, and we know their IP addresses, we can associate them with the Kubernetes endpoint.