feat: network configuration improvements on the NoCloud platform

* support for bonding
* added interface selection by MAC address
* added routes management

Signed-off-by: Andrei Kvapil <kvapss@gmail.com>
Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrei Kvapil 2023-07-26 02:47:02 +02:00 committed by Andrey Smirnov
parent 5adeb5042f
commit 10f958cf41
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
6 changed files with 430 additions and 110 deletions

View File

@ -16,12 +16,15 @@ import (
"path/filepath"
"strings"
"github.com/cosi-project/runtime/pkg/safe"
"github.com/cosi-project/runtime/pkg/state"
"github.com/siderolabs/go-blockdevice/blockdevice/filesystem"
"github.com/siderolabs/go-blockdevice/blockdevice/probe"
"golang.org/x/sys/unix"
yaml "gopkg.in/yaml.v3"
networkadapter "github.com/siderolabs/talos/internal/app/machined/pkg/adapters/network"
networkctrl "github.com/siderolabs/talos/internal/app/machined/pkg/controllers/network"
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime"
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1/platform/errors"
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1/platform/internal/netutils"
@ -45,7 +48,7 @@ type NetworkConfig struct {
Config []struct {
Mac string `yaml:"mac_address,omitempty"`
Interfaces string `yaml:"name,omitempty"`
MTU string `yaml:"mtu,omitempty"`
MTU uint32 `yaml:"mtu,omitempty"`
Subnets []struct {
Address string `yaml:"address,omitempty"`
Netmask string `yaml:"netmask,omitempty"`
@ -70,25 +73,34 @@ type Ethernet struct {
Address []string `yaml:"addresses,omitempty"`
Gateway4 string `yaml:"gateway4,omitempty"`
Gateway6 string `yaml:"gateway6,omitempty"`
MTU int `yaml:"mtu,omitempty"`
MTU uint32 `yaml:"mtu,omitempty"`
NameServers struct {
Search []string `yaml:"search,omitempty"`
Address []string `yaml:"addresses,omitempty"`
} `yaml:"nameservers,omitempty"`
Routes []struct {
To string `yaml:"to,omitempty"`
Via string `yaml:"via,omitempty"`
Metric string `yaml:"metric,omitempty"`
Table uint32 `yaml:"table,omitempty"`
} `yaml:"routes,omitempty"`
RoutingPolicy []struct { // TODO
From string `yaml:"froom,omitempty"`
Table uint32 `yaml:"table,omitempty"`
} `yaml:"routing-policy,omitempty"`
}
// Bonds holds bonding interface info.
type Bonds struct {
Interfaces []string `yaml:"interfaces,omitempty"`
Address []string `yaml:"addresses,omitempty"`
NameServers struct {
Search []string `yaml:"search,omitempty"`
Address []string `yaml:"addresses,omitempty"`
} `yaml:"nameservers,omitempty"`
Params []struct {
Ethernet `yaml:",inline"`
Interfaces []string `yaml:"interfaces,omitempty"`
Params struct {
Mode string `yaml:"mode,omitempty"`
LACPRate string `yaml:"lacp-rate,omitempty"`
HashPolicy string `yaml:"transmit-hash-policy,omitempty"`
MIIMon uint32 `yaml:"mii-monitor-interval,omitempty"`
UpDelay uint32 `yaml:"up-delay,omitempty"`
DownDelay uint32 `yaml:"down-delay,omitempty"`
} `yaml:"parameters,omitempty"`
}
@ -333,11 +345,11 @@ func (n *Nocloud) applyNetworkConfigV1(config *NetworkConfig, networkConfig *run
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: family,
Priority: 1024,
Priority: networkctrl.DefaultRouteMetric,
}
if family == nethelpers.FamilyInet6 {
route.Priority = 2048
route.Priority = 2 * networkctrl.DefaultRouteMetric
}
route.Normalize()
@ -363,119 +375,241 @@ func (n *Nocloud) applyNetworkConfigV1(config *NetworkConfig, networkConfig *run
}
//nolint:gocyclo
func (n *Nocloud) applyNetworkConfigV2(config *NetworkConfig, networkConfig *runtime.PlatformNetworkConfig) error {
func applyNetworkConfigV2Ethernet(name string, eth Ethernet, networkConfig *runtime.PlatformNetworkConfig, dnsIPs *[]netip.Addr) error {
if eth.DHCPv4 {
networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{
Operator: network.OperatorDHCP4,
LinkName: name,
RequireUp: true,
DHCP4: network.DHCP4OperatorSpec{
RouteMetric: uint32(networkctrl.DefaultRouteMetric),
},
ConfigLayer: network.ConfigPlatform,
})
}
if eth.DHCPv6 {
networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{
Operator: network.OperatorDHCP6,
LinkName: name,
RequireUp: true,
DHCP6: network.DHCP6OperatorSpec{
RouteMetric: uint32(networkctrl.DefaultRouteMetric),
},
ConfigLayer: network.ConfigPlatform,
})
}
for _, addr := range eth.Address {
ipPrefix, err := netip.ParsePrefix(addr)
if err != nil {
return err
}
family := nethelpers.FamilyInet4
if ipPrefix.Addr().Is6() {
family = nethelpers.FamilyInet6
}
networkConfig.Addresses = append(networkConfig.Addresses,
network.AddressSpecSpec{
ConfigLayer: network.ConfigPlatform,
LinkName: name,
Address: ipPrefix,
Scope: nethelpers.ScopeGlobal,
Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent),
Family: family,
},
)
}
if eth.Gateway4 != "" {
gw, err := netip.ParseAddr(eth.Gateway4)
if err != nil {
return err
}
route := network.RouteSpecSpec{
ConfigLayer: network.ConfigPlatform,
Gateway: gw,
OutLinkName: name,
Table: nethelpers.TableMain,
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: nethelpers.FamilyInet4,
Priority: networkctrl.DefaultRouteMetric,
}
route.Normalize()
networkConfig.Routes = append(networkConfig.Routes, route)
}
if eth.Gateway6 != "" {
gw, err := netip.ParseAddr(eth.Gateway6)
if err != nil {
return err
}
route := network.RouteSpecSpec{
ConfigLayer: network.ConfigPlatform,
Gateway: gw,
OutLinkName: name,
Table: nethelpers.TableMain,
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: nethelpers.FamilyInet6,
Priority: 2 * networkctrl.DefaultRouteMetric,
}
route.Normalize()
networkConfig.Routes = append(networkConfig.Routes, route)
}
for _, addr := range eth.NameServers.Address {
if ip, err := netip.ParseAddr(addr); err == nil {
*dnsIPs = append(*dnsIPs, ip)
} else {
return err
}
}
for _, route := range eth.Routes {
gw, err := netip.ParseAddr(route.Via)
if err != nil {
return fmt.Errorf("failed to parse route gateway: %w", err)
}
dest, err := netip.ParsePrefix(route.To)
if err != nil {
return fmt.Errorf("failed to parse route destination: %w", err)
}
family := nethelpers.FamilyInet4
if gw.Is6() {
family = nethelpers.FamilyInet6
}
route := network.RouteSpecSpec{
ConfigLayer: network.ConfigPlatform,
Destination: dest,
Gateway: gw,
OutLinkName: name,
Table: nethelpers.RoutingTable(route.Table),
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: family,
Priority: 1024,
}
route.Normalize()
networkConfig.Routes = append(networkConfig.Routes, route)
}
return nil
}
//nolint:gocyclo
func (n *Nocloud) applyNetworkConfigV2(config *NetworkConfig, st state.State, networkConfig *runtime.PlatformNetworkConfig) error {
var dnsIPs []netip.Addr
hostInterfaces, err := safe.StateListAll[*network.LinkStatus](context.TODO(), st)
if err != nil {
return fmt.Errorf("error listing host interfaces: %w", err)
}
for name, eth := range config.Ethernets {
if !strings.HasPrefix(name, "eth") {
continue
var bondSlave network.BondSlave
for bondName, bond := range config.Bonds {
for _, iface := range bond.Interfaces {
if iface == name {
bondSlave.MasterName = bondName
bondSlave.SlaveIndex = 1
}
}
}
if eth.Match.HWAddr != "" {
var availableMACAddresses []string
macAddressMatched := false
hostInterfaceIter := safe.IteratorFromList(hostInterfaces)
for hostInterfaceIter.Next() {
macAddress := hostInterfaceIter.Value().TypedSpec().PermanentAddr.String()
if macAddress == eth.Match.HWAddr {
name = hostInterfaceIter.Value().Metadata().ID()
macAddressMatched = true
break
}
availableMACAddresses = append(availableMACAddresses, macAddress)
}
if !macAddressMatched {
log.Printf("vmware: no link with matching MAC address %q (available %v), defaulted to use name %s instead", eth.Match.HWAddr, availableMACAddresses, name)
}
}
networkConfig.Links = append(networkConfig.Links, network.LinkSpecSpec{
Name: name,
Up: true,
MTU: uint32(eth.MTU),
MTU: eth.MTU,
ConfigLayer: network.ConfigPlatform,
BondSlave: bondSlave,
})
if eth.DHCPv4 {
networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{
Operator: network.OperatorDHCP4,
LinkName: name,
RequireUp: true,
DHCP4: network.DHCP4OperatorSpec{
RouteMetric: 1024,
},
ConfigLayer: network.ConfigPlatform,
})
err := applyNetworkConfigV2Ethernet(name, eth, networkConfig, &dnsIPs)
if err != nil {
return err
}
}
for name, bond := range config.Bonds {
mode, err := nethelpers.BondModeByName(bond.Params.Mode)
if err != nil {
return fmt.Errorf("invalid mode: %w", err)
}
if eth.DHCPv6 {
networkConfig.Operators = append(networkConfig.Operators, network.OperatorSpecSpec{
Operator: network.OperatorDHCP6,
LinkName: name,
RequireUp: true,
DHCP6: network.DHCP6OperatorSpec{
RouteMetric: 1024,
},
ConfigLayer: network.ConfigPlatform,
})
hashPolicy, err := nethelpers.BondXmitHashPolicyByName(bond.Params.HashPolicy)
if err != nil {
return fmt.Errorf("invalid transmit-hash-policy: %w", err)
}
for _, addr := range eth.Address {
ipPrefix, err := netip.ParsePrefix(addr)
if err != nil {
return err
}
family := nethelpers.FamilyInet4
if ipPrefix.Addr().Is6() {
family = nethelpers.FamilyInet6
}
networkConfig.Addresses = append(networkConfig.Addresses,
network.AddressSpecSpec{
ConfigLayer: network.ConfigPlatform,
LinkName: name,
Address: ipPrefix,
Scope: nethelpers.ScopeGlobal,
Flags: nethelpers.AddressFlags(nethelpers.AddressPermanent),
Family: family,
},
)
lacpRate, err := nethelpers.LACPRateByName(bond.Params.LACPRate)
if err != nil {
return fmt.Errorf("invalid lacp-rate: %w", err)
}
if eth.Gateway4 != "" {
gw, err := netip.ParseAddr(eth.Gateway4)
if err != nil {
return err
}
route := network.RouteSpecSpec{
ConfigLayer: network.ConfigPlatform,
Gateway: gw,
OutLinkName: name,
Table: nethelpers.TableMain,
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: nethelpers.FamilyInet4,
Priority: 1024,
}
route.Normalize()
networkConfig.Routes = append(networkConfig.Routes, route)
bondLink := network.LinkSpecSpec{
ConfigLayer: network.ConfigPlatform,
Name: name,
Logical: true,
Up: true,
MTU: bond.Ethernet.MTU,
Kind: network.LinkKindBond,
Type: nethelpers.LinkEther,
BondMaster: network.BondMasterSpec{
Mode: mode,
HashPolicy: hashPolicy,
MIIMon: bond.Params.MIIMon,
UpDelay: bond.Params.UpDelay,
DownDelay: bond.Params.DownDelay,
LACPRate: lacpRate,
},
}
if eth.Gateway6 != "" {
gw, err := netip.ParseAddr(eth.Gateway6)
if err != nil {
return err
}
networkadapter.BondMasterSpec(&bondLink.BondMaster).FillDefaults()
networkConfig.Links = append(networkConfig.Links, bondLink)
route := network.RouteSpecSpec{
ConfigLayer: network.ConfigPlatform,
Gateway: gw,
OutLinkName: name,
Table: nethelpers.TableMain,
Protocol: nethelpers.ProtocolStatic,
Type: nethelpers.TypeUnicast,
Family: nethelpers.FamilyInet6,
Priority: 2048,
}
route.Normalize()
networkConfig.Routes = append(networkConfig.Routes, route)
}
for _, addr := range eth.NameServers.Address {
if ip, err := netip.ParseAddr(addr); err == nil {
dnsIPs = append(dnsIPs, ip)
} else {
return err
}
err = applyNetworkConfigV2Ethernet(name, bond.Ethernet, networkConfig, &dnsIPs)
if err != nil {
return err
}
}

View File

@ -30,7 +30,7 @@ func (n *Nocloud) Name() string {
}
// ParseMetadata converts nocloud metadata to platform network config.
func (n *Nocloud) ParseMetadata(unmarshalledNetworkConfig *NetworkConfig, metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) {
func (n *Nocloud) ParseMetadata(unmarshalledNetworkConfig *NetworkConfig, st state.State, metadata *MetadataConfig) (*runtime.PlatformNetworkConfig, error) {
networkConfig := &runtime.PlatformNetworkConfig{}
if metadata.Hostname != "" {
@ -51,7 +51,7 @@ func (n *Nocloud) ParseMetadata(unmarshalledNetworkConfig *NetworkConfig, metada
return nil, err
}
case 2:
if err := n.applyNetworkConfigV2(unmarshalledNetworkConfig, networkConfig); err != nil {
if err := n.applyNetworkConfigV2(unmarshalledNetworkConfig, st, networkConfig); err != nil {
return nil, err
}
default:
@ -101,8 +101,8 @@ func (n *Nocloud) KernelArgs() procfs.Parameters {
// NetworkConfiguration implements the runtime.Platform interface.
//
//nolint:gocyclo
func (n *Nocloud) NetworkConfiguration(ctx context.Context, r state.State, ch chan<- *runtime.PlatformNetworkConfig) error {
metadataConfigDl, metadataNetworkConfigDl, _, metadata, err := n.acquireConfig(ctx, r)
func (n *Nocloud) NetworkConfiguration(ctx context.Context, st state.State, ch chan<- *runtime.PlatformNetworkConfig) error {
metadataConfigDl, metadataNetworkConfigDl, _, metadata, err := n.acquireConfig(ctx, st)
if stderrors.Is(err, errors.ErrNoConfigSource) {
err = nil
}
@ -123,7 +123,7 @@ func (n *Nocloud) NetworkConfiguration(ctx context.Context, r state.State, ch ch
}
}
networkConfig, err := n.ParseMetadata(&unmarshalledNetworkConfig, metadata)
networkConfig, err := n.ParseMetadata(&unmarshalledNetworkConfig, st, metadata)
if err != nil {
return err
}

View File

@ -5,15 +5,21 @@
package nocloud_test
import (
"context"
_ "embed"
"fmt"
"testing"
"github.com/cosi-project/runtime/pkg/state"
"github.com/cosi-project/runtime/pkg/state/impl/inmem"
"github.com/cosi-project/runtime/pkg/state/impl/namespaced"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime/v1alpha1/platform/nocloud"
"github.com/siderolabs/talos/pkg/machinery/nethelpers"
"github.com/siderolabs/talos/pkg/machinery/resources/network"
)
//go:embed testdata/metadata-v1.yaml
@ -50,6 +56,16 @@ func TestParseMetadata(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
n := &nocloud.Nocloud{}
st := state.WrapCore(namespaced.NewState(inmem.Build))
eth1 := network.NewLinkStatus(network.NamespaceName, "eth1")
eth1.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf8}
require.NoError(t, st.Create(context.TODO(), eth1))
eth2 := network.NewLinkStatus(network.NamespaceName, "eth2")
eth2.TypedSpec().PermanentAddr = nethelpers.HardwareAddr{0x68, 0x05, 0xca, 0xb8, 0xf1, 0xf9}
require.NoError(t, st.Create(context.TODO(), eth2))
var m nocloud.NetworkConfig
require.NoError(t, yaml.Unmarshal(tt.raw, &m))
@ -59,7 +75,7 @@ func TestParseMetadata(t *testing.T) {
InstanceID: "0",
}
networkConfig, err := n.ParseMetadata(&m, &mc)
networkConfig, err := n.ParseMetadata(&m, st, &mc)
require.NoError(t, err)
marshaled, err := yaml.Marshal(networkConfig)

View File

@ -11,6 +11,12 @@ addresses:
scope: global
flags: permanent
layer: platform
- address: 10.10.4.140/29
linkName: bond0
family: inet4
scope: global
flags: permanent
layer: platform
links:
- name: eth0
logical: false
@ -19,6 +25,48 @@ links:
kind: ""
type: netrom
layer: platform
- name: eth1
logical: false
up: true
mtu: 0
kind: ""
type: netrom
masterName: bond0
slaveIndex: 1
layer: platform
- name: eth2
logical: false
up: true
mtu: 0
kind: ""
type: netrom
masterName: bond0
slaveIndex: 1
layer: platform
- name: bond0
logical: true
up: true
mtu: 1500
kind: bond
type: ether
bondMaster:
mode: 802.3ad
xmitHashPolicy: layer3+4
lacpRate: fast
arpValidate: none
arpAllTargets: any
primaryReselect: always
failOverMac: 0
miimon: 100
updelay: 200
downdelay: 200
resendIgmp: 1
lpInterval: 1
packetsPerSlave: 1
numPeerNotif: 1
tlbLogicalLb: 1
adActorSysPrio: 65535
layer: platform
routes:
- family: inet4
dst: ""
@ -44,6 +92,42 @@ routes:
flags: ""
protocol: static
layer: platform
- family: inet4
dst: 10.0.0.0/8
src: ""
gateway: 10.10.4.147
outLinkName: bond0
table: unspec
priority: 1024
scope: global
type: unicast
flags: ""
protocol: static
layer: platform
- family: inet4
dst: 192.168.0.0/16
src: ""
gateway: 10.10.4.147
outLinkName: bond0
table: unspec
priority: 1024
scope: global
type: unicast
flags: ""
protocol: static
layer: platform
- family: inet4
dst: 188.42.208.0/21
src: ""
gateway: 10.10.4.147
outLinkName: bond0
table: unspec
priority: 1024
scope: global
type: unicast
flags: ""
protocol: static
layer: platform
hostnames:
- hostname: talos
domainname: fqdn
@ -51,6 +135,8 @@ hostnames:
resolvers:
- dnsServers:
- 8.8.8.8
- 1.1.1.1
- 2.2.2.2
layer: platform
timeServers: []
operators:

View File

@ -12,3 +12,38 @@ ethernets:
nameservers:
search: [foo.local, bar.local]
addresses: [8.8.8.8]
ext1:
match:
macaddress: 68:05:ca:b8:f1:f8
ext2:
match:
macaddress: 68:05:ca:b8:f1:f9
bonds:
bond0:
interfaces:
- ext1
- ext2
macaddress: e4:3d:1a:4d:6a:28
mtu: 1500
parameters:
mode: 802.3ad
mii-monitor-interval: 100
down-delay: 200
up-delay: 200
lacp-rate: fast
transmit-hash-policy: layer3+4
addresses:
- 10.10.4.140/29
nameservers:
addresses:
- 1.1.1.1
- 2.2.2.2
routes:
- to: 10.0.0.0/8
via: 10.10.4.147
- to: 192.168.0.0/16
via: 10.10.4.147
- to: 188.42.208.0/21
via: 10.10.4.147

View File

@ -0,0 +1,49 @@
version: 2
ethernets:
eth0:
match:
macaddress: '00:20:6e:1f:f9:a8'
dhcp4: true
addresses:
- 192.168.14.2/24
- 2001:1::1/64
gateway4: 192.168.14.1
gateway6: 2001:1::2
nameservers:
search: [foo.local, bar.local]
addresses: [8.8.8.8]
ext1:
match:
macaddress: f4:33:01:16:3d:90
ext2:
match:
macaddress: 50:a5:1b:e8:7b:db
bonds:
bond0:
interfaces:
- ext1
- ext2
macaddress: e4:3d:1a:4d:6a:28
mtu: 1500
parameters:
mode: 802.3ad
mii-monitor-interval: 100
down-delay: 200
up-delay: 200
lacp-rate: fast
transmit-hash-policy: layer3+4
addresses:
- 10.10.4.140/29
nameservers:
addresses:
- 1.1.1.1
- 2.2.2.2
routes:
- to: 10.0.0.0/8
via: 10.10.4.147
- to: 192.168.0.0/16
via: 10.10.4.147
- to: 188.42.208.0/21
via: 10.10.4.147