feat: generate Flannel CNI manifest from upstream

Fixes #6730

`go generate`-based step downloads the upstream manifest, transforms it
to match our requirements, and it is compiled in as the Flannel
manifest.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2023-03-13 19:27:44 +04:00
parent 6656d35eca
commit 02b0ff35ee
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
7 changed files with 471 additions and 190 deletions

View File

@ -284,6 +284,7 @@ COPY --from=generate-build /api/resource/*.pb.go /pkg/machinery/api/resource/
COPY --from=generate-build /api/resource/config/*.pb.go /pkg/machinery/api/resource/config/
COPY --from=generate-build /api/resource/network/*.pb.go /pkg/machinery/api/resource/network/
COPY --from=generate-build /api/inspect/*.pb.go /pkg/machinery/api/inspect/
COPY --from=go-generate /src/pkg/flannel/ /pkg/flannel/
COPY --from=go-generate /src/pkg/machinery/resources/ /pkg/machinery/resources/
COPY --from=go-generate /src/pkg/machinery/config/types/v1alpha1/ /pkg/machinery/config/types/v1alpha1/
COPY --from=go-generate /src/pkg/machinery/nethelpers/ /pkg/machinery/nethelpers/

View File

@ -4,6 +4,8 @@
package k8s
import "github.com/siderolabs/talos/pkg/flannel"
// kube-apiserver configuration:
var kubeSystemEncryptionConfigTemplate = []byte(`apiVersion: v1
@ -452,195 +454,6 @@ spec:
protocol: TCP
`)
var flannelTemplate = []byte(`apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
k8s-app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "1.0.0",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
{{- $hasIPv4 := false }}
{{- range $cidr := .PodCIDRs }}
{{- if contains $cidr "." }}
{{- $hasIPv4 = true }}
"Network": "{{ $cidr }}",
{{- else }}
"IPv6Network": "{{ $cidr }}",
"EnableIPv6": true,
{{- end }}
{{- end }}
{{- if not $hasIPv4 }}
"EnableIPv4": false,
{{- end }}
"Backend": {
"Type": "vxlan",
"Port": 4789
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: kube-flannel
namespace: kube-system
labels:
tier: node
k8s-app: flannel
spec:
selector:
matchLabels:
tier: node
k8s-app: flannel
template:
metadata:
labels:
tier: node
k8s-app: flannel
spec:
serviceAccountName: flannel
initContainers:
- name: install-config
image: {{ .FlannelImage }}
command:
- cp
args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
volumeMounts:
- name: cni
mountPath: /etc/cni/net.d
- name: flannel-cfg
mountPath: /etc/kube-flannel/
- name: install-cni
image: {{ .FlannelCNIImage }}
command: ["/install-cni.sh"]
volumeMounts:
- name: host-cni-bin
mountPath: /host/opt/cni/bin/
containers:
- name: kube-flannel
image: {{ .FlannelImage }}
command:
- /opt/bin/flanneld
args:
- --ip-masq
- --kube-subnet-mgr
securityContext:
privileged: true
capabilities:
add: ["NET_ADMIN", "NET_RAW"]
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
volumeMounts:
- name: run
mountPath: /run/flannel
- name: flannel-cfg
mountPath: /etc/kube-flannel/
hostNetwork: true
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
volumes:
- name: run
hostPath:
path: /run/flannel
- name: cni
hostPath:
path: /etc/cni/net.d
- name: flannel-cfg
configMap:
name: kube-flannel-cfg
- name: host-cni-bin
hostPath:
path: /opt/cni/bin
updateStrategy:
rollingUpdate:
maxUnavailable: 1
type: RollingUpdate
`)
// podSecurityPolicy is the default PSP.
var podSecurityPolicy = []byte(`kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
@ -717,6 +530,8 @@ spec:
targetPort: {{ .ApidPort }}
`)
var flannelTemplate = flannel.Template
// talosServiceAccountCRDTemplate is the template of the CRD which
// allows injecting Talos with credentials into the Kubernetes cluster.
var talosServiceAccountCRDTemplate = []byte(`apiVersion: apiextensions.k8s.io/v1

8
pkg/flannel/flannel.go Normal file
View File

@ -0,0 +1,8 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Package flannel provides flannel default manifest.
package flannel
//go:generate go run ./gen.go

237
pkg/flannel/gen.go Normal file
View File

@ -0,0 +1,237 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build generate
package main
import (
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"strings"
"github.com/siderolabs/gen/slices"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
rbacv1 "k8s.io/api/rbac/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/scheme"
"sigs.k8s.io/yaml"
"github.com/siderolabs/talos/pkg/machinery/constants"
)
const sourceURL = "https://raw.githubusercontent.com/flannel-io/flannel/%s/Documentation/kube-flannel.yml"
const configMap = `apiVersion: v1
kind: ConfigMap
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
k8s-app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "1.0.0",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
{{- $hasIPv4 := false }}
{{- range $cidr := .PodCIDRs }}
{{- if contains $cidr "." }}
{{- $hasIPv4 = true }}
"Network": "{{ $cidr }}",
{{- else }}
"IPv6Network": "{{ $cidr }}",
"EnableIPv6": true,
{{- end }}
{{- end }}
{{- if not $hasIPv4 }}
"EnableIPv4": false,
{{- end }}
"Backend": {
"Type": "vxlan",
"Port": 4789
}
}
---
`
func marshal(out io.Writer, obj runtime.Object) {
m, err := yaml.Marshal(obj)
if err != nil {
log.Fatal(err)
}
m = regexp.MustCompile(` +creationTimestamp: null\n`).ReplaceAll(m, nil)
m = regexp.MustCompile(`status:\n( .+\n)+`).ReplaceAll(m, nil)
fmt.Fprintf(out, "%s---\n", string(m))
}
func main() {
url := fmt.Sprintf(sourceURL, constants.FlannelVersion)
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
if resp.StatusCode != http.StatusOK {
log.Fatalf("unexpected status code: %d", resp.StatusCode)
}
manifest, err := io.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
out, err := os.Create("template.go")
if err != nil {
log.Fatal(err)
}
defer out.Close()
fmt.Fprintf(out, `// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Code generated from the manifest %s DO NOT EDIT
package flannel
// Template is a flannel manifest template.
var Template = []byte(`+"`", url)
decoder := scheme.Codecs.UniversalDeserializer()
for _, resourceYAML := range strings.Split(string(manifest), "---") {
if len(resourceYAML) == 0 {
continue
}
obj, groupVersionKind, err := decoder.Decode(
[]byte(resourceYAML),
nil,
nil)
if err != nil {
log.Fatal(err)
}
switch groupVersionKind.Kind {
case "Namespace":
continue
case "ClusterRole":
marshal(out, obj)
case "ClusterRoleBinding":
crb := obj.(*rbacv1.ClusterRoleBinding)
crb.Subjects[0].Namespace = "kube-system"
crb.CreationTimestamp = metav1.Time{}
marshal(out, obj)
case "ServiceAccount":
sa := obj.(*corev1.ServiceAccount)
sa.Namespace = "kube-system"
marshal(out, obj)
case "ConfigMap":
fmt.Fprint(out, configMap)
case "DaemonSet":
ds := obj.(*appsv1.DaemonSet)
ds.Namespace = "kube-system"
ds.Name = "kube-flannel"
ds.Status = appsv1.DaemonSetStatus{}
ds.Labels["k8s-app"] = "flannel"
delete(ds.Labels, "app")
ds.Spec.Template.Labels["k8s-app"] = "flannel"
delete(ds.Spec.Template.Labels, "app")
ds.Spec.Template.Spec.Tolerations = append(ds.Spec.Template.Spec.Tolerations,
corev1.Toleration{
Effect: "NoExecute",
Operator: "Exists",
})
ds.Spec.Selector.MatchLabels["k8s-app"] = "flannel"
ds.Spec.Selector.MatchLabels["tier"] = "node"
delete(ds.Spec.Selector.MatchLabels, "app")
ds.Spec.Template.Spec.Containers[0].Image = "{{ .FlannelImage }}"
ds.Spec.Template.Spec.Volumes = slices.FilterInPlace(ds.Spec.Template.Spec.Volumes, func(v corev1.Volume) bool {
return v.Name != "xtables-lock"
})
ds.Spec.Template.Spec.Containers[0].VolumeMounts = slices.FilterInPlace(
ds.Spec.Template.Spec.Containers[0].VolumeMounts, func(v corev1.VolumeMount) bool {
return v.Name != "xtables-lock"
})
ds.Spec.Template.Spec.InitContainers = []corev1.Container{
{
Name: "install-config",
Image: "{{ .FlannelImage }}",
Command: []string{"cp"},
Args: []string{
"-f",
"/etc/kube-flannel/cni-conf.json",
"/etc/cni/net.d/10-flannel.conflist",
},
VolumeMounts: []corev1.VolumeMount{
{
Name: "cni",
MountPath: "/etc/cni/net.d",
},
{
Name: "flannel-cfg",
MountPath: "/etc/kube-flannel/",
},
},
},
{
Name: "install-cni",
Image: "{{ .FlannelCNIImage }}",
Command: []string{"/install-cni.sh"},
VolumeMounts: []corev1.VolumeMount{
{
Name: "cni-plugin",
MountPath: "/host/opt/cni/bin/",
},
},
},
}
marshal(out, obj)
default:
log.Fatalf("unknown resource kind: %q", groupVersionKind.Kind)
}
}
fmt.Fprint(out, "`)\n")
}

216
pkg/flannel/template.go Normal file
View File

@ -0,0 +1,216 @@
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
// Code generated from the manifest https://raw.githubusercontent.com/flannel-io/flannel/v0.20.2/Documentation/kube-flannel.yml DO NOT EDIT
package flannel
// Template is a flannel manifest template.
var Template = []byte(`apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: flannel
rules:
- apiGroups:
- ""
resources:
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- nodes/status
verbs:
- patch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: flannel
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: flannel
subjects:
- kind: ServiceAccount
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: flannel
namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
name: kube-flannel-cfg
namespace: kube-system
labels:
tier: node
k8s-app: flannel
data:
cni-conf.json: |
{
"name": "cbr0",
"cniVersion": "1.0.0",
"plugins": [
{
"type": "flannel",
"delegate": {
"hairpinMode": true,
"isDefaultGateway": true
}
},
{
"type": "portmap",
"capabilities": {
"portMappings": true
}
}
]
}
net-conf.json: |
{
{{- $hasIPv4 := false }}
{{- range $cidr := .PodCIDRs }}
{{- if contains $cidr "." }}
{{- $hasIPv4 = true }}
"Network": "{{ $cidr }}",
{{- else }}
"IPv6Network": "{{ $cidr }}",
"EnableIPv6": true,
{{- end }}
{{- end }}
{{- if not $hasIPv4 }}
"EnableIPv4": false,
{{- end }}
"Backend": {
"Type": "vxlan",
"Port": 4789
}
}
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
k8s-app: flannel
tier: node
name: kube-flannel
namespace: kube-system
spec:
selector:
matchLabels:
k8s-app: flannel
tier: node
template:
metadata:
labels:
k8s-app: flannel
tier: node
spec:
affinity:
nodeAffinity:
requiredDuringSchedulingIgnoredDuringExecution:
nodeSelectorTerms:
- matchExpressions:
- key: kubernetes.io/os
operator: In
values:
- linux
containers:
- args:
- --ip-masq
- --kube-subnet-mgr
command:
- /opt/bin/flanneld
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: EVENT_QUEUE_DEPTH
value: "5000"
image: '{{ .FlannelImage }}'
name: kube-flannel
resources:
limits:
cpu: 100m
memory: 50Mi
requests:
cpu: 100m
memory: 50Mi
securityContext:
capabilities:
add:
- NET_ADMIN
- NET_RAW
privileged: false
volumeMounts:
- mountPath: /run/flannel
name: run
- mountPath: /etc/kube-flannel/
name: flannel-cfg
hostNetwork: true
initContainers:
- args:
- -f
- /etc/kube-flannel/cni-conf.json
- /etc/cni/net.d/10-flannel.conflist
command:
- cp
image: '{{ .FlannelImage }}'
name: install-config
resources: {}
volumeMounts:
- mountPath: /etc/cni/net.d
name: cni
- mountPath: /etc/kube-flannel/
name: flannel-cfg
- command:
- /install-cni.sh
image: '{{ .FlannelCNIImage }}'
name: install-cni
resources: {}
volumeMounts:
- mountPath: /host/opt/cni/bin/
name: cni-plugin
priorityClassName: system-node-critical
serviceAccountName: flannel
tolerations:
- effect: NoSchedule
operator: Exists
- effect: NoExecute
operator: Exists
volumes:
- hostPath:
path: /run/flannel
name: run
- hostPath:
path: /opt/cni/bin
name: cni-plugin
- hostPath:
path: /etc/cni/net.d
name: cni
- configMap:
name: kube-flannel-cfg
name: flannel-cfg
updateStrategy: {}
---
`)

View File

@ -10,6 +10,7 @@ import (
criconfig "github.com/containerd/containerd/pkg/cri/config"
"github.com/siderolabs/talos/pkg/machinery/config"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/version"
)
@ -37,7 +38,7 @@ func List(config config.Provider) Versions {
images.Etcd = config.Cluster().Etcd().Image()
images.CoreDNS = config.Cluster().CoreDNS().Image()
images.Flannel = "ghcr.io/siderolabs/flannel:v0.20.2" // mirrored from docker.io/flannelcni/flannel
images.Flannel = fmt.Sprintf("ghcr.io/siderolabs/flannel:%s", constants.FlannelVersion) // mirrored from docker.io/flannelcni/flannel
images.FlannelCNI = fmt.Sprintf("ghcr.io/siderolabs/install-cni:%s", version.ExtrasVersion)
images.Kubelet = config.Machine().Kubelet().Image()
images.KubeAPIServer = config.Cluster().APIServer().Image()

View File

@ -817,6 +817,9 @@ const (
// DashboardTTY is the number of the TTY device (/dev/ttyN) for dashboard.
DashboardTTY = 2
// FlannelVersion is the version of flannel to use.
FlannelVersion = "v0.20.2"
)
// See https://linux.die.net/man/3/klogctl