feat: add docker-os make target, Kubeadm.ExtraArgs, and a dev Makefile (#446)

Signed-off-by: leigh capili <leigh@null.net>

Dockerfile

@@ -186,6 +186,19 @@ RUN ["/toolchain/bin/tar", "-cvpzf", "/rootfs.tar.gz", "."]
 FROM scratch AS rootfs
 COPY --from=rootfs-build /rootfs.tar.gz /rootfs.tar.gz
 
+# The docker-os target generates a docker image that can be used to run Talos
+# in containers.
+
+FROM rootfs-build AS docker-rootfs-build
+# workaround docker overwriting our /etc symlink
+RUN rm /rootfs/etc
+RUN mv /rootfs/var/etc /rootfs/etc
+RUN ln -s /etc /rootfs/var/etc
+FROM scratch AS docker-os
+COPY --from=docker-rootfs-build /rootfs /
+COPY --from=initramfs-build /init /sbin/init
+ENTRYPOINT ["/sbin/init"]
+
 # The installer target generates an image that can be used to install Talos to
 # various environments.
 

Makefile

@@ -107,6 +107,17 @@ rootfs: buildkitd hyperkube etcd coredns pause osd trustd proxyd blockd
 		--frontend-opt target=$@ \
 		$(COMMON_ARGS)
 
+docker-os: buildkitd
+	@buildctl --addr $(BUILDKIT_HOST) \
+		build \
+		--exporter=docker \
+		--exporter-opt output=build/$@.tar \
+		--exporter-opt name=docker.io/autonomy/talos-os:$(TAG) \
+		--exporter-opt push=$(PUSH) \
+		--frontend-opt target=$@ \
+		$(COMMON_ARGS)
+	@docker load < build/$@.tar
+
 installer: buildkitd
 	@mkdir -p build
 	@buildctl --addr $(BUILDKIT_HOST) \

dev/.gitignore

@@ -0,0 +1,5 @@
+pki/
+config/*
+!config/.*
+*.kmsg
+*.logs/

dev/Makefile

@@ -0,0 +1,125 @@
+SHELL := /bin/bash
+
+ORG = talos.local
+K_ORG = talos.k8s.local
+USER = dev
+CA_LENGTH = $$(( 24 * 365 * 1 ))
+K_CA_LENGTH = $$(( 24 * 365 * 1 ))
+USER_LENGTH = $$(( 24 * 30 * 5 ))
+
+NODE = talos-os-1
+
+GIT_SHORT_SHA = $(shell git rev-parse --short=7 HEAD)
+GIT_DIRTY = $(shell [[ -n `git status -s` ]] && echo -dirty)
+TAG = $(GIT_SHORT_SHA)$(GIT_DIRTY)
+# TAG = $(shell gitmeta image tag)  # TODO: this tool is slow
+
+OS_IMAGE_NAME ?= autonomy/talos-os
+
+UNAME_S := $(shell uname -s)
+ifeq ($(UNAME_S),Linux)
+OSCTL ?= ../build/osctl-linux-amd64
+endif
+ifeq ($(UNAME_S),Darwin)
+OSCTL ?= ../build/osctl-darwin-amd64
+endif
+
+
+DOCKER_COMPOSE ?= IMAGE=$(OS_IMAGE_NAME) TAG=$(TAG) NODE=$(NODE) docker-compose
+.PHONY: os-up os-down os-clean
+os-up :
+	touch os1.kmsg
+	mkdir -p os1.logs/pods/ && chmod -R 777 os1.logs  # easier to manage pod logs
+	$(DOCKER_COMPOSE) up -d
+	tail -f os1.kmsg
+os-down :
+	$(DOCKER_COMPOSE) down
+os-clean :
+	$(DOCKER_COMPOSE) down -v
+	rm -rf os1.kmsg os1.logs || true
+
+
+.PHONY: config config-clean
+config : config/$(NODE).yaml $(HOME)/.talos/config
+config-clean:
+	rm config/* || true  # does not delete dotfiles
+
+
+pki/ :
+	@mkdir -p pki/
+
+# OS CA PKI
+pki/$(ORG).key : pki/
+	# gen OS CA
+	@$(OSCTL) gen ca \
+		--hours "$(CA_LENGTH)" \
+		--organization "pki/$(ORG)"
+pki/$(ORG).crt : pki/$(ORG).key
+pki/$(ORG).sha256 : pki/$(ORG).key
+
+# user PKI
+pki/$(USER).key : pki/
+	# gen user key
+	@$(OSCTL) gen key \
+		--name "pki/$(USER)"
+pki/$(USER).csr : pki/$(USER).key
+	# gen user CSR
+	@$(OSCTL) gen csr \
+		--ip "127.0.0.1" \
+		--key "pki/$(USER).key"
+pki/$(USER).crt : pki/$(USER).csr pki/$(ORG).crt
+	# gen user crt
+	@$(OSCTL) gen crt \
+		--hours "$(USER_LENGTH)" \
+		--ca "pki/$(ORG)" \
+		--csr "pki/$(USER).csr" \
+		--name "pki/$(USER)"
+
+# k8s CA PKI
+pki/$(K_ORG).key : pki/
+	# gen k8s CA
+	@$(OSCTL) gen ca \
+		--rsa \
+		--hours "$(K_CA_LENGTH)" \
+		--organization "pki/$(K_ORG)"
+pki/$(K_ORG).crt : pki/$(K_ORG).key
+pki/$(K_ORG).sha256 : pki/$(K_ORG).key
+
+# PKI cleanup
+.PHONY: pki-clean-all pki-clean-os pki-clean-user pki-clean-k8s
+pki-clean-all:
+	rm -rf pki || true
+pki-clean-os:
+	rm -f pki/$(ORG)* || true
+pki-clean-user:
+	rm -f pki/$(USER)* || true
+pki-clean-k8s:
+	rm -f pki/$(K_ORG)* || true
+
+
+config/$(NODE).yaml : pki/$(ORG).crt pki/$(ORG).key pki/$(K_ORG).crt pki/$(K_ORG).key
+	# inject base os config
+	@cp config/.base.node.yaml "config/$(NODE).yaml"
+	@chmod 600 "config/$(NODE).yaml"
+	@$(OSCTL) inject os \
+		--crt "pki/$(ORG).crt" \
+		--key "pki/$(ORG).key" \
+		"config/$(NODE).yaml"
+	@$(OSCTL) inject kubernetes \
+		--crt "pki/$(K_ORG).crt" \
+		--key "pki/$(K_ORG).key" \
+		"config/$(NODE).yaml"
+
+$(HOME)/.talos/config : pki/$(ORG).crt pki/$(USER).crt pki/$(USER).key
+	# configure dev osctl
+	@touch $${HOME}/.talos/config
+	@$(OSCTL) config add "talos-local" \
+		--ca "pki/$(ORG).crt" \
+		--crt "pki/$(USER).crt" \
+		--key "pki/$(USER).key"
+	@$(OSCTL) config context "talos-local"
+	@$(OSCTL) config target "127.0.0.1"
+
+
+.PHONY: clean
+clean : config-clean os-clean pki-clean-all

dev/config/.base.node.yaml

@@ -0,0 +1,25 @@
+services:
+  init:
+    cni: flannel
+  kubeadm:
+    configuration: |
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: InitConfiguration
+      apiVersion: kubeadm.k8s.io/v1beta1
+      nodeRegistration:
+        criSocket: /run/containerd/containerd.sock
+      ---
+      apiVersion: kubeadm.k8s.io/v1beta1
+      kind: ClusterConfiguration
+      networking:
+        dnsDomain: cluster.local
+      ---
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      kind: KubeletConfiguration
+      failSwapOn: false  # necessary for some docker hosts
+    extraArgs:
+      - --ignore-preflight-errors=cri,kubeletversion,numcpu,requiredipvskernelmodulesavailable,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,Swap
+        # additions to talos default are:  FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,Swap
+  trustd:
+    username: 'dev'
+    password: 'talos_trust_dev'

dev/docker-compose.yaml

@@ -0,0 +1,42 @@
+version: '3.7'
+
+services:
+  os1:
+    image: ${IMAGE:-autonomy/talos-os}:${TAG}
+    privileged: true
+    security_opt:
+      - seccomp:unconfined
+    entrypoint: /sbin/init
+    command: --switch-root
+    volumes:
+      # logging hacks
+      - ./os1.kmsg:/dev/kmsg
+      - ./os1.logs/:/var/log/
+      # os config
+      - ./config/${NODE:-talos-os-1}.yaml:/run/userdata.yaml
+      # state dirs
+      - os1_containerd:/var/lib/containerd/
+      - os1_etcd:/var/lib/etcd/
+      - os1_kubelet:/var/lib/kubelet/
+      - os1_kubernetes:/etc/kubernetes/
+      # kubeadm mounts os-release, but init --switch-root does not create
+      - ./os-release:/etc/os-release
+      # kubelet is missing cni mount
+      - os1_cni:/etc/cni/
+    tmpfs:
+      - /run
+    # sysctls:
+    #   net.bridge.bridge-nf-call-iptables: 1
+    #   # this proc sysctl path is missing in the container
+    #   # kubeadm errors on pre-flight -- modified init to ignore it
+    ports:
+      - "50000:50000" # osd
+      - "6443:6443" # kube-api-server
+
+volumes:
+  os1_containerd:
+  os1_etcd:
+  os1_kubelet:
+  os1_kubernetes:
+  os1_ssl:
+  os1_cni:

dev/os-release

@@ -0,0 +1,6 @@
+NAME="Talos"
+ID=dev-container
+VERSION_ID=dev-container
+PRETTY_NAME="Talos (dev-container) by Autonomy"
+HOME_URL="https://talos.autonomy.io/"
+BUG_REPORT_URL="https://github.com/autonomy/talos/issues"

internal/app/init/pkg/system/services/kubeadm.go

@@ -131,6 +131,8 @@ func (k *Kubeadm) Start(data *userdata.UserData) error {
 		args.ProcessArgs = []string{"kubeadm", "join", "--config=/etc/kubernetes/kubeadm-config.yaml", ignore}
 	}
 
+	args.ProcessArgs = append(args.ProcessArgs, data.Services.Kubeadm.ExtraArgs...)
+
 	// Set the mounts.
 	// nolint: dupl
 	mounts := []specs.Mount{

internal/pkg/userdata/userdata.go

@@ -119,6 +119,7 @@ type Kubeadm struct {
 	CommonServiceOptions `yaml:",inline"`
 
 	Configuration runtime.Object `yaml:"configuration"`
+	ExtraArgs     []string       `yaml:"extraArgs,omitempty"`
 	bootstrap     bool
 	controlPlane  bool
 }
@@ -126,9 +127,12 @@ type Kubeadm struct {
 // MarshalYAML implements the yaml.Marshaler interface.
 func (kdm *Kubeadm) MarshalYAML() (interface{}, error) {
 	var aux struct {
-		Configuration string `yaml:"configuration,omitempty"`
+		Configuration string   `yaml:"configuration,omitempty"`
+		ExtraArgs     []string `yaml:"extraArgs,omitempty"`
 	}
 
+	aux.ExtraArgs = kdm.ExtraArgs
+
 	b, err := configutil.MarshalKubeadmConfigObject(kdm.Configuration)
 	if err != nil {
 		return nil, err
@@ -154,13 +158,16 @@ func (kdm *Kubeadm) MarshalYAML() (interface{}, error) {
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
 func (kdm *Kubeadm) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	var aux struct {
-		Configuration string `yaml:"configuration,omitempty"`
+		Configuration string   `yaml:"configuration,omitempty"`
+		ExtraArgs     []string `yaml:"extraArgs,omitempty"`
 	}
 
 	if err := unmarshal(&aux); err != nil {
 		return err
 	}
 
+	kdm.ExtraArgs = aux.ExtraArgs
+
 	b := []byte(aux.Configuration)
 
 	gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b)

internal/pkg/userdata/userdata_test.go

@@ -74,6 +74,12 @@ services:
       mode: ipvs
       ipvs:
         scheduler: lc
+      ---
+      apiVersion: kubelet.config.k8s.io/v1beta1
+      kind: KubeletConfiguration
+      failSwapOn: false
+    extraArgs:
+    - --ignore-preflight-errors=cri,kubeletversion,numcpu,requiredipvskernelmodulesavailable,Swap
   trustd:
     image: docker.io/autonomy/trustd:latest
     username: '5H7iU_9\u003cxizSRD'