diff --git a/Dockerfile b/Dockerfile index ba0e4b31c..f299bdf89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -186,6 +186,19 @@ RUN ["/toolchain/bin/tar", "-cvpzf", "/rootfs.tar.gz", "."] FROM scratch AS rootfs COPY --from=rootfs-build /rootfs.tar.gz /rootfs.tar.gz +# The docker-os target generates a docker image that can be used to run Talos +# in containers. + +FROM rootfs-build AS docker-rootfs-build +# workaround docker overwriting our /etc symlink +RUN rm /rootfs/etc +RUN mv /rootfs/var/etc /rootfs/etc +RUN ln -s /etc /rootfs/var/etc +FROM scratch AS docker-os +COPY --from=docker-rootfs-build /rootfs / +COPY --from=initramfs-build /init /sbin/init +ENTRYPOINT ["/sbin/init"] + # The installer target generates an image that can be used to install Talos to # various environments. diff --git a/Makefile b/Makefile index f6f29705f..4f9e05e4b 100644 --- a/Makefile +++ b/Makefile @@ -107,6 +107,17 @@ rootfs: buildkitd hyperkube etcd coredns pause osd trustd proxyd blockd --frontend-opt target=$@ \ $(COMMON_ARGS) +docker-os: buildkitd + @buildctl --addr $(BUILDKIT_HOST) \ + build \ + --exporter=docker \ + --exporter-opt output=build/$@.tar \ + --exporter-opt name=docker.io/autonomy/talos-os:$(TAG) \ + --exporter-opt push=$(PUSH) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + @docker load < build/$@.tar + installer: buildkitd @mkdir -p build @buildctl --addr $(BUILDKIT_HOST) \ diff --git a/dev/.gitignore b/dev/.gitignore new file mode 100644 index 000000000..e93a1555f --- /dev/null +++ b/dev/.gitignore @@ -0,0 +1,5 @@ +pki/ +config/* +!config/.* +*.kmsg +*.logs/ diff --git a/dev/Makefile b/dev/Makefile new file mode 100644 index 000000000..6570b8ef0 --- /dev/null +++ b/dev/Makefile @@ -0,0 +1,125 @@ +SHELL := /bin/bash + +ORG = talos.local +K_ORG = talos.k8s.local +USER = dev +CA_LENGTH = $$(( 24 * 365 * 1 )) +K_CA_LENGTH = $$(( 24 * 365 * 1 )) +USER_LENGTH = $$(( 24 * 30 * 5 )) + +NODE = talos-os-1 + +GIT_SHORT_SHA = $(shell git rev-parse --short=7 HEAD) +GIT_DIRTY = $(shell [[ -n `git status -s` ]] && echo -dirty) +TAG = $(GIT_SHORT_SHA)$(GIT_DIRTY) +# TAG = $(shell gitmeta image tag) # TODO: this tool is slow + +OS_IMAGE_NAME ?= autonomy/talos-os + +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +OSCTL ?= ../build/osctl-linux-amd64 +endif +ifeq ($(UNAME_S),Darwin) +OSCTL ?= ../build/osctl-darwin-amd64 +endif + + +DOCKER_COMPOSE ?= IMAGE=$(OS_IMAGE_NAME) TAG=$(TAG) NODE=$(NODE) docker-compose +.PHONY: os-up os-down os-clean +os-up : + touch os1.kmsg + mkdir -p os1.logs/pods/ && chmod -R 777 os1.logs # easier to manage pod logs + $(DOCKER_COMPOSE) up -d + tail -f os1.kmsg +os-down : + $(DOCKER_COMPOSE) down +os-clean : + $(DOCKER_COMPOSE) down -v + rm -rf os1.kmsg os1.logs || true + + +.PHONY: config config-clean +config : config/$(NODE).yaml $(HOME)/.talos/config +config-clean: + rm config/* || true # does not delete dotfiles + + +pki/ : + @mkdir -p pki/ + +# OS CA PKI +pki/$(ORG).key : pki/ + # gen OS CA + @$(OSCTL) gen ca \ + --hours "$(CA_LENGTH)" \ + --organization "pki/$(ORG)" +pki/$(ORG).crt : pki/$(ORG).key +pki/$(ORG).sha256 : pki/$(ORG).key + +# user PKI +pki/$(USER).key : pki/ + # gen user key + @$(OSCTL) gen key \ + --name "pki/$(USER)" +pki/$(USER).csr : pki/$(USER).key + # gen user CSR + @$(OSCTL) gen csr \ + --ip "127.0.0.1" \ + --key "pki/$(USER).key" +pki/$(USER).crt : pki/$(USER).csr pki/$(ORG).crt + # gen user crt + @$(OSCTL) gen crt \ + --hours "$(USER_LENGTH)" \ + --ca "pki/$(ORG)" \ + --csr "pki/$(USER).csr" \ + --name "pki/$(USER)" + +# k8s CA PKI +pki/$(K_ORG).key : pki/ + # gen k8s CA + @$(OSCTL) gen ca \ + --rsa \ + --hours "$(K_CA_LENGTH)" \ + --organization "pki/$(K_ORG)" +pki/$(K_ORG).crt : pki/$(K_ORG).key +pki/$(K_ORG).sha256 : pki/$(K_ORG).key + +# PKI cleanup +.PHONY: pki-clean-all pki-clean-os pki-clean-user pki-clean-k8s +pki-clean-all: + rm -rf pki || true +pki-clean-os: + rm -f pki/$(ORG)* || true +pki-clean-user: + rm -f pki/$(USER)* || true +pki-clean-k8s: + rm -f pki/$(K_ORG)* || true + + +config/$(NODE).yaml : pki/$(ORG).crt pki/$(ORG).key pki/$(K_ORG).crt pki/$(K_ORG).key + # inject base os config + @cp config/.base.node.yaml "config/$(NODE).yaml" + @chmod 600 "config/$(NODE).yaml" + @$(OSCTL) inject os \ + --crt "pki/$(ORG).crt" \ + --key "pki/$(ORG).key" \ + "config/$(NODE).yaml" + @$(OSCTL) inject kubernetes \ + --crt "pki/$(K_ORG).crt" \ + --key "pki/$(K_ORG).key" \ + "config/$(NODE).yaml" + +$(HOME)/.talos/config : pki/$(ORG).crt pki/$(USER).crt pki/$(USER).key + # configure dev osctl + @touch $${HOME}/.talos/config + @$(OSCTL) config add "talos-local" \ + --ca "pki/$(ORG).crt" \ + --crt "pki/$(USER).crt" \ + --key "pki/$(USER).key" + @$(OSCTL) config context "talos-local" + @$(OSCTL) config target "127.0.0.1" + + +.PHONY: clean +clean : config-clean os-clean pki-clean-all diff --git a/dev/config/.base.node.yaml b/dev/config/.base.node.yaml new file mode 100644 index 000000000..6785fdccb --- /dev/null +++ b/dev/config/.base.node.yaml @@ -0,0 +1,25 @@ +services: + init: + cni: flannel + kubeadm: + configuration: | + apiVersion: kubeadm.k8s.io/v1beta1 + kind: InitConfiguration + apiVersion: kubeadm.k8s.io/v1beta1 + nodeRegistration: + criSocket: /run/containerd/containerd.sock + --- + apiVersion: kubeadm.k8s.io/v1beta1 + kind: ClusterConfiguration + networking: + dnsDomain: cluster.local + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + failSwapOn: false # necessary for some docker hosts + extraArgs: + - --ignore-preflight-errors=cri,kubeletversion,numcpu,requiredipvskernelmodulesavailable,FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,Swap + # additions to talos default are: FileContent--proc-sys-net-bridge-bridge-nf-call-iptables,Swap + trustd: + username: 'dev' + password: 'talos_trust_dev' diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml new file mode 100644 index 000000000..0f23da2b4 --- /dev/null +++ b/dev/docker-compose.yaml @@ -0,0 +1,42 @@ +version: '3.7' + +services: + os1: + image: ${IMAGE:-autonomy/talos-os}:${TAG} + privileged: true + security_opt: + - seccomp:unconfined + entrypoint: /sbin/init + command: --switch-root + volumes: + # logging hacks + - ./os1.kmsg:/dev/kmsg + - ./os1.logs/:/var/log/ + # os config + - ./config/${NODE:-talos-os-1}.yaml:/run/userdata.yaml + # state dirs + - os1_containerd:/var/lib/containerd/ + - os1_etcd:/var/lib/etcd/ + - os1_kubelet:/var/lib/kubelet/ + - os1_kubernetes:/etc/kubernetes/ + # kubeadm mounts os-release, but init --switch-root does not create + - ./os-release:/etc/os-release + # kubelet is missing cni mount + - os1_cni:/etc/cni/ + tmpfs: + - /run + # sysctls: + # net.bridge.bridge-nf-call-iptables: 1 + # # this proc sysctl path is missing in the container + # # kubeadm errors on pre-flight -- modified init to ignore it + ports: + - "50000:50000" # osd + - "6443:6443" # kube-api-server + +volumes: + os1_containerd: + os1_etcd: + os1_kubelet: + os1_kubernetes: + os1_ssl: + os1_cni: diff --git a/dev/os-release b/dev/os-release new file mode 100644 index 000000000..d6f6b1ec7 --- /dev/null +++ b/dev/os-release @@ -0,0 +1,6 @@ +NAME="Talos" +ID=dev-container +VERSION_ID=dev-container +PRETTY_NAME="Talos (dev-container) by Autonomy" +HOME_URL="https://talos.autonomy.io/" +BUG_REPORT_URL="https://github.com/autonomy/talos/issues" diff --git a/internal/app/init/pkg/system/services/kubeadm.go b/internal/app/init/pkg/system/services/kubeadm.go index 6486189bb..8ca3c8068 100644 --- a/internal/app/init/pkg/system/services/kubeadm.go +++ b/internal/app/init/pkg/system/services/kubeadm.go @@ -131,6 +131,8 @@ func (k *Kubeadm) Start(data *userdata.UserData) error { args.ProcessArgs = []string{"kubeadm", "join", "--config=/etc/kubernetes/kubeadm-config.yaml", ignore} } + args.ProcessArgs = append(args.ProcessArgs, data.Services.Kubeadm.ExtraArgs...) + // Set the mounts. // nolint: dupl mounts := []specs.Mount{ diff --git a/internal/pkg/userdata/userdata.go b/internal/pkg/userdata/userdata.go index a4c562c24..d008d9bbf 100644 --- a/internal/pkg/userdata/userdata.go +++ b/internal/pkg/userdata/userdata.go @@ -119,6 +119,7 @@ type Kubeadm struct { CommonServiceOptions `yaml:",inline"` Configuration runtime.Object `yaml:"configuration"` + ExtraArgs []string `yaml:"extraArgs,omitempty"` bootstrap bool controlPlane bool } @@ -126,9 +127,12 @@ type Kubeadm struct { // MarshalYAML implements the yaml.Marshaler interface. func (kdm *Kubeadm) MarshalYAML() (interface{}, error) { var aux struct { - Configuration string `yaml:"configuration,omitempty"` + Configuration string `yaml:"configuration,omitempty"` + ExtraArgs []string `yaml:"extraArgs,omitempty"` } + aux.ExtraArgs = kdm.ExtraArgs + b, err := configutil.MarshalKubeadmConfigObject(kdm.Configuration) if err != nil { return nil, err @@ -154,13 +158,16 @@ func (kdm *Kubeadm) MarshalYAML() (interface{}, error) { // UnmarshalYAML implements the yaml.Unmarshaler interface. func (kdm *Kubeadm) UnmarshalYAML(unmarshal func(interface{}) error) error { var aux struct { - Configuration string `yaml:"configuration,omitempty"` + Configuration string `yaml:"configuration,omitempty"` + ExtraArgs []string `yaml:"extraArgs,omitempty"` } if err := unmarshal(&aux); err != nil { return err } + kdm.ExtraArgs = aux.ExtraArgs + b := []byte(aux.Configuration) gvks, err := kubeadmutil.GroupVersionKindsFromBytes(b) diff --git a/internal/pkg/userdata/userdata_test.go b/internal/pkg/userdata/userdata_test.go index e4b68b27e..c8e46797a 100644 --- a/internal/pkg/userdata/userdata_test.go +++ b/internal/pkg/userdata/userdata_test.go @@ -74,6 +74,12 @@ services: mode: ipvs ipvs: scheduler: lc + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + failSwapOn: false + extraArgs: + - --ignore-preflight-errors=cri,kubeletversion,numcpu,requiredipvskernelmodulesavailable,Swap trustd: image: docker.io/autonomy/trustd:latest username: '5H7iU_9\u003cxizSRD'