feat: use new pkgs for initramfs and rootfs

This brings in the newly compiled libraries and binaries from our new
pkg builds.

Signed-off-by: Andrew Rynhard <andrew@andrewrynhard.com>

.drone.yml

@@ -97,7 +97,7 @@ steps:
       BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
       BINDIR: /usr/local/bin
     commands:
-      - make osctl-darwin-amd64
+      - make osctl-darwin
     depends_on:
       - lint
 
@@ -108,7 +108,7 @@ steps:
       BUILDKIT_HOST: tcp://buildkitd.ci.svc:1234
       BINDIR: /usr/local/bin
     commands:
-      - make osctl-linux-amd64
+      - make osctl-linux
     depends_on:
       - lint
 

Dockerfile

@@ -1,39 +1,46 @@
 # syntax = docker/dockerfile:1.1-experimental
 
-ARG KERNEL_IMAGE
-ARG TOOLCHAIN_IMAGE
-ARG ROOTFS_IMAGE
-ARG INITRAMFS_IMAGE
+ARG TOOLS
+FROM $TOOLS AS tools
+ENV PATH /toolchain/bin
+RUN ["/toolchain/bin/mkdir", "/bin", "/tmp"]
+RUN ["/toolchain/bin/ln", "-svf", "/toolchain/bin/bash", "/bin/sh"]
+RUN ["/toolchain/bin/ln", "-svf", "/toolchain/etc/ssl", "/etc/ssl"]
+RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /toolchain/bin v1.16.0
 
-# The proto target generates code from protobuf service definitions.
+# The build target creates a container that will be used to build Talos source
+# code.
 
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS proto-build
-WORKDIR /osd
-COPY ./internal/app/osd/proto ./proto
-RUN protoc -I/usr/local/include -I./proto --go_out=plugins=grpc:proto proto/api.proto
-WORKDIR /trustd
-COPY ./internal/app/trustd/proto ./proto
-RUN protoc -I/usr/local/include -I./proto --go_out=plugins=grpc:proto proto/api.proto
-WORKDIR /init
-COPY ./internal/app/init/proto ./proto
-RUN protoc -I/usr/local/include -I./proto --go_out=plugins=grpc:proto proto/api.proto
-
-FROM scratch AS proto
-COPY --from=proto-build /osd/proto/api.pb.go /internal/app/osd/proto/
-COPY --from=proto-build /trustd/proto/api.pb.go /internal/app/trustd/proto/
-COPY --from=proto-build /init/proto/api.pb.go /internal/app/init/proto/
-
-# The base provides a common image to build the Talos source code.
-
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS base
-ENV GOPATH /toolchain/gopath
-RUN mkdir -p ${GOPATH}
+FROM scratch AS build
+COPY --from=tools / /
+SHELL ["/toolchain/bin/bash", "-c"]
+ENV PATH /toolchain/bin:/toolchain/go/bin
 ENV GO111MODULE on
 ENV GOPROXY https://proxy.golang.org
 ENV CGO_ENABLED 0
 WORKDIR /src
+
+# The generate target generates code from protobuf service definitions.
+
+FROM build AS generate-build
+WORKDIR /osd
+COPY ./internal/app/osd/proto ./proto
+RUN protoc -I./proto --go_out=plugins=grpc:proto proto/api.proto
+WORKDIR /trustd
+COPY ./internal/app/trustd/proto ./proto
+RUN protoc -I./proto --go_out=plugins=grpc:proto proto/api.proto
+WORKDIR /init
+COPY ./internal/app/init/proto ./proto
+RUN protoc -I./proto --go_out=plugins=grpc:proto proto/api.proto
+FROM scratch AS generate
+COPY --from=generate-build /osd/proto/api.pb.go /internal/app/osd/proto/
+COPY --from=generate-build /trustd/proto/api.pb.go /internal/app/trustd/proto/
+COPY --from=generate-build /init/proto/api.pb.go /internal/app/init/proto/
+
+# The base target provides a container that can be used to build all Talos
+# assets.
+
+FROM build AS base
 COPY ./go.mod ./
 COPY ./go.sum ./
 RUN go mod download
@@ -41,11 +48,36 @@ RUN go mod verify
 COPY ./cmd ./cmd
 COPY ./pkg ./pkg
 COPY ./internal ./internal
-COPY --from=proto /internal/app ./internal/app
+COPY --from=generate /internal/app ./internal/app
 RUN go list -mod=readonly all >/dev/null
 RUN ! go mod tidy -v 2>&1 | grep .
 
-# The osd target builds the osd binary.
+# The init target builds the init binary.
+
+FROM base AS init-build
+ARG SHA
+ARG TAG
+ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
+WORKDIR /src/internal/app/init
+RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Talos -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /init
+RUN chmod +x /init
+FROM scratch AS init
+COPY --from=init-build /init /init
+
+# The ntpd target builds the ntpd image.
+
+FROM base AS ntpd-build
+ARG SHA
+ARG TAG
+ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
+WORKDIR /src/internal/app/ntpd
+RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /ntpd
+RUN chmod +x /ntpd
+FROM scratch AS ntpd
+COPY --from=ntpd-build /ntpd /ntpd
+ENTRYPOINT ["/ntpd"]
+
+# The osd target builds the osd image.
 
 FROM base AS osd-build
 ARG SHA
@@ -54,34 +86,22 @@ ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
 WORKDIR /src/internal/app/osd
 RUN --mount=type=cache,target=/root/.cache go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osd
 RUN chmod +x /osd
-
 FROM scratch AS osd
 COPY --from=osd-build /osd /osd
 ENTRYPOINT ["/osd"]
 
-# The osctl targets build the osctl binaries.
+# The proxyd target builds the proxyd image.
 
-FROM base AS osctl-linux-amd64-build
+FROM base AS proxyd-build
 ARG SHA
 ARG TAG
 ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
-WORKDIR /src/cmd/osctl
-RUN --mount=type=cache,target=/root/.cache GOOS=linux GOARCH=amd64 go build -a -ldflags "-s -w -linkmode external -extldflags \"-static\" -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-linux-amd64
-RUN chmod +x /osctl-linux-amd64
-
-FROM scratch AS osctl-linux-amd64
-COPY --from=osctl-linux-amd64-build /osctl-linux-amd64 /osctl-linux-amd64
-
-FROM base AS osctl-darwin-amd64-build
-ARG SHA
-ARG TAG
-ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
-WORKDIR /src/cmd/osctl
-RUN --mount=type=cache,target=/root/.cache GOOS=darwin GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-darwin-amd64
-RUN chmod +x /osctl-darwin-amd64
-
-FROM scratch AS osctl-darwin-amd64
-COPY --from=osctl-darwin-amd64-build /osctl-darwin-amd64 /osctl-darwin-amd64
+WORKDIR /src/internal/app/proxyd
+RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /proxyd
+RUN chmod +x /proxyd
+FROM scratch AS proxyd
+COPY --from=proxyd-build /proxyd /proxyd
+ENTRYPOINT ["/proxyd"]
 
 # The trustd target builds the trustd image.
 
@@ -92,134 +112,106 @@ ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
 WORKDIR /src/internal/app/trustd
 RUN --mount=type=cache,target=/root/.cache go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /trustd
 RUN chmod +x /trustd
-
 FROM scratch AS trustd
 COPY --from=trustd-build /trustd /trustd
 ENTRYPOINT ["/trustd"]
 
-# The proxyd target builds the proxyd image.
+# The osctl targets build the osctl binaries.
 
-FROM base AS proxyd-build
+FROM base AS osctl-linux-build
 ARG SHA
 ARG TAG
 ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
-WORKDIR /src/internal/app/proxyd
-RUN --mount=type=cache,target=/root/.cache go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /proxyd
-RUN chmod +x /proxyd
+WORKDIR /src/cmd/osctl
+RUN GOOS=linux GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-linux-amd64
+RUN chmod +x /osctl-linux-amd64
+FROM scratch AS osctl-linux
+COPY --from=osctl-linux-build /osctl-linux-amd64 /osctl-linux-amd64
 
-FROM scratch AS proxyd
-COPY --from=proxyd-build /proxyd /proxyd
-ENTRYPOINT ["/proxyd"]
-
-# The ntpd target builds the ntpd image.
-
-FROM base AS ntpd-build
+FROM base AS osctl-darwin-build
 ARG SHA
 ARG TAG
 ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
-WORKDIR /src/internal/app/ntpd
-RUN --mount=type=cache,target=/root/.cache go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Server -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /ntpd
-RUN chmod +x /ntpd
-
-FROM scratch AS ntpd
-COPY --from=ntpd-build /ntpd /ntpd
-ENTRYPOINT ["/ntpd"]
-
-# The binaries target allows for parallel compilation of all binaries.
-
-FROM scratch AS binaries-build
-COPY --from=init / /
-COPY --from=osd / /
-COPY --from=trustd / /
-COPY --from=proxyd / /
-COPY --from=ntpd / /
-COPY --from=osctl-linux-amd64 / /
-COPY --from=osctl-darwin-amd64 / /
-
-FROM scratch AS binaries
-COPY --from=binaries-build /osctl-linux-amd64 /osctl-linux-amd64
-COPY --from=binaries-build /osctl-darwin-amd64 /osctl-darwin-amd64
+WORKDIR /src/cmd/osctl
+RUN GOOS=darwin GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-darwin-amd64
+RUN chmod +x /osctl-darwin-amd64
+FROM scratch AS osctl-darwin
+COPY --from=osctl-darwin-build /osctl-darwin-amd64 /osctl-darwin-amd64
 
 # The kernel target is the linux kernel.
 
-ARG KERNEL_IMAGE
-FROM ${KERNEL_IMAGE} as kernel
+FROM scratch AS kernel
+COPY --from=docker.io/autonomy/kernel:2ac99a0 /boot/vmlinuz /vmlinuz
 
-# The initramfs target creates the compressed initramfs.
+# The initramfs target provides the Talos initramfs image.
 
-FROM base AS init-build
-ARG SHA
-ARG TAG
-ARG VERSION_PKG="github.com/talos-systems/talos/internal/pkg/version"
-WORKDIR /src/internal/app/init
-RUN --mount=type=cache,target=/root/.cache go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Talos -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /init
-RUN chmod +x /init
+FROM tools AS initramfs-build
+COPY --from=docker.io/autonomy/fhs:8467184 / /rootfs
+COPY --from=docker.io/autonomy/ca-certificates:20f39f7 / /rootfs
+COPY --from=docker.io/autonomy/dosfstools:767dee6 / /rootfs
+COPY --from=docker.io/autonomy/musl:9bc7430 / /rootfs
+COPY --from=docker.io/autonomy/syslinux:85e1f9c / /rootfs
+COPY --from=docker.io/autonomy/xfsprogs:5e50579 / /rootfs
+COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
+RUN cleanup.sh /rootfs
 
-FROM scratch AS init
-COPY --from=init-build /init /init
+FROM scratch AS initramfs-base
+COPY --from=initramfs-build /rootfs /
+COPY --from=init /init /init
 
-ARG INITRAMFS_IMAGE
-FROM ${INITRAMFS_IMAGE} AS initramfs-build
-WORKDIR /
-COPY --from=init-build /init /init
-
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS initramfs-archive
-COPY --from=initramfs-build / /initramfs
+FROM build AS initramfs-archive
+COPY --from=initramfs-base / /initramfs
 WORKDIR /initramfs
 RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz
 
 FROM scratch AS initramfs
 COPY --from=initramfs-archive /initramfs.xz /initramfs.xz
 
-# The rootfs target creates the root filesystem archive.
+# The rootfs target provides the Talos rootfs image.
 
-ARG ROOTFS_IMAGE
-FROM ${ROOTFS_IMAGE} AS rootfs-build
-COPY --from=kernel /modules /
-COPY images /usr/images
+FROM tools AS rootfs-build
+COPY --from=docker.io/autonomy/fhs:8467184 / /rootfs
+COPY --from=docker.io/autonomy/ca-certificates:20f39f7 / /rootfs
+COPY --from=docker.io/autonomy/containerd:03821f9 / /rootfs
+COPY --from=docker.io/autonomy/cni:063e06f / /rootfs
+COPY --from=docker.io/autonomy/dosfstools:767dee6 / /rootfs
+COPY --from=docker.io/autonomy/eudev:05186a8 / /rootfs
+COPY --from=docker.io/autonomy/iptables:a7aa58f / /rootfs
+COPY --from=docker.io/autonomy/libressl:3fca2cf / /rootfs
+COPY --from=docker.io/autonomy/libseccomp:80ea634 / /rootfs
+COPY --from=docker.io/autonomy/musl:9bc7430 / /rootfs
+COPY --from=docker.io/autonomy/runc:c79f79d / /rootfs
+COPY --from=docker.io/autonomy/socat:032c783 / /rootfs
+COPY --from=docker.io/autonomy/syslinux:85e1f9c / /rootfs
+COPY --from=docker.io/autonomy/xfsprogs:5e50579 / /rootfs
+COPY --from=docker.io/autonomy/images:150048d / /rootfs
+COPY --from=docker.io/autonomy/kubeadm:8607389 / /rootfs
+COPY --from=docker.io/autonomy/crictl:ddbeea1 / /rootfs
+COPY --from=docker.io/autonomy/base:f9a4941 /toolchain/lib/libblkid.* /rootfs/lib
+COPY --from=docker.io/autonomy/base:f9a4941 /toolchain/lib/libuuid.* /rootfs/lib
+COPY --from=docker.io/autonomy/base:f9a4941 /toolchain/lib/libkmod.* /rootfs/lib
+COPY --from=docker.io/autonomy/kernel:2ac99a0 /lib/modules /rootfs/lib/modules
+COPY images/*.tar /rootfs/usr/images
+COPY ./hack/cleanup.sh /toolchain/bin/cleanup.sh
+RUN cleanup.sh /rootfs
 
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS rootfs-archive
-COPY --from=rootfs-build / /rootfs
+FROM scratch AS rootfs-base
+COPY --from=rootfs-build /rootfs /
+
+FROM build AS rootfs-archive
+COPY --from=rootfs-base / /rootfs
 WORKDIR /rootfs
-RUN tar -cvpzf /rootfs.tar.gz .
+RUN tar -cpzf /rootfs.tar.gz .
 
 FROM scratch AS rootfs
 COPY --from=rootfs-archive /rootfs.tar.gz /rootfs.tar.gz
 
-# The test target performs tests on the source code.
-
-FROM base AS test
-COPY --from=rootfs-build / /rootfs
-ENV PATH /rootfs/bin:$PATH
-COPY hack/golang/test.sh /bin
-
-# The lint target performs linting on the codebase.
-
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS golangci-lint
-RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /toolchain/bin v1.16.0
-
-FROM base AS lint
-COPY hack/golang/golangci-lint.yaml .
-COPY --from=golangci-lint /toolchain/bin/golangci-lint /toolchain/bin/
-RUN golangci-lint run --config golangci-lint.yaml
-
 # The talos target generates a docker image that can be used to run Talos
 # in containers.
 
-ARG TOOLCHAIN_IMAGE
-FROM ${TOOLCHAIN_IMAGE} AS talos-build
-COPY --from=rootfs-build / /rootfs
-# A workaround docker overwriting our /etc symlink.
-RUN rm /rootfs/etc
-RUN mv /rootfs/var/etc /rootfs/etc
-RUN ln -s /etc /rootfs/var/etc
-
 FROM scratch AS talos
-COPY --from=talos-build /rootfs /
-COPY --from=init-build /init /init
+COPY --from=rootfs-base / /
+COPY --from=init /init /init
 ENTRYPOINT ["/init"]
 
 # The installer target generates an image that can be used to install Talos to
@@ -239,10 +231,23 @@ COPY --from=hashicorp/packer:1.4.2 /bin/packer /bin/packer
 COPY hack/installer/packer.json /packer.json
 COPY hack/installer/entrypoint.sh /bin/entrypoint.sh
 COPY --from=kernel /vmlinuz /usr/install/vmlinuz
-COPY --from=initramfs-build /usr/lib/syslinux/ /usr/lib/syslinux
+COPY --from=initramfs-base /usr/lib/syslinux/ /usr/lib/syslinux
 COPY --from=initramfs /initramfs.xz /usr/install/initramfs.xz
 COPY --from=rootfs /rootfs.tar.gz /usr/install/rootfs.tar.gz
-COPY --from=osctl-linux-amd64-build /osctl-linux-amd64 /bin/osctl
+COPY --from=osctl-linux-build /osctl-linux-amd64 /bin/osctl
 ARG TAG
 ENV VERSION ${TAG}
 ENTRYPOINT ["entrypoint.sh"]
+
+# The test target performs tests on the source code.
+
+FROM base AS test
+RUN unlink /etc/ssl
+COPY --from=rootfs-base / /
+COPY hack/golang/test.sh /bin
+
+# The lint target performs linting on the source code.
+
+FROM base AS lint
+COPY hack/golang/golangci-lint.yaml .
+RUN golangci-lint run --config golangci-lint.yaml

Makefile

@@ -1,7 +1,4 @@
-KERNEL_IMAGE ?= autonomy/kernel:87a888a
-TOOLCHAIN_IMAGE ?= autonomy/toolchain:6cf146a
-ROOTFS_IMAGE ?= autonomy/rootfs-base:6cf146a
-INITRAMFS_IMAGE ?= autonomy/initramfs-base:6cf146a
+TOOLS ?= autonomy/tools:b4e3778
 
 # TODO(andrewrynhard): Move this logic to a shell script.
 BUILDKIT_VERSION ?= v0.5.0
@@ -46,10 +43,7 @@ COMMON_ARGS = --progress=plain
 COMMON_ARGS += --frontend=dockerfile.v0
 COMMON_ARGS += --local context=.
 COMMON_ARGS += --local dockerfile=.
-COMMON_ARGS += --opt build-arg:KERNEL_IMAGE=$(KERNEL_IMAGE)
-COMMON_ARGS += --opt build-arg:TOOLCHAIN_IMAGE=$(TOOLCHAIN_IMAGE)
-COMMON_ARGS += --opt build-arg:ROOTFS_IMAGE=$(ROOTFS_IMAGE)
-COMMON_ARGS += --opt build-arg:INITRAMFS_IMAGE=$(INITRAMFS_IMAGE)
+COMMON_ARGS += --opt build-arg:TOOLS=$(TOOLS)
 COMMON_ARGS += --opt build-arg:SHA=$(SHA)
 COMMON_ARGS += --opt build-arg:TAG=$(TAG)
 
@@ -161,8 +155,8 @@ installer: buildkitd
 		$(COMMON_ARGS)
 	@docker load < build/$@.tar
 
-.PHONY: proto
-proto: buildkitd
+.PHONY: generate
+generate: buildkitd
 	$(BINDIR)/buildctl --addr $(BUILDKIT_HOST) \
 		build \
     --output type=local,dest=./ \
@@ -240,15 +234,15 @@ lint: buildkitd
 		$(COMMON_ARGS)
 
 .PHONY: osctl-linux-amd64
-osctl-linux-amd64: buildkitd
+osctl-linux: buildkitd
 	@$(BINDIR)/buildctl --addr $(BUILDKIT_HOST) \
 		build \
     --output type=local,dest=build \
 		--opt target=$@ \
 		$(COMMON_ARGS)
 
-.PHONY: osctl-darwin-amd64
-osctl-darwin-amd64: buildkitd
+.PHONY: osctl-darwin
+osctl-darwin: buildkitd
 	@$(BINDIR)/buildctl --addr $(BUILDKIT_HOST) \
 		build \
     --output type=local,dest=build \

hack/cleanup.sh

@@ -0,0 +1,34 @@
+#!/toolchain/bin/bash
+
+export PATH=/toolchain/bin
+
+PREFIX="${1}"
+
+function remove_symlinks() {
+    set +e
+    for l in $(find ${PREFIX} -type l); do
+        readlink $l | grep -q /toolchain
+        if [ $? == 0 ]; then
+            unlink $l
+        fi
+    done
+    set -e
+}
+
+# Remove any symlinks that might have been need at build time.
+remove_symlinks
+
+# Remove any archives as we do not need them since everything is dynamically linked.
+find ${PREFIX} -type f -name \*.a -print0 | xargs -0 rm -rf || true
+find ${PREFIX} -type f -name \*.la -print0 | xargs -0 rm -rf || true
+# Strip debug symbols from all libraries and binaries.
+find ${PREFIX}/{lib,usr/lib} -type f \( -name \*.so* -a ! -name \*dbg \) -exec strip --strip-unneeded {} ';' || true
+find ${PREFIX}/{bin,sbin,usr/bin,usr/sbin} -type f -exec strip --strip-all {} ';' || true
+
+# Remove header files, man files, and any other non-rutime dependencies.
+rm -rf ${PREFIX}/{lib,usr/lib}/pkgconfig/ \
+       ${PREFIX}/{include,usr/include}/* \
+       ${PREFIX}/{share,usr/share}/* \
+       ${PREFIX}/lib/gconv/ \
+       ${PREFIX}/usr/libexec/getconf \
+       ${PREFIX}/var/db

hack/golang/test.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/sh
 
 set -e
 

internal/app/init/internal/rootfs/rootfs.go

@@ -66,6 +66,12 @@ func Prepare(s string, inContainer bool, data *userdata.UserData) (err error) {
 		}
 	}
 
+	// Create required directories that are not part of FHS.
+	for _, p := range []string{"/etc/kubernetes/manifests", "/etc/cni", "/var/lib/kubelet", "/var/log/pods", "/usr/libexec/kubernetes"} {
+		if err = os.MkdirAll(filepath.Join(s, p), 0700); err != nil {
+			return err
+		}
+	}
 	// Create /etc/os-release.
 	if err = etc.OSRelease(s); err != nil {
 		return

internal/app/init/main.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -188,6 +189,28 @@ func initram() (err error) {
 	return nil
 }
 
+func createOverlay(path string) error {
+	log.Printf("mounting overlay for %s\n", path)
+
+	parts := strings.Split(path, "/")
+	prefix := strings.Join(parts[1:], "-")
+	diff := fmt.Sprintf("/var/%s-diff", prefix)
+	workdir := fmt.Sprintf("/var/%s-workdir", prefix)
+
+	for _, s := range []string{diff, workdir} {
+		if err := os.MkdirAll(s, 0700); err != nil {
+			return err
+		}
+	}
+
+	opts := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", path, diff, workdir)
+	if err := unix.Mount("overlay", path, "overlay", 0, opts); err != nil {
+		return errors.Errorf("error creating overlay mount to %s: %v", path, err)
+	}
+
+	return nil
+}
+
 // nolint: gocyclo
 func root() (err error) {
 	if !*inContainer {
@@ -195,6 +218,12 @@ func root() (err error) {
 		if _, err = kmsg("[talos]"); err != nil {
 			return fmt.Errorf("failed to setup logging to /dev/kmsg: %v", err)
 		}
+
+		for _, overlay := range []string{"/etc/kubernetes", "/etc/cni", "/usr/libexec/kubernetes", "/usr/etc/udev", "/opt"} {
+			if err = createOverlay(overlay); err != nil {
+				return err
+			}
+		}
 	}
 
 	// Read the user data.

internal/app/init/pkg/system/runner/containerd/containerd_test.go

@@ -68,7 +68,7 @@ func (suite *ContainerdSuite) SetupSuite() {
 	args := &runner.Args{
 		ID: "containerd",
 		ProcessArgs: []string{
-			"/rootfs/bin/containerd",
+			"/bin/containerd",
 			"--address", suite.containerdAddress,
 			"--state", stateDir,
 			"--root", rootDir,
@@ -79,7 +79,7 @@ func (suite *ContainerdSuite) SetupSuite() {
 		&userdata.UserData{},
 		args,
 		runner.WithLogPath(suite.tmpDir),
-		runner.WithEnv([]string{"PATH=/rootfs/bin:" + constants.PATH}),
+		runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}),
 	)
 	suite.Require().NoError(suite.containerdRunner.Open(context.Background()))
 	suite.containerdWg.Add(1)
@@ -341,13 +341,13 @@ func (suite *ContainerdSuite) TestStopSigKill() {
 func (suite *ContainerdSuite) TestImportSuccess() {
 	reqs := []*containerdrunner.ImportRequest{
 		{
-			Path: "/rootfs/usr/images/osd.tar",
+			Path: "/usr/images/osd.tar",
 			Options: []containerd.ImportOpt{
 				containerd.WithIndexName("testtalos/osd"),
 			},
 		},
 		{
-			Path: "/rootfs/usr/images/proxyd.tar",
+			Path: "/usr/images/proxyd.tar",
 			Options: []containerd.ImportOpt{
 				containerd.WithIndexName("testtalos/proxyd"),
 			},
@@ -367,13 +367,13 @@ func (suite *ContainerdSuite) TestImportSuccess() {
 func (suite *ContainerdSuite) TestImportFail() {
 	reqs := []*containerdrunner.ImportRequest{
 		{
-			Path: "/rootfs/usr/images/osd.tar",
+			Path: "/usr/images/osd.tar",
 			Options: []containerd.ImportOpt{
 				containerd.WithIndexName("testtalos/osd2"),
 			},
 		},
 		{
-			Path: "/rootfs/usr/images/nothere.tar",
+			Path: "/usr/images/nothere.tar",
 			Options: []containerd.ImportOpt{
 				containerd.WithIndexName("testtalos/nothere"),
 			},
@@ -387,7 +387,7 @@ func TestContainerdSuite(t *testing.T) {
 	if os.Getuid() != 0 {
 		t.Skip("can't run the test as non-root")
 	}
-	_, err := os.Stat("/rootfs/bin/containerd")
+	_, err := os.Stat("/bin/containerd")
 	if err != nil {
 		t.Skip("containerd binary is not available, skipping the test")
 	}

internal/app/init/pkg/system/runner/process/process_test.go

@@ -47,7 +47,7 @@ func (suite *ProcessSuite) TearDownSuite() {
 func (suite *ProcessSuite) TestRunSuccess() {
 	r := process.NewRunner(&userdata.UserData{}, &runner.Args{
 		ID:          "test",
-		ProcessArgs: []string{"/bin/bash", "-c", "exit 0"},
+		ProcessArgs: []string{"/bin/sh", "-c", "exit 0"},
 	}, runner.WithLogPath(suite.tmpDir))
 
 	suite.Assert().NoError(r.Open(context.Background()))
@@ -61,7 +61,7 @@ func (suite *ProcessSuite) TestRunSuccess() {
 func (suite *ProcessSuite) TestRunLogs() {
 	r := process.NewRunner(&userdata.UserData{}, &runner.Args{
 		ID:          "logtest",
-		ProcessArgs: []string{"/bin/bash", "-c", "echo -n \"Test 1\nTest 2\n\""},
+		ProcessArgs: []string{"/bin/sh", "-c", "echo -n \"Test 1\nTest 2\n\""},
 	}, runner.WithLogPath(suite.tmpDir))
 
 	suite.Assert().NoError(r.Open(context.Background()))
@@ -88,7 +88,7 @@ func (suite *ProcessSuite) TestRunRestartFailed() {
 
 	r := restart.New(process.NewRunner(&userdata.UserData{}, &runner.Args{
 		ID:          "restarter",
-		ProcessArgs: []string{"/bin/bash", "-c", "echo \"ran\"; test -f " + testFile},
+		ProcessArgs: []string{"/bin/sh", "-c", "echo \"ran\"; test -f " + testFile},
 	}, runner.WithLogPath(suite.tmpDir)), restart.WithType(restart.UntilSuccess), restart.WithRestartInterval(time.Millisecond))
 
 	suite.Assert().NoError(r.Open(context.Background()))
@@ -128,7 +128,7 @@ func (suite *ProcessSuite) TestStopFailingAndRestarting() {
 
 	r := restart.New(process.NewRunner(&userdata.UserData{}, &runner.Args{
 		ID:          "endless",
-		ProcessArgs: []string{"/bin/bash", "-c", "test -f " + testFile},
+		ProcessArgs: []string{"/bin/sh", "-c", "test -f " + testFile},
 	}, runner.WithLogPath(suite.tmpDir)), restart.WithType(restart.Forever), restart.WithRestartInterval(5*time.Millisecond))
 
 	suite.Assert().NoError(r.Open(context.Background()))
@@ -169,7 +169,7 @@ func (suite *ProcessSuite) TestStopFailingAndRestarting() {
 func (suite *ProcessSuite) TestStopSigKill() {
 	r := process.NewRunner(&userdata.UserData{}, &runner.Args{
 		ID:          "nokill",
-		ProcessArgs: []string{"/bin/bash", "-c", "trap -- '' SIGTERM; while :; do :; done"},
+		ProcessArgs: []string{"/bin/sh", "-c", "trap -- '' SIGTERM; while :; do :; done"},
 	},
 		runner.WithLogPath(suite.tmpDir),
 		runner.WithGracefulShutdownTimeout(10*time.Millisecond),

internal/app/init/pkg/system/services/kubeadm.go

@@ -39,25 +39,6 @@ func (k *Kubeadm) ID(data *userdata.UserData) string {
 // PreFunc implements the Service interface.
 // nolint: gocyclo
 func (k *Kubeadm) PreFunc(ctx context.Context, data *userdata.UserData) (err error) {
-	requiredMounts := []string{
-		"/dev/disk/by-path",
-		"/etc/kubernetes",
-		"/etc/kubernetes/manifests",
-		"/lib/modules",
-		"/run",
-		"/sys/fs/cgroup",
-		"/usr/libexec/kubernetes",
-		"/var/lib/containerd",
-		"/var/lib/kubelet",
-		"/var/log/pods",
-	}
-
-	for _, dir := range requiredMounts {
-		if err = os.MkdirAll(dir, os.ModeDir); err != nil {
-			return errors.Wrapf(err, "create %s", dir)
-		}
-	}
-
 	reqs := []*containerd.ImportRequest{
 		{
 			Path: "/usr/images/hyperkube.tar",

internal/app/init/pkg/system/services/osd.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"os"
 
 	containerdapi "github.com/containerd/containerd"
 	"github.com/containerd/containerd/oci"
@@ -35,10 +34,6 @@ func (o *OSD) ID(data *userdata.UserData) string {
 
 // PreFunc implements the Service interface.
 func (o *OSD) PreFunc(ctx context.Context, data *userdata.UserData) error {
-	if err := os.MkdirAll("/etc/kubernetes", os.ModeDir); err != nil {
-		return err
-	}
-
 	return containerd.Import(constants.SystemContainerdNamespace, &containerd.ImportRequest{
 		Path: "/usr/images/osd.tar",
 		Options: []containerdapi.ImportOpt{

internal/app/init/pkg/system/services/proxyd.go

@@ -9,7 +9,6 @@ import (
 	"context"
 	"fmt"
 	"net"
-	"os"
 
 	containerdapi "github.com/containerd/containerd"
 	"github.com/containerd/containerd/oci"
@@ -35,10 +34,6 @@ func (p *Proxyd) ID(data *userdata.UserData) string {
 
 // PreFunc implements the Service interface.
 func (p *Proxyd) PreFunc(ctx context.Context, data *userdata.UserData) error {
-	if err := os.MkdirAll("/etc/kubernetes", os.ModeDir); err != nil {
-		return err
-	}
-
 	return containerd.Import(constants.SystemContainerdNamespace, &containerd.ImportRequest{
 		Path: "/usr/images/proxyd.tar",
 		Options: []containerdapi.ImportOpt{

internal/app/init/pkg/system/services/udevd.go

@@ -28,7 +28,7 @@ func (c *Udevd) ID(data *userdata.UserData) string {
 // PreFunc implements the Service interface.
 func (c *Udevd) PreFunc(ctx context.Context, data *userdata.UserData) error {
 	cmd := exec.Command(
-		"/bin/udevadm",
+		"/sbin/udevadm",
 		"hwdb",
 		"--update",
 	)
@@ -56,7 +56,7 @@ func (c *Udevd) Runner(data *userdata.UserData) (runner.Runner, error) {
 	args := &runner.Args{
 		ID: c.ID(data),
 		ProcessArgs: []string{
-			"/bin/udevd",
+			"/sbin/udevd",
 			"--resolve-names=never",
 			"-D",
 		},

internal/app/init/pkg/system/services/udevd_trigger.go

@@ -51,7 +51,7 @@ func (c *UdevdTrigger) Runner(data *userdata.UserData) (runner.Runner, error) {
 	args := &runner.Args{
 		ID: c.ID(data),
 		ProcessArgs: []string{
-			"/bin/udevadm",
+			"/sbin/udevadm",
 			"trigger",
 		},
 	}

internal/pkg/containers/containerd/containerd_test.go

@@ -85,7 +85,7 @@ func (suite *ContainerdSuite) SetupSuite() {
 	args := &runner.Args{
 		ID: "containerd",
 		ProcessArgs: []string{
-			"/rootfs/bin/containerd",
+			"/bin/containerd",
 			"--address", suite.containerdAddress,
 			"--state", stateDir,
 			"--root", rootDir,
@@ -96,7 +96,7 @@ func (suite *ContainerdSuite) SetupSuite() {
 		&userdata.UserData{},
 		args,
 		runner.WithLogPath(suite.tmpDir),
-		runner.WithEnv([]string{"PATH=/rootfs/bin:" + constants.PATH}),
+		runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}),
 	)
 	suite.Require().NoError(suite.containerdRunner.Open(context.Background()))
 	suite.containerdWg.Add(1)
@@ -343,7 +343,7 @@ func TestContainerdSuite(t *testing.T) {
 	if os.Getuid() != 0 {
 		t.Skip("can't run the test as non-root")
 	}
-	_, err := os.Stat("/rootfs/bin/containerd")
+	_, err := os.Stat("/bin/containerd")
 	if err != nil {
 		t.Skip("containerd binary is not available, skipping the test")
 	}

internal/pkg/containers/cri/cri_test.go

@@ -72,7 +72,7 @@ func (suite *CRISuite) SetupSuite() {
 	args := &runner.Args{
 		ID: "containerd",
 		ProcessArgs: []string{
-			"/rootfs/bin/containerd",
+			"/bin/containerd",
 			"--address", suite.containerdAddress,
 			"--state", stateDir,
 			"--root", rootDir,
@@ -83,7 +83,7 @@ func (suite *CRISuite) SetupSuite() {
 		&userdata.UserData{},
 		args,
 		runner.WithLogPath(suite.tmpDir),
-		runner.WithEnv([]string{"PATH=/rootfs/bin:" + constants.PATH}),
+		runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}),
 	)
 	suite.Require().NoError(suite.containerdRunner.Open(context.Background()))
 	suite.containerdWg.Add(1)
@@ -257,7 +257,7 @@ func TestCRISuite(t *testing.T) {
 	if os.Getuid() != 0 {
 		t.Skip("can't run the test as non-root")
 	}
-	_, err := os.Stat("/rootfs/bin/containerd")
+	_, err := os.Stat("/bin/containerd")
 	if err != nil {
 		t.Skip("containerd binary is not available, skipping the test")
 	}

internal/pkg/cri/cri_test.go

@@ -62,7 +62,7 @@ func (suite *CRISuite) SetupSuite() {
 	args := &runner.Args{
 		ID: "containerd",
 		ProcessArgs: []string{
-			"/rootfs/bin/containerd",
+			"/bin/containerd",
 			"--address", suite.containerdAddress,
 			"--state", stateDir,
 			"--root", rootDir,
@@ -73,7 +73,7 @@ func (suite *CRISuite) SetupSuite() {
 		&userdata.UserData{},
 		args,
 		runner.WithLogPath(suite.tmpDir),
-		runner.WithEnv([]string{"PATH=/rootfs/bin:" + constants.PATH}),
+		runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}),
 	)
 	suite.Require().NoError(suite.containerdRunner.Open(context.Background()))
 	suite.containerdWg.Add(1)
@@ -200,7 +200,7 @@ func TestCRISuite(t *testing.T) {
 	if os.Getuid() != 0 {
 		t.Skip("can't run the test as non-root")
 	}
-	_, err := os.Stat("/rootfs/bin/containerd")
+	_, err := os.Stat("/bin/containerd")
 	if err != nil {
 		t.Skip("containerd binary is not available, skipping the test")
 	}