From ee7df39925e9902d0101fe2115c41067730d0e52 Mon Sep 17 00:00:00 2001 From: Andrew Rynhard Date: Sat, 19 Jan 2019 01:58:26 -0800 Subject: [PATCH] chore: use buildkitd for builds (#320) Signed-off-by: Andrew Rynhard --- .dockerignore | 4 +- .gitignore | 2 +- Dockerfile | 135 ++++++------ Makefile | 201 ++++++++++-------- hack/golang/test.sh | 0 hack/scripts/cleanup.sh | 0 hack/scripts/symlink.sh | 0 images/.keep | 0 internal/app/init/main.go | 102 ++++----- .../pkg/system/runner/containerd/import.go | 75 +++++++ .../app/init/pkg/system/services/blockd.go | 3 +- internal/app/init/pkg/system/services/osd.go | 3 +- .../app/init/pkg/system/services/proxyd.go | 3 +- .../app/init/pkg/system/services/trustd.go | 3 +- 14 files changed, 312 insertions(+), 219 deletions(-) mode change 100644 => 100755 hack/golang/test.sh mode change 100644 => 100755 hack/scripts/cleanup.sh mode change 100644 => 100755 hack/scripts/symlink.sh create mode 100644 images/.keep create mode 100644 internal/app/init/pkg/system/runner/containerd/import.go diff --git a/.dockerignore b/.dockerignore index e94a177bb..d57629732 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,6 @@ ** -!bin !hack -!images -!installer +!images/*.tar !internal !vendor !web diff --git a/.gitignore b/.gitignore index 259e388df..8ee577472 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ build cache -images +images/*.tar vendor .vscode diff --git a/Dockerfile b/Dockerfile index ce8f2fd6d..c6d5fb9a0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ # syntax=docker/dockerfile:experimental -ARG TOOLCHAIN_VERSION ARG KERNEL_VERSION +ARG TOOLCHAIN_VERSION ARG GOLANG_VERSION # The proto target will generate code based on service definitions. @@ -33,6 +33,20 @@ FROM autonomy/toolchain:${TOOLCHAIN_VERSION} AS base # ca-certificates RUN mkdir -p /etc/ssl/certs RUN ln -s /toolchain/etc/ssl/certs/ca-certificates /etc/ssl/certs/ca-certificates +# fhs +COPY hack/scripts/fhs.sh /bin +RUN fhs.sh /rootfs +# xfsprogs +WORKDIR /tmp/xfsprogs +RUN curl -L https://www.kernel.org/pub/linux/utils/fs/xfs/xfsprogs/xfsprogs-4.18.0.tar.xz | tar -xJ --strip-components=1 +RUN make \ + DEBUG=-DNDEBUG \ + INSTALL_USER=0 \ + INSTALL_GROUP=0 \ + LOCAL_CONFIGURE_OPTIONS="--prefix=/" +RUN make install DESTDIR=/rootfs +# libuuid +RUN cp /toolchain/lib/libuuid.* /rootfs/lib # gcompat WORKDIR /tmp/gcompat RUN curl -L https://github.com/AdelieLinux/gcompat/archive/0.3.0.tar.gz | tar -xz --strip-components=1 @@ -49,35 +63,15 @@ RUN ln -s lib /lib64 ENV GO111MODULE on ENV CGO_ENABLED 0 WORKDIR /src -COPY ./ ./ +COPY ./internal ./internal +COPY ./go.mod ./ +COPY ./go.sum ./ COPY --from=proto /osd/proto/api.pb.go ./internal/app/osd/proto COPY --from=proto /trustd/proto/api.pb.go ./internal/app/trustd/proto COPY --from=proto /blockd/proto/api.pb.go ./internal/app/blockd/proto RUN go mod download RUN go mod verify -# The common target creates a filesystem that contains requirements common to -# the initramfs and rootfs. - -ARG TOOLCHAIN_VERSION -FROM base AS common -# fhs -COPY hack/scripts/fhs.sh . -RUN ./fhs.sh /rootfs -# xfsprogs -RUN mkdir -p /etc/ssl/certs -WORKDIR /tmp/xfsprogs -RUN curl -L https://www.kernel.org/pub/linux/utils/fs/xfs/xfsprogs/xfsprogs-4.18.0.tar.xz | tar -xJ --strip-components=1 -RUN make \ - DEBUG=-DNDEBUG \ - INSTALL_USER=0 \ - INSTALL_GROUP=0 \ - LOCAL_CONFIGURE_OPTIONS="--prefix=/" -RUN make install DESTDIR=/rootfs -# libuuid -RUN cp /toolchain/lib/libuuid.* /rootfs/lib -WORKDIR /src - # The udevd target builds the udevd binary. FROM base AS udevd-build @@ -97,10 +91,30 @@ ENTRYPOINT ["/udevd"] ARG KERNEL_VERSION FROM autonomy/kernel:${KERNEL_VERSION} as kernel +# The initramfs target creates an initramfs. + +FROM base AS initramfs-build +ARG SHA +ARG TAG +ARG VERSION_PKG="github.com/autonomy/talos/internal/pkg/version" +WORKDIR /src/internal/app/init +RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Talos -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /init +RUN chmod +x /init +WORKDIR /initramfs +RUN cp /init ./ +COPY --from=base /rootfs ./ +WORKDIR /src +COPY hack/scripts/cleanup.sh /bin +RUN cleanup.sh /initramfs +WORKDIR /initramfs +RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz +FROM scratch AS initramfs +COPY --from=initramfs-build /initramfs.xz /initramfs.xz + # The rootfs target creates a root filesysyem with only what is required to run # Kubernetes. -FROM common AS rootfs +FROM base AS rootfs-build # libseccomp WORKDIR /toolchain/usr/local/src/libseccomp RUN curl -L https://github.com/seccomp/libseccomp/releases/download/v2.3.3/libseccomp-2.3.3.tar.gz | tar --strip-components=1 -xz @@ -136,37 +150,20 @@ RUN curl -L https://github.com/containernetworking/plugins/releases/download/v0. # kubeadm RUN curl --retry 3 --retry-delay 60 -L https://storage.googleapis.com/kubernetes-release/release/v1.13.2/bin/linux/amd64/kubeadm -o /rootfs/bin/kubeadm RUN chmod +x /rootfs/bin/kubeadm -# images -COPY images /rootfs/usr/images # udevd COPY --from=udevd-build /udevd /rootfs/bin/udevd +# images +COPY images /rootfs/usr/images # cleanup WORKDIR /src -RUN chmod +x ./hack/scripts/cleanup.sh -RUN ./hack/scripts/cleanup.sh /rootfs -COPY ./hack/scripts/symlink.sh /bin -RUN chmod +x ./hack/scripts/symlink.sh -RUN ./hack/scripts/symlink.sh /rootfs +COPY hack/scripts/cleanup.sh /bin +RUN cleanup.sh /rootfs +COPY hack/scripts/symlink.sh /bin +RUN symlink.sh /rootfs WORKDIR /rootfs RUN ["/toolchain/bin/tar", "-cvpzf", "/rootfs.tar.gz", "."] - -# The initramfs target creates an initramfs. - -FROM base AS initramfs -ARG SHA -ARG TAG -ARG VERSION_PKG="github.com/autonomy/talos/internal/pkg/version" -WORKDIR /src/internal/app/init -RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Talos -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /init -RUN chmod +x /init -WORKDIR /initramfs -RUN cp /init ./ -COPY --from=common /rootfs ./ -WORKDIR /src -RUN chmod +x ./hack/scripts/cleanup.sh -RUN ./hack/scripts/cleanup.sh /initramfs -WORKDIR /initramfs -RUN set -o pipefail && find . 2>/dev/null | cpio -H newc -o | xz -v -C crc32 -0 -e -T 0 -z >/initramfs.xz +FROM scratch AS rootfs +COPY --from=rootfs-build /rootfs.tar.gz /rootfs.tar.gz # The installer target generates an image that can be used to install Talos to # various environments. @@ -183,37 +180,39 @@ RUN curl -L https://releases.hashicorp.com/packer/1.3.1/packer_1.3.1_linux_amd64 && unzip -d /tmp /tmp/packer.zip \ && mv /tmp/packer /bin \ && rm /tmp/packer.zip -COPY ./hack/installer/packer.json /packer.json -COPY ./hack/installer/entrypoint.sh /bin/entrypoint.sh -RUN chmod +x /bin/entrypoint.sh +COPY hack/installer/packer.json /packer.json +COPY hack/installer/entrypoint.sh /bin/entrypoint.sh ARG TAG ENV VERSION ${TAG} ENTRYPOINT ["entrypoint.sh"] # The test target performs tests on the codebase. -FROM common AS test +FROM base AS test # xfsprogs is required by the tests ENV PATH /rootfs/bin:$PATH -RUN chmod +x ./hack/golang/test.sh -RUN ./hack/golang/test.sh --short -RUN ./hack/golang/test.sh +COPY hack/golang/test.sh /bin +RUN test.sh --short +RUN test.sh # The lint target performs linting on the codebase. -FROM common AS lint +FROM base AS lint RUN curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | bash -s -- -b /toolchain/bin v1.12.5 -RUN golangci-lint run --config ./hack/golang/golangci-lint.yaml +COPY hack/golang/golangci-lint.yaml . +RUN golangci-lint run --config golangci-lint.yaml # The docs target generates a static website containing documentation. -FROM base as docs +FROM base as docs-build RUN curl -L https://github.com/gohugoio/hugo/releases/download/v0.49.2/hugo_0.49.2_Linux-64bit.tar.gz | tar -xz -C /bin WORKDIR /web COPY ./web ./ RUN mkdir /docs RUN hugo --destination=/docs --verbose RUN echo "talos.autonomy.io" > /docs/CNAME +FROM scratch AS docs +COPY --from=docs-build /docs /docs # The osd target builds the osd binary. @@ -231,15 +230,25 @@ ENTRYPOINT ["/osd"] # The osctl target builds the osctl binaries. -FROM base AS osctl +FROM base AS osctl-linux-amd64-build ARG SHA ARG TAG ARG VERSION_PKG="github.com/autonomy/talos/internal/pkg/version" WORKDIR /src/internal/app/osctl -RUN go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-linux-amd64 -RUN GOOS=darwin GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-darwin-amd64 +RUN GOOS=linux GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-linux-amd64 RUN chmod +x /osctl-linux-amd64 +FROM scratch AS osctl-linux-amd64 +COPY --from=osctl-linux-amd64-build /osctl-linux-amd64 /osctl-linux-amd64 + +FROM base AS osctl-darwin-amd64-build +ARG SHA +ARG TAG +ARG VERSION_PKG="github.com/autonomy/talos/internal/pkg/version" +WORKDIR /src/internal/app/osctl +RUN GOOS=darwin GOARCH=amd64 go build -a -ldflags "-s -w -X ${VERSION_PKG}.Name=Client -X ${VERSION_PKG}.SHA=${SHA} -X ${VERSION_PKG}.Tag=${TAG}" -o /osctl-darwin-amd64 RUN chmod +x /osctl-darwin-amd64 +FROM scratch AS osctl-darwin-amd64 +COPY --from=osctl-darwin-amd64-build /osctl-darwin-amd64 /osctl-darwin-amd64 # The trustd target builds the trustd binary. diff --git a/Makefile b/Makefile index 2ef17b4ce..ebcb2c64f 100644 --- a/Makefile +++ b/Makefile @@ -1,71 +1,136 @@ SHA := $(shell gitmeta git sha) TAG := $(shell gitmeta image tag) BUILT := $(shell gitmeta built) +PUSH := $(shell gitmeta pushable) -COMMON_APP_ARGS := -f ./Dockerfile --build-arg TOOLCHAIN_VERSION=397b293 --build-arg KERNEL_VERSION=65ec2e6 --build-arg GOLANG_VERSION=1.11.4 --build-arg SHA=$(SHA) --build-arg TAG=$(TAG) . +TOOLCHAIN_VERSION ?= 397b293 +KERNEL_VERSION ?= 65ec2e6 +GOLANG_VERSION ?= 1.11.4 -export DOCKER_BUILDKIT := 1 +COMMON_ARGS := --frontend=dockerfile.v0 +COMMON_ARGS += --local context=. +COMMON_ARGS += --local dockerfile=. +COMMON_ARGS += --frontend-opt build-arg:KERNEL_VERSION=$(KERNEL_VERSION) +COMMON_ARGS += --frontend-opt build-arg:TOOLCHAIN_VERSION=$(TOOLCHAIN_VERSION) +COMMON_ARGS += --frontend-opt build-arg:GOLANG_VERSION=$(GOLANG_VERSION) +COMMON_ARGS += --frontend-opt build-arg:SHA=$(SHA) +COMMON_ARGS += --frontend-opt build-arg:TAG=$(TAG) -all: enforce rootfs initramfs osctl test lint docs installer +all: enforce rootfs initramfs kernel osctl-linux-amd64 osctl-darwin-amd64 test lint docs installer enforce: @docker run --rm -it -v $(PWD):/src -w /src autonomy/conform:latest common: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) + @buildctl build \ + --exporter=docker \ + --exporter-opt output=build/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + @docker load < build/$@.tar -osd: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) +kernel: + @buildctl build \ + --exporter=local \ + --exporter-opt output=build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) -osctl: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - @docker run --rm -it -v $(PWD)/build:/build autonomy/$@:$(TAG) cp /osctl-linux-amd64 /build - @docker run --rm -it -v $(PWD)/build:/build autonomy/$@:$(TAG) cp /osctl-darwin-amd64 /build +initramfs: + @buildctl build \ + --exporter=local \ + --exporter-opt output=build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) -trustd: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) +rootfs: hyperkube etcd coredns pause osd trustd proxyd blockd + @buildctl build \ + --exporter=local \ + --exporter-opt output=build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) -proxyd: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) +installer: + @buildctl build \ + --exporter=docker \ + --exporter-opt output=build/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --exporter-opt push=$(PUSH) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + @docker load < build/$@.tar + @docker run --rm -it -v /dev:/dev -v $(PWD)/build:/out --privileged autonomy/$@:$(TAG) image -l -blockd: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - -udevd: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) \ +.PHONY: docs +docs: + @rm -rf ./docs + @buildctl build \ + --exporter=local \ + --exporter-opt output=. \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) test: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) + @buildctl build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) lint: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) + @buildctl build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +osctl-linux-amd64: + @buildctl build \ + --exporter=local \ + --exporter-opt output=build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +osctl-darwin-amd64: + @buildctl build \ + --exporter=local \ + --exporter-opt output=build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +udevd: + @buildctl build \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +osd: + @buildctl build \ + --exporter=docker \ + --exporter-opt output=images/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +trustd: + @buildctl build \ + --exporter=docker \ + --exporter-opt output=images/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +proxyd: + @buildctl build \ + --exporter=docker \ + --exporter-opt output=images/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) + +blockd: + @buildctl build \ + --exporter=docker \ + --exporter-opt output=images/$@.tar \ + --exporter-opt name=docker.io/autonomy/$@:$(TAG) \ + --frontend-opt target=$@ \ + $(COMMON_ARGS) hyperkube: @docker pull k8s.gcr.io/$@:v1.13.2 @@ -83,45 +148,9 @@ pause: @docker pull k8s.gcr.io/$@:3.1 @docker save k8s.gcr.io/$@:3.1 -o ./images/$@.tar -rootfs: hyperkube etcd coredns pause osd trustd proxyd blockd - @docker save autonomy/osd:$(TAG) -o ./images/osd.tar - @docker save autonomy/trustd:$(TAG) -o ./images/trustd.tar - @docker save autonomy/proxyd:$(TAG) -o ./images/proxyd.tar - @docker save autonomy/blockd:$(TAG) -o ./images/blockd.tar - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - @docker run --rm -it -v $(PWD)/build:/build autonomy/$@:$(TAG) cp /rootfs.tar.gz /build - -initramfs: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - @docker run --rm -it -v $(PWD)/build:/build autonomy/$@:$(TAG) cp /initramfs.xz /build - -.PHONY: docs -docs: - @docker build \ - -t autonomy/$@:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - @rm -rf ./docs - @docker run --rm -it -v $(PWD):/out autonomy/$@:$(TAG) cp -R /docs /out - -installer: - @docker build \ - -t autonomy/talos:$(TAG) \ - --target=$@ \ - $(COMMON_APP_ARGS) - @docker run --rm -it -v $(PWD)/build:/build autonomy/talos:$(TAG) cp /generated/boot/vmlinuz /build - @docker run --rm -it -v /dev:/dev -v $(PWD)/build:/out --privileged autonomy/talos:$(TAG) image -l - deps: @GO111MODULES=on CGO_ENABLED=0 go get -u github.com/autonomy/gitmeta - @GO111MODULES=on CGO_ENABLED=0 go get -u github.com/autonomy/conform clean: - go clean -modcache - rm -rf build vendor + -go clean -modcache + -rm -rf build vendor diff --git a/hack/golang/test.sh b/hack/golang/test.sh old mode 100644 new mode 100755 diff --git a/hack/scripts/cleanup.sh b/hack/scripts/cleanup.sh old mode 100644 new mode 100755 diff --git a/hack/scripts/symlink.sh b/hack/scripts/symlink.sh old mode 100644 new mode 100755 diff --git a/images/.keep b/images/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/internal/app/init/main.go b/internal/app/init/main.go index 2eaaa6fad..c54d46a45 100644 --- a/internal/app/init/main.go +++ b/internal/app/init/main.go @@ -5,25 +5,21 @@ package main import ( - "context" "flag" "fmt" "log" "os" - "sync" "time" "github.com/autonomy/talos/internal/app/init/internal/platform" "github.com/autonomy/talos/internal/app/init/internal/rootfs" "github.com/autonomy/talos/internal/app/init/internal/rootfs/mount" "github.com/autonomy/talos/internal/app/init/pkg/system" - "github.com/autonomy/talos/internal/app/init/pkg/system/conditions" + ctrdrunner "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" "github.com/autonomy/talos/internal/app/init/pkg/system/services" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" "github.com/containerd/containerd" - "github.com/containerd/containerd/defaults" - "github.com/containerd/containerd/namespaces" criconstants "github.com/containerd/cri/pkg/constants" "github.com/pkg/errors" @@ -149,7 +145,33 @@ func startSystemServices(data *userdata.UserData) { svcs := system.Services(data) // Import the system images. - if err := importImages([]string{"/usr/images/blockd.tar", "/usr/images/osd.tar", "/usr/images/proxyd.tar", "/usr/images/trustd.tar"}, constants.SystemContainerdNamespace); err != nil { + reqs := []*ctrdrunner.ImportRequest{ + { + Path: "/usr/images/blockd.tar", + Options: []containerd.ImportOpt{ + containerd.WithIndexName("talos/blockd"), + }, + }, + { + Path: "/usr/images/osd.tar", + Options: []containerd.ImportOpt{ + containerd.WithIndexName("talos/osd"), + }, + }, + { + Path: "/usr/images/proxyd.tar", + Options: []containerd.ImportOpt{ + containerd.WithIndexName("talos/proxyd"), + }, + }, + { + Path: "/usr/images/trustd.tar", + Options: []containerd.ImportOpt{ + containerd.WithIndexName("talos/trustd"), + }, + }, + } + if err := ctrdrunner.Import(constants.SystemContainerdNamespace, reqs...); err != nil { panic(err) } @@ -173,7 +195,22 @@ func startKubernetesServices(data *userdata.UserData) { svcs := system.Services(data) // Import the Kubernetes images. - if err := importImages([]string{"/usr/images/hyperkube.tar", "/usr/images/etcd.tar", "/usr/images/coredns.tar", "/usr/images/pause.tar"}, criconstants.K8sContainerdNamespace); err != nil { + + reqs := []*ctrdrunner.ImportRequest{ + { + Path: "/usr/images/hyperkube.tar", + }, + { + Path: "/usr/images/etcd.tar", + }, + { + Path: "/usr/images/coredns.tar", + }, + { + Path: "/usr/images/pause.tar", + }, + } + if err := ctrdrunner.Import(criconstants.K8sContainerdNamespace, reqs...); err != nil { panic(err) } @@ -184,57 +221,6 @@ func startKubernetesServices(data *userdata.UserData) { ) } -func importImages(files []string, namespace string) (err error) { - _, err = conditions.WaitForFileToExist(defaults.DefaultAddress)() - if err != nil { - return err - } - - ctx := namespaces.WithNamespace(context.Background(), namespace) - client, err := containerd.New(defaults.DefaultAddress) - if err != nil { - return err - } - // nolint: errcheck - defer client.Close() - - var wg sync.WaitGroup - - wg.Add(len(files)) - - for _, file := range files { - go func(wg *sync.WaitGroup, f string) { - defer wg.Done() - - tarball, err := os.Open(f) - if err != nil { - panic(err) - } - - imgs, err := client.Import(ctx, tarball) - if err != nil { - panic(err) - } - if err = tarball.Close(); err != nil { - panic(err) - } - - for _, img := range imgs { - image := containerd.NewImage(client, img) - log.Printf("unpacking %s (%s)\n", img.Name, img.Target.Digest) - err = image.Unpack(ctx, containerd.DefaultSnapshotter) - if err != nil { - panic(err) - } - } - }(&wg, file) - } - - wg.Wait() - - return nil -} - func recovery() { if r := recover(); r != nil { log.Printf("recovered from: %+v\n", r) diff --git a/internal/app/init/pkg/system/runner/containerd/import.go b/internal/app/init/pkg/system/runner/containerd/import.go new file mode 100644 index 000000000..62c84789a --- /dev/null +++ b/internal/app/init/pkg/system/runner/containerd/import.go @@ -0,0 +1,75 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package containerd + +import ( + "context" + "log" + "os" + "sync" + + "github.com/autonomy/talos/internal/app/init/pkg/system/conditions" + "github.com/containerd/containerd" + "github.com/containerd/containerd/defaults" + "github.com/containerd/containerd/namespaces" +) + +// ImportRequest represents an image import request. +type ImportRequest struct { + Path string + Options []containerd.ImportOpt +} + +// Import imports the images specified by the import requests. +func Import(namespace string, reqs ...*ImportRequest) (err error) { + _, err = conditions.WaitForFileToExist(defaults.DefaultAddress)() + if err != nil { + return err + } + + ctx := namespaces.WithNamespace(context.Background(), namespace) + client, err := containerd.New(defaults.DefaultAddress) + if err != nil { + return err + } + // nolint: errcheck + defer client.Close() + + var wg sync.WaitGroup + + wg.Add(len(reqs)) + + for _, req := range reqs { + go func(wg *sync.WaitGroup, r *ImportRequest) { + defer wg.Done() + + tarball, err := os.Open(r.Path) + if err != nil { + panic(err) + } + + imgs, err := client.Import(ctx, tarball, r.Options...) + if err != nil { + panic(err) + } + if err = tarball.Close(); err != nil { + panic(err) + } + + for _, img := range imgs { + image := containerd.NewImage(client, img) + log.Printf("unpacking %s (%s)\n", img.Name, img.Target.Digest) + err = image.Unpack(ctx, containerd.DefaultSnapshotter) + if err != nil { + panic(err) + } + } + }(&wg, req) + } + + wg.Wait() + + return nil +} diff --git a/internal/app/init/pkg/system/services/blockd.go b/internal/app/init/pkg/system/services/blockd.go index bac325f0d..446951969 100644 --- a/internal/app/init/pkg/system/services/blockd.go +++ b/internal/app/init/pkg/system/services/blockd.go @@ -14,7 +14,6 @@ import ( "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" - "github.com/autonomy/talos/internal/pkg/version" "github.com/containerd/containerd/oci" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -44,7 +43,7 @@ func (t *Blockd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (t *Blockd) Start(data *userdata.UserData) error { - image := "docker.io/autonomy/blockd:" + version.Tag + image := "talos/blockd" // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/osd.go b/internal/app/init/pkg/system/services/osd.go index d5c26d22a..19b47a840 100644 --- a/internal/app/init/pkg/system/services/osd.go +++ b/internal/app/init/pkg/system/services/osd.go @@ -14,7 +14,6 @@ import ( "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" - "github.com/autonomy/talos/internal/pkg/version" "github.com/containerd/containerd/defaults" "github.com/containerd/containerd/oci" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -45,7 +44,7 @@ func (o *OSD) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc { } func (o *OSD) Start(data *userdata.UserData) error { - image := "docker.io/autonomy/osd:" + version.Tag + image := "talos/osd" // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/proxyd.go b/internal/app/init/pkg/system/services/proxyd.go index dfcab3064..6053e8374 100644 --- a/internal/app/init/pkg/system/services/proxyd.go +++ b/internal/app/init/pkg/system/services/proxyd.go @@ -13,7 +13,6 @@ import ( "github.com/autonomy/talos/internal/app/init/pkg/system/runner" "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" "github.com/autonomy/talos/internal/pkg/userdata" - "github.com/autonomy/talos/internal/pkg/version" "github.com/containerd/containerd/oci" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -43,7 +42,7 @@ func (p *Proxyd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (p *Proxyd) Start(data *userdata.UserData) error { - image := "docker.io/autonomy/proxyd:" + version.Tag + image := "talos/proxyd" // Set the process arguments. args := runner.Args{ diff --git a/internal/app/init/pkg/system/services/trustd.go b/internal/app/init/pkg/system/services/trustd.go index 4fc2014bc..f37558805 100644 --- a/internal/app/init/pkg/system/services/trustd.go +++ b/internal/app/init/pkg/system/services/trustd.go @@ -13,7 +13,6 @@ import ( "github.com/autonomy/talos/internal/app/init/pkg/system/runner/containerd" "github.com/autonomy/talos/internal/pkg/constants" "github.com/autonomy/talos/internal/pkg/userdata" - "github.com/autonomy/talos/internal/pkg/version" "github.com/containerd/containerd/oci" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -43,7 +42,7 @@ func (t *Trustd) ConditionFunc(data *userdata.UserData) conditions.ConditionFunc } func (t *Trustd) Start(data *userdata.UserData) error { - image := "docker.io/autonomy/trustd:" + version.Tag + image := "talos/trustd" // Set the process arguments. args := runner.Args{