Add distroless builder and refactor work with images

build.py

@@ -1,20 +1,28 @@
 #!/usr/bin/python3
 
 import argparse
+import functools
 import json
-import os
 import re
 import subprocess
 import textwrap
 from graphlib import TopologicalSorter
 from pathlib import Path
 
+import tomli
 from jinja2 import Template
 
 
 ORG_DIR = Path("org")
 
 
+class Image:
+    def __init__(self, canonical_name):
+        self.canonical_name = canonical_name
+        self.path = ORG_DIR / canonical_name
+        self.base_name = re.sub("^[^/]+/", "", canonical_name)
+
+
 class Tasks:
     def __init__(self, tasks):
         if tasks is None:
@@ -22,12 +30,16 @@ class Tasks:
         else:
             self._tasks = json.loads(Path(tasks).read_text())
 
-    def get(self, branch, image):
+    def get(self, branch, image: Image):
         if self._tasks is None:
             return []
         else:
             if branch_tasks := self._tasks.get(branch):
-                return [n for n, i in branch_tasks.items() if image in i or len(i) == 0]
+                return [
+                    n
+                    for n, i in branch_tasks.items()
+                    if image.canonical_name in i or len(i) == 0
+                ]
 
 
 class Tags:
@@ -39,27 +51,55 @@ class Tags:
             self._tags = json.loads(tags_file.read_text())
         self._latest = latest
 
-    def tags(self, branch, image):
+    def tags(self, branch, image: Image):
         if self._tags is None:
             tags = [branch]
         else:
-            tags = self._tags[image][branch]
+            tags = self._tags[image.canonical_name][branch]
         if branch == self._latest:
             tags.append("latest")
         return tags
 
 
+class Distroless:
+    def __init__(self, distrolessfile, renderer):
+        dd = tomli.loads(distrolessfile.read_text())
+
+        self.raw_from = dd["from"]
+        self.from_ = renderer(dd["from"])
+
+        self.file_lists = dd.get("file-lists", [])
+        self.files = dd.get("files", [])
+        self.packages = dd.get("packages", [])
+
+        self.builder_install_packages = dd.get("builder-install-packages")
+
+        self.timezone = dd.get("timezone")
+
+        self.copy = dd.get("copy", {})
+
+        self.config_options = []
+        for option in ["cmd", "entrypoint", "user"]:
+            if value := dd.get(option):
+                self.config_options.append(f"--{option}={value}")
+
+
 class DockerBuilder:
-    def make_from_re(self):
+    def make_image_re(self):
         registry = r"(?P<registry>[\w.:]+)"
         organization = r"(?P<organization>\w+)"
-        name = r"(?P<name>\w+)"
+        name = r"(?P<name>[-.\w]+)"
         tag = r"(?P<tag>[\w.]+)"
-        return f"^FROM (:?{registry}/)?(:?{organization}/)?{name}(:?:{tag})?$"
+        return rf"(:?{registry}/)?(:?{organization}/)?{name}(:?:{tag})?"
+
+    def make_dockerfile_from_re(self):
+        image_re = self.make_image_re()
+        return rf"^\s*FROM\s+{image_re}$"
 
     def __init__(
         self,
         registry,
+        branch,
         organization,
         overwrite_organization,
         latest,
@@ -68,10 +108,12 @@ class DockerBuilder:
         tasks: Tasks,
         tags: Tags,
     ):
-        self.from_re = re.compile(self.make_from_re())
+        self.image_re = re.compile(self.make_image_re())
+        self.dockerfile_from_re = re.compile(self.make_dockerfile_from_re())
         self.org_dir = ORG_DIR
         self.images_dir = ORG_DIR / organization
         self.registry = registry
+        self.branch = branch
         self.organization = organization
         if overwrite_organization:
             self.overwrite_organization = overwrite_organization
@@ -82,20 +124,18 @@ class DockerBuilder:
         self.images_info = images_info
         self.tasks = tasks
         self.tags = tags
-
-    def full_image(self, image):
-        return f"{self.organization}/{image}"
+        self.distrolesses = {}
 
     def forall_images(consume_result):
         def forall_images_decorator(f):
             def wrapped(self, *args, **kwargs):
-                for image in self.images_dir.iterdir():
-                    image_name = "/".join(image.parts[1:])
+                for image_path in self.images_dir.iterdir():
+                    image = Image("/".join(image_path.parts[1:]))
                     local_kwargs = {
-                        "image_name": image_name,
                         "image": image,
-                        "dockerfile": image / "Dockerfile",
-                        "dockerfile_template": image / "Dockerfile.template",
+                        "dockerfile": image_path / "Dockerfile",
+                        "dockerfile_template": image_path / "Dockerfile.template",
+                        "distrolessfile": image_path / "distroless.toml",
                     }
                     new_kwargs = kwargs | local_kwargs
                     yield f(self, *args, **new_kwargs)
@@ -116,7 +156,9 @@ class DockerBuilder:
         if kwargs["dockerfile"].exists():
             kwargs["dockerfile"].unlink()
 
-    def render_template(self, template: str, branch: str, install_pakages=None) -> str:
+    def render_template(
+        self, template: str, organization: str, install_pakages=None
+    ) -> str:
         if self.registry:
             registry = self.registry.rstrip("/") + "/"
             alt_image = "alt/alt"
@@ -125,18 +167,18 @@ class DockerBuilder:
             alt_image = "alt"
         rendered = Template(template).render(
             alt_image=alt_image,
-            branch=branch,
+            branch=self.branch,
             install_pakages=install_pakages,
-            organization=self.overwrite_organization,
+            organization=organization,
             registry=registry,
         )
 
         return rendered
 
     @forall_images(consume_result=True)
-    def render_dockerfiles(self, branch, **kwargs):
+    def render_dockerfiles(self, **kwargs):
         def install_pakages(*names):
-            tasks = self.tasks.get(branch, kwargs["image_name"])
+            tasks = self.tasks.get(self.branch, kwargs["image"].canonical_name)
             linux32 = '$([ "$(rpm --eval %_host_cpu)" = i586 ] && echo linux32)'
             if tasks:
                 apt_repo = "\\\n    apt-get install apt-repo -y && \\"
@@ -159,37 +201,67 @@ class DockerBuilder:
         dockerfile_template = kwargs["dockerfile_template"]
         if dockerfile_template.exists():
             rendered = self.render_template(
-                dockerfile_template.read_text(), branch, install_pakages
+                dockerfile_template.read_text(),
+                self.overwrite_organization,
+                install_pakages,
             )
             kwargs["dockerfile"].write_text(rendered + "\n")
 
+    @forall_images(consume_result=True)
+    def load_distrolesses(self, **kwargs):
+        renderer = functools.partial(
+            self.render_template,
+            organization=self.overwrite_organization,
+        )
+        distrolessfile = kwargs["distrolessfile"]
+        canonical_name = "/".join(distrolessfile.parts[-3:-1])
+        if distrolessfile.exists():
+            self.distrolesses[canonical_name] = Distroless(distrolessfile, renderer)
+
     @forall_images(consume_result=False)
     def get_requires(self, **kwargs):
         requires = set()
+        dockerfile_template = kwargs["dockerfile_template"]
+        distrolessfile = kwargs["distrolessfile"]
+        canonical_name = kwargs["image"].canonical_name
 
-        for line in kwargs["dockerfile"].read_text().splitlines():
-            if match := re.match(self.from_re, line):
+        if dockerfile_template.exists():
+            for line in dockerfile_template.read_text().splitlines():
+                if not re.match(r"\s*FROM", line):
+                    continue
+                line = self.render_template(line, self.organization)
+                if match := re.match(self.dockerfile_from_re, line):
+                    from_image = match.groupdict()
+                    if from_image["name"] != "scratch":
+                        requires.add(
+                            f"{from_image['organization']}/{from_image['name']}"
+                        )
+        elif distrolessfile.exists():
+            requires.add("alt/distroless-builder")
+            raw_from = self.distrolesses[canonical_name].raw_from
+            from_ = self.render_template(raw_from, self.organization)
+            if match := re.match(self.image_re, from_):
                 from_image = match.groupdict()
-                if from_image["organization"] == self.organization:
-                    requires.add(from_image["name"])
+                if from_image["name"] != "scratch":
+                    requires.add(f"{from_image['organization']}/{from_image['name']}")
 
-        return (kwargs["image"].name, requires)
+        return (canonical_name, requires)
 
     def get_build_order(self):
         requires = {}
-        for image, image_requires in self.get_requires():
-            requires[image] = image_requires
+        for canonical_name, image_requires in self.get_requires():
+            requires[canonical_name] = image_requires
         ts = TopologicalSorter(requires)
-        return ts.static_order()
+        return (Image(i) for i in ts.static_order())
 
-    def render_full_tag(self, image, tag):
+    def render_full_tag(self, image: Image, tag: str):
         if self.registry:
             registry = self.registry.rstrip("/") + "/"
         else:
             registry = ""
         if tag:
             tag = f":{tag}"
-        return f"{registry}{self.overwrite_organization}/{image}{tag}"
+        return f"{registry}{self.overwrite_organization}/{image.base_name}{tag}"
 
     def run(self, cmd, *args, **kwargs):
         if "check" not in kwargs:
@@ -200,50 +272,235 @@ class DockerBuilder:
             pre_cmd = []
         subprocess.run(pre_cmd + cmd, *args, **kwargs)
 
-    def podman_build(self, image, arches, branch):
-        full_image = self.full_image(image)
-        if self.images_info.skip_branch(full_image, branch):
+    def distroless_build(self, image: Image, arches):
+        def distroless_build_arch(arch, manifest):
+            distroless_builder = self.render_full_tag(
+                Image("alt/distroless-builder"), self.branch
+            )
+            distroless = self.distrolesses[image.canonical_name]
+            builder = f"distroless-builder-{arch}"
+            new = f"distroless-new-{arch}"
+            run = functools.partial(self.run, cwd=image.path)
+            run(
+                ["buildah", "rm", builder, new],
+                check=False,
+                stderr=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+            )
+            run(
+                [
+                    "buildah",
+                    "from",
+                    "--arch",
+                    arch,
+                    "--name",
+                    builder,
+                    distroless_builder,
+                ]
+            )
+            run(["buildah", "from", "--arch", arch, "--name", new, distroless.from_])
+
+            if packages := distroless.builder_install_packages:
+                run(["buildah", "run", builder, "apt-get", "update"])
+                run(
+                    ["buildah", "run", builder, "apt-get", "reinstall", "-y"] + packages
+                )
+
+            if timezone := distroless.timezone:
+                run(
+                    [
+                        "buildah",
+                        "run",
+                        builder,
+                        "ln",
+                        "-s",
+                        f"/usr/share/zoneinfo/{timezone}",
+                        "/etc/localtime",
+                    ]
+                )
+
+            files_options = []
+            file_lists_options = []
+            packages_options = []
+            if distroless.files:
+                files_options = ["-f"] + distroless.files
+            if file_lists := distroless.file_lists:
+                file_lists_options = ["-l"]
+                file_lists_options.extend([f"file-lists/{f}" for f in file_lists])
+                for file_list in file_lists:
+                    run(
+                        [
+                            "buildah",
+                            "copy",
+                            builder,
+                            f"./{file_list}",
+                            f"file-lists/{file_list}",
+                        ]
+                    )
+            if distroless.packages:
+                packages_options = ["-p"] + distroless.packages
+
+            run(
+                [
+                    "buildah",
+                    "run",
+                    builder,
+                    "./distroless-builder.py",
+                    "add",
+                    "--clean",
+                ]
+                + files_options
+                + file_lists_options
+                + packages_options
+            )
+
+            run(["buildah", "run", builder, "./distroless-builder.py", "tar"])
+
+            run(
+                [
+                    "buildah",
+                    "add",
+                    "--from",
+                    builder,
+                    new,
+                    "/usr/src/distroless/distroless.tar",
+                    "/",
+                ]
+            )
+
+            for local_file, image_file in distroless.copy.items():
+                run(
+                    [
+                        "buildah",
+                        "copy",
+                        new,
+                        f"./{local_file}",
+                        image_file,
+                    ]
+                )
+
+            run(["buildah", "config"] + distroless.config_options + [new])
+
+            run(["buildah", "commit", "--rm", "--manifest", manifest, new])
+            run(
+                ["buildah", "rm", builder],
+                check=False,
+                stderr=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+            )
+
+        if self.images_info.skip_branch(image.canonical_name, self.branch):
             return
 
-        msg = "Building image {} for branch {} and {} arches".format(
-            self.full_image(image),
-            branch,
+        build_arches = set(arches) - set(
+            self.images_info.skip_arches(image.canonical_name)
+        )
+        tags = self.tags.tags(self.branch, image.canonical_name)
+        manifest = self.render_full_tag(image, tags[0])
+
+        msg = "Building image {} for {} arches".format(
+            manifest,
             arches,
         )
         print(msg)
 
-        build_arches = set(arches) - set(self.images_info.skip_arches(full_image))
-        platforms = ",".join([f"linux/{a}" for a in build_arches])
-        tags = self.tags.tags(branch, full_image)
-        full_name = self.render_full_tag(image, tags[0])
-
+        rm_image_cmd = [
+            "podman",
+            "image",
+            "rm",
+            "--force",
+            manifest,
+        ]
+        self.run(
+            rm_image_cmd,
+            check=False,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+        )
         rm_manifest_cmd = [
             "podman",
             "manifest",
             "rm",
-            full_name,
+            manifest,
         ]
-        self.run(rm_manifest_cmd, check=False)
+        self.run(
+            rm_manifest_cmd,
+            check=False,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+        )
+
+        for arch in build_arches:
+            distroless_build_arch(arch, manifest)
+
+        for tag in tags[1:]:
+            other_manifest = self.render_full_tag(image, tag)
+            tag_cmd = ["podman", "tag", manifest, other_manifest]
+            self.run(tag_cmd)
+
+    def podman_build(self, image: Image, arches):
+        if self.images_info.skip_branch(image.canonical_name, self.branch):
+            return
+
+        build_arches = set(arches) - set(
+            self.images_info.skip_arches(image.canonical_name)
+        )
+        platforms = ",".join([f"linux/{a}" for a in build_arches])
+        tags = self.tags.tags(self.branch, image.canonical_name)
+        manifest = self.render_full_tag(image, tags[0])
+
+        msg = "Building image {} for {} arches".format(
+            manifest,
+            arches,
+        )
+        print(msg)
+
+        rm_image_cmd = [
+            "podman",
+            "image",
+            "rm",
+            "--force",
+            manifest,
+        ]
+        self.run(
+            rm_image_cmd,
+            check=False,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+        )
+        rm_manifest_cmd = [
+            "podman",
+            "manifest",
+            "rm",
+            manifest,
+        ]
+        self.run(
+            rm_manifest_cmd,
+            check=False,
+            stderr=subprocess.DEVNULL,
+            stdout=subprocess.DEVNULL,
+        )
         build_cmd = [
             "podman",
             "build",
-            f"--manifest={full_name}",
+            "--rm",
+            "--force-rm",
+            f"--manifest={manifest}",
             f"--platform={platforms}",
             ".",
         ]
-        self.run(build_cmd, cwd=self.images_dir / image)
+        self.run(build_cmd, cwd=image.path)
 
         for tag in tags[1:]:
-            other_full_name = self.render_full_tag(image, tag)
-            tag_cmd = ["podman", "tag", full_name, other_full_name]
+            other_manifest = self.render_full_tag(image, tag)
+            tag_cmd = ["podman", "tag", manifest, other_manifest]
             self.run(tag_cmd)
 
-    def podman_push(self, image, branch, sign=None):
-        full_image = self.full_image(image)
-        if self.images_info.skip_branch(full_image, branch):
+    def podman_push(self, image: Image, sign=None):
+        if self.images_info.skip_branch(image.canonical_name, self.branch):
             return
 
-        tags = self.tags.tags(branch, full_image)
+        tags = self.tags.tags(self.branch, image.canonical_name)
         manifests = [self.render_full_tag(image, t) for t in tags]
 
         for manifest in manifests:
@@ -271,20 +528,20 @@ class ImagesInfo:
 
         self._info = info
 
-    def skip_arch(self, image, arch):
-        info = self._info.get(image, {})
+    def skip_arch(self, canonical_name, arch):
+        info = self._info.get(canonical_name, {})
         return arch in info.get("skip-arches", [])
 
-    def skip_arches(self, image):
-        info = self._info.get(image, {})
+    def skip_arches(self, canonical_name):
+        info = self._info.get(canonical_name, {})
         return info.get("skip-arches", [])
 
-    def skip_branch(self, image, branch):
-        info = self._info.get(image, {})
+    def skip_branch(self, canonical_name, branch):
+        info = self._info.get(canonical_name, {})
         return branch in info.get("skip-branches", [])
 
-    def skip_branches(self, image):
-        info = self._info.get(image, {})
+    def skip_branches(self, canonical_name):
+        info = self._info.get(canonical_name, {})
         return info.get("skip-branches", [])
 
 
@@ -408,12 +665,14 @@ def parse_args():
 
 def main():
     args = parse_args()
+    arches = args.arches
     images_info = ImagesInfo()
     tags = Tags(args.tags, args.latest)
     for organization in args.organizations:
         for branch in args.branches:
             db = DockerBuilder(
                 args.registry,
+                branch,
                 organization,
                 args.overwrite_organization,
                 args.latest,
@@ -425,16 +684,20 @@ def main():
             if "remove_dockerfiles" in args.stages:
                 db.remove_dockerfiles()
             if "render_dockerfiles" in args.stages:
-                db.render_dockerfiles(branch)
+                db.render_dockerfiles()
+            db.load_distrolesses()
             for image in db.get_build_order():
-                if f"{organization}/{image}" not in args.images:
+                if image.canonical_name not in args.images:
                     continue
 
                 if "build" in args.stages:
-                    db.podman_build(image, args.arches, branch)
+                    if image.canonical_name in db.distrolesses:
+                        db.distroless_build(image, arches)
+                    else:
+                        db.podman_build(image, arches)
 
                 if "push" in args.stages:
-                    db.podman_push(image, branch, args.sign)
+                    db.podman_push(image, args.sign)
 
 
 if __name__ == "__main__":