Allow arch in remote

cloud_build/cloud_build.py

@@ -1,6 +1,6 @@
 #!/usr/bin/python3
 
-from typing import Dict, List, Union, Optional
+from typing import Dict, List, Set, Tuple, Union, Optional
 from pathlib import Path
 
 import contextlib
@@ -10,6 +10,7 @@ import logging
 import os
 import re
 import shutil
+import string
 import subprocess
 import time
 
@@ -76,10 +77,10 @@ class CB:
         self.checksum_command = 'sha256sum'
 
         if built_images_dir:
-            self.images_dir = Path(built_images_dir).absolute()
+            self._images_dir = Path(built_images_dir).absolute()
             self.no_build = True
         else:
-            self.images_dir = data_dir / 'images'
+            self._images_dir = data_dir / 'images'
             self.no_build = False
         self.work_dir = data_dir / 'work'
         self.out_dir = data_dir / 'out'
@@ -122,6 +123,56 @@ class CB:
             pass
         self.lock_file.close()
 
+    @property
+    def _remote_formaters(self) -> Set[str]:
+        return {
+            key
+            for tup in string.Formatter().parse(self._remote)
+            if (key := tup[1]) is not None
+        }
+
+    @property
+    def is_remote_arch(self) -> bool:
+        return 'arch' in self._remote_formaters
+
+    @property
+    def is_remote_branch(self) -> bool:
+        return 'branch' in self._remote_formaters
+
+    def images_dirs_remotes_list(self) -> List[Tuple[Path, str]]:
+        images_dirs_list = []
+        images_dir = self._images_dir
+        if self.is_remote_branch:
+            for branch in self.branches:
+                if self.is_remote_arch:
+                    for arch in self.arches_by_branch(branch):
+                        remote = self._remote.format(branch=branch, arch=arch)
+                        pair = (images_dir / branch / arch, remote)
+                        images_dirs_list.append(pair)
+                else:
+                    remote = self._remote.format(branch=branch)
+                    images_dirs_list.append((images_dir / branch, remote))
+        else:
+            if self.is_remote_arch:
+                for arch in self.all_arches:
+                    remote = self._remote.format(arch=arch)
+                    images_dirs_list.append((images_dir / arch, remote))
+            else:
+                images_dirs_list.append((images_dir, self._remote))
+
+        return images_dirs_list
+
+    def images_dirs_list(self) -> List[Path]:
+        return [pair[0] for pair in self.images_dirs_remotes_list()]
+
+    def images_dir(self, branch: str, arch: str) -> Path:
+        images_dir = self._images_dir
+        if self.is_remote_branch:
+            images_dir = images_dir / branch
+        if self.is_remote_arch:
+            images_dir = images_dir / arch
+        return images_dir
+
     def expand_path(self, path: PathLike):
         result = os.path.expanduser(os.path.expandvars(path))
         if isinstance(path, Path):
@@ -221,8 +272,9 @@ class CB:
         self.log.error(err)
         raise err
 
-    def remote(self, branch: str) -> str:
+    def remote(self, branch: str, arch: str) -> str:
-        return self._remote.format(branch=branch)
+        # import pdb; pdb.set_trace()
+        return self._remote.format(branch=branch, arch=arch)
 
     def repository_url(self, branch: str, arch: str) -> str:
         url = self._branches[branch]['arches'][arch].get('repository_url')
@@ -276,8 +328,9 @@ class CB:
                 value = getattr(self, attr)
                 if isinstance(value, str) or isinstance(value, os.PathLike):
                     os.makedirs(value, exist_ok=True)
-        for branch in self.branches:
+
-            os.makedirs(self.images_dir / branch, exist_ok=True)
+        for images_dir in self.images_dirs_list():
+            os.makedirs(images_dir, exist_ok=True)
 
     def generate_apt_files(self) -> None:
         apt_dir = self.work_dir / 'apt'
@@ -386,6 +439,13 @@ Dir::Etc::preferencesparts "/var/empty";
     def arches_by_branch(self, branch: str) -> List[str]:
         return list(self._branches[branch]['arches'].keys())
 
+    @property
+    def all_arches(self) -> List[str]:
+        arches: Set[str] = set()
+        for branch in self.branches:
+            arches |= set(self.arches_by_branch(branch))
+        return list(arches)
+
     def branding_by_branch(self, branch: str) -> str:
         return self._branches[branch].get('branding', '')
 
@@ -590,8 +650,7 @@ Dir::Etc::preferencesparts "/var/empty";
         kind: str
     ) -> Path:
         path = (
-            self.images_dir
+            self.images_dir(branch, arch)
-            / branch
             / f'alt-{branch.lower()}-{image}-{arch}.{kind}'
         )
         return path
@@ -602,9 +661,8 @@ Dir::Etc::preferencesparts "/var/empty";
         os.link(src, dst)
 
     def clear_images_dir(self):
-        for branch in self.branches:
+        for images_dir in self.images_dirs_list():
-            directory = self.images_dir / branch
+            for path in images_dir.iterdir():
-            for path in directory.iterdir():
                 os.unlink(path)
 
     def remove_old_tarballs(self):
@@ -683,17 +741,23 @@ Dir::Etc::preferencesparts "/var/empty";
         self.remove_old_tarballs()
 
     def copy_external_files(self):
-        if self.external_files:
+        if not self.external_files:
+            return
+
         for branch in os.listdir(self.external_files):
             if branch not in self.branches:
                 self.error(f'Unknown branch {branch} in external_files')
-
+            arches = self.arches_by_branch(branch)
-                with self.pushd(self.external_files / branch):
+            for arch in os.listdir(self.external_files / branch):
+                if arch not in arches:
+                    self.error(f'Unknown arch {arch} in external_files')
+                with self.pushd(self.external_files / branch / arch):
                     for image in os.listdir():
-                        self.info(f'Copy external image {image} in {branch}')
+                        msg = f'Copy external file {image} in {branch}/{arch}'
+                        self.info(msg)
                         self.copy_image(
                             image,
-                            self.images_dir / branch / image,
+                            self.images_dir(branch, arch) / image,
                             rewrite=True,
                         )
 
@@ -702,8 +766,8 @@ Dir::Etc::preferencesparts "/var/empty";
             self.error('Pass key to config file for sign')
 
         sum_file = self.checksum_command.upper()
-        for branch in self.branches:
+        for images_dir in self.images_dirs_list():
-            with self.pushd(self.images_dir / branch):
+            with self.pushd(images_dir):
                 files = [f
                          for f in os.listdir()
                          if not f.startswith(sum_file)]
@@ -736,13 +800,12 @@ Dir::Etc::preferencesparts "/var/empty";
             self.call(cmd(command))
 
     def sync(self, create_remote_dirs: bool = False) -> None:
-        for branch in self.branches:
+        for images_dir, remote in self.images_dirs_remotes_list():
-            remote = self.remote(branch)
             if create_remote_dirs:
                 os.makedirs(remote, exist_ok=True)
             cmd = [
                 'rsync',
-                f'{self.images_dir}/{branch}/',
+                f'{images_dir}/',
                 '-rv',
                 remote,
             ]

tests/test_integration_images.py

@@ -1,3 +1,4 @@
+from collections import defaultdict
 from contextlib import ExitStack
 from pathlib import Path
 from unittest import TestCase
@@ -6,25 +7,68 @@ from unittest import mock
 import os
 import shutil
 
+from parameterized import parameterized, parameterized_class  # type: ignore
+
+import yaml
+
 from cloud_build import CB
 from tests.call import Call
 
 
+def change(origin, new, key, value):
+    with open(origin) as f:
+        cfg = yaml.safe_load(f)
+
+    with open(new, 'w') as f:
+        yaml.safe_dump(cfg | {key: value}, f)
+
+
+def get_class_name(cls, num, params_dict):
+    result = cls.__name__
+    if branch := params_dict['branch']:
+        result = f'{result}_{parameterized.to_safe_name(branch)}'
+    if arch := params_dict['arch']:
+        result = f'{result}_{parameterized.to_safe_name(arch)}'
+    return f'{result}_{num}'
+
+
+@parameterized_class([
+    {'branch': 'branch', 'arch': ''},
+    {'branch': 'branch', 'arch': 'arch'},
+    {'branch': '', 'arch': 'arch'},
+    {'branch': '', 'arch': ''},
+], class_name_func=get_class_name)
 class TestIntegrationImages(TestCase):
     def setUp(self):
-        self.images = self.__class__.images
+        self._images = self.__class__._images
+        self.images_dir = self.__class__.images_dir
 
     @classmethod
     def setUpClass(cls):
         cls.work_dir = Path('/tmp/cloud-build')
-        os.makedirs(cls.work_dir / 'external_files/p9', exist_ok=True)
+        os.makedirs(cls.work_dir / 'external_files/p9/x86_64', exist_ok=True)
-        (cls.work_dir / 'external_files/p9/README').write_text('README')
+        (cls.work_dir / 'external_files/p9/x86_64/README').write_text('README')
+        config = cls.work_dir / 'config.yaml'
+        branch = cls.branch
+        arch = cls.arch
+        remote = Path('/tmp/cloud-build/images')
+        if branch:
+            remote = remote / '{branch}'
+        if arch:
+            remote = remote / '{arch}'
+        remote = (remote / 'cloud').as_posix()
+        change(
+            'tests/test_integration_images.yaml',
+            config,
+            'remote',
+            remote,
+        )
 
         with ExitStack() as stack:
             stack.enter_context(mock.patch('subprocess.call', Call()))
 
             cloud_build = CB(
-                config='tests/test_integration_images.yaml',
+                config=config,
                 data_dir=(cls.work_dir / 'cloud_build').as_posix(),
             )
             cloud_build.create_images(no_tests=True)
@@ -33,9 +77,54 @@ class TestIntegrationImages(TestCase):
             cloud_build.sync(create_remote_dirs=True)
 
         images_dir = cls.work_dir / 'images'
-        cls.images = {}
+        cls.images_dir = images_dir
+        images = defaultdict(lambda: defaultdict(list))
+
+        if branch:
             for branch in os.listdir(images_dir):
-            cls.images[branch] = os.listdir(images_dir / branch / 'cloud')
+                if arch:
+                    for arch in os.listdir(images_dir / branch):
+                        images[branch][arch] = os.listdir(
+                            images_dir / branch / arch / 'cloud'
+                        )
+                else:
+                    images[branch]['arch'] = os.listdir(
+                        images_dir / branch / 'cloud'
+                    )
+        elif arch:
+            for arch in os.listdir(images_dir):
+                images['branch'][arch] = os.listdir(
+                    images_dir / arch / 'cloud'
+                )
+        else:
+            images['branch']['arch'] = os.listdir(
+                images_dir / 'cloud'
+            )
+        cls._images = images
+
+    def image_path(self, branch, arch, image) -> Path:
+        images_dir = self.images_dir
+        if self.branch:  # type: ignore
+            images_dir = images_dir / branch
+        if self.arch:  # type: ignore
+            images_dir = images_dir / arch
+
+        return images_dir / 'cloud' / image
+
+    def images(self, branch, arch) -> list:
+        if not self.branch:  # type: ignore
+            branch = 'branch'
+        if not self.arch:  # type: ignore
+            arch = 'arch'
+
+        return self._images[branch][arch]
+
+    def images_lists(self) -> list:
+        result = []
+        for branch_value in self._images.values():
+            for arch_value in branch_value.values():
+                result.append(arch_value)
+        return result
 
     @classmethod
     def tearDownClass(cls):
@@ -43,28 +132,45 @@ class TestIntegrationImages(TestCase):
 
     def test_arch_ppc64le(self):
         self.assertIn('alt-p9-rootfs-minimal-ppc64le.tar.xz',
-                      self.images['p9'])
+                      self.images('p9', 'ppc64le'))
 
     def test_branches(self):
-        self.assertCountEqual(self.images.keys(), ['Sisyphus', 'p9', 'p8'])
+        if self.branch:
+            self.assertCountEqual(self._images.keys(),
+                                  ['Sisyphus', 'p9', 'p8'])
+        else:
+            self.assertCountEqual(self._images.keys(), ['branch'])
 
     def test_build_cloud(self):
-        self.assertIn('alt-p9-cloud-x86_64.qcow2', self.images['p9'])
+        self.assertIn('alt-p9-cloud-x86_64.qcow2', self.images('p9', 'x86_64'))
 
     def test_exclude_arches(self):
-        self.assertNotIn('alt-p8-cloud-x86_64.qcow2', self.images['p8'])
+        self.assertNotIn('alt-p8-cloud-x86_64.qcow2',
+                         self.images('p8', 'x86_64'))
 
     def test_exclude_branches(self):
-        self.assertNotIn('alt-p9-cloud-ppc64le.qcow2', self.images['p9'])
+        self.assertNotIn('alt-p9-cloud-ppc64le.qcow2',
+                         self.images('p9', 'ppc64le'))
 
     def test_external_files(self):
-        self.assertIn('README', self.images['p9'])
+        self.assertIn('README', self.images('p9', 'x86_64'))
 
     def test_verification_files(self):
-        for images in self.images.values():
+        for images_list in self.images_lists():
-            self.assertIn('SHA256SUMS', images)
+            self.assertIn('SHA256SUMS', images_list)
-            self.assertIn('SHA256SUMS.gpg', images)
+            self.assertIn('SHA256SUMS.gpg', images_list)
+
+        number_of_images = len(self.image_path(
+            'p9',
+            'x86_64',
+            'SHA256SUMS',
+        ).read_text().splitlines())
+        index = bool(self.branch) * 2 + bool(self.arch)
+        expected_numbers = [52, 17, 21, 7]
+        self.assertEqual(number_of_images, expected_numbers[index])
 
     def test_number_of_images(self):
-        number_of_images = sum(len(self.images[b]) for b in self.images.keys())
+        number_of_images = sum(len(lst) for lst in self.images_lists())
-        self.assertEqual(number_of_images, 64)
+        index = bool(self.branch) * 2 + bool(self.arch)
+        expected_numbers = [56, 72, 64, 96]
+        self.assertEqual(number_of_images, expected_numbers[index])

tests/test_rebuild.py

@@ -55,7 +55,6 @@ class TestRebuild(TestCase):
         image = (
             self.data_dir
             / 'images'
-            / 'Sisyphus'
             / 'alt-sisyphus-rootfs-minimal-x86_64.tar.xz'
         )
         msg = 'Do not create image when rebuild'