Allow arch in remote
This commit is contained in:
Mikhail Gordeev
parent
d59dc3a036
commit
4498ec109e
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/python3
|
||||
|
||||
from typing import Dict, List, Union, Optional
|
||||
from typing import Dict, List, Set, Tuple, Union, Optional
|
||||
from pathlib import Path
|
||||
|
||||
import contextlib
|
||||
@ -10,6 +10,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
@ -76,10 +77,10 @@ class CB:
|
||||
self.checksum_command = 'sha256sum'
|
||||
|
||||
if built_images_dir:
|
||||
self.images_dir = Path(built_images_dir).absolute()
|
||||
self._images_dir = Path(built_images_dir).absolute()
|
||||
self.no_build = True
|
||||
else:
|
||||
self.images_dir = data_dir / 'images'
|
||||
self._images_dir = data_dir / 'images'
|
||||
self.no_build = False
|
||||
self.work_dir = data_dir / 'work'
|
||||
self.out_dir = data_dir / 'out'
|
||||
@ -122,6 +123,56 @@ class CB:
|
||||
pass
|
||||
self.lock_file.close()
|
||||
|
||||
@property
|
||||
def _remote_formaters(self) -> Set[str]:
|
||||
return {
|
||||
key
|
||||
for tup in string.Formatter().parse(self._remote)
|
||||
if (key := tup[1]) is not None
|
||||
}
|
||||
|
||||
@property
|
||||
def is_remote_arch(self) -> bool:
|
||||
return 'arch' in self._remote_formaters
|
||||
|
||||
@property
|
||||
def is_remote_branch(self) -> bool:
|
||||
return 'branch' in self._remote_formaters
|
||||
|
||||
def images_dirs_remotes_list(self) -> List[Tuple[Path, str]]:
|
||||
images_dirs_list = []
|
||||
images_dir = self._images_dir
|
||||
if self.is_remote_branch:
|
||||
for branch in self.branches:
|
||||
if self.is_remote_arch:
|
||||
for arch in self.arches_by_branch(branch):
|
||||
remote = self._remote.format(branch=branch, arch=arch)
|
||||
pair = (images_dir / branch / arch, remote)
|
||||
images_dirs_list.append(pair)
|
||||
else:
|
||||
remote = self._remote.format(branch=branch)
|
||||
images_dirs_list.append((images_dir / branch, remote))
|
||||
else:
|
||||
if self.is_remote_arch:
|
||||
for arch in self.all_arches:
|
||||
remote = self._remote.format(arch=arch)
|
||||
images_dirs_list.append((images_dir / arch, remote))
|
||||
else:
|
||||
images_dirs_list.append((images_dir, self._remote))
|
||||
|
||||
return images_dirs_list
|
||||
|
||||
def images_dirs_list(self) -> List[Path]:
|
||||
return [pair[0] for pair in self.images_dirs_remotes_list()]
|
||||
|
||||
def images_dir(self, branch: str, arch: str) -> Path:
|
||||
images_dir = self._images_dir
|
||||
if self.is_remote_branch:
|
||||
images_dir = images_dir / branch
|
||||
if self.is_remote_arch:
|
||||
images_dir = images_dir / arch
|
||||
return images_dir
|
||||
|
||||
def expand_path(self, path: PathLike):
|
||||
result = os.path.expanduser(os.path.expandvars(path))
|
||||
if isinstance(path, Path):
|
||||
@ -221,8 +272,9 @@ class CB:
|
||||
self.log.error(err)
|
||||
raise err
|
||||
|
||||
def remote(self, branch: str) -> str:
|
||||
return self._remote.format(branch=branch)
|
||||
def remote(self, branch: str, arch: str) -> str:
|
||||
# import pdb; pdb.set_trace()
|
||||
return self._remote.format(branch=branch, arch=arch)
|
||||
|
||||
def repository_url(self, branch: str, arch: str) -> str:
|
||||
url = self._branches[branch]['arches'][arch].get('repository_url')
|
||||
@ -276,8 +328,9 @@ class CB:
|
||||
value = getattr(self, attr)
|
||||
if isinstance(value, str) or isinstance(value, os.PathLike):
|
||||
os.makedirs(value, exist_ok=True)
|
||||
for branch in self.branches:
|
||||
os.makedirs(self.images_dir / branch, exist_ok=True)
|
||||
|
||||
for images_dir in self.images_dirs_list():
|
||||
os.makedirs(images_dir, exist_ok=True)
|
||||
|
||||
def generate_apt_files(self) -> None:
|
||||
apt_dir = self.work_dir / 'apt'
|
||||
@ -386,6 +439,13 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
def arches_by_branch(self, branch: str) -> List[str]:
|
||||
return list(self._branches[branch]['arches'].keys())
|
||||
|
||||
@property
|
||||
def all_arches(self) -> List[str]:
|
||||
arches: Set[str] = set()
|
||||
for branch in self.branches:
|
||||
arches |= set(self.arches_by_branch(branch))
|
||||
return list(arches)
|
||||
|
||||
def branding_by_branch(self, branch: str) -> str:
|
||||
return self._branches[branch].get('branding', '')
|
||||
|
||||
@ -590,8 +650,7 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
kind: str
|
||||
) -> Path:
|
||||
path = (
|
||||
self.images_dir
|
||||
/ branch
|
||||
self.images_dir(branch, arch)
|
||||
/ f'alt-{branch.lower()}-{image}-{arch}.{kind}'
|
||||
)
|
||||
return path
|
||||
@ -602,9 +661,8 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
os.link(src, dst)
|
||||
|
||||
def clear_images_dir(self):
|
||||
for branch in self.branches:
|
||||
directory = self.images_dir / branch
|
||||
for path in directory.iterdir():
|
||||
for images_dir in self.images_dirs_list():
|
||||
for path in images_dir.iterdir():
|
||||
os.unlink(path)
|
||||
|
||||
def remove_old_tarballs(self):
|
||||
@ -683,17 +741,23 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
self.remove_old_tarballs()
|
||||
|
||||
def copy_external_files(self):
|
||||
if self.external_files:
|
||||
for branch in os.listdir(self.external_files):
|
||||
if branch not in self.branches:
|
||||
self.error(f'Unknown branch {branch} in external_files')
|
||||
if not self.external_files:
|
||||
return
|
||||
|
||||
with self.pushd(self.external_files / branch):
|
||||
for branch in os.listdir(self.external_files):
|
||||
if branch not in self.branches:
|
||||
self.error(f'Unknown branch {branch} in external_files')
|
||||
arches = self.arches_by_branch(branch)
|
||||
for arch in os.listdir(self.external_files / branch):
|
||||
if arch not in arches:
|
||||
self.error(f'Unknown arch {arch} in external_files')
|
||||
with self.pushd(self.external_files / branch / arch):
|
||||
for image in os.listdir():
|
||||
self.info(f'Copy external image {image} in {branch}')
|
||||
msg = f'Copy external file {image} in {branch}/{arch}'
|
||||
self.info(msg)
|
||||
self.copy_image(
|
||||
image,
|
||||
self.images_dir / branch / image,
|
||||
self.images_dir(branch, arch) / image,
|
||||
rewrite=True,
|
||||
)
|
||||
|
||||
@ -702,8 +766,8 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
self.error('Pass key to config file for sign')
|
||||
|
||||
sum_file = self.checksum_command.upper()
|
||||
for branch in self.branches:
|
||||
with self.pushd(self.images_dir / branch):
|
||||
for images_dir in self.images_dirs_list():
|
||||
with self.pushd(images_dir):
|
||||
files = [f
|
||||
for f in os.listdir()
|
||||
if not f.startswith(sum_file)]
|
||||
@ -736,13 +800,12 @@ Dir::Etc::preferencesparts "/var/empty";
|
||||
self.call(cmd(command))
|
||||
|
||||
def sync(self, create_remote_dirs: bool = False) -> None:
|
||||
for branch in self.branches:
|
||||
remote = self.remote(branch)
|
||||
for images_dir, remote in self.images_dirs_remotes_list():
|
||||
if create_remote_dirs:
|
||||
os.makedirs(remote, exist_ok=True)
|
||||
cmd = [
|
||||
'rsync',
|
||||
f'{self.images_dir}/{branch}/',
|
||||
f'{images_dir}/',
|
||||
'-rv',
|
||||
remote,
|
||||
]
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
from collections import defaultdict
|
||||
from contextlib import ExitStack
|
||||
from pathlib import Path
|
||||
from unittest import TestCase
|
||||
@ -6,25 +7,68 @@ from unittest import mock
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from parameterized import parameterized, parameterized_class # type: ignore
|
||||
|
||||
import yaml
|
||||
|
||||
from cloud_build import CB
|
||||
from tests.call import Call
|
||||
|
||||
|
||||
def change(origin, new, key, value):
|
||||
with open(origin) as f:
|
||||
cfg = yaml.safe_load(f)
|
||||
|
||||
with open(new, 'w') as f:
|
||||
yaml.safe_dump(cfg | {key: value}, f)
|
||||
|
||||
|
||||
def get_class_name(cls, num, params_dict):
|
||||
result = cls.__name__
|
||||
if branch := params_dict['branch']:
|
||||
result = f'{result}_{parameterized.to_safe_name(branch)}'
|
||||
if arch := params_dict['arch']:
|
||||
result = f'{result}_{parameterized.to_safe_name(arch)}'
|
||||
return f'{result}_{num}'
|
||||
|
||||
|
||||
@parameterized_class([
|
||||
{'branch': 'branch', 'arch': ''},
|
||||
{'branch': 'branch', 'arch': 'arch'},
|
||||
{'branch': '', 'arch': 'arch'},
|
||||
{'branch': '', 'arch': ''},
|
||||
], class_name_func=get_class_name)
|
||||
class TestIntegrationImages(TestCase):
|
||||
def setUp(self):
|
||||
self.images = self.__class__.images
|
||||
self._images = self.__class__._images
|
||||
self.images_dir = self.__class__.images_dir
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.work_dir = Path('/tmp/cloud-build')
|
||||
os.makedirs(cls.work_dir / 'external_files/p9', exist_ok=True)
|
||||
(cls.work_dir / 'external_files/p9/README').write_text('README')
|
||||
os.makedirs(cls.work_dir / 'external_files/p9/x86_64', exist_ok=True)
|
||||
(cls.work_dir / 'external_files/p9/x86_64/README').write_text('README')
|
||||
config = cls.work_dir / 'config.yaml'
|
||||
branch = cls.branch
|
||||
arch = cls.arch
|
||||
remote = Path('/tmp/cloud-build/images')
|
||||
if branch:
|
||||
remote = remote / '{branch}'
|
||||
if arch:
|
||||
remote = remote / '{arch}'
|
||||
remote = (remote / 'cloud').as_posix()
|
||||
change(
|
||||
'tests/test_integration_images.yaml',
|
||||
config,
|
||||
'remote',
|
||||
remote,
|
||||
)
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(mock.patch('subprocess.call', Call()))
|
||||
|
||||
cloud_build = CB(
|
||||
config='tests/test_integration_images.yaml',
|
||||
config=config,
|
||||
data_dir=(cls.work_dir / 'cloud_build').as_posix(),
|
||||
)
|
||||
cloud_build.create_images(no_tests=True)
|
||||
@ -33,9 +77,54 @@ class TestIntegrationImages(TestCase):
|
||||
cloud_build.sync(create_remote_dirs=True)
|
||||
|
||||
images_dir = cls.work_dir / 'images'
|
||||
cls.images = {}
|
||||
for branch in os.listdir(images_dir):
|
||||
cls.images[branch] = os.listdir(images_dir / branch / 'cloud')
|
||||
cls.images_dir = images_dir
|
||||
images = defaultdict(lambda: defaultdict(list))
|
||||
|
||||
if branch:
|
||||
for branch in os.listdir(images_dir):
|
||||
if arch:
|
||||
for arch in os.listdir(images_dir / branch):
|
||||
images[branch][arch] = os.listdir(
|
||||
images_dir / branch / arch / 'cloud'
|
||||
)
|
||||
else:
|
||||
images[branch]['arch'] = os.listdir(
|
||||
images_dir / branch / 'cloud'
|
||||
)
|
||||
elif arch:
|
||||
for arch in os.listdir(images_dir):
|
||||
images['branch'][arch] = os.listdir(
|
||||
images_dir / arch / 'cloud'
|
||||
)
|
||||
else:
|
||||
images['branch']['arch'] = os.listdir(
|
||||
images_dir / 'cloud'
|
||||
)
|
||||
cls._images = images
|
||||
|
||||
def image_path(self, branch, arch, image) -> Path:
|
||||
images_dir = self.images_dir
|
||||
if self.branch: # type: ignore
|
||||
images_dir = images_dir / branch
|
||||
if self.arch: # type: ignore
|
||||
images_dir = images_dir / arch
|
||||
|
||||
return images_dir / 'cloud' / image
|
||||
|
||||
def images(self, branch, arch) -> list:
|
||||
if not self.branch: # type: ignore
|
||||
branch = 'branch'
|
||||
if not self.arch: # type: ignore
|
||||
arch = 'arch'
|
||||
|
||||
return self._images[branch][arch]
|
||||
|
||||
def images_lists(self) -> list:
|
||||
result = []
|
||||
for branch_value in self._images.values():
|
||||
for arch_value in branch_value.values():
|
||||
result.append(arch_value)
|
||||
return result
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
@ -43,28 +132,45 @@ class TestIntegrationImages(TestCase):
|
||||
|
||||
def test_arch_ppc64le(self):
|
||||
self.assertIn('alt-p9-rootfs-minimal-ppc64le.tar.xz',
|
||||
self.images['p9'])
|
||||
self.images('p9', 'ppc64le'))
|
||||
|
||||
def test_branches(self):
|
||||
self.assertCountEqual(self.images.keys(), ['Sisyphus', 'p9', 'p8'])
|
||||
if self.branch:
|
||||
self.assertCountEqual(self._images.keys(),
|
||||
['Sisyphus', 'p9', 'p8'])
|
||||
else:
|
||||
self.assertCountEqual(self._images.keys(), ['branch'])
|
||||
|
||||
def test_build_cloud(self):
|
||||
self.assertIn('alt-p9-cloud-x86_64.qcow2', self.images['p9'])
|
||||
self.assertIn('alt-p9-cloud-x86_64.qcow2', self.images('p9', 'x86_64'))
|
||||
|
||||
def test_exclude_arches(self):
|
||||
self.assertNotIn('alt-p8-cloud-x86_64.qcow2', self.images['p8'])
|
||||
self.assertNotIn('alt-p8-cloud-x86_64.qcow2',
|
||||
self.images('p8', 'x86_64'))
|
||||
|
||||
def test_exclude_branches(self):
|
||||
self.assertNotIn('alt-p9-cloud-ppc64le.qcow2', self.images['p9'])
|
||||
self.assertNotIn('alt-p9-cloud-ppc64le.qcow2',
|
||||
self.images('p9', 'ppc64le'))
|
||||
|
||||
def test_external_files(self):
|
||||
self.assertIn('README', self.images['p9'])
|
||||
self.assertIn('README', self.images('p9', 'x86_64'))
|
||||
|
||||
def test_verification_files(self):
|
||||
for images in self.images.values():
|
||||
self.assertIn('SHA256SUMS', images)
|
||||
self.assertIn('SHA256SUMS.gpg', images)
|
||||
for images_list in self.images_lists():
|
||||
self.assertIn('SHA256SUMS', images_list)
|
||||
self.assertIn('SHA256SUMS.gpg', images_list)
|
||||
|
||||
number_of_images = len(self.image_path(
|
||||
'p9',
|
||||
'x86_64',
|
||||
'SHA256SUMS',
|
||||
).read_text().splitlines())
|
||||
index = bool(self.branch) * 2 + bool(self.arch)
|
||||
expected_numbers = [52, 17, 21, 7]
|
||||
self.assertEqual(number_of_images, expected_numbers[index])
|
||||
|
||||
def test_number_of_images(self):
|
||||
number_of_images = sum(len(self.images[b]) for b in self.images.keys())
|
||||
self.assertEqual(number_of_images, 64)
|
||||
number_of_images = sum(len(lst) for lst in self.images_lists())
|
||||
index = bool(self.branch) * 2 + bool(self.arch)
|
||||
expected_numbers = [56, 72, 64, 96]
|
||||
self.assertEqual(number_of_images, expected_numbers[index])
|
||||
|
||||
@ -55,7 +55,6 @@ class TestRebuild(TestCase):
|
||||
image = (
|
||||
self.data_dir
|
||||
/ 'images'
|
||||
/ 'Sisyphus'
|
||||
/ 'alt-sisyphus-rootfs-minimal-x86_64.tar.xz'
|
||||
)
|
||||
msg = 'Do not create image when rebuild'
|
||||
|
||||
Loading…
Reference in New Issue
Block a user