Add option to control time that chache lives

cloud_build/cloud_build.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python3
 
 from typing import Dict, List, Union
-
 from pathlib import Path
+
 import contextlib
 import datetime
 import fcntl
@@ -10,6 +10,7 @@ import logging
 import os
 import re
 import subprocess
+import time
 
 import yaml
 
@@ -75,7 +76,6 @@ class CB:
         self.work_dir = data_dir / 'work'
         self.out_dir = data_dir / 'out'
 
-        self.date = datetime.date.today().strftime('%Y%m%d')
         self.service_default_state = 'enabled'
         self.created_scripts: List[Path] = []
         self._build_errors: List[BuildError] = []
@@ -165,6 +165,17 @@ class CB:
         if self.external_files:
             self.external_files = self.expand_path(Path(self.external_files))
 
+        rebuild_after = cfg.get('rebuild_after', {'days': 1})
+        try:
+            self.rebuild_after = datetime.timedelta(**rebuild_after)
+        except TypeError as e:
+            m = re.match(r"'([^']+)'", str(e))
+            if m:
+                arg = m.groups()[0]
+                raise Error(f'Invalid key `{arg}` passed to rebuild_after')
+            else:
+                raise
+
         self._packages = cfg.get('packages', {})
         self._services = cfg.get('services', {})
         self._scripts = cfg.get('scripts', {})
@@ -464,6 +475,17 @@ Dir::Etc::preferencesparts "/var/empty";
         else:
             self.error(BuildError(target, arch))
 
+    def should_rebuild(self, tarball):
+        if not os.path.exists(tarball):
+            rebuild = True
+        else:
+            lived = time.time() - os.path.getmtime(tarball)
+            delta = datetime.timedelta(seconds=lived)
+            rebuild = delta > self.rebuild_after
+            if rebuild:
+                os.unlink(tarball)
+        return rebuild
+
     def build_tarball(
         self,
         target: str,
@@ -476,10 +498,11 @@ Dir::Etc::preferencesparts "/var/empty";
         target = f'{target}_{self.escape_branch(branch)}'
         image = re.sub(r'.*/', '', target)
         full_target = f'{target}.{kind}'
-        tarball = self.out_dir / f'{image}-{self.date}-{arch}.{kind}'
+        tarball_name = f'{image}-{arch}.{kind}'
+        tarball_path = self.out_dir / tarball_name
         apt_dir = self.work_dir / 'apt'
         with self.pushd(self.work_dir / 'mkimage-profiles'):
-            if tarball.exists():
+            if not self.should_rebuild(tarball_path):
                 self.info(f'Skip building of {full_target} {arch}')
             else:
                 cmd = [
@@ -487,17 +510,19 @@ Dir::Etc::preferencesparts "/var/empty";
                     f'APTCONF={apt_dir}/apt.conf.{branch}.{arch}',
                     f'ARCH={arch}',
                     f'IMAGE_OUTDIR={self.out_dir}',
+                    f'IMAGE_OUTFILE={tarball_name}',
                     full_target,
                 ]
                 self.info(f'Begin building of {full_target} {arch}')
                 self.call(cmd)
-                if os.path.exists(tarball):
+
+                if os.path.exists(tarball_path):
                     self.info(f'End building of {full_target} {arch}')
                 else:
                     self.build_failed(full_target, arch)
-                    tarball = None
+                    tarball_path = None
 
-        return tarball
+        return tarball_path
 
     def image_path(
         self,
@@ -525,7 +550,9 @@ Dir::Etc::preferencesparts "/var/empty";
     def remove_old_tarballs(self):
         with self.pushd(self.out_dir):
             for tb in os.listdir():
-                if not re.search(f'-{self.date}-', tb):
+                lived = time.time() - os.path.getmtime(tb)
+                delta = datetime.timedelta(seconds=lived)
+                if delta > self.rebuild_after:
                     os.unlink(tb)
 
     def ensure_scripts(self, image):

example-config.yaml

@@ -7,6 +7,12 @@ log_level: info
 bad_arches:
 - armh
 
+rebuild_after:
+  weeks: 0
+  days: 0
+  hours: 24
+  minutes: 10
+
 external_files: ~/external_files
 
 images:

tests/call.py

@@ -1,7 +1,6 @@
 from pathlib import Path
 from collections.abc import Iterable, Callable
 
-import datetime
 import os
 import random
 import re
@@ -157,8 +156,7 @@ def make(args):
 
     match = re.match(r'.*/([-\w]*)\.(.*)', args[-1])
     target, kind = match.groups()
-    date = datetime.date.today().strftime('%Y%m%d')
-    image = out_dir / f'{target}-{date}-{arch}.{kind}'
+    image = out_dir / f'{target}-{arch}.{kind}'
     image.write_bytes(
         bytes(random.randint(0, 255) for x in range(random.randint(32, 128)))
     )

tests/test_errors.py

@@ -96,3 +96,8 @@ class TestErrors(TestCase):
                 regex,
                 cloud_build.create_images
             )
+
+    def test_rebuild_after_format(self):
+        regex = 'years.*rebuild_after'
+        self.kwargs.update(config='tests/test_rebuild_after_format.yaml')
+        self.assertRaisesRegex(Error, regex, CB, **self.kwargs)

tests/test_rebuild.py

@@ -0,0 +1,50 @@
+from pathlib import Path
+from unittest import TestCase
+from unittest import mock
+
+import os
+import shutil
+import tempfile
+import time
+
+from cloud_build import CB
+from cloud_build import BuildError
+
+import tests.call as call
+
+
+DS = {'make': [call.return_d(0), call.nop_d]}
+
+
+class TestErrors(TestCase):
+    def setUp(self):
+        self.data_dir = Path(tempfile.mkdtemp(prefix='cloud_build'))
+        self.cb = CB(
+            config='tests/test_rebuild.yaml',
+            data_dir=self.data_dir,
+            no_tests=True,
+            create_remote_dirs=True,
+        )
+
+    def tearDown(self):
+        shutil.rmtree(self.data_dir)
+
+    @mock.patch('subprocess.call', call.Call(decorators=DS))
+    def test_do_rebuild(self):
+        tarball = self.data_dir / 'out/docker_Sisyphus-x86_64.tar.xz'
+        tarball.touch()
+        two_hours_ago = time.time() - 2*60*60
+        os.utime(tarball, times=(two_hours_ago, two_hours_ago))
+        msg = 'Do not try to rebuild with outdated cache'
+        with self.assertRaises(BuildError, msg=msg):
+            self.cb.create_images()
+
+    @mock.patch('subprocess.call', call.Call(decorators=DS))
+    def test_dont_rebuild(self):
+        tarball = self.data_dir / 'out/docker_Sisyphus-x86_64.tar.xz'
+        tarball.touch()
+        msg = 'Try to rebuild with valid cache'
+        try:
+            self.cb.create_images()
+        except BuildError:
+            self.fail(msg)

tests/test_rebuild.yaml

@@ -0,0 +1,18 @@
+---
+remote: '/var/empty'
+key: 0x00000000
+
+rebuild_after:
+  hours: 1
+
+images:
+  rootfs-minimal:
+    target: ve/docker
+    kinds:
+    - tar.xz
+
+branches:
+  Sisyphus:
+    arches:
+      x86_64:
+...

tests/test_rebuild_after_format.yaml

@@ -0,0 +1,19 @@
+---
+remote: '/var/empty'
+key: 0x00000000
+
+rebuild_after:
+  days: 3
+  years: 10
+
+images:
+  rootfs-minimal:
+    target: ve/docker
+    kinds:
+    - tar.xz
+
+branches:
+  Sisyphus:
+    arches:
+      x86_64:
+...