From 29b230107ea0db881255e7cc551de5215eb11d59 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 1 May 2012 20:51:29 -0400 Subject: [PATCH] ostbuild: Break out git-mirror from resolve Other bits --- Makefile-ostbuild.am | 1 + .../pyostbuild/builtin_build_components.py | 1 + src/ostbuild/pyostbuild/builtin_git_mirror.py | 63 ++++++++++++++++ src/ostbuild/pyostbuild/builtin_resolve.py | 71 ++----------------- src/ostbuild/pyostbuild/main.py | 1 + src/ostbuild/pyostbuild/vcs.py | 63 +++++++++++++++- 6 files changed, 134 insertions(+), 66 deletions(-) create mode 100755 src/ostbuild/pyostbuild/builtin_git_mirror.py diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 15b53c93..40695582 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -30,6 +30,7 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ src/ostbuild/pyostbuild/builtin_compile_one.py \ src/ostbuild/pyostbuild/builtin_pull_components.py \ + src/ostbuild/pyostbuild/builtin_git_mirror.py \ src/ostbuild/pyostbuild/builtin_prefix.py \ src/ostbuild/pyostbuild/builtin_resolve.py \ src/ostbuild/pyostbuild/builtin_modify_snapshot.py \ diff --git a/src/ostbuild/pyostbuild/builtin_build_components.py b/src/ostbuild/pyostbuild/builtin_build_components.py index af1ba7c4..6adcbc7c 100755 --- a/src/ostbuild/pyostbuild/builtin_build_components.py +++ b/src/ostbuild/pyostbuild/builtin_build_components.py @@ -96,6 +96,7 @@ class OstbuildBuildComponents(builtins.Builtin): log("No previous build for '%s' found" % (name, )) checkoutdir = os.path.join(self.workdir, 'checkouts') + fileutil.ensure_dir(checkoutdir) component_src = os.path.join(checkoutdir, basename) run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path, '--clean', '--overwrite', basename], cwd=checkoutdir) diff --git a/src/ostbuild/pyostbuild/builtin_git_mirror.py b/src/ostbuild/pyostbuild/builtin_git_mirror.py new file mode 100755 index 00000000..5d827de2 --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_git_mirror.py @@ -0,0 +1,63 @@ +# Copyright (C) 2011,2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# ostbuild-compile-one-make wraps systems that implement the GNOME build API: +# http://people.gnome.org/~walters/docs/build-api.txt + +import os,sys,stat,subprocess,tempfile,re,shutil +import argparse +from StringIO import StringIO +import json + +from . import builtins +from .ostbuildlog import log, fatal +from . import vcs +from .subprocess_helpers import run_sync, run_sync_get_output +from . import buildutil + +class OstbuildGitMirror(builtins.Builtin): + name = "git-mirror" + short_description = "Update internal git mirror for one or more components" + + def __init__(self): + builtins.Builtin.__init__(self) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--prefix') + parser.add_argument('--src-snapshot') + parser.add_argument('--fetch', action='store_true') + parser.add_argument('components', nargs='*') + + args = parser.parse_args(argv) + self.parse_config() + self.parse_snapshot(args.prefix, args.src_snapshot) + + if len(args.components) == 0: + components = self.snapshot['components'].keys() + else: + components = args.components + for name in components: + component = self.snapshot['components'][name] + src = component['src'] + (keytype, uri) = vcs.parse_src_key(src) + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch']) + + if args.fetch: + run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) + +builtins.register(OstbuildGitMirror) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 2c39b051..96116be4 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -27,6 +27,7 @@ from . import builtins from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output from . import ostbuildrc +from . import vcs from . import jsondb from . import buildutil from . import kvfile @@ -39,64 +40,6 @@ class OstbuildResolve(builtins.Builtin): def __init__(self): builtins.Builtin.__init__(self) - def _ensure_vcs_mirror(self, name, keytype, uri, branch): - # FIXME - remove "name" from parameter list here - hash uri? - mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri) - tmp_mirror = mirror + '.tmp' - if os.path.isdir(tmp_mirror): - shutil.rmtree(tmp_mirror) - if not os.path.isdir(mirror): - run_sync(['git', 'clone', '--mirror', uri, tmp_mirror]) - run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror) - os.rename(tmp_mirror, mirror) - if branch is None: - return mirror - last_fetch_path = mirror + '.%s-lastfetch' % (name, ) - if os.path.exists(last_fetch_path): - f = open(last_fetch_path) - last_fetch_contents = f.read() - f.close() - last_fetch_contents = last_fetch_contents.strip() - else: - last_fetch_contents = None - current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) - current_vcs_version = current_vcs_version.strip() - if current_vcs_version != last_fetch_contents: - log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version)) - tmp_checkout = buildutil.get_mirrordir(self.mirrordir, keytype, uri, prefix='_tmp-checkouts') - if os.path.isdir(tmp_checkout): - shutil.rmtree(tmp_checkout) - parent = os.path.dirname(tmp_checkout) - if not os.path.isdir(parent): - os.makedirs(parent) - run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout]) - run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout) - submodules = [] - submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout) - submodule_status_lines = submodules_status_text.split('\n') - for line in submodule_status_lines: - if line == '': continue - line = line[1:] - (sub_checksum, sub_name) = line.split(' ', 1) - sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', - 'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout) - self._ensure_vcs_mirror(name + '-' + sub_name, keytype, sub_url, sub_checksum) - shutil.rmtree(tmp_checkout) - f = open(last_fetch_path, 'w') - f.write(current_vcs_version + '\n') - f.close() - return mirror - - def _parse_src_key(self, srckey): - idx = srckey.find(':') - if idx < 0: - raise ValueError("Invalid SRC uri=%s" % (srckey, )) - keytype = srckey[:idx] - if keytype not in ['git']: - raise ValueError("Unsupported SRC uri=%s" % (srckey, )) - uri = srckey[idx+1:] - return (keytype, uri) - def _resolve_component_meta(self, component_meta): result = dict(component_meta) orig_src = component_meta['src'] @@ -160,29 +103,29 @@ class OstbuildResolve(builtins.Builtin): break if not found: fatal("Unknown component %r" % (component_name, )) - (keytype, uri) = self._parse_src_key(component['src']) - mirrordir = self._ensure_vcs_mirror(component_name, keytype, uri, None) + (keytype, uri) = vcs.parse_src_key(component['src']) + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, None) log("Running git fetch for %s" % (component['name'], )) run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) else: fetch_components = [] global_patches_meta = self._resolve_component_meta(self.manifest['patches']) - (keytype, uri) = self._parse_src_key(global_patches_meta['src']) - mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch']) + (keytype, uri) = vcs.parse_src_key(global_patches_meta['src']) + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch']) revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch']) global_patches_meta['revision'] = revision unique_component_names = set() for component in component_source_list: - (keytype, uri) = self._parse_src_key(component['src']) + (keytype, uri) = vcs.parse_src_key(component['src']) name = component['name'] if name in unique_component_names: fatal("Duplicate component name '%s'" % (name, )) unique_component_names.add(name) - mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch']) + mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch']) revision = buildutil.get_git_version_describe(mirrordir, component['branch']) component['revision'] = revision diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index 507cb1ef..8437769c 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -29,6 +29,7 @@ from . import builtin_checkout from . import builtin_chroot_compile_one from . import builtin_compose from . import builtin_compile_one +from . import builtin_git_mirror from . import builtin_pull_components from . import builtin_prefix from . import builtin_resolve diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py index 7811f5ee..43adbbdd 100755 --- a/src/ostbuild/pyostbuild/vcs.py +++ b/src/ostbuild/pyostbuild/vcs.py @@ -21,6 +21,8 @@ import urlparse import shutil from .subprocess_helpers import run_sync_get_output, run_sync +from . import buildutil +from .ostbuildlog import log, fatal def get_mirrordir(mirrordir, keytype, uri, prefix=''): assert keytype == 'git' @@ -61,10 +63,10 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): if not os.path.isdir(tmp_dest): run_sync(['git', 'clone', '-q', '--origin', 'localmirror', '--no-checkout', module_mirror, tmp_dest]) + run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest) else: - run_sync(['git', 'fetch'], cwd=tmp_dest) + run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest) run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest) - run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest) run_sync(['git', 'submodule', 'init'], cwd=tmp_dest) have_submodules = _fixup_submodule_references(mirrordir, tmp_dest) if have_submodules: @@ -76,3 +78,60 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): def clean(keytype, checkoutdir): assert keytype in ('git', 'dirty-git') run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir) + +def parse_src_key(srckey): + idx = srckey.find(':') + if idx < 0: + raise ValueError("Invalid SRC uri=%s" % (srckey, )) + keytype = srckey[:idx] + if keytype not in ['git']: + raise ValueError("Unsupported SRC uri=%s" % (srckey, )) + uri = srckey[idx+1:] + return (keytype, uri) + +def ensure_vcs_mirror(mirrordir, keytype, uri, branch): + mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) + tmp_mirror = mirror + '.tmp' + if os.path.isdir(tmp_mirror): + shutil.rmtree(tmp_mirror) + if not os.path.isdir(mirror): + run_sync(['git', 'clone', '--mirror', uri, tmp_mirror]) + run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror) + os.rename(tmp_mirror, mirror) + if branch is None: + return mirror + last_fetch_path = mirror + '.lastfetch' + if os.path.exists(last_fetch_path): + f = open(last_fetch_path) + last_fetch_contents = f.read() + f.close() + last_fetch_contents = last_fetch_contents.strip() + else: + last_fetch_contents = None + current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) + current_vcs_version = current_vcs_version.strip() + if current_vcs_version != last_fetch_contents: + log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version)) + tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts') + if os.path.isdir(tmp_checkout): + shutil.rmtree(tmp_checkout) + parent = os.path.dirname(tmp_checkout) + if not os.path.isdir(parent): + os.makedirs(parent) + run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout]) + run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout) + submodules = [] + submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout) + submodule_status_lines = submodules_status_text.split('\n') + for line in submodule_status_lines: + if line == '': continue + line = line[1:] + (sub_checksum, sub_name) = line.split(' ', 1) + sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', + 'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout) + ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum) + shutil.rmtree(tmp_checkout) + f = open(last_fetch_path, 'w') + f.write(current_vcs_version + '\n') + f.close() + return mirror