From f4aba2a3390547993a32fadf2c94ccdb32ab99f7 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Sun, 13 May 2012 15:53:55 -0400 Subject: [PATCH] ostbuild: Tighten build process significantly First, "resolve" now just picks git commits. We don't expand config-opts and patches, nor do we generate tree contents. This makes the generated files *much* more human editable. Next, fold "build-components" and "compose" into just "build". One never really wants to just build components. This lets us eliminate binary snapshots as a concept; instead we always have a combination of source snapshot and component/ refs. --- Makefile-ostbuild.am | 6 +- ...n_build_components.py => builtin_build.py} | 277 ++++++++++-------- src/ostbuild/pyostbuild/builtin_checkout.py | 8 +- .../pyostbuild/builtin_chroot_compile_one.py | 48 ++- src/ostbuild/pyostbuild/builtin_compose.py | 113 ------- ..._tree_to_src.py => builtin_import_tree.py} | 8 +- .../pyostbuild/builtin_modify_snapshot.py | 63 ---- src/ostbuild/pyostbuild/builtin_resolve.py | 117 +------- src/ostbuild/pyostbuild/builtins.py | 39 ++- src/ostbuild/pyostbuild/main.py | 5 +- 10 files changed, 212 insertions(+), 472 deletions(-) rename src/ostbuild/pyostbuild/{builtin_build_components.py => builtin_build.py} (52%) delete mode 100755 src/ostbuild/pyostbuild/builtin_compose.py rename src/ostbuild/pyostbuild/{builtin_tree_to_src.py => builtin_import_tree.py} (92%) delete mode 100755 src/ostbuild/pyostbuild/builtin_modify_snapshot.py diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 7a0f52be..dd1b2c73 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -24,20 +24,18 @@ pyostbuilddir=$(libdir)/ostbuild/pyostbuild pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/buildutil.py \ src/ostbuild/pyostbuild/builtin_branch_prefix.py \ - src/ostbuild/pyostbuild/builtin_build_components.py \ + src/ostbuild/pyostbuild/builtin_build.py \ src/ostbuild/pyostbuild/builtin_checkout.py \ - src/ostbuild/pyostbuild/builtin_compose.py \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ src/ostbuild/pyostbuild/builtin_compile_one.py \ src/ostbuild/pyostbuild/builtin_deploy_qemu.py \ src/ostbuild/pyostbuild/builtin_deploy_root.py \ + src/ostbuild/pyostbuild/builtin_import_tree.py \ src/ostbuild/pyostbuild/builtin_pull_components.py \ src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \ src/ostbuild/pyostbuild/builtin_git_mirror.py \ src/ostbuild/pyostbuild/builtin_prefix.py \ src/ostbuild/pyostbuild/builtin_resolve.py \ - src/ostbuild/pyostbuild/builtin_modify_snapshot.py \ - src/ostbuild/pyostbuild/builtin_tree_to_src.py \ src/ostbuild/pyostbuild/builtin_init.py \ src/ostbuild/pyostbuild/builtin_status.py \ src/ostbuild/pyostbuild/builtins.py \ diff --git a/src/ostbuild/pyostbuild/builtin_build_components.py b/src/ostbuild/pyostbuild/builtin_build.py similarity index 52% rename from src/ostbuild/pyostbuild/builtin_build_components.py rename to src/ostbuild/pyostbuild/builtin_build.py index b1b01024..1dd69dc0 100755 --- a/src/ostbuild/pyostbuild/builtin_build_components.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -37,9 +37,9 @@ from . import vcs class BuildOptions(object): pass -class OstbuildBuildComponents(builtins.Builtin): - name = "build-components" - short_description = "Build multiple components from given source snapshot" +class OstbuildBuild(builtins.Builtin): + name = "build" + short_description = "Build multiple components and generate trees" def __init__(self): builtins.Builtin.__init__(self) @@ -62,19 +62,20 @@ class OstbuildBuildComponents(builtins.Builtin): run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True) fatal("Exiting after debug shell") - def _build_one_component(self, basename, component, architecture): + def _build_one_component(self, component, architecture): + basename = component['name'] branch = component['branch'] - name = '%s/%s' % (basename, architecture) - buildname = 'components/%s' % (name, ) + archname = '%s/%s' % (basename, architecture) + buildname = 'components/%s' % (archname, ) current_vcs_version = component.get('revision') + expanded_component = self.expand_component(component) + # TODO - deduplicate this with chroot_compile_one current_meta_io = StringIO() - meta_copy = dict(component) - meta_copy['name'] = basename # Note we have to match the name here - json.dump(meta_copy, current_meta_io, indent=4, sort_keys=True) + json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True) current_metadata_text = current_meta_io.getvalue() sha = hashlib.sha256() sha.update(current_metadata_text) @@ -90,7 +91,7 @@ class OstbuildBuildComponents(builtins.Builtin): none_on_error=True) if (current_vcs_version is not None and previous_build_version is not None): - log("Previous build of '%s' is %s" % (name, previous_build_version)) + log("Previous build of '%s' is %s" % (archname, previous_build_version)) previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo, 'cat', previous_build_version, @@ -102,33 +103,33 @@ class OstbuildBuildComponents(builtins.Builtin): if current_meta_digest == previous_meta_digest: log("Metadata is unchanged from previous") - return False + return previous_build_version else: previous_metadata = json.loads(previous_metadata_text) previous_vcs_version = previous_metadata['revision'] if current_vcs_version == previous_vcs_version: log("Metadata differs; VCS version unchanged") if self.buildopts.skip_vcs_matches: - return False - for k,v in meta_copy.iteritems(): + return previous_build_version + for k,v in expanded_component.iteritems(): previous_v = previous_metadata.get(k) if v != previous_v: log("Key %r differs: old: %r new: %r" % (k, previous_v, v)) else: log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) else: - log("No previous build for '%s' found" % (name, )) + log("No previous build for '%s' found" % (archname, )) checkoutdir = os.path.join(self.workdir, 'checkouts') - fileutil.ensure_dir(checkoutdir) - component_src = os.path.join(checkoutdir, basename) + component_src = os.path.join(checkoutdir, archname) + fileutil.ensure_parent_dir(component_src) run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path, '--checkoutdir=' + component_src, '--clean', '--overwrite', basename]) artifact_meta = dict(component) - logdir = os.path.join(self.workdir, 'logs', name) + logdir = os.path.join(self.workdir, 'logs', archname) fileutil.ensure_dir(logdir) log_path = os.path.join(logdir, 'compile.log') if os.path.isfile(log_path): @@ -139,7 +140,7 @@ class OstbuildBuildComponents(builtins.Builtin): log("Logging to %s" % (log_path, )) f = open(log_path, 'w') chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture]) + chroot_args.extend(['--name=' + basename, '--arch=' + architecture]) if self.buildopts.shell_on_failure: ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -162,76 +163,87 @@ class OstbuildBuildComponents(builtins.Builtin): f.close() args.append('--statoverride=' + statoverride_path) - component_resultdir = os.path.join(self.workdir, 'results', name) + component_resultdir = os.path.join(self.workdir, 'results', archname) run_sync(args, cwd=component_resultdir) if statoverride_path is not None: os.unlink(statoverride_path) - return True - def _resolve_refs(self, refs): - args = ['ostree', '--repo=' + self.repo, 'rev-parse'] - args.extend(refs) - output = run_sync_get_output(args) - return output.split('\n') + return run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', buildname]) - def _save_bin_snapshot(self, components, component_architectures): - bin_snapshot = dict(self.snapshot) + def _compose_one_target(self, target, component_build_revs): + base = target['base'] + base_name = 'bases/%s' % (base['name'], ) - del bin_snapshot['00ostree-src-snapshot-version'] - bin_snapshot['00ostree-bin-snapshot-version'] = 1 + compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) + if os.path.isdir(compose_rootdir): + shutil.rmtree(compose_rootdir) + os.mkdir(compose_rootdir) - for target in bin_snapshot['targets']: - base = target['base'] - base_name = 'bases/%s' % (base['name'], ) - if 'ostree-revision' not in target: - base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', base_name]) - base['ostree-revision'] = base_revision + base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', base_name]) - if 'architecture-buildroots2' in bin_snapshot: - for arch,buildroot in bin_snapshot['architecture-buildroots2'].iteritems(): - name = buildroot['name'] - if 'ostree-revision' not in buildroot: - rev = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', name]) - buildroot['ostree-revision'] = rev + compose_contents = [(base_name, base_revision, '/')] + for tree_content in target['contents']: + name = tree_content['name'] + rev = component_build_revs[name] + subtrees = tree_content['trees'] + for subpath in subtrees: + compose_contents.append((name, rev, subpath)) - component_revisions = bin_snapshot.get('component-revisions', {}) + (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') + related_f = os.fdopen(related_fd, 'w') + resolved_refs = {} + for (name, branch, subpath) in compose_contents: + resolved_refs[name] = branch + for (name, rev) in resolved_refs.iteritems(): + related_f.write(name) + related_f.write(' ') + related_f.write(rev) + related_f.write('\n') + related_f.close() - component_refs_to_resolve = [] - for name in components.iterkeys(): - for architecture in component_architectures[name]: - archname = '%s/%s' % (name, architecture) - if archname not in component_revisions: - component_refs_to_resolve.append('components/' + archname) + (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') + contents_f = os.fdopen(contents_fd, 'w') + for (name, branch, subpath) in compose_contents: + contents_f.write(branch) + contents_f.write('\0') + contents_f.write(subpath) + contents_f.write('\0') + contents_f.close() - if len(component_refs_to_resolve) > 0: - resolved_refs = self._resolve_refs(component_refs_to_resolve) - for name,rev in zip(component_refs_to_resolve, resolved_refs): - assert name.startswith('components/') - archname = name[len('components/'):] - component_revisions[archname] = rev + run_sync(['ostree', '--repo=' + self.repo, + 'checkout', '--user-mode', '--no-triggers', '--union', + '--from-file=' + contents_tmppath, compose_rootdir]) + os.unlink(contents_tmppath) - bin_snapshot['component-revisions'] = component_revisions + contents_path = os.path.join(compose_rootdir, 'contents.json') + f = open(contents_path, 'w') + json.dump(self.snapshot, f, indent=4, sort_keys=True) + f.close() - path = self.get_bin_snapshot_db().store(bin_snapshot) - log("Binary snapshot: %s" % (path, )) + treename = 'trees/%s' % (target['name'], ) + + run_sync(['ostree', '--repo=' + self.repo, + 'commit', '-b', treename, '-s', 'Compose', + '--owner-uid=0', '--owner-gid=0', '--no-xattrs', + '--related-objects-file=' + related_tmppath, + '--skip-if-unchanged'], cwd=compose_rootdir) + os.unlink(related_tmppath) + shutil.rmtree(compose_rootdir) def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--force-rebuild', action='store_true') - parser.add_argument('--skip-vcs-matches', action='store_true') parser.add_argument('--prefix') parser.add_argument('--src-snapshot') - parser.add_argument('--compose', action='store_true') + parser.add_argument('--force-rebuild', action='store_true') + parser.add_argument('--skip-vcs-matches', action='store_true') + parser.add_argument('--no-compose', action='store_true') parser.add_argument('--compose-only', action='store_true') - parser.add_argument('--start-at') parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') parser.add_argument('components', nargs='*') - - args = parser.parse_args(argv) self.args = args @@ -239,15 +251,7 @@ class OstbuildBuildComponents(builtins.Builtin): self.parse_config() self.parse_snapshot(args.prefix, args.src_snapshot) - component_revisions = self.snapshot.get('component-revisions', {}) - - if component_revisions is not None: - snapshot_type = "source+binary" - else: - snapshot_type = "source" - - log("Using %s snapshot: %s" % (snapshot_type, - os.path.basename(self.snapshot_path), )) + log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), )) self.buildopts = BuildOptions() self.buildopts.shell_on_failure = args.shell_on_failure @@ -256,68 +260,87 @@ class OstbuildBuildComponents(builtins.Builtin): self.force_build_components = set() - required_components = {} - component_architectures = {} - for target in self.snapshot['targets']: - for tree_content in target['contents']: - (name, arch) = tree_content['name'].rsplit('/', 1) - required_components[name] = self.snapshot['components'][name] - if name not in component_architectures: - component_architectures[name] = set([arch]) - else: - component_architectures[name].add(arch) + components = self.snapshot['components'] - build_component_order = [] - if len(args.components) == 0: - tsorted = buildutil.tsort_components(required_components, 'build-depends') - tsorted.reverse() - build_component_order = tsorted - else: - if args.start_at is not None: - fatal("Can't specify --start-at with component list") - for name in args.components: - found = False - component = self.snapshot['components'].get(name) - if component is None: - fatal("Unknown component %r" % (name, )) - build_component_order.append(name) - self.force_build_components.add(name) + prefix = self.snapshot['prefix'] + base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix) - start_at_index = -1 - if args.start_at is not None: - for i,component_name in enumerate(build_component_order): - if component_name == args.start_at: - start_at_index = i - break - if start_at_index == -1: - fatal("Unknown component %r specified for --start-at" % (args.start_at, )) - else: - start_at_index = 0 + architectures = self.snapshot['architectures'] + + component_to_arches = {} + + runtime_components = [] + devel_components = [] + + for component in components: + name = component['name'] + + is_runtime = component.get('component', 'runtime') == 'runtime' + + if is_runtime: + runtime_components.append(component) + devel_components.append(component) + + is_noarch = component.get('noarch', False) + if is_noarch: + # Just use the first specified architecture + component_arches = [architectures[0]] + else: + component_arches = component.get('architectures', architectures) + component_to_arches[name] = component_arches + + for name in args.components: + component = components.get(name) + if component is None: + fatal("Unknown component %r" % (name, )) + self.force_build_components.add(name) components_to_build = [] component_skipped_count = 0 + component_build_revs = {} + if not args.compose_only: - for component_name in build_component_order[start_at_index:]: - component = required_components[component_name] - architectures = component_architectures[component_name] + for component in components: for architecture in architectures: - archname = '%s/%s' % (component_name, architecture) - if (component_revisions is not None and - archname in component_revisions): - component_skipped_count += 1 - else: - components_to_build.append((component_name, component, architecture)) + components_to_build.append((component, architecture)) log("%d components to build" % (len(components_to_build), )) - if component_skipped_count > 0: - log("%d components skipped due to existing component-revisions" % (component_skipped_count, )) - for (component_name, component, architecture) in components_to_build: - self._build_one_component(component_name, component, architecture) + for (component, architecture) in components_to_build: + archname = '%s/%s' % (component['name'], architecture) + build_rev = self._build_one_component(component, architecture) + component_build_revs[archname] = build_rev - self._save_bin_snapshot(required_components, component_architectures) + targets_list = [] + for target_component_type in ['runtime', 'devel']: + for architecture in architectures: + target = {} + targets_list.append(target) + target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type) - if args.compose or args.compose_only: - run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix]) - -builtins.register(OstbuildBuildComponents) + base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type) + target['base'] = {'name': base_ref} + + if target_component_type == 'runtime': + target_components = runtime_components + else: + target_components = devel_components + + contents = [] + for component in target_components: + builds_for_component = component_to_arches[component['name']] + if architecture not in builds_for_component: + continue + binary_name = '%s/%s' % (component['name'], architecture) + component_ref = {'name': binary_name} + if target_component_type == 'runtime': + component_ref['trees'] = ['/runtime'] + else: + component_ref['trees'] = ['/runtime', '/devel', '/doc'] + contents.append(component_ref) + target['contents'] = contents + + for target in targets_list: + self._compose_one_target(target, component_build_revs) + +builtins.register(OstbuildBuild) diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py index e9c11427..93415488 100755 --- a/src/ostbuild/pyostbuild/builtin_checkout.py +++ b/src/ostbuild/pyostbuild/builtin_checkout.py @@ -60,7 +60,7 @@ class OstbuildCheckout(builtins.Builtin): component_name = args.component found = False - component = self.get_component_meta(component_name) + component = self.get_expanded_component(component_name) (keytype, uri) = buildutil.parse_src_key(component['src']) is_local = (keytype == 'local') @@ -82,9 +82,9 @@ class OstbuildCheckout(builtins.Builtin): else: checkoutdir = os.path.join(os.getcwd(), component_name) fileutil.ensure_parent_dir(checkoutdir) - vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, - component['revision'], - overwrite=args.overwrite) + vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, + component['revision'], + overwrite=args.overwrite) if args.clean: if is_local: diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 463fc934..9edab0cb 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -34,6 +34,8 @@ class OstbuildChrootCompileOne(builtins.Builtin): short_description = "Build artifacts from the current source directory in a chroot" def _resolve_refs(self, refs): + if len(refs) == 0: + return [] args = ['ostree', '--repo=' + self.repo, 'rev-parse'] args.extend(refs) output = run_sync_get_output(args) @@ -52,32 +54,27 @@ class OstbuildChrootCompileOne(builtins.Builtin): shutil.rmtree(rootdir_tmp) components = self.snapshot['components'] - dependencies = buildutil.build_depends(component_name, components) - component = components.get(component_name) + component = None + build_dependencies = [] + for component in components: + if component['name'] == component_name: + break + build_dependencies.append(component) ref_to_rev = {} - arch_buildroot_name = None - arch_buildroot_rev = None - if 'architecture-buildroots2' in self.snapshot: - buildroots = self.snapshot['architecture-buildroots2'] - arch_buildroot = buildroots[architecture] - arch_buildroot_name = arch_buildroot['name'] - arch_buildroot_rev = arch_buildroot.get('ostree-revision') - else: - buildroots = self.snapshot['architecture-buildroots'] - arch_rev_suffix = buildroots[architecture] - arch_buildroot_name = 'bases/' + arch_rev_suffix + arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'], + self.snapshot['prefix'], + architecture) - if arch_buildroot_rev is None: - arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', - arch_buildroot_name]).strip() + arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', + arch_buildroot_name]).strip() ref_to_rev[arch_buildroot_name] = arch_buildroot_rev checkout_trees = [(arch_buildroot_name, '/')] refs_to_resolve = [] - for dependency_name in dependencies: - buildname = 'components/%s/%s' % (dependency_name, architecture) + for dependency in build_dependencies: + buildname = 'components/%s/%s' % (dependency['name'], architecture) refs_to_resolve.append(buildname) checkout_trees.append((buildname, '/runtime')) checkout_trees.append((buildname, '/devel')) @@ -152,7 +149,6 @@ class OstbuildChrootCompileOne(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--pristine', action='store_true') parser.add_argument('--prefix') parser.add_argument('--snapshot', required=True) parser.add_argument('--name') @@ -169,15 +165,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): else: component_name = self.get_component_from_cwd() - components = self.snapshot['components'] - component = components.get(component_name) - if component is None: - fatal("Couldn't find component '%s' in manifest" % (component_name, )) - self.metadata = dict(component) - self.metadata['name'] = component_name - if not args.pristine: - self.metadata['src'] = 'dirty:worktree' - self.metadata['revision'] = 'dirty-worktree' + component = self.get_expanded_component(component_name) workdir = self.workdir @@ -202,7 +190,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): fileutil.ensure_dir(sourcedir) output_metadata = open('_ostbuild-meta.json', 'w') - json.dump(self.metadata, output_metadata, indent=4, sort_keys=True) + json.dump(component, output_metadata, indent=4, sort_keys=True) output_metadata.close() chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name) @@ -229,7 +217,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json') recorded_meta_f = open(recorded_meta_path, 'w') - json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True) + json.dump(component, recorded_meta_f, indent=4, sort_keys=True) recorded_meta_f.close() builtins.register(OstbuildChrootCompileOne) diff --git a/src/ostbuild/pyostbuild/builtin_compose.py b/src/ostbuild/pyostbuild/builtin_compose.py deleted file mode 100755 index 6886bfc6..00000000 --- a/src/ostbuild/pyostbuild/builtin_compose.py +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from .subprocess_helpers import run_sync_monitor_log_file -from . import ostbuildrc -from . import buildutil -from . import fileutil -from . import kvfile -from . import odict -from . import vcs - -class OstbuildCompose(builtins.Builtin): - name = "compose" - short_description = "Build complete trees from components" - - def __init__(self): - builtins.Builtin.__init__(self) - - def _compose_one_target(self, bin_snapshot, target): - components = bin_snapshot['component-revisions'] - base = target['base'] - base_name = 'bases/%s' % (base['name'], ) - base_revision = target['base']['ostree-revision'] - - compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) - if os.path.isdir(compose_rootdir): - shutil.rmtree(compose_rootdir) - os.mkdir(compose_rootdir) - - compose_contents = [(base_revision, '/')] - for tree_content in target['contents']: - name = tree_content['name'] - rev = components[name] - subtrees = tree_content['trees'] - for subpath in subtrees: - compose_contents.append((rev, subpath)) - - (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - related_f = os.fdopen(related_fd, 'w') - contents_f = os.fdopen(contents_fd, 'w') - for (branch, subpath) in compose_contents: - related_f.write(' ') - related_f.write(branch) - related_f.write('\n') - contents_f.write(branch) - contents_f.write('\0') - contents_f.write(subpath) - contents_f.write('\0') - related_f.close() - contents_f.close() - - run_sync(['ostree', '--repo=' + self.repo, - 'checkout', '--user-mode', '--no-triggers', '--union', - '--from-file=' + contents_tmppath, compose_rootdir]) - os.unlink(contents_tmppath) - - contents_path = os.path.join(compose_rootdir, 'contents.json') - f = open(contents_path, 'w') - json.dump(bin_snapshot, f, indent=4, sort_keys=True) - f.close() - - run_sync(['ostree', '--repo=' + self.repo, - 'commit', '-b', target['name'], '-s', 'Compose', - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--related-objects-file=' + related_tmppath, - '--skip-if-unchanged'], cwd=compose_rootdir) - os.unlink(related_tmppath) - shutil.rmtree(compose_rootdir) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--bin-snapshot') - - args = parser.parse_args(argv) - self.args = args - - self.parse_config() - self.parse_bin_snapshot(args.prefix, args.bin_snapshot) - - log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), )) - - for target in self.bin_snapshot['targets']: - log("Composing target %r from %u components" % (target['name'], - len(target['contents']))) - self._compose_one_target(self.bin_snapshot, target) - -builtins.register(OstbuildCompose) diff --git a/src/ostbuild/pyostbuild/builtin_tree_to_src.py b/src/ostbuild/pyostbuild/builtin_import_tree.py similarity index 92% rename from src/ostbuild/pyostbuild/builtin_tree_to_src.py rename to src/ostbuild/pyostbuild/builtin_import_tree.py index b2df3b1a..b24b0ae9 100755 --- a/src/ostbuild/pyostbuild/builtin_tree_to_src.py +++ b/src/ostbuild/pyostbuild/builtin_import_tree.py @@ -28,9 +28,9 @@ from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output from . import buildutil -class OstbuildTreeToSrc(builtins.Builtin): - name = "tree-to-src" - short_description = "Turn a tree into a source snapshot" +class OstbuildImportTree(builtins.Builtin): + name = "import-tree" + short_description = "Extract source data from tree" def __init__(self): builtins.Builtin.__init__(self) @@ -63,4 +63,4 @@ class OstbuildTreeToSrc(builtins.Builtin): path = db.store(snapshot) log("Source snapshot: %s" % (path, )) -builtins.register(OstbuildTreeToSrc) +builtins.register(OstbuildImportTree) diff --git a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py deleted file mode 100755 index b438a9ae..00000000 --- a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,stat,subprocess,tempfile,re,shutil -from StringIO import StringIO -import json -import select,time -import argparse - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output - -class OstbuildModifySnapshot(builtins.Builtin): - name = "modify-snapshot" - short_description = "Change the current source snapshot" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--src-snapshot') - - args = parser.parse_args(argv) - - self.parse_config() - self.parse_snapshot(args.prefix, args.src_snapshot) - - component_name = self.get_component_from_cwd() - current_meta = self.get_component_meta(component_name) - - new_snapshot = dict(self.snapshot) - new_meta = dict(current_meta) - if 'patches' in new_meta: - del new_meta['patches'] - new_meta['src'] = "dirty-git:%s" % (os.getcwd(), ) - new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD']) - - new_snapshot['components'][component_name] = new_meta - - db = self.get_src_snapshot_db() - path = db.store(new_snapshot) - log("Replaced %s with %s %s" % (component_name, new_meta['src'], - new_meta['revision'])) - log("New source snapshot: %s" % (path, )) - -builtins.register(OstbuildModifySnapshot) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 721a5800..d5b5fd58 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -35,7 +35,7 @@ from . import odict class OstbuildResolve(builtins.Builtin): name = "resolve" - short_description = "Download the source code for a given manifest" + short_description = "Expand git revisions in source to exact targets" def __init__(self): builtins.Builtin.__init__(self) @@ -88,8 +88,8 @@ class OstbuildResolve(builtins.Builtin): self.prefix = self.manifest['prefix'] snapshot = copy.deepcopy(self.manifest) - component_source_list = map(self._resolve_component_meta, self.manifest['components']) - del snapshot['components'] + components = map(self._resolve_component_meta, self.manifest['components']) + snapshot['components'] = components if args.fetch: if len(args.components) == 0: @@ -98,7 +98,7 @@ class OstbuildResolve(builtins.Builtin): fetch_components = args.components for component_name in fetch_components: found = False - for component in component_source_list: + for component in components: if component['name'] == component_name: found = True break @@ -112,6 +112,7 @@ class OstbuildResolve(builtins.Builtin): fetch_components = [] global_patches_meta = self._resolve_component_meta(self.manifest['patches']) + snapshot['patches'] = global_patches_meta (keytype, uri) = vcs.parse_src_key(global_patches_meta['src']) mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch']) if args.fetch_patches: @@ -120,7 +121,7 @@ class OstbuildResolve(builtins.Builtin): global_patches_meta['revision'] = revision unique_component_names = set() - for component in component_source_list: + for component in components: (keytype, uri) = vcs.parse_src_key(component['src']) name = component['name'] @@ -133,112 +134,6 @@ class OstbuildResolve(builtins.Builtin): component['branch']) component['revision'] = revision - config_opts = list(self.manifest['config-opts']) - config_opts.extend(component.get('config-opts', [])) - component['config-opts'] = config_opts - - patch_files = component.get('patches') - if patch_files is not None: - component['patches'] = copy.deepcopy(global_patches_meta) - component['patches']['files'] = patch_files - - manifest_architectures = snapshot['architectures'] - - ostree_prefix = snapshot['prefix'] - base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix) - - snapshot['architecture-buildroots'] = {} - for architecture in manifest_architectures: - snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture) - # Lame bit neeeded because I didn't have enough foresight to use an object - # for this in the first place, and I don't want to break backwards compatibility - # right now. - snapshot['architecture-buildroots2'] = {} - for architecture in manifest_architectures: - snapshot['architecture-buildroots2'][architecture] = {'name': 'bases/%s-%s-devel' % (base_prefix, architecture)} - - components_by_name = {} - component_ordering = [] - build_prev_component = None - runtime_prev_component = None - runtime_components = [] - devel_components = [] - - builds = {} - - for component in component_source_list: - base_name = component['name'] - name = '%s/%s' % (ostree_prefix, base_name) - component['name'] = name - - components_by_name[name] = component - - if build_prev_component is not None: - component['build-depends'] = [build_prev_component['name']] - build_prev_component = component - - is_runtime = component.get('component', 'runtime') == 'runtime' - - if runtime_prev_component is not None: - component['runtime-depends'] = [runtime_prev_component['name']] - - if is_runtime: - runtime_prev_component = component - runtime_components.append(component) - devel_components.append(component) - - is_noarch = component.get('noarch', False) - if is_noarch: - # Just use the first specified architecture - component_arches = [manifest_architectures[0]] - else: - component_arches = component.get('architectures', manifest_architectures) - builds[name] = component_arches - - # We expanded these keys - del snapshot['config-opts'] - del snapshot['vcsconfig'] - del snapshot['patches'] - del snapshot['architectures'] - - targets_list = [] - snapshot['targets'] = targets_list - for target_component_type in ['runtime', 'devel']: - for architecture in manifest_architectures: - target = {} - targets_list.append(target) - target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type) - - base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type) - target['base'] = {'name': base_ref} - - if target_component_type == 'runtime': - target_components = runtime_components - else: - target_components = devel_components - - contents = [] - for component in target_components: - builds_for_component = builds[component['name']] - if architecture not in builds_for_component: - continue - binary_name = '%s/%s' % (component['name'], architecture) - component_ref = {'name': binary_name} - if target_component_type == 'runtime': - component_ref['trees'] = ['/runtime'] - else: - component_ref['trees'] = ['/runtime', '/devel', '/doc'] - contents.append(component_ref) - target['contents'] = contents - - for component in components_by_name.itervalues(): - del component['name'] - snapshot['components'] = components_by_name - - snapshot['00ostree-src-snapshot-version'] = 0 - - current_time = time.time() - src_db = self.get_src_snapshot_db() path = src_db.store(snapshot) log("Source snapshot: %s" % (path, )) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index 109be653..7d6dfb4a 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -129,19 +129,31 @@ class Builtin(object): '/_ostbuild-meta.json']) return json.loads(text) - def get_component_meta(self, name): - assert self.repo is not None - - if self.snapshot is not None: - return self.snapshot['components'][name] - - meta = self._meta_cache.get(name) - if meta is None: - content = self.get_component_snapshot(name) - meta = self.get_component_meta_from_revision(content['ostree-revision']) - self._meta_cache[name] = meta + def expand_component(self, component): + meta = dict(component) + global_patchmeta = self.snapshot.get('patches') + if global_patchmeta is not None: + component_patch_files = component.get('patches', []) + if len(component_patch_files) > 0: + patches = dict(global_patchmeta) + patches['files'] = component_patch_files + meta['patches'] = patches + config_opts = self.snapshot.get('config-opts', []) + config_opts.extend(component.get('config-opts', [])) + meta['config-opts'] = config_opts return meta + def get_component(self, name): + assert self.repo is not None + assert self.snapshot is not None + for component in self.snapshot['components']: + if component['name'] == name: + return component + fatal("Couldn't find component '%s' in manifest" % (component_name, )) + + def get_expanded_component(self, name): + return self.expand_component(self.get_component(name)) + def get_prefix(self): if self.prefix is None: path = os.path.expanduser('~/.config/ostbuild-prefix') @@ -197,9 +209,10 @@ class Builtin(object): else: self.snapshot_path = path self.snapshot = json.load(open(self.snapshot_path)) - src_ver = self.snapshot['00ostree-src-snapshot-version'] + key = '00ostbuild-manifest-version' + src_ver = self.snapshot[key] if src_ver != 0: - fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0" % (src_ver, )) + fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, )) def parse_bin_snapshot(self, prefix, path): self.parse_prefix(prefix) diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index 1c01bbc8..a86cb951 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -22,21 +22,20 @@ import sys import argparse from . import builtins -from . import builtin_build_components +from . import builtin_build from . import builtin_branch_prefix from . import builtin_checkout from . import builtin_chroot_compile_one -from . import builtin_compose from . import builtin_compile_one from . import builtin_deploy_root from . import builtin_deploy_qemu +from . import builtin_import_tree from . import builtin_git_mirror from . import builtin_pull_components from . import builtin_privhelper_deploy_qemu from . import builtin_prefix from . import builtin_resolve from . import builtin_modify_snapshot -from . import builtin_tree_to_src from . import builtin_init from . import builtin_status