From 6e2efc61347c8426bdadae446fababb7e5c0cd7c Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Mon, 14 May 2012 20:47:14 -0400 Subject: [PATCH] ostbuild: Add import-tree This is the new workflow: $ ostbuild import-tree wip-gnome-menus-3.6 $ $EDITOR ~/build/ostbuild/snapshots/wip-gnome-menus-3.6-2012.1-abcd.json $ ostbuild build --- src/ostbuild/pyostbuild/builtin_build.py | 104 ++++++++++++------ .../pyostbuild/builtin_chroot_compile_one.py | 12 +- .../pyostbuild/builtin_import_tree.py | 50 +++++++-- src/ostbuild/pyostbuild/builtins.py | 50 +++------ 4 files changed, 128 insertions(+), 88 deletions(-) diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 1dd69dc0..b4b932b0 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -66,8 +66,8 @@ class OstbuildBuild(builtins.Builtin): basename = component['name'] branch = component['branch'] - archname = '%s/%s' % (basename, architecture) - buildname = 'components/%s' % (archname, ) + buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture) + build_ref = 'components/%s' % (buildname, ) current_vcs_version = component.get('revision') @@ -86,12 +86,12 @@ class OstbuildBuild(builtins.Builtin): previous_build_version = None else: previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', buildname], + 'rev-parse', build_ref], stderr=open('/dev/null', 'w'), none_on_error=True) if (current_vcs_version is not None and previous_build_version is not None): - log("Previous build of '%s' is %s" % (archname, previous_build_version)) + log("Previous build of '%s' is %s" % (buildname, previous_build_version)) previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo, 'cat', previous_build_version, @@ -118,10 +118,10 @@ class OstbuildBuild(builtins.Builtin): else: log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) else: - log("No previous build for '%s' found" % (archname, )) + log("No previous build for '%s' found" % (buildname, )) checkoutdir = os.path.join(self.workdir, 'checkouts') - component_src = os.path.join(checkoutdir, archname) + component_src = os.path.join(checkoutdir, buildname) fileutil.ensure_parent_dir(component_src) run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path, '--checkoutdir=' + component_src, @@ -129,7 +129,7 @@ class OstbuildBuild(builtins.Builtin): artifact_meta = dict(component) - logdir = os.path.join(self.workdir, 'logs', archname) + logdir = os.path.join(self.workdir, 'logs', buildname) fileutil.ensure_dir(logdir) log_path = os.path.join(logdir, 'compile.log') if os.path.isfile(log_path): @@ -137,10 +137,16 @@ class OstbuildBuild(builtins.Builtin): saved_name = os.path.join(logdir, 'compile-prev.log') os.rename(log_path, saved_name) + component_resultdir = os.path.join(self.workdir, 'results', buildname) + if os.path.isdir(component_resultdir): + shutil.rmtree(component_resultdir) + fileutil.ensure_dir(component_resultdir) + log("Logging to %s" % (log_path, )) f = open(log_path, 'w') chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--name=' + basename, '--arch=' + architecture]) + chroot_args.extend(['--name=' + basename, '--arch=' + architecture, + '--resultdir=' + component_resultdir]) if self.buildopts.shell_on_failure: ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -149,7 +155,7 @@ class OstbuildBuild(builtins.Builtin): run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src) args = ['ostree', '--repo=' + self.repo, - 'commit', '-b', buildname, '-s', 'Build', + 'commit', '-b', build_ref, '-s', 'Build', '--owner-uid=0', '--owner-gid=0', '--no-xattrs', '--skip-if-unchanged'] @@ -163,50 +169,66 @@ class OstbuildBuild(builtins.Builtin): f.close() args.append('--statoverride=' + statoverride_path) - component_resultdir = os.path.join(self.workdir, 'results', archname) - run_sync(args, cwd=component_resultdir) if statoverride_path is not None: os.unlink(statoverride_path) + if os.path.islink(component_src): + os.unlink(component_src) + else: + shutil.rmtree(component_src) + shutil.rmtree(component_resultdir) + return run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', buildname]) + 'rev-parse', build_ref]) def _compose_one_target(self, target, component_build_revs): base = target['base'] base_name = 'bases/%s' % (base['name'], ) + runtime_name = 'bases/%s' % (base['runtime'], ) + devel_name = 'bases/%s' % (base['devel'], ) compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) if os.path.isdir(compose_rootdir): shutil.rmtree(compose_rootdir) os.mkdir(compose_rootdir) + related_refs = {} + base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', base_name]) - compose_contents = [(base_name, base_revision, '/')] - for tree_content in target['contents']: - name = tree_content['name'] - rev = component_build_revs[name] - subtrees = tree_content['trees'] - for subpath in subtrees: - compose_contents.append((name, rev, subpath)) + runtime_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', runtime_name]) + related_refs[runtime_name] = runtime_revision + devel_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', devel_name]) + related_refs[devel_name] = devel_revision + + for name,rev in component_build_revs.iteritems(): + build_ref = 'components/%s/%s' % (self.snapshot['prefix'], name) + related_refs[build_ref] = rev (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') related_f = os.fdopen(related_fd, 'w') - resolved_refs = {} - for (name, branch, subpath) in compose_contents: - resolved_refs[name] = branch - for (name, rev) in resolved_refs.iteritems(): + for (name, rev) in related_refs.iteritems(): related_f.write(name) related_f.write(' ') related_f.write(rev) related_f.write('\n') related_f.close() + compose_contents = [(base_revision, '/')] + for tree_content in target['contents']: + name = tree_content['name'] + rev = component_build_revs[name] + subtrees = tree_content['trees'] + for subpath in subtrees: + compose_contents.append((rev, subpath)) + (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') contents_f = os.fdopen(contents_fd, 'w') - for (name, branch, subpath) in compose_contents: + for (branch, subpath) in compose_contents: contents_f.write(branch) contents_f.write('\0') contents_f.write(subpath) @@ -224,12 +246,15 @@ class OstbuildBuild(builtins.Builtin): f.close() treename = 'trees/%s' % (target['name'], ) - - run_sync(['ostree', '--repo=' + self.repo, - 'commit', '-b', treename, '-s', 'Compose', - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--related-objects-file=' + related_tmppath, - '--skip-if-unchanged'], cwd=compose_rootdir) + + child_args = ['ostree', '--repo=' + self.repo, + 'commit', '-b', treename, '-s', 'Compose', + '--owner-uid=0', '--owner-gid=0', '--no-xattrs', + '--related-objects-file=' + related_tmppath, + ] + if not self.buildopts.no_skip_if_unchanged: + child_args.append('--skip-if-unchanged') + run_sync(child_args, cwd=compose_rootdir) os.unlink(related_tmppath) shutil.rmtree(compose_rootdir) @@ -240,6 +265,7 @@ class OstbuildBuild(builtins.Builtin): parser.add_argument('--force-rebuild', action='store_true') parser.add_argument('--skip-vcs-matches', action='store_true') parser.add_argument('--no-compose', action='store_true') + parser.add_argument('--no-skip-if-unchanged', action='store_true') parser.add_argument('--compose-only', action='store_true') parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') @@ -257,6 +283,7 @@ class OstbuildBuild(builtins.Builtin): self.buildopts.shell_on_failure = args.shell_on_failure self.buildopts.force_rebuild = args.force_rebuild self.buildopts.skip_vcs_matches = args.skip_vcs_matches + self.buildopts.no_skip_if_unchanged = args.no_skip_if_unchanged self.force_build_components = set() @@ -290,10 +317,8 @@ class OstbuildBuild(builtins.Builtin): component_to_arches[name] = component_arches for name in args.components: - component = components.get(name) - if component is None: - fatal("Unknown component %r" % (name, )) - self.force_build_components.add(name) + component = self.get_component(name) + self.force_build_components.add(component['name']) components_to_build = [] component_skipped_count = 0 @@ -318,8 +343,15 @@ class OstbuildBuild(builtins.Builtin): targets_list.append(target) target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type) - base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type) - target['base'] = {'name': base_ref} + runtime_ref = '%s-%s-runtime' % (base_prefix, architecture) + buildroot_ref = '%s-%s-devel' % (base_prefix, architecture) + if target_component_type == 'runtime': + base_ref = runtime_ref + else: + base_ref = buildroot_ref + target['base'] = {'name': base_ref, + 'runtime': runtime_ref, + 'devel': buildroot_ref} if target_component_type == 'runtime': target_components = runtime_components diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 9edab0cb..9ff1122c 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -63,8 +63,10 @@ class OstbuildChrootCompileOne(builtins.Builtin): ref_to_rev = {} + prefix = self.snapshot['prefix'] + arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'], - self.snapshot['prefix'], + prefix, architecture) arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', @@ -74,7 +76,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): checkout_trees = [(arch_buildroot_name, '/')] refs_to_resolve = [] for dependency in build_dependencies: - buildname = 'components/%s/%s' % (dependency['name'], architecture) + buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture) refs_to_resolve.append(buildname) checkout_trees.append((buildname, '/runtime')) checkout_trees.append((buildname, '/devel')) @@ -152,6 +154,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): parser.add_argument('--prefix') parser.add_argument('--snapshot', required=True) parser.add_argument('--name') + parser.add_argument('--resultdir') parser.add_argument('--arch', required=True) parser.add_argument('--debug-shell', action='store_true') @@ -177,10 +180,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): shutil.rmtree(child_tmpdir) fileutil.ensure_dir(child_tmpdir) - resultdir = os.path.join(self.workdir, 'results', component_name, args.arch) - if os.path.isdir(resultdir): - shutil.rmtree(resultdir) - fileutil.ensure_dir(resultdir) + resultdir = args.resultdir rootdir = self._compose_buildroot(component_name, args.arch) diff --git a/src/ostbuild/pyostbuild/builtin_import_tree.py b/src/ostbuild/pyostbuild/builtin_import_tree.py index b24b0ae9..e83d59af 100755 --- a/src/ostbuild/pyostbuild/builtin_import_tree.py +++ b/src/ostbuild/pyostbuild/builtin_import_tree.py @@ -30,7 +30,7 @@ from . import buildutil class OstbuildImportTree(builtins.Builtin): name = "import-tree" - short_description = "Extract source data from tree" + short_description = "Extract source data from tree into new prefix" def __init__(self): builtins.Builtin.__init__(self) @@ -45,22 +45,52 @@ class OstbuildImportTree(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') parser.add_argument('--tree') + parser.add_argument('new_prefix') args = parser.parse_args(argv) self.parse_config() - if args.prefix: - self.prefix = args.prefix + self.parse_snapshot_from_current() - if args.tree: - self.load_bin_snapshot_from_path(args.tree) - else: - self.load_bin_snapshot_from_current() + log("Loading source from tree %r" % (self.snapshot_path, )) + + related_objects = run_sync_get_output(['ostree', '--repo='+ self.repo, + 'show', '--print-related', + self.active_branch_checksum]) + ref_to_revision = {} + for line in StringIO(related_objects): + line = line.strip() + (ref, revision) = line.split(' ', 1) + ref_to_revision[ref] = revision + + (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-import-tree-') + f = os.fdopen(fd, 'w') + for (ref, rev) in ref_to_revision.iteritems(): + if ref.startswith('components/'): + ref = ref[len('components/'):] + (prefix, subref) = ref.split('/', 1) + newref = 'components/%s/%s' % (args.new_prefix, subref) + elif ref.startswith('bases/'): + # hack + base_key = '/' + self.snapshot['prefix'] + '-' + replace_key = '/' + args.new_prefix + '-' + newref = ref.replace(base_key, replace_key) + else: + fatal("Unhandled ref %r; expected components/ or bases/" % (ref, )) + + f.write('%s %s\n' % (newref, rev)) + f.close() + + run_sync(['ostree', '--repo=' + self.repo, + 'write-refs'], stdin=open(tmppath)) + + self.snapshot['prefix'] = args.new_prefix + + run_sync(['ostbuild', 'prefix', args.new_prefix]) + self.prefix = args.new_prefix - snapshot = self.bin_snapshot_to_src(self.bin_snapshot) db = self.get_src_snapshot_db() - path = db.store(snapshot) + path = db.store(self.snapshot) log("Source snapshot: %s" % (path, )) builtins.register(OstbuildImportTree) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index 7d6dfb4a..62a20b3d 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -92,27 +92,6 @@ class Builtin(object): self.snapshot_dir = os.path.join(self.workdir, 'snapshots') self.patchdir = os.path.join(self.workdir, 'patches') - def load_bin_snapshot_from_path(self, path): - self.bin_snapshot_path = os.path.join(path, 'ostree-meta', 'contents.json') - if not os.path.exists(self.bin_snapshot_path): - self.bin_snapshot_path = os.path.join(path, 'contents.json') - self.bin_snapshot = json.load(open(self.bin_snapshot_path)) - bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version'] - if bin_ver != 1: - fatal("Unhandled 00ostree-bin-snapshot-version %r, expected 1" % (bin_ver, )) - - def load_bin_snapshot_from_current(self): - if self.ostree_dir is None: - fatal("/ostree directory not found") - repo_path = os.path.join(self.ostree_dir, 'repo') - if not os.path.isdir(repo_path): - fatal("Repository '%s' doesn't exist" % (repo_path, )) - self.repo = repo_path - if self.active_branch is None: - fatal("No \"current\" link found") - tree_path = os.path.join(self.ostree_dir, self.active_branch) - self.load_bin_snapshot_from_path(tree_path) - def get_component_snapshot(self, name): found = False for content in self.active_branch_contents['contents']: @@ -138,7 +117,7 @@ class Builtin(object): patches = dict(global_patchmeta) patches['files'] = component_patch_files meta['patches'] = patches - config_opts = self.snapshot.get('config-opts', []) + config_opts = list(self.snapshot.get('config-opts', [])) config_opts.extend(component.get('config-opts', [])) meta['config-opts'] = config_opts return meta @@ -213,21 +192,20 @@ class Builtin(object): src_ver = self.snapshot[key] if src_ver != 0: fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, )) + if self.prefix is None: + self.prefix = self.snapshot['prefix'] - def parse_bin_snapshot(self, prefix, path): - self.parse_prefix(prefix) - self._init_repo() - if path is None: - latest_path = self.get_bin_snapshot_db().get_latest_path() - if latest_path is None: - raise Exception("No binary snapshot found for prefix %r" % (self.prefix, )) - self.bin_snapshot_path = latest_path - else: - self.bin_snapshot_path = path - self.bin_snapshot = json.load(open(self.bin_snapshot_path)) - bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version'] - if bin_ver != 1: - fatal("Unhandled 00ostree-bin-snapshot-version %r, expected 1" % (bin_ver, )) + def parse_snapshot_from_current(self): + if self.ostree_dir is None: + fatal("/ostree directory not found") + repo_path = os.path.join(self.ostree_dir, 'repo') + if not os.path.isdir(repo_path): + fatal("Repository '%s' doesn't exist" % (repo_path, )) + self.repo = repo_path + if self.active_branch is None: + fatal("No \"current\" link found") + tree_path = os.path.join(self.ostree_dir, self.active_branch) + self.parse_snapshot(None, os.path.join(tree_path, 'contents.json')) def execute(self, args): raise NotImplementedError()