From 717cec46c342371321b1a26cefd6b8c7fa3dddae Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 6 Apr 2012 15:12:57 -0400 Subject: [PATCH] ostbuild: Major rework --- Makefile-ostbuild.am | 11 +- gnomeos/3.4/gnomeos-3.4-src.json | 15 +- gnomeos/README-build.md | 42 ++++- gnomeos/gnomeos-install.sh | 20 ++- gnomeos/gnomeos-qemu-create.sh | 74 ++++++++ gnomeos/gnomeos-qemu-pull.sh | 70 ++++++++ ...meos-clone-qemu.sh => gnomeos-qemu-run.sh} | 54 +----- gnomeos/gnomeos-setup.sh | 5 + src/ostbuild/pyostbuild/buildutil.py | 2 +- src/ostbuild/pyostbuild/builtin_bin_to_src.py | 76 ++++++++ ...ry_content.py => builtin_branch_prefix.py} | 54 +++--- ...n_build.py => builtin_build_components.py} | 150 ++++++++-------- src/ostbuild/pyostbuild/builtin_checkout.py | 45 +++-- .../pyostbuild/builtin_chroot_compile_one.py | 146 +++++++++++----- .../pyostbuild/builtin_chroot_run_triggers.py | 47 ----- .../pyostbuild/builtin_compile_one.py | 25 +-- src/ostbuild/pyostbuild/builtin_compose.py | 110 ++++++++++++ .../pyostbuild/builtin_modify_snapshot.py | 63 +++++++ src/ostbuild/pyostbuild/builtin_prefix.py | 73 ++++++++ .../pyostbuild/builtin_pull_components.py | 70 ++++++++ src/ostbuild/pyostbuild/builtin_resolve.py | 103 ++++++----- src/ostbuild/pyostbuild/builtins.py | 165 ++++++++++++++++-- src/ostbuild/pyostbuild/jsondb.py | 110 ++++++++++++ src/ostbuild/pyostbuild/main.py | 10 +- src/ostbuild/pyostbuild/ostbuildlog.py | 10 +- src/ostbuild/pyostbuild/subprocess_helpers.py | 6 +- src/ostbuild/pyostbuild/vcs.py | 2 +- 27 files changed, 1205 insertions(+), 353 deletions(-) create mode 100755 gnomeos/gnomeos-qemu-create.sh create mode 100755 gnomeos/gnomeos-qemu-pull.sh rename gnomeos/{gnomeos-clone-qemu.sh => gnomeos-qemu-run.sh} (55%) create mode 100755 src/ostbuild/pyostbuild/builtin_bin_to_src.py rename src/ostbuild/pyostbuild/{builtin_query_content.py => builtin_branch_prefix.py} (53%) rename src/ostbuild/pyostbuild/{builtin_build.py => builtin_build_components.py} (64%) delete mode 100755 src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py create mode 100755 src/ostbuild/pyostbuild/builtin_compose.py create mode 100755 src/ostbuild/pyostbuild/builtin_modify_snapshot.py create mode 100755 src/ostbuild/pyostbuild/builtin_prefix.py create mode 100755 src/ostbuild/pyostbuild/builtin_pull_components.py create mode 100644 src/ostbuild/pyostbuild/jsondb.py diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index 08eb8c03..a2279163 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -22,18 +22,23 @@ bin_SCRIPTS += ostbuild pyostbuilddir=$(libdir)/ostbuild/pyostbuild pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/buildutil.py \ - src/ostbuild/pyostbuild/builtin_build.py \ + src/ostbuild/pyostbuild/builtin_bin_to_src.py \ + src/ostbuild/pyostbuild/builtin_branch_prefix.py \ + src/ostbuild/pyostbuild/builtin_build_components.py \ src/ostbuild/pyostbuild/builtin_checkout.py \ + src/ostbuild/pyostbuild/builtin_compose.py \ src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ - src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \ src/ostbuild/pyostbuild/builtin_compile_one.py \ - src/ostbuild/pyostbuild/builtin_query_content.py \ + src/ostbuild/pyostbuild/builtin_pull_components.py \ + src/ostbuild/pyostbuild/builtin_prefix.py \ src/ostbuild/pyostbuild/builtin_resolve.py \ + src/ostbuild/pyostbuild/builtin_modify_snapshot.py \ src/ostbuild/pyostbuild/builtin_status.py \ src/ostbuild/pyostbuild/builtins.py \ src/ostbuild/pyostbuild/filemonitor.py \ src/ostbuild/pyostbuild/fileutil.py \ src/ostbuild/pyostbuild/__init__.py \ + src/ostbuild/pyostbuild/jsondb.py \ src/ostbuild/pyostbuild/kvfile.py \ src/ostbuild/pyostbuild/main.py \ src/ostbuild/pyostbuild/mainloop.py \ diff --git a/gnomeos/3.4/gnomeos-3.4-src.json b/gnomeos/3.4/gnomeos-3.4-src.json index a229770c..f58a5ea2 100644 --- a/gnomeos/3.4/gnomeos-3.4-src.json +++ b/gnomeos/3.4/gnomeos-3.4-src.json @@ -1,7 +1,10 @@ { - "name-prefix": "gnomeos-3.4", + "prefix": "gnomeos-3.4", "architectures": ["i686"], - "base-prefix": "yocto/gnomeos-3.4", + "base": { + "name": "yocto", + "src": "cgwalters:poky" + }, "config-opts": ["--disable-static", "--disable-silent-rules"], @@ -25,13 +28,15 @@ "patches": {"src": "git:/src/ostree", "branch": "wip/ostbuild-v2", - "prefix": "gnomeos/3.4"}, + "subdir": "gnomeos/3.4"}, "components": [ - {"src": "cgwalters:ginitscripts"}, + {"src": "cgwalters:ginitscripts", + "noarch": true}, {"src": "gnome:gtk-doc-stub", - "component": "devel"}, + "component": "devel", + "noarch": true}, {"src": "git:git://github.com/atgreen/libffi.git"}, diff --git a/gnomeos/README-build.md b/gnomeos/README-build.md index bd59261f..0146c2e2 100644 --- a/gnomeos/README-build.md +++ b/gnomeos/README-build.md @@ -16,6 +16,9 @@ base "runtime", and one "devel" with all of the development tools like gcc. We then import that into an OSTree branch e.g. "bases/yocto/gnomeos-3.4-i686-devel". +At present, it's still (mostly) possible to put this data on an ext4 +filesystem and boot into it. + We also have a Yocto recipe "ostree-native" which generates (as you might guess) a native binary of ostree. That binary is used to import into an "archive mode" OSTree repository. You can see it in @@ -26,26 +29,59 @@ can use "ostbuild" which uses "linux-user-chroot" to chroot inside, run a build on a source tree, and outputs binaries, which we then add to the build tree for the next module, and so on. +The final result of all of this is that the OSTree repository gains +new commits (which can be downloaded by clients), while still +retaining old build history. + +Yocto details +------------- + +I have a branch of Yocto here: + +https://github.com/cgwalters/poky + +It has a collection of patches on top of the "Edison" release of +Yocto, some of which are hacky, others upstreamable. The most +important part though are the modifications to commit the generated +root filesystem into OSTree. + ostbuild details ---------------- The simple goal of ostbuild is that it only takes as input a -"manifest" which is basically just a list of components to build. A -component is a pure metadata file which includes the git repository +"manifest" which is basically just a list of components to build. You +can see this here: + +http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json + +A component is a pure metadata file which includes the git repository URL and branch name, as well as ./configure flags (--enable-foo). There is no support for building from "tarballs" - I want the ability to review all of the code that goes in, and to efficiently store -source code updates. +source code updates. It's also just significantly easier from an +implementation perspective, versus having to maintain a version +control abstraction layer. The result of a build of a component is an OSTree branch like "artifacts/gnomeos-3.4-i686-devel/libxslt/master". Then, a "compose" process merges together the individual filesystem trees into the final branches (e.g. gnomeos-3.4-i686-devel). +Doing local builds +------------------ + +This is where you want to modify one (or a few) components on top of +what comes from the ostree.gnome.org server, and test the result +locally. I'm working on this. + Doing a full build on your system --------------------------------- +Following this process is equivalent to what we have set up on the +ostree.gnome.org build server. It will generate a completely new +repository. + srcdir=/src builddir=/src/build diff --git a/gnomeos/gnomeos-install.sh b/gnomeos/gnomeos-install.sh index 99394be6..c37e25af 100755 --- a/gnomeos/gnomeos-install.sh +++ b/gnomeos/gnomeos-install.sh @@ -46,17 +46,19 @@ if ! test -d /ostree/repo/objects; then mkdir -p /ostree $SRCDIR/gnomeos-setup.sh /ostree + + cd /ostree + + ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel} + ostree-pull --repo=repo gnome + for branch in runtime devel; do + ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch} + done + ln -sf ${BRANCH_PREFIX}runtime current + + cd ${WORKDIR} fi -cd /ostree - -ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel} -ostree-pull --repo=repo gnome -for branch in runtime devel; do - ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch} -done -ln -sf ${BRANCH_PREFIX}runtime current - uname=$(uname -r) if test -d /etc/grub.d; then diff --git a/gnomeos/gnomeos-qemu-create.sh b/gnomeos/gnomeos-qemu-create.sh new file mode 100755 index 00000000..bc0e6a57 --- /dev/null +++ b/gnomeos/gnomeos-qemu-create.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# -*- indent-tabs-mode: nil; -*- +# Create ostree-qemu.img file in the current directory, suitable +# for booting via qemu. +# +# Copyright (C) 2011,2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -e +set -x + +SRCDIR=`dirname $0` +WORKDIR=`pwd` + +if test $(id -u) != 0; then + cat < +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +set -e +set -x + +SRCDIR=`dirname $0` +WORKDIR=`pwd` + +if test $(id -u) != 0; then + cat < +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# ostbuild-compile-one-make wraps systems that implement the GNOME build API: +# http://people.gnome.org/~walters/docs/build-api.txt + +import os,sys,stat,subprocess,tempfile,re,shutil +import argparse +from StringIO import StringIO +import json + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from . import buildutil + +class OstbuildBinToSrc(builtins.Builtin): + name = "bin-to-src" + short_description = "Turn a binary snapshot into a source snapshot" + + def __init__(self): + builtins.Builtin.__init__(self) + + def bin_snapshot_to_src(self, bin_snapshot): + del bin_snapshot['00ostree-bin-snapshot-version'] + + src_snapshot = dict(bin_snapshot) + src_snapshot['00ostree-src-snapshot-version'] = 0 + + all_architectures = src_snapshot['architecture-buildroots'].keys() + # Arbitrarily take first architecture + first_arch = all_architectures[0] + + bin_components = src_snapshot['components'] + src_components = {} + src_snapshot['components'] = src_components + for archname,rev in bin_components.iteritems(): + (name, arch) = archname.rsplit('/', 1) + if arch != first_arch: + continue + meta = dict(self.get_component_meta_from_revision(rev)) + del meta['name'] + src_components[name] = meta + + for target in src_snapshot['targets']: + del target['base']['ostree-revision'] + + return src_snapshot + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--prefix') + parser.add_argument('--bin-snapshot') + + args = parser.parse_args(argv) + self.parse_config() + self.parse_bin_snapshot(args.prefix, args.bin_snapshot) + + snapshot = self.bin_snapshot_to_src(self.bin_snapshot) + json.dump(snapshot, sys.stdout, indent=4, sort_keys=True) + +builtins.register(OstbuildBinToSrc) diff --git a/src/ostbuild/pyostbuild/builtin_query_content.py b/src/ostbuild/pyostbuild/builtin_branch_prefix.py similarity index 53% rename from src/ostbuild/pyostbuild/builtin_query_content.py rename to src/ostbuild/pyostbuild/builtin_branch_prefix.py index 89f58396..5aaf9399 100755 --- a/src/ostbuild/pyostbuild/builtin_query_content.py +++ b/src/ostbuild/pyostbuild/builtin_branch_prefix.py @@ -28,40 +28,44 @@ from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output from . import buildutil -class OstbuildQueryContent(builtins.Builtin): - name = "query-content" - short_description = "Output metadata from a component" +class OstbuildBranchPrefix(builtins.Builtin): + name = "prefix-branch" + short_description = "Copy current source snapshot to new prefix" def __init__(self): builtins.Builtin.__init__(self) def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--branch', required=True) - parser.add_argument('--component') + parser.add_argument('--prefix') + parser.add_argument('--src-snapshot') + parser.add_argument('newprefix') args = parser.parse_args(argv) - self.args = args self.parse_config() + self.parse_snapshot(args.prefix, args.src_snapshot) - contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'cat', args.branch, 'contents.json']) + if args.newprefix == self.prefix: + fatal("Specified prefix %r matches active prefix" % (args.newprefix, )) + + db = self.create_db('src-snapshot', prefix=args.newprefix) + + log("Branching from source snapshot %r" % (self.snapshot_path, )) + + orig_prefix = self.prefix + + forked_snapshot = dict(self.snapshot) + forked_snapshot['prefix'] = args.newprefix + + for target in forked_snapshot['targets']: + name = target['name'] + if not name.startswith(orig_prefix): + fatal("Mismatched name %r in snapshot" % (name, )) + target['name'] = name.replace(orig_prefix, args.newprefix) - if args.component is None: - sys.stdout.write(contents_json_text) - else: - contents = json.loads(contents_json_text) - contents_list = contents['contents'] - found = False - for content in contents_list: - if content['name'] != args.component: - found = True - break - if not found: - fatal("Unknown component '%s'" % (args.component, )) - ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'cat', content['ostree-revision'], - '/_ostbuild-meta.json']) - sys.stdout.write(ostbuildmeta_json) + db.store(forked_snapshot) -builtins.register(OstbuildQueryContent) + run_sync(['ostbuild', 'prefix', args.newprefix], + log_initiation=False, log_success=False) + +builtins.register(OstbuildBranchPrefix) diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build_components.py similarity index 64% rename from src/ostbuild/pyostbuild/builtin_build.py rename to src/ostbuild/pyostbuild/builtin_build_components.py index 772aaca8..5e35c0f0 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build_components.py @@ -36,9 +36,9 @@ from . import vcs class BuildOptions(object): pass -class OstbuildBuild(builtins.Builtin): - name = "build" - short_description = "Rebuild all artifacts from the given manifest" +class OstbuildBuildComponents(builtins.Builtin): + name = "build-components" + short_description = "Build multiple components from given source snapshot" def __init__(self): builtins.Builtin.__init__(self) @@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin): args = ['setarch', architecture] else: args = [] - args.extend(['ostbuild', 'chroot-compile-one']) + args.extend(['ostbuild', 'chroot-compile-one', + '--snapshot=' + self.snapshot_path]) return args def _launch_debug_shell(self, architecture, buildroot, cwd=None): @@ -60,10 +61,10 @@ class OstbuildBuild(builtins.Builtin): run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True) fatal("Exiting after debug shell") - def _build_one_component(self, name, component): + def _build_one_component(self, basename, component, architecture): branch = component['branch'] - architecture = component['architecture'] + name = '%s/%s' % (basename, architecture) buildname = 'components/%s' % (name, ) current_vcs_version = component['revision'] @@ -73,7 +74,7 @@ class OstbuildBuild(builtins.Builtin): stderr=open('/dev/null', 'w'), none_on_error=True) if previous_build_version is not None: - log("Previous build of '%s' is %s" % (buildname, previous_build_version)) + log("Previous build of '%s' is %s" % (name, previous_build_version)) previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo, 'cat', previous_build_version, @@ -92,19 +93,15 @@ class OstbuildBuild(builtins.Builtin): else: log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) else: - log("No previous build for '%s' found" % (buildname, )) + log("No previous build for '%s' found" % (name, )) checkoutdir = os.path.join(self.workdir, 'src') - component_src = os.path.join(checkoutdir, name) - run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir) + component_src = os.path.join(checkoutdir, basename) + run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path, + '--clean', '--overwrite', basename], cwd=checkoutdir) artifact_meta = dict(component) - metadata_path = os.path.join(component_src, '_ostbuild-meta.json') - f = open(metadata_path, 'w') - json.dump(artifact_meta, f, indent=4, sort_keys=True) - f.close() - logdir = os.path.join(self.workdir, 'logs', name) fileutil.ensure_dir(logdir) log_path = os.path.join(logdir, 'compile.log') @@ -116,7 +113,7 @@ class OstbuildBuild(builtins.Builtin): log("Logging to %s" % (log_path, )) f = open(log_path, 'w') chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--pristine', '--name=' + name]) + chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture]) if self.buildopts.shell_on_failure: ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -146,69 +143,48 @@ class OstbuildBuild(builtins.Builtin): os.unlink(statoverride_path) return True - def _compose(self, target): - base_name = 'bases/%s' % (target['base']['name'], ) - branch_to_rev = {} - branch_to_subtrees = {} - - contents = [base_name] - branch_to_subtrees[base_name] = ['/'] - base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', base_name]) - - branch_to_rev[base_name] = base_revision - + def _resolve_refs(self, refs): args = ['ostree', '--repo=' + self.repo, 'rev-parse'] - for component in target['contents']: - name = component['name'] - contents.append(name) - args.append('components/%s' % (name, )) - branch_to_subtrees[name] = component['trees'] - branch_revs_text = run_sync_get_output(args) - branch_revs = branch_revs_text.split('\n') + args.extend(refs) + output = run_sync_get_output(args) + return output.split('\n') - for (content, rev) in zip(target['contents'], branch_revs): - name = content['name'] - branch_to_rev[name] = rev - - compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) - if os.path.isdir(compose_rootdir): - shutil.rmtree(compose_rootdir) - os.mkdir(compose_rootdir) + def _save_bin_snapshot(self, components, component_architectures): + bin_snapshot = dict(self.snapshot) - resolved_base = dict(target['base']) - resolved_base['ostree-revision'] = base_revision - resolved_contents = list(target['contents']) - for component in resolved_contents: - component['ostree-revision'] = branch_to_rev[component['name']] - metadata = {'source': 'ostbuild compose v0', - 'base': resolved_base, - 'contents': resolved_contents} + del bin_snapshot['00ostree-src-snapshot-version'] + bin_snapshot['00ostree-bin-snapshot-version'] = 0 - for branch in contents: - branch_rev = branch_to_rev[branch] - subtrees = branch_to_subtrees[branch] - for subtree in subtrees: - run_sync(['ostree', '--repo=' + self.repo, - 'checkout', '--user-mode', - '--union', '--subpath=' + subtree, - branch_rev, compose_rootdir]) + for target in bin_snapshot['targets']: + base = target['base'] + base_name = 'bases/%s' % (base['name'], ) + base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', base_name]) + base['ostree-revision'] = base_revision - contents_path = os.path.join(compose_rootdir, 'contents.json') - f = open(contents_path, 'w') - json.dump(metadata, f, indent=4, sort_keys=True) - f.close() + component_refs = [] + for name in components.iterkeys(): + for architecture in component_architectures[name]: + component_refs.append('components/%s/%s' % (name, architecture)) - run_sync(['ostree', '--repo=' + self.repo, - 'commit', '-b', target['name'], '-s', 'Compose', - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--skip-if-unchanged'], cwd=compose_rootdir) + new_components = {} + resolved_refs = self._resolve_refs(component_refs) + for name,rev in zip(components.iterkeys(), resolved_refs): + for architecture in component_architectures[name]: + archname = '%s/%s' % (name, architecture) + new_components[archname] = rev + + bin_snapshot['components'] = new_components + + path = self.get_bin_snapshot_db().store(bin_snapshot) + log("Binary snapshot: %s" % (path, )) def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--skip-built', action='store_true') - parser.add_argument('--recompose', action='store_true') - parser.add_argument('--skip-compose', action='store_true') + parser.add_argument('--prefix') + parser.add_argument('--src-snapshot') + parser.add_argument('--compose', action='store_true') parser.add_argument('--start-at') parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') @@ -218,17 +194,28 @@ class OstbuildBuild(builtins.Builtin): self.args = args self.parse_config() - self.parse_snapshot() + self.parse_snapshot(args.prefix, args.src_snapshot) + + log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), )) self.buildopts = BuildOptions() self.buildopts.shell_on_failure = args.shell_on_failure self.buildopts.skip_built = args.skip_built + required_components = {} + component_architectures = {} + for target in self.snapshot['targets']: + for tree_content in target['contents']: + (name, arch) = tree_content['name'].rsplit('/', 1) + required_components[name] = self.snapshot['components'][name] + if name not in component_architectures: + component_architectures[name] = set([arch]) + else: + component_architectures[name].add(arch) + build_component_order = [] - if args.recompose: - pass - elif len(args.components) == 0: - tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends') + if len(args.components) == 0: + tsorted = buildutil.tsort_components(required_components, 'build-depends') tsorted.reverse() build_component_order = tsorted else: @@ -253,11 +240,14 @@ class OstbuildBuild(builtins.Builtin): start_at_index = 0 for component_name in build_component_order[start_at_index:]: - component = self.snapshot['components'].get(component_name) - self._build_one_component(component_name, component) + component = required_components[component_name] + architectures = component_architectures[component_name] + for architecture in architectures: + self._build_one_component(component_name, component, architecture) - if not args.skip_compose: - for target in self.snapshot['targets']: - self._compose(target) + self._save_bin_snapshot(required_components, component_architectures) + + if args.compose: + run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix]) -builtins.register(OstbuildBuild) +builtins.register(OstbuildBuildComponents) diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py index 553e505f..27281bea 100755 --- a/src/ostbuild/pyostbuild/builtin_checkout.py +++ b/src/ostbuild/pyostbuild/builtin_checkout.py @@ -40,6 +40,9 @@ class OstbuildCheckout(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--overwrite', action='store_true') + parser.add_argument('--prefix') + parser.add_argument('--snapshot') + parser.add_argument('-a', '--active-tree', action='store_true') parser.add_argument('--clean', action='store_true') parser.add_argument('components', nargs='*') @@ -47,28 +50,41 @@ class OstbuildCheckout(builtins.Builtin): self.args = args self.parse_config() - self.parse_snapshot() if len(args.components) > 0: checkout_components = args.components else: checkout_components = [os.path.basename(os.getcwd())] + if args.active_tree: + self.parse_active_branch() + else: + self.parse_snapshot(args.prefix, args.snapshot) + for component_name in checkout_components: found = False - component = self.snapshot['components'].get(component_name) - if component is None: - fatal("Unknown component %r" % (component_name, )) + component = self.get_component_meta(component_name) (keytype, uri) = buildutil.parse_src_key(component['src']) checkoutdir = os.path.join(os.getcwd(), component_name) fileutil.ensure_parent_dir(checkoutdir) - component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, - component['revision'], - overwrite=args.overwrite) + is_dirty = (keytype == 'dirty-git') + if is_dirty: + # Kind of a hack, but... + if os.path.lexists(checkoutdir): + os.unlink(checkoutdir) + os.symlink(uri, checkoutdir) + else: + vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, + component['revision'], + overwrite=args.overwrite) + if args.clean: - vcs.clean(keytype, checkoutdir) + if is_dirty: + log("note: ignoring --clean argument due to \"dirty-git:\" specification") + else: + vcs.clean(keytype, checkoutdir) patches = component.get('patches') if patches is not None: @@ -78,15 +94,20 @@ class OstbuildCheckout(builtins.Builtin): self.patchdir, patches['branch'], overwrite=True) - patch_prefix = patches.get('prefix', None) - if patch_prefix is not None: - patchdir = os.path.join(self.patchdir, patch_prefix) + patch_subdir = patches.get('subdir', None) + if patch_subdir is not None: + patchdir = os.path.join(self.patchdir, patch_subdir) else: patchdir = self.patchdir for patch in patches['files']: patch_path = os.path.join(patchdir, patch) run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir) + + metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json') + f = open(metadata_path, 'w') + json.dump(component, f, indent=4, sort_keys=True) + f.close() - print "Checked out: %r" % (component_src, ) + log("Checked out: %r" % (checkoutdir, )) builtins.register(OstbuildCheckout) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 0c6a96ca..9bb8c988 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -18,7 +18,9 @@ import os,sys,re,subprocess,tempfile,shutil from StringIO import StringIO import argparse +import time import json +import hashlib from . import builtins from . import buildutil @@ -31,47 +33,133 @@ class OstbuildChrootCompileOne(builtins.Builtin): name = "chroot-compile-one" short_description = "Build artifacts from the current source directory in a chroot" - def _compose_buildroot(self, component_name, dirpath): + def _resolve_refs(self, refs): + args = ['ostree', '--repo=' + self.repo, 'rev-parse'] + args.extend(refs) + output = run_sync_get_output(args) + return output.split('\n') + + def _compose_buildroot(self, component_name, architecture): + starttime = time.time() + + rootdir_prefix = os.path.join(self.workdir, 'roots') + rootdir = os.path.join(rootdir_prefix, component_name) + fileutil.ensure_parent_dir(rootdir) + + # Clean up any leftover root dir + rootdir_tmp = rootdir + '.tmp' + if os.path.isdir(rootdir_tmp): + shutil.rmtree(rootdir_tmp) + components = self.snapshot['components'] dependencies = buildutil.build_depends(component_name, components) component = components.get(component_name) - base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'], - component['architecture'], - 'devel') + buildroots = self.snapshot['architecture-buildroots'] + base_devel_name = 'bases/' + buildroots[architecture] + + refs_to_resolve = [base_devel_name] checkout_trees = [(base_devel_name, '/')] for dependency_name in dependencies: - buildname = 'components/%s' % (dependency_name, ) + buildname = 'components/%s/%s' % (dependency_name, architecture) + refs_to_resolve.append(buildname) checkout_trees.append((buildname, '/runtime')) checkout_trees.append((buildname, '/devel')) + resolved_refs = self._resolve_refs(refs_to_resolve) + ref_to_rev = {} + for ref,rev in zip(refs_to_resolve, resolved_refs): + ref_to_rev[ref] = rev + link_cache_dir = os.path.join(self.workdir, 'link-cache') fileutil.ensure_dir(link_cache_dir) - for (branch, rootpath) in checkout_trees: - run_sync(['ostree', '--repo=' + self.repo, - 'checkout', '--user-mode', '--link-cache=' + link_cache_dir, - '--union', '--subpath=' + rootpath, - branch, dirpath]) + sha = hashlib.sha256() + + (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-') + f = os.fdopen(fd, 'w') + for (branch, subpath) in checkout_trees: + f.write(ref_to_rev[branch]) + f.write('\0') + f.write(subpath) + f.write('\0') + f.close() + + f = open(tmppath) + buf = f.read(8192) + while buf != '': + sha.update(buf) + buf = f.read(8192) + f.close() + + new_root_cacheid = sha.hexdigest() + + rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid') + + if os.path.isdir(rootdir): + if os.path.isfile(rootdir_cache_path): + f = open(rootdir_cache_path) + prev_cache_id = f.read().strip() + f.close() + if prev_cache_id == new_root_cacheid: + log("Reusing previous buildroot") + os.unlink(tmppath) + return rootdir + else: + log("New buildroot differs from previous") + + shutil.rmtree(rootdir) + + os.mkdir(rootdir_tmp) + + if len(checkout_trees) > 0: + log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees), + checkout_trees[-1][0])) + + link_cache_dir = os.path.join(self.workdir, 'link-cache') + fileutil.ensure_dir(link_cache_dir) + + run_sync(['ostree', '--repo=' + self.repo, + 'checkout', '--link-cache=' + link_cache_dir, + '--user-mode', '--union', '--from-stdin', rootdir_tmp], + stdin=open(tmppath)) + + os.unlink(tmppath); + + builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild') + os.mkdir(builddir_tmp) + os.mkdir(os.path.join(builddir_tmp, 'source')) + os.mkdir(os.path.join(builddir_tmp, 'results')) + os.rename(rootdir_tmp, rootdir) + + f = open(rootdir_cache_path, 'w') + f.write(new_root_cacheid) + f.write('\n') + f.close() + + endtime = time.time() + log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),)) + + return rootdir def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) parser.add_argument('--pristine', action='store_true') + parser.add_argument('--prefix') + parser.add_argument('--snapshot', required=True) parser.add_argument('--name') + parser.add_argument('--arch', required=True) parser.add_argument('--debug-shell', action='store_true') args = parser.parse_args(argv) self.parse_config() - self.parse_snapshot() + self.parse_snapshot(args.prefix, args.snapshot) if args.name: component_name = args.name else: - cwd = os.getcwd() - parent = os.path.dirname(cwd) - parentparent = os.path.dirname(parent) - component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd])) + component_name = self.get_component_from_cwd() components = self.snapshot['components'] component = components.get(component_name) @@ -93,37 +181,16 @@ class OstbuildChrootCompileOne(builtins.Builtin): shutil.rmtree(child_tmpdir) fileutil.ensure_dir(child_tmpdir) - resultdir = os.path.join(self.workdir, 'results', component_name) + resultdir = os.path.join(self.workdir, 'results', component_name, args.arch) if os.path.isdir(resultdir): shutil.rmtree(resultdir) fileutil.ensure_dir(resultdir) - rootdir_prefix = os.path.join(workdir, 'roots') - fileutil.ensure_dir(rootdir_prefix) - rootdir = os.path.join(rootdir_prefix, component_name) - fileutil.ensure_parent_dir(rootdir) - if os.path.isdir(rootdir): - shutil.rmtree(rootdir) - - rootdir_tmp = rootdir + '.tmp' - builddir = os.path.join(rootdir, 'ostbuild'); - if os.path.isdir(rootdir_tmp): - shutil.rmtree(rootdir_tmp) - os.mkdir(rootdir_tmp) - - self._compose_buildroot(component_name, rootdir_tmp) + rootdir = self._compose_buildroot(component_name, args.arch) - child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp] - run_sync(child_args) - - builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild') - os.mkdir(builddir_tmp) - os.mkdir(os.path.join(builddir_tmp, 'source')) - os.mkdir(os.path.join(builddir_tmp, 'results')) - os.rename(rootdir_tmp, rootdir) log("Checked out buildroot: %s" % (rootdir, )) - sourcedir=os.path.join(builddir, 'source', component_name) + sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name) fileutil.ensure_dir(sourcedir) output_metadata = open('_ostbuild-meta.json', 'w') @@ -148,7 +215,6 @@ class OstbuildChrootCompileOne(builtins.Builtin): 'compile-one', '--ostbuild-resultdir=/ostbuild/results', '--ostbuild-meta=_ostbuild-meta.json']) - child_args.extend(self.metadata.get('config-opts', [])) env_copy = dict(buildutil.BUILD_ENV) env_copy['PWD'] = chroot_sourcedir run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py b/src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py deleted file mode 100755 index b9455996..00000000 --- a/src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,re,subprocess,tempfile,shutil -from StringIO import StringIO -import argparse -import json - -from . import builtins -from . import buildutil -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync - -class OstbuildChrootRunTriggers(builtins.Builtin): - name = "chroot-run-triggers" - short_description = "Run ostree-run-triggers inside a chroot" - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('root') - - args = parser.parse_args(argv) - - child_args = buildutil.get_base_user_chroot_args() - child_args.extend(['--mount-proc', '/proc', - '--mount-bind', '/dev', '/dev', - args.root, - '/usr/bin/ostree-run-triggers']) - env_copy = dict(buildutil.BUILD_ENV) - env_copy['PWD'] = '/' - run_sync(child_args, env=env_copy) - -builtins.register(OstbuildChrootRunTriggers) diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index f703d0d7..eef08dfe 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin): def execute(self, args): self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )] + + starttime = time.time() uname=os.uname() kernel=uname[0].lower() @@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin): '--infodir=' + os.path.join(PREFIX, 'share', 'info')] self.makeargs = ['make'] - self.ostbuild_resultdir=None - self.ostbuild_meta=None + self.ostbuild_resultdir='_ostbuild-results' + self.ostbuild_meta_path='_ostbuild-meta.json' chdir = None + opt_install = False for arg in args: if arg.startswith('--ostbuild-resultdir='): self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):] elif arg.startswith('--ostbuild-meta='): - self.ostbuild_meta=arg[len('--ostbuild-meta='):] + self.ostbuild_meta_path=arg[len('--ostbuild-meta='):] elif arg.startswith('--chdir='): os.chdir(arg[len('--chdir='):]) - elif arg.startswith('--'): - self.configargs.append(arg) else: self.makeargs.append(arg) - if self.ostbuild_resultdir is None: - fatal("Must specify --ostbuild-resultdir=") - if self.ostbuild_meta is None: - fatal("Must specify --ostbuild-meta=") - - f = open(self.ostbuild_meta) + f = open(self.ostbuild_meta_path) self.metadata = json.load(f) f.close() + self.configargs.extend(self.metadata.get('config-opts', [])) + if self.metadata.get('rm-configure', False): configure_path = 'configure' if os.path.exists(configure_path): @@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin): except OSError, e: pass + endtime = time.time() + + log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),)) + log("Results placed in %s" % (self.ostbuild_resultdir, )) + def _install_and_unlink(self, src, dest): statsrc = os.lstat(src) dirname = os.path.dirname(dest) diff --git a/src/ostbuild/pyostbuild/builtin_compose.py b/src/ostbuild/pyostbuild/builtin_compose.py new file mode 100755 index 00000000..52dbb8f8 --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_compose.py @@ -0,0 +1,110 @@ +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os,sys,subprocess,tempfile,re,shutil +import argparse +import time +import urlparse +import json +from StringIO import StringIO + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from .subprocess_helpers import run_sync_monitor_log_file +from . import ostbuildrc +from . import buildutil +from . import fileutil +from . import kvfile +from . import odict +from . import vcs + +class OstbuildCompose(builtins.Builtin): + name = "compose" + short_description = "Build complete trees from components" + + def __init__(self): + builtins.Builtin.__init__(self) + + def _compose_one_target(self, bin_snapshot, target): + components = bin_snapshot['components'] + base = target['base'] + base_name = 'bases/%s' % (base['name'], ) + base_revision = target['base']['ostree-revision'] + + compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) + if os.path.isdir(compose_rootdir): + shutil.rmtree(compose_rootdir) + os.mkdir(compose_rootdir) + + compose_contents = [(base_revision, '/')] + for tree_content in target['contents']: + name = tree_content['name'] + rev = components[name] + subtrees = tree_content['trees'] + for subpath in subtrees: + compose_contents.append((rev, subpath)) + + (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') + f = os.fdopen(fd, 'w') + for (branch, subpath) in compose_contents: + f.write(branch) + f.write('\0') + f.write(subpath) + f.write('\0') + f.close() + + link_cache_dir = os.path.join(self.workdir, 'link-cache') + fileutil.ensure_dir(link_cache_dir) + + run_sync(['ostree', '--repo=' + self.repo, + 'checkout', '--link-cache=' + link_cache_dir, + '--user-mode', '--no-triggers', + '--union', '--from-stdin', compose_rootdir], + stdin=open(tmppath)) + os.unlink(tmppath) + + contents_path = os.path.join(compose_rootdir, 'contents.json') + f = open(contents_path, 'w') + json.dump(bin_snapshot, f, indent=4, sort_keys=True) + f.close() + + run_sync(['ostree', '--repo=' + self.repo, + 'commit', '-b', target['name'], '-s', 'Compose', + '--owner-uid=0', '--owner-gid=0', '--no-xattrs', + '--skip-if-unchanged'], cwd=compose_rootdir) + shutil.rmtree(compose_rootdir) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--prefix') + parser.add_argument('--bin-snapshot') + + args = parser.parse_args(argv) + self.args = args + + self.parse_config() + self.parse_bin_snapshot(args.prefix, args.bin_snapshot) + + log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), )) + + for target in self.bin_snapshot['targets']: + log("Composing target %r from %u components" % (target['name'], + len(target['contents']))) + self._compose_one_target(self.bin_snapshot, target) + +builtins.register(OstbuildCompose) diff --git a/src/ostbuild/pyostbuild/builtin_modify_snapshot.py b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py new file mode 100755 index 00000000..b438a9ae --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_modify_snapshot.py @@ -0,0 +1,63 @@ +# Copyright (C) 2011,2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os,sys,stat,subprocess,tempfile,re,shutil +from StringIO import StringIO +import json +import select,time +import argparse + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output + +class OstbuildModifySnapshot(builtins.Builtin): + name = "modify-snapshot" + short_description = "Change the current source snapshot" + + def __init__(self): + builtins.Builtin.__init__(self) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('--prefix') + parser.add_argument('--src-snapshot') + + args = parser.parse_args(argv) + + self.parse_config() + self.parse_snapshot(args.prefix, args.src_snapshot) + + component_name = self.get_component_from_cwd() + current_meta = self.get_component_meta(component_name) + + new_snapshot = dict(self.snapshot) + new_meta = dict(current_meta) + if 'patches' in new_meta: + del new_meta['patches'] + new_meta['src'] = "dirty-git:%s" % (os.getcwd(), ) + new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD']) + + new_snapshot['components'][component_name] = new_meta + + db = self.get_src_snapshot_db() + path = db.store(new_snapshot) + log("Replaced %s with %s %s" % (component_name, new_meta['src'], + new_meta['revision'])) + log("New source snapshot: %s" % (path, )) + +builtins.register(OstbuildModifySnapshot) diff --git a/src/ostbuild/pyostbuild/builtin_prefix.py b/src/ostbuild/pyostbuild/builtin_prefix.py new file mode 100755 index 00000000..4ffa9814 --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_prefix.py @@ -0,0 +1,73 @@ +# Copyright (C) 2011,2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +# ostbuild-compile-one-make wraps systems that implement the GNOME build API: +# http://people.gnome.org/~walters/docs/build-api.txt + +import os,sys,stat,subprocess,tempfile,re,shutil +import argparse +from StringIO import StringIO +import json + +from . import builtins +from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output +from . import buildutil + +class OstbuildPrefix(builtins.Builtin): + name = "prefix" + short_description = "Display or modify \"prefix\" (build target)" + + def __init__(self): + builtins.Builtin.__init__(self) + + def _set_prefix(self, prefix): + f = open(self.path, 'w') + f.write(prefix) + f.write('\n') + f.close() + log("Prefix is now %r" % (prefix, )) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('-a', '--active', action='store_true') + parser.add_argument('prefix', nargs='?', default=None) + + args = parser.parse_args(argv) + + self.path = os.path.expanduser('~/.config/ostbuild-prefix') + if args.prefix is None and not args.active: + if os.path.exists(self.path): + f = open(self.path) + print "%s" % (f.read().strip(), ) + f.close() + else: + log("No currently active prefix") + elif args.prefix is not None and args.active: + fatal("Can't specify -a with prefix") + elif args.prefix is not None: + self._set_prefix(args.prefix) + else: + assert args.active + + self.parse_active_branch() + + active_prefix = self.active_branch_contents['prefix'] + + self._set_prefix(active_prefix) + +builtins.register(OstbuildPrefix) diff --git a/src/ostbuild/pyostbuild/builtin_pull_components.py b/src/ostbuild/pyostbuild/builtin_pull_components.py new file mode 100755 index 00000000..7464b1ed --- /dev/null +++ b/src/ostbuild/pyostbuild/builtin_pull_components.py @@ -0,0 +1,70 @@ +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os,sys,subprocess,tempfile,re,shutil +import copy +import argparse +import json +import time +import urlparse +from StringIO import StringIO + +from . import builtins +from .ostbuildlog import log, fatal +from . import ostbuildrc +from . import buildutil +from .subprocess_helpers import run_sync, run_sync_get_output +from . import kvfile +from . import odict + +class OstbuildPullComponents(builtins.Builtin): + name = "pull-components" + short_description = "Download the component data for active branch" + + def __init__(self): + builtins.Builtin.__init__(self) + + def execute(self, argv): + parser = argparse.ArgumentParser(description=self.short_description) + parser.add_argument('targets', nargs='*') + + args = parser.parse_args(argv) + + self.parse_active_branch() + + if len(args.targets) == 0: + targets = [self.active_branch] + else: + targets = args.targets + + tree_contents_list = [] + for target in targets: + tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json') + tree_contents = json.load(open(tree_contents_path)) + tree_contents_list.append(tree_contents) + revisions = set() + for tree_contents in tree_contents_list: + for content_item in tree_contents['contents']: + revisions.add(content_item['ostree-revision']) + args = ['ostree-pull', '--repo=' + self.repo] + # FIXME FIXME - don't hardcode origin here + args.append('gnome') + for revision in revisions: + args.append(revision) + run_sync(args) + +builtins.register(OstbuildPullComponents) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 3486396d..4ddd1ad0 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil import copy import argparse import json +import time import urlparse from StringIO import StringIO @@ -26,6 +27,7 @@ from . import builtins from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output from . import ostbuildrc +from . import jsondb from . import buildutil from . import kvfile from . import odict @@ -38,6 +40,7 @@ class OstbuildResolve(builtins.Builtin): builtins.Builtin.__init__(self) def _ensure_vcs_mirror(self, name, keytype, uri, branch): + # FIXME - remove "name" from parameter list here - hash uri? mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri) tmp_mirror = mirror + '.tmp' if os.path.isdir(tmp_mirror): @@ -135,6 +138,7 @@ class OstbuildResolve(builtins.Builtin): self.args = args self.parse_config() + self.repo = ostbuildrc.get_key('repo') manifest_path = self.ostbuildrc.get_key('manifest') self.manifest = json.load(open(manifest_path)) @@ -192,52 +196,52 @@ class OstbuildResolve(builtins.Builtin): component['patches'] = copy.deepcopy(global_patches_meta) component['patches']['files'] = patch_files - name_prefix = snapshot['name-prefix'] - base_prefix = snapshot['base-prefix'] - manifest_architectures = snapshot['architectures'] + ostree_prefix = snapshot['prefix'] + base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix) + + snapshot['architecture-buildroots'] = {} + for architecture in manifest_architectures: + snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture) + components_by_name = {} component_ordering = [] - build_prev_component_by_arch = {} - runtime_prev_component_by_arch = {} - runtime_components_by_arch = {} - devel_components_by_arch = {} - for architecture in manifest_architectures: - runtime_components_by_arch[architecture] = [] - devel_components_by_arch[architecture] = [] + build_prev_component = None + runtime_prev_component = None + runtime_components = [] + devel_components = [] + + builds = {} for component in component_source_list: - component_architectures = component.get('architectures', manifest_architectures) - for architecture in component_architectures: - component_binary = copy.deepcopy(component) - source_name = component['name'] - binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name) - component_binary['name'] = binary_name - component_binary['architecture'] = architecture + base_name = component['name'] + name = '%s/%s' % (ostree_prefix, base_name) + component['name'] = name - components_by_name[binary_name] = component_binary + components_by_name[name] = component - prev_component = build_prev_component_by_arch.get(architecture) - if prev_component is not None: - component_binary['build-depends'] = [prev_component['name']] - build_prev_component_by_arch[architecture] = component_binary + if build_prev_component is not None: + component['build-depends'] = [build_prev_component['name']] + build_prev_component = component - is_runtime = component.get('component', 'runtime') == 'runtime' + is_runtime = component.get('component', 'runtime') == 'runtime' - prev_component = runtime_prev_component_by_arch.get(architecture) - if prev_component is not None: - component_binary['runtime-depends'] = [prev_component['name']] + if runtime_prev_component is not None: + component['runtime-depends'] = [runtime_prev_component['name']] - if is_runtime: - runtime_prev_component_by_arch[architecture] = component_binary + if is_runtime: + runtime_prev_component = component + runtime_components.append(component) + devel_components.append(component) - if is_runtime: - runtime_components_by_arch[architecture].append(component_binary) - devel_components_by_arch[architecture].append(component_binary) - - if 'architectures' in component_binary: - del component_binary['architectures'] + is_noarch = component.get('noarch', False) + if is_noarch: + # Just use the first specified architecture + component_arches = [manifest_architectures[0]] + else: + component_arches = component.get('architectures', manifest_architectures) + builds[name] = component_arches # We expanded these keys del snapshot['config-opts'] @@ -247,26 +251,27 @@ class OstbuildResolve(builtins.Builtin): targets_list = [] snapshot['targets'] = targets_list - for architecture in manifest_architectures: - for target_component_type in ['runtime', 'devel']: + for target_component_type in ['runtime', 'devel']: + for architecture in manifest_architectures: target = {} targets_list.append(target) - target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type) + target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type) base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type) - base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', 'bases/%s' % (base_ref, )]) target['base'] = {'name': base_ref} if target_component_type == 'runtime': - target_components = runtime_components_by_arch[architecture] + target_components = runtime_components else: - target_components = devel_components_by_arch[architecture] + target_components = devel_components contents = [] for component in target_components: - name = component['name'] - component_ref = {'name': name} + builds_for_component = builds[component['name']] + if architecture not in builds_for_component: + continue + binary_name = '%s/%s' % (component['name'], architecture) + component_ref = {'name': binary_name} if target_component_type == 'runtime': component_ref['trees'] = ['/runtime'] else: @@ -278,10 +283,12 @@ class OstbuildResolve(builtins.Builtin): del component['name'] snapshot['components'] = components_by_name - out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, )) - f = open(out_snapshot, 'w') - json.dump(snapshot, f, indent=4, sort_keys=True) - f.close() - print "Created: %s" % (out_snapshot, ) + snapshot['00ostree-src-snapshot-version'] = 0 + + current_time = time.time() + + src_db = self.get_src_snapshot_db() + path = src_db.store(snapshot) + log("Source snapshot: %s" % (path, )) builtins.register(OstbuildResolve) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index 64f7be4e..c71431fd 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -19,11 +19,15 @@ import os import sys +import stat import argparse import json from . import ostbuildrc +from . import fileutil +from . import jsondb from .ostbuildlog import log, fatal +from .subprocess_helpers import run_sync, run_sync_get_output _all_builtins = {} @@ -31,26 +35,167 @@ class Builtin(object): name = None short_description = None + def __init__(self): + self._meta_cache = {} + self.prefix = None + self.manifest = None + self.snapshot = None + self.bin_snapshot = None + self.repo = None + self.ostree_dir = self._find_ostree_dir() + (self.active_branch, self.active_branch_checksum) = self._find_active_branch() + self._src_snapshots = None + self._bin_snapshots = None + + def _find_ostree_dir(self): + for path in ['/ostree', '/sysroot/ostree']: + if os.path.isdir(path): + return path + return None + + def _find_active_branch(self): + if self.ostree_dir is None: + return (None, None) + current_path = os.path.join(self.ostree_dir, 'current') + while True: + try: + target = os.path.join(self.ostree_dir, current_path) + stbuf = os.lstat(target) + except OSError, e: + current_path = None + break + if not stat.S_ISLNK(stbuf.st_mode): + break + current_path = os.readlink(target) + if current_path is not None: + basename = os.path.basename(current_path) + return basename.rsplit('-', 1) + else: + return (None, None) + + def get_component_from_cwd(self): + cwd = os.getcwd() + parent = os.path.dirname(cwd) + parentparent = os.path.dirname(parent) + return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd])) + def parse_config(self): self.ostbuildrc = ostbuildrc - self.repo = ostbuildrc.get_key('repo') + self.mirrordir = ostbuildrc.get_key('mirrordir') if not os.path.isdir(self.mirrordir): fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, )) self.workdir = ostbuildrc.get_key('workdir') if not os.path.isdir(self.workdir): fatal("Specified workdir '%s' is not a directory" % (self.workdir, )) + + self.snapshot_dir = os.path.join(self.workdir, 'snapshots') self.patchdir = os.path.join(self.workdir, 'patches') - def parse_manifest(self): - self.manifest_path = ostbuildrc.get_key('manifest') - self.manifest = json.load(open(self.manifest_path)) - self.name_prefix = self.manifest['name-prefix'] + def parse_active_branch(self): + if self.ostree_dir is None: + fatal("/ostree directory not found") + repo_path = os.path.join(self.ostree_dir, 'repo') + if not os.path.isdir(repo_path): + fatal("Repository '%s' doesn't exist" % (repo_path, )) + self.repo = repo_path + if self.active_branch is None: + fatal("No \"current\" link found") + branch_path = os.path.join(self.ostree_dir, self.active_branch) + contents_path = os.path.join(branch_path, 'contents.json') + f = open(contents_path) + self.active_branch_contents = json.load(f) + f.close() - def parse_snapshot(self): - self.parse_manifest() - snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, )) - self.snapshot = json.load(open(snapshot_path)) + def get_component_snapshot(self, name): + found = False + for content in self.active_branch_contents['contents']: + if content['name'] == name: + found = True + break + if not found: + fatal("Unknown component '%s'" % (name, )) + return content + + def get_component_meta_from_revision(self, revision): + text = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'cat', revision, + '/_ostbuild-meta.json']) + return json.loads(text) + + def get_component_meta(self, name): + assert self.repo is not None + + if self.snapshot is not None: + return self.snapshot['components'][name] + + meta = self._meta_cache.get(name) + if meta is None: + content = self.get_component_snapshot(name) + meta = self.get_component_meta_from_revision(content['ostree-revision']) + self._meta_cache[name] = meta + return meta + + def get_prefix(self): + if self.prefix is None: + path = os.path.expanduser('~/.config/ostbuild-prefix') + if not os.path.exists(path): + fatal("No prefix set; use \"ostbuild prefix\" to set one") + f = open(path) + self.prefix = f.read().strip() + f.close() + return self.prefix + + def create_db(self, dbsuffix, prefix=None): + if prefix is None: + target_prefix = self.get_prefix() + else: + target_prefix = prefix + name = '%s-%s' % (target_prefix, dbsuffix) + fileutil.ensure_dir(self.snapshot_dir) + return jsondb.JsonDB(self.snapshot_dir, prefix=name) + + def get_src_snapshot_db(self): + if self._src_snapshots is None: + self._src_snapshots = self.create_db('src-snapshot') + return self._src_snapshots + + def get_bin_snapshot_db(self): + if self._bin_snapshots is None: + self._bin_snapshots = self.create_db('bin-snapshot') + return self._bin_snapshots + + def parse_snapshot(self, prefix, path): + if prefix is not None: + self.prefix = prefix + self.repo = ostbuildrc.get_key('repo') + if path is None: + latest_path = self.get_src_snapshot_db().get_latest_path() + if latest_path is None: + raise Exception("No source snapshot found for prefix %r" % (self.prefix, )) + self.snapshot_path = latest_path + else: + self.snapshot_path = path + self.snapshot = json.load(open(self.snapshot_path)) + src_ver = self.snapshot['00ostree-src-snapshot-version'] + if src_ver != 0: + fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver) + + def parse_bin_snapshot(self, prefix, path): + if prefix is not None: + self.prefix = prefix + self.repo = ostbuildrc.get_key('repo') + if path is None: + latest_path = self.get_bin_snapshot_db().get_latest_path() + if latest_path is None: + raise Exception("No binary snapshot found for prefix %r" % (self.prefix, )) + self.bin_snapshot_path = latest_path + else: + self.bin_snapshot_path = path + self.bin_snapshot = json.load(open(self.bin_snapshot_path)) + bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version'] + if bin_ver != 0: + fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver) def execute(self, args): raise NotImplementedError() @@ -65,4 +210,4 @@ def get(name): return None def get_all(): - return _all_builtins.itervalues() + return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name)) diff --git a/src/ostbuild/pyostbuild/jsondb.py b/src/ostbuild/pyostbuild/jsondb.py new file mode 100644 index 00000000..aecee359 --- /dev/null +++ b/src/ostbuild/pyostbuild/jsondb.py @@ -0,0 +1,110 @@ +# +# Copyright (C) 2012 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os +import stat +import time +import tempfile +import re +import shutil +import hashlib +import json + +class JsonDB(object): + def __init__(self, dirpath, prefix): + self._dirpath = dirpath + self._prefix = prefix + self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$') + + def _cmp_match_by_version(self, a, b): + # Note this is a reversed comparison; bigger is earlier + a_major = a[0] + a_minor = a[1] + b_major = b[0] + b_minor = b[1] + + c = cmp(b_major, a_major) + if c == 0: + return cmp(b_minor, a_minor) + return 0 + + def _get_all(self): + result = [] + for fname in os.listdir(self._dirpath): + if not (fname.startswith(self._prefix) and fname.endswith('.json')): + continue + + path = os.path.join(self._dirpath, fname) + match = self._version_csum_re.search(fname) + if not match: + raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum", + path) + result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname)) + result.sort(self._cmp_match_by_version) + return result + + def get_latest(self): + path = self.get_latest_path() + if path is None: + return None + return json.load(open(path)) + + def get_latest_path(self): + files = self._get_all() + if len(files) == 0: + return None + return os.path.join(self._dirpath, files[0][3]) + + def store(self, obj): + files = self._get_all() + if len(files) == 0: + latest = None + else: + latest = files[0] + + current_time = time.gmtime() + + (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-') + os.close(fd) + f = open(tmppath, 'w') + json.dump(obj, f, indent=4, sort_keys=True) + f.close() + + csum = hashlib.sha256() + f = open(tmppath) + buf = f.read(8192) + while buf != '': + csum.update(buf) + buf = f.read(8192) + f.close() + digest = csum.hexdigest() + + if latest is not None: + if digest == latest[2]: + return latest[3] + latest_version = (latest[0], latest[1]) + else: + latest_version = (current_time.tm_year, 0) + target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year, + latest_version[1] + 1, digest) + target_path = os.path.join(self._dirpath, target_name) + os.rename(tmppath, target_path) + return target_path + + + diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py index 842cb5bc..63827c1b 100755 --- a/src/ostbuild/pyostbuild/main.py +++ b/src/ostbuild/pyostbuild/main.py @@ -22,13 +22,17 @@ import sys import argparse from . import builtins -from . import builtin_build +from . import builtin_bin_to_src +from . import builtin_build_components +from . import builtin_branch_prefix from . import builtin_checkout from . import builtin_chroot_compile_one -from . import builtin_chroot_run_triggers +from . import builtin_compose from . import builtin_compile_one -from . import builtin_query_content +from . import builtin_pull_components +from . import builtin_prefix from . import builtin_resolve +from . import builtin_modify_snapshot from . import builtin_status def usage(ecode): diff --git a/src/ostbuild/pyostbuild/ostbuildlog.py b/src/ostbuild/pyostbuild/ostbuildlog.py index a4af2a6a..0f852298 100755 --- a/src/ostbuild/pyostbuild/ostbuildlog.py +++ b/src/ostbuild/pyostbuild/ostbuildlog.py @@ -20,12 +20,16 @@ import os import sys -def log(msg): - fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg) +def log(msg, prefix=None): + if prefix is None: + prefix_target = '' + else: + prefix_target = prefix + fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg) sys.stdout.write(fullmsg) sys.stdout.flush() def fatal(msg): - log(msg) + log(msg, prefix="FATAL: ") sys.exit(1) diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py index 116d0fbc..37549007 100755 --- a/src/ostbuild/pyostbuild/subprocess_helpers.py +++ b/src/ostbuild/pyostbuild/subprocess_helpers.py @@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none return None def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False, - log_success=True, log_initiation=True, stdout=None, + log_success=True, log_initiation=True, stdin=None, stdout=None, stderr=None): if log_initiation: log("running: %s" % (subprocess.list2cmdline(args),)) env_copy = _get_env_for_cwd(cwd, env) - if keep_stdin: + if stdin is not None: + stdin_target = stdin + elif keep_stdin: stdin_target = sys.stdin else: stdin_target = open('/dev/null', 'r') diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py index 10c7e397..07161224 100755 --- a/src/ostbuild/pyostbuild/vcs.py +++ b/src/ostbuild/pyostbuild/vcs.py @@ -71,5 +71,5 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): return dest def clean(keytype, checkoutdir): - assert keytype == 'git' + assert keytype in ('git', 'dirty-git') run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)