diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am index f45eb0a2..cc5ed5ec 100644 --- a/Makefile-ostbuild.am +++ b/Makefile-ostbuild.am @@ -32,6 +32,7 @@ pyostbuild_PYTHON = \ src/ostbuild/pyostbuild/builtin_status.py \ src/ostbuild/pyostbuild/builtins.py \ src/ostbuild/pyostbuild/filemonitor.py \ + src/ostbuild/pyostbuild/fileutil.py \ src/ostbuild/pyostbuild/__init__.py \ src/ostbuild/pyostbuild/kvfile.py \ src/ostbuild/pyostbuild/main.py \ diff --git a/gnomeos/3.4/manifest.json b/gnomeos/3.4/manifest.json index 79dcdbc7..16ad7249 100644 --- a/gnomeos/3.4/manifest.json +++ b/gnomeos/3.4/manifest.json @@ -1,7 +1,7 @@ { "name-prefix": "gnomeos-3.4", "architectures": ["i686"], - "base-prefix": "bases/yocto/gnomeos-3.4", + "base-prefix": "yocto/gnomeos-3.4", "config-opts": ["--disable-static", "--disable-silent-rules"], diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py index 79f0129f..814e5e57 100755 --- a/src/ostbuild/pyostbuild/buildutil.py +++ b/src/ostbuild/pyostbuild/buildutil.py @@ -19,6 +19,7 @@ import os import re import urlparse import tempfile +import StringIO from .subprocess_helpers import run_sync_get_output @@ -75,23 +76,51 @@ def get_git_version_describe(dirpath, commit=None): version = run_sync_get_output(args, cwd=dirpath) return version.strip() -def manifest_target(manifest): - name = manifest['name'] - is_runtime = name.endswith('-runtime') - # HACK - we should really name builds just like e.g. gnomeos-3.4-i686 - if is_runtime: - return name[:-len('-runtime')] + '-devel' - return name +def ref_to_unix_name(ref): + return ref.replace('/', '.') -def manifest_buildname(manifest, component): - return 'artifacts/%s/%s/%s' % (manifest_target(manifest), - component['name'], - component['branch']) +def tsort_components(components, key): + (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-') + f = os.fdopen(fd, 'w') + for name,component in components.iteritems(): + build_prev = component.get(key) + if (build_prev is not None and len(build_prev) > 0): + for dep_name in build_prev: + f.write('%s %s\n' % (name, dep_name)) + f.close() + + output = run_sync_get_output(['tsort', path]) + os.unlink(path) + output_stream = StringIO.StringIO(output) + result = [] + for line in output_stream: + result.append(line.strip()) + return result -def manifest_buildroot_name(manifest, component): - return 'buildroots/%s/%s/%s' % (manifest_target (manifest), - component['name'], - component['branch']) +def _recurse_depends(depkey, component_name, components, dep_names): + component = components[component_name] + depends = component.get(depkey) + if (depends is None or len(depends) == 0): + return + for depname in depends: + dep_names.add(depname) + _recurse_depends(depkey, depname, components, dep_names) + +def _sorted_depends(deptype, component_name, components): + dep_names = set() + _recurse_depends(deptype, component_name, components, dep_names) + dep_components = {} + for component_name in dep_names: + dep_components[component_name] = components[component_name] + result = tsort_components(dep_components, deptype) + result.reverse() + return result + +def build_depends(component_name, components): + return _sorted_depends('build-depends', component_name, components) + +def runtime_depends(component_name, components): + return _sorted_depends('runtime-depends', component_name, components) def find_component_in_manifest(manifest, component_name): for component in manifest['components']: diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py index 24736609..88073bb6 100755 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ b/src/ostbuild/pyostbuild/builtin_build.py @@ -28,6 +28,7 @@ from .subprocess_helpers import run_sync, run_sync_get_output from .subprocess_helpers import run_sync_monitor_log_file from . import ostbuildrc from . import buildutil +from . import fileutil from . import kvfile from . import odict from . import vcs @@ -91,16 +92,13 @@ class OstbuildBuild(builtins.Builtin): return result - def _build_one_component(self, meta): - name = meta['name'] - branch = meta['branch'] - architecture = meta['arch'] + def _build_one_component(self, name, component): + branch = component['branch'] + architecture = component['architecture'] - target = buildutil.manifest_target(self.manifest) - buildname = buildutil.manifest_buildname(self.manifest, meta) - buildroot_name = buildutil.manifest_buildroot_name(self.manifest, meta) + buildname = 'components/%s' % (name, ) - current_vcs_version = meta['revision'] + current_vcs_version = component['revision'] previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', buildname], @@ -130,31 +128,27 @@ class OstbuildBuild(builtins.Builtin): checkoutdir = os.path.join(self.workdir, 'src') component_src = os.path.join(checkoutdir, name) - run_sync(['ostbuild', 'checkout', '--overwrite', '--manifest=' + self.manifest_path, name], cwd=checkoutdir) + run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir) - artifact_meta = dict(meta) + artifact_meta = dict(component) metadata_path = os.path.join(component_src, '_ostbuild-meta.json') f = open(metadata_path, 'w') json.dump(artifact_meta, f, indent=4, sort_keys=True) f.close() - logdir = os.path.join(self.workdir, 'logs', 'compile', name) - old_logdir = os.path.join(self.workdir, 'old-logs', 'compile', name) - if not os.path.isdir(logdir): - os.makedirs(logdir) - if not os.path.isdir(old_logdir): - os.makedirs(old_logdir) - log_path = os.path.join(logdir, '%s.log' % (name, )) + logdir = os.path.join(self.workdir, 'logs', name) + fileutil.ensure_dir(logdir) + log_path = os.path.join(logdir, 'compile.log') if os.path.isfile(log_path): curtime = int(time.time()) - saved_name = '%s-%d.log' % (name, int(time.time()),) - os.rename(log_path, os.path.join(old_logdir, saved_name)) + saved_name = os.path.join(logdir, 'compile-prev.log') + os.rename(log_path, saved_name) log("Logging to %s" % (log_path, )) f = open(log_path, 'w') chroot_args = self._get_ostbuild_chroot_args(architecture) - chroot_args.extend(['--pristine', '--manifest=' + self.manifest_path]) + chroot_args.extend(['--pristine', '--name=' + name]) if self.buildopts.shell_on_failure: ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) if ecode != 0: @@ -184,53 +178,45 @@ class OstbuildBuild(builtins.Builtin): os.unlink(statoverride_path) return True - def _compose(self, components): - base = self.manifest['base'] - base_branch = base['branch'] - base_revision = base['revision'] - # HACK - manifest_build_name = self.manifest['name'] - is_runtime = manifest_build_name.endswith('-runtime') - + def _compose(self, target): + base_name = 'bases/%s' % (target['base']['name'], ) branch_to_rev = {} branch_to_subtrees = {} - component_branches = [] - for component in components: - branch = buildutil.manifest_buildname(self.manifest, component) - component_branches.append(branch) + contents = [base_name] + branch_to_subtrees[base_name] = ['/'] + base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, + 'rev-parse', base_name]) - args = ['ostree', '--repo=' + self.repo, - 'rev-parse'] - args.extend(component_branches) + branch_to_rev[base_name] = base_revision + + args = ['ostree', '--repo=' + self.repo, 'rev-parse'] + for component in target['contents']: + name = component['name'] + contents.append(name) + args.append('components/%s' % (name, )) + branch_to_subtrees[name] = component['trees'] branch_revs_text = run_sync_get_output(args) branch_revs = branch_revs_text.split('\n') - for (branch, rev) in zip(component_branches, branch_revs): - branch_to_rev[branch] = rev - - contents = [base_branch] - branch_to_subtrees[base_branch] = ['/'] - branch_to_rev[base_branch] = base_revision + for (content, rev) in zip(target['contents'], branch_revs): + name = content['name'] + branch_to_rev[name] = rev - for branch in component_branches: - contents.append(branch) - subtrees = ['/runtime'] - branch_to_subtrees[branch] = subtrees - if not is_runtime: - # For now just hardcode docs going in devel - subtrees.append('/doc') - subtrees.append('/devel') - - compose_rootdir = os.path.join(self.workdir, 'roots', self.manifest['name']) + compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) if os.path.isdir(compose_rootdir): shutil.rmtree(compose_rootdir) os.mkdir(compose_rootdir) - metadata_contents = [] + resolved_base = dict(target['base']) + resolved_base['ostree-revision'] = base_revision + resolved_contents = list(target['contents']) + for component in resolved_contents: + component['ostree-revision'] = branch_to_rev[component['name']] metadata = {'source': 'ostbuild compose v0', - 'base': base, - 'contents': metadata_contents} + 'base': resolved_base, + 'contents': resolved_contents} + for branch in contents: branch_rev = branch_to_rev[branch] subtrees = branch_to_subtrees[branch] @@ -239,18 +225,14 @@ class OstbuildBuild(builtins.Builtin): 'checkout', '--user-mode', '--union', '--subpath=' + subtree, branch_rev, compose_rootdir]) - branch_meta = {'name': branch, - 'rev': branch_rev, - 'subtrees': subtrees} - metadata_contents.append(branch_meta) - contents_path = os.path.join(compose_rootdir, 'contents.json') + contents_path = os.path.join(compose_rootdir, 'manifest.json') f = open(contents_path, 'w') json.dump(metadata, f, indent=4, sort_keys=True) f.close() run_sync(['ostree', '--repo=' + self.repo, - 'commit', '-b', self.manifest['name'], '-s', 'Compose', + 'commit', '-b', target['name'], '-s', 'Compose', '--owner-uid=0', '--owner-gid=0', '--no-xattrs', '--skip-if-unchanged'], cwd=compose_rootdir) @@ -259,7 +241,6 @@ class OstbuildBuild(builtins.Builtin): parser.add_argument('--skip-built', action='store_true') parser.add_argument('--recompose', action='store_true') parser.add_argument('--start-at') - parser.add_argument('--manifest', required=True) parser.add_argument('--shell-on-failure', action='store_true') parser.add_argument('--debug-shell', action='store_true') parser.add_argument('components', nargs='*') @@ -268,37 +249,33 @@ class OstbuildBuild(builtins.Builtin): self.args = args self.parse_config() + self.parse_components_and_targets() self.buildopts = BuildOptions() self.buildopts.shell_on_failure = args.shell_on_failure self.buildopts.skip_built = args.skip_built - self.manifest_path = args.manifest - self.manifest = json.load(open(args.manifest)) - - components = self.manifest['components'] + build_component_order = [] if args.recompose: - build_components = [] + pass elif len(args.components) == 0: - build_components = components + tsorted = buildutil.tsort_components(self.components, 'build-depends') + tsorted.reverse() + build_component_order = tsorted else: - build_components = [] + if args.start_at is not None: + fatal("Can't specify --start-at with component list") for name in args.components: found = False - for child in components: - if child['name'] == name: - found = True - build_components.append(child) - break - if not found: + component = self.components.get(name) + if component is None: fatal("Unknown component %r" % (name, )) + build_component_order.append(name) start_at_index = -1 if args.start_at is not None: - if build_components != components: - fatal("Can't specify --start-at with component list") - for i,component in enumerate(build_components): - if component['name'] == args.start_at: + for i,component_name in enumerate(build_component_order): + if component_name == args.start_at: start_at_index = i break if start_at_index == -1: @@ -306,10 +283,11 @@ class OstbuildBuild(builtins.Builtin): else: start_at_index = 0 - for component in build_components[start_at_index:]: - index = components.index(component) - self._build_one_component(component) + for component_name in build_component_order[start_at_index:]: + component = self.components.get(component_name) + self._build_one_component(component_name, component) - self._compose(components) + for target in self.targets['targets']: + self._compose(target) builtins.register(OstbuildBuild) diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py index d45a742e..b768fe62 100755 --- a/src/ostbuild/pyostbuild/builtin_checkout.py +++ b/src/ostbuild/pyostbuild/builtin_checkout.py @@ -26,6 +26,7 @@ from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output from . import ostbuildrc from . import buildutil +from . import fileutil from . import odict from . import vcs @@ -38,16 +39,15 @@ class OstbuildCheckout(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--manifest', required=True) parser.add_argument('--overwrite', action='store_true') + parser.add_argument('--clean', action='store_true') parser.add_argument('components', nargs='*') args = parser.parse_args(argv) self.args = args self.parse_config() - - self.manifest = json.load(open(args.manifest)) + self.parse_components_and_targets() if len(args.components) > 0: checkout_components = args.components @@ -56,17 +56,20 @@ class OstbuildCheckout(builtins.Builtin): for component_name in checkout_components: found = False - component = buildutil.find_component_in_manifest(self.manifest, - component_name) + component = self.components.get(component_name) if component is None: fatal("Unknown component %r" % (component_name, )) (keytype, uri) = buildutil.parse_src_key(component['src']) - checkoutdir = os.path.join(os.getcwd(), component['name']) + checkoutdir = os.path.join(os.getcwd(), component_name) + fileutil.ensure_parent_dir(checkoutdir) component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, component['revision'], overwrite=args.overwrite) + if args.clean: + vcs.clean(keytype, checkoutdir) + patches = component.get('patches') if patches is not None: (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src']) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py index 6569a39d..a77d6cfc 100755 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py @@ -22,6 +22,7 @@ import json from . import builtins from . import buildutil +from . import fileutil from .ostbuildlog import log, fatal from .subprocess_helpers import run_sync, run_sync_get_output @@ -29,16 +30,16 @@ class OstbuildChrootCompileOne(builtins.Builtin): name = "chroot-compile-one" short_description = "Build artifacts from the current source directory in a chroot" - def _compose_buildroot(self, component, dirpath): - components = self.manifest['components'] - index = components.index(component) - dependencies = components[:index] + def _compose_buildroot(self, component_name, dirpath): + dependencies = buildutil.build_depends(component_name, self.components) + component = self.components.get(component_name) - base = self.manifest['base'] - base_revision = base['revision'] - checkout_trees = [(base_revision, '/')] - for dep in dependencies: - buildname = buildutil.manifest_buildname(self.manifest, dep) + base_devel_name = 'bases/%s-%s-%s' % (self.manifest['base-prefix'], + component['architecture'], + 'devel') + checkout_trees = [(base_devel_name, '/')] + for dependency_name in dependencies: + buildname = 'components/%s' % (dependency_name, ) checkout_trees.append((buildname, '/runtime')) checkout_trees.append((buildname, '/devel')) @@ -50,31 +51,32 @@ class OstbuildChrootCompileOne(builtins.Builtin): def execute(self, argv): parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--manifest', required=True) parser.add_argument('--pristine', action='store_true') + parser.add_argument('--name') parser.add_argument('--debug-shell', action='store_true') args = parser.parse_args(argv) self.parse_config() + self.parse_components_and_targets() - component_name = os.path.basename(os.getcwd()) - self.manifest = json.load(open(args.manifest)) + if args.name: + component_name = args.name + else: + cwd = os.getcwd() + parent = os.path.dirname(cwd) + parentparent = os.path.dirname(parent) + component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd])) - component = buildutil.find_component_in_manifest(self.manifest, component_name) - self.metadata = component + component = self.components.get(component_name) if component is None: fatal("Couldn't find component '%s' in manifest" % (component_name, )) + self.metadata = dict(component) + self.metadata['name'] = component_name if not args.pristine: self.metadata['src'] = 'dirty:worktree' self.metadata['revision'] = 'dirty-worktree' - architecture = os.uname()[4] - - if 'name' not in self.metadata: - sys.stderr.write('Missing required key "%s" in metadata' % (k, )) - sys.exit(1) - workdir = self.workdir log("Using working directory: %s" % (workdir, )) @@ -83,17 +85,17 @@ class OstbuildChrootCompileOne(builtins.Builtin): if os.path.isdir(child_tmpdir): log("Cleaning up previous tmpdir: %r" % (child_tmpdir, )) shutil.rmtree(child_tmpdir) - os.mkdir(child_tmpdir) + fileutil.ensure_dir(child_tmpdir) - resultdir = os.path.join(self.workdir, 'results', component['name']) + resultdir = os.path.join(self.workdir, 'results', component_name) if os.path.isdir(resultdir): shutil.rmtree(resultdir) - os.makedirs(resultdir) + fileutil.ensure_dir(resultdir) rootdir_prefix = os.path.join(workdir, 'roots') - if not os.path.isdir(rootdir_prefix): - os.makedirs(rootdir_prefix) - rootdir = os.path.join(rootdir_prefix, component['name']) + fileutil.ensure_dir(rootdir_prefix) + rootdir = os.path.join(rootdir_prefix, component_name) + fileutil.ensure_parent_dir(rootdir) if os.path.isdir(rootdir): shutil.rmtree(rootdir) @@ -103,7 +105,7 @@ class OstbuildChrootCompileOne(builtins.Builtin): shutil.rmtree(rootdir_tmp) os.mkdir(rootdir_tmp) - self._compose_buildroot(component, rootdir_tmp) + self._compose_buildroot(component_name, rootdir_tmp) child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp] run_sync(child_args) @@ -115,15 +117,14 @@ class OstbuildChrootCompileOne(builtins.Builtin): os.rename(rootdir_tmp, rootdir) log("Checked out buildroot: %s" % (rootdir, )) - sourcedir=os.path.join(builddir, 'source', self.metadata['name']) - if not os.path.isdir(sourcedir): - os.mkdir(sourcedir) + sourcedir=os.path.join(builddir, 'source', component_name) + fileutil.ensure_dir(sourcedir) output_metadata = open('_ostbuild-meta.json', 'w') json.dump(self.metadata, output_metadata, indent=4, sort_keys=True) output_metadata.close() - chroot_sourcedir = os.path.join('/ostbuild', 'source', self.metadata['name']) + chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name) ostbuild_user_chroot_path = buildutil.find_user_chroot_path() diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py index 03805ca3..f703d0d7 100755 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ b/src/ostbuild/pyostbuild/builtin_compile_one.py @@ -103,10 +103,6 @@ class OstbuildCompileOne(builtins.Builtin): self.metadata = json.load(f) f.close() - for k in ['name', 'revision']: - if k not in self.metadata: - fatal('Missing required key "%s" in metadata' % (k, )) - if self.metadata.get('rm-configure', False): configure_path = 'configure' if os.path.exists(configure_path): @@ -180,10 +176,7 @@ class OstbuildCompileOne(builtins.Builtin): run_sync(args, cwd=builddir) - name = self.metadata['name'] - assert ',' not in name - - tempdir = tempfile.mkdtemp(prefix='ostbuild-%s-' % (name,)) + tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), )) self.tempfiles.append(tempdir) args = ['make', 'install', 'DESTDIR=' + tempdir] run_sync(args, cwd=builddir) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py index 3f4b566f..6defbae4 100755 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ b/src/ostbuild/pyostbuild/builtin_resolve.py @@ -16,6 +16,7 @@ # Boston, MA 02111-1307, USA. import os,sys,subprocess,tempfile,re,shutil +import copy import argparse import json import urlparse @@ -138,16 +139,18 @@ class OstbuildResolve(builtins.Builtin): manifest_path = self.ostbuildrc.get_key('manifest') self.manifest = json.load(open(manifest_path)) - self.resolved_components = map(self._resolve_component_meta, self.manifest['components']) + snapshot = copy.deepcopy(self.manifest) + component_source_list = map(self._resolve_component_meta, self.manifest['components']) + del snapshot['components'] if args.fetch: if len(args.components) == 0: - fetch_components = map(lambda x: x['name'], self.resolved_components) + fetch_components = map(lambda x: x['name'], component_source_list) else: fetch_components = args.components for component_name in fetch_components: found = False - for component in self.resolved_components: + for component in component_source_list: if component['name'] == component_name: found = True break @@ -159,76 +162,132 @@ class OstbuildResolve(builtins.Builtin): run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) else: fetch_components = [] - + global_patches_meta = self._resolve_component_meta(self.manifest['patches']) (keytype, uri) = self._parse_src_key(global_patches_meta['src']) mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch']) revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch']) global_patches_meta['revision'] = revision - for component in self.resolved_components: + unique_component_names = set() + for component in component_source_list: (keytype, uri) = self._parse_src_key(component['src']) name = component['name'] + + if name in unique_component_names: + fatal("Duplicate component name '%s'" % (name, )) + unique_component_names.add(name) + mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch']) revision = buildutil.get_git_version_describe(mirrordir, component['branch']) component['revision'] = revision - if 'component' not in component: - component['component'] = 'runtime' - config_opts = list(self.manifest['config-opts']) config_opts.extend(component.get('config-opts', [])) component['config-opts'] = config_opts patch_files = component.get('patches') if patch_files is not None: - component['patches'] = dict(global_patches_meta) + component['patches'] = copy.deepcopy(global_patches_meta) component['patches']['files'] = patch_files - self.manifest['components'] = self.resolved_components + name_prefix = snapshot['name-prefix'] + del snapshot['name-prefix'] + base_prefix = snapshot['base-prefix'] + del snapshot['base-prefix'] + + manifest_architectures = snapshot['architectures'] + + components_by_name = {} + component_ordering = [] + build_prev_component_by_arch = {} + runtime_prev_component_by_arch = {} + runtime_components_by_arch = {} + devel_components_by_arch = {} + for architecture in manifest_architectures: + runtime_components_by_arch[architecture] = [] + devel_components_by_arch[architecture] = [] + + for component in component_source_list: + component_architectures = component.get('architectures', manifest_architectures) + for architecture in component_architectures: + component_binary = copy.deepcopy(component) + source_name = component['name'] + binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name) + component_binary['name'] = binary_name + component_binary['architecture'] = architecture + + components_by_name[binary_name] = component_binary + + prev_component = build_prev_component_by_arch.get(architecture) + if prev_component is not None: + component_binary['build-depends'] = [prev_component['name']] + build_prev_component_by_arch[architecture] = component_binary + + is_runtime = component.get('component', 'runtime') == 'runtime' + + prev_component = runtime_prev_component_by_arch.get(architecture) + if prev_component is not None: + component_binary['runtime-depends'] = [prev_component['name']] + + if is_runtime: + runtime_prev_component_by_arch[architecture] = component_binary + + if is_runtime: + runtime_components_by_arch[architecture].append(component_binary) + devel_components_by_arch[architecture].append(component_binary) + + if 'architectures' in component_binary: + del component_binary['architectures'] # We expanded these keys - del self.manifest['config-opts'] - del self.manifest['vcsconfig'] - del self.manifest['patches'] + del snapshot['config-opts'] + del snapshot['vcsconfig'] + del snapshot['patches'] + del snapshot['architectures'] - manifest_architectures = self.manifest['architectures'] - del self.manifest['architectures'] + targets_json = {} + targets_list = [] + targets_json['targets'] = targets_list for architecture in manifest_architectures: - arch_manifest = dict(self.manifest) - for component in arch_manifest['components']: - component['arch'] = architecture + for target_component_type in ['runtime', 'devel']: + target = {} + targets_list.append(target) + target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type) - runtime_components = filter(lambda x: x.get('component', 'runtime') == 'runtime', arch_manifest['components']) - devel_components = arch_manifest['components'] - for component in arch_manifest['components']: - if 'component' in component: - del component['component'] - - for component_type in ['runtime', 'devel']: - snapshot = dict(arch_manifest) - if component_type == 'runtime': - snapshot['components'] = runtime_components - else: - snapshot['components'] = devel_components - - name_prefix = snapshot['name-prefix'] - del snapshot['name-prefix'] - base_prefix = snapshot['base-prefix'] - del snapshot['base-prefix'] - - snapshot['name'] = '%s-%s-%s' % (name_prefix, architecture, component_type) - - base_ref = '%s-%s-%s' % (base_prefix, architecture, component_type) + base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type) base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', base_ref]) - snapshot['base'] = {'branch': base_ref, - 'revision': base_revision} - out_snapshot = os.path.join(self.workdir, snapshot['name'] + '.snapshot') - f = open(out_snapshot, 'w') - json.dump(snapshot, f, indent=4, sort_keys=True) - f.close() - print "Created: %s" % (out_snapshot, ) + 'rev-parse', 'bases/%s' % (base_ref, )]) + target['base'] = {'name': base_ref} + + if target_component_type == 'runtime': + target_components = runtime_components_by_arch[architecture] + else: + target_components = devel_components_by_arch[architecture] + + contents = [] + for component in target_components: + name = component['name'] + component_ref = {'name': name} + if target_component_type == 'runtime': + component_ref['trees'] = ['/runtime'] + else: + component_ref['trees'] = ['/runtime', '/devel', '/doc'] + contents.append(component_ref) + target['contents'] = contents + out_targets = os.path.join(self.workdir, '%s-targets.json' % (name_prefix, )) + f = open(out_targets, 'w') + json.dump(targets_json, f, indent=4, sort_keys=True) + f.close() + print "Created: %s" % (out_targets, ) + + out_components = os.path.join(self.workdir, '%s-components.json' % (name_prefix, )) + f = open(out_components, 'w') + for component in components_by_name.itervalues(): + del component['name'] + json.dump(components_by_name, f, indent=4, sort_keys=True) + f.close() + print "Created: %s" % (out_components, ) builtins.register(OstbuildResolve) diff --git a/src/ostbuild/pyostbuild/builtin_status.py b/src/ostbuild/pyostbuild/builtin_status.py index 4b85c9d5..f4585d51 100755 --- a/src/ostbuild/pyostbuild/builtin_status.py +++ b/src/ostbuild/pyostbuild/builtin_status.py @@ -42,14 +42,14 @@ class OstbuildStatus(builtins.Builtin): args = parser.parse_args(argv) self.parse_config() - self.manifest = json.load(open(args.manifest)) + self.parse_components_and_targets() - for component in self.manifest['components']: - branch = buildutil.manifest_buildname(self.manifest, component) + for name,component in self.components.iteritems(): + buildname = 'components/%s' % (name, ) build_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, 'show', '--print-metadata-key=ostbuild-artifact-version', - branch], + buildname], none_on_error=True) if build_revision is None: build_revision = '(not built)' @@ -57,7 +57,7 @@ class OstbuildStatus(builtins.Builtin): build_status = '(needs build)' else: build_status = 'ok' - sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(component['name'], + sys.stdout.write('{:<40} {:<95} {:<10}\n'.format(name, build_revision, build_status)) builtins.register(OstbuildStatus) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py index 4a2baee6..60f52781 100755 --- a/src/ostbuild/pyostbuild/builtins.py +++ b/src/ostbuild/pyostbuild/builtins.py @@ -20,6 +20,7 @@ import os import sys import argparse +import json from . import ostbuildrc from .ostbuildlog import log, fatal @@ -41,6 +42,18 @@ class Builtin(object): fatal("Specified workdir '%s' is not a directory" % (self.workdir, )) self.patchdir = os.path.join(self.workdir, 'patches') + def parse_manifest(self): + self.manifest_path = ostbuildrc.get_key('manifest') + self.manifest = json.load(open(self.manifest_path)) + self.name_prefix = self.manifest['name-prefix'] + + def parse_components_and_targets(self): + self.parse_manifest() + components_path = os.path.join(self.workdir, '%s-components.json' % (self.name_prefix, )) + self.components = json.load(open(components_path)) + targets_path = os.path.join(self.workdir, '%s-targets.json' % (self.name_prefix, )) + self.targets = json.load(open(targets_path)) + def execute(self, args): raise NotImplementedError() diff --git a/src/ostbuild/pyostbuild/fileutil.py b/src/ostbuild/pyostbuild/fileutil.py new file mode 100644 index 00000000..d9ae91fa --- /dev/null +++ b/src/ostbuild/pyostbuild/fileutil.py @@ -0,0 +1,26 @@ +# +# Copyright (C) 2011 Colin Walters +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place - Suite 330, +# Boston, MA 02111-1307, USA. + +import os + +def ensure_dir(path): + if not os.path.isdir(path): + os.makedirs(path) + +def ensure_parent_dir(path): + ensure_dir(os.path.dirname(path)) diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py index 2479d62e..10c7e397 100755 --- a/src/ostbuild/pyostbuild/vcs.py +++ b/src/ostbuild/pyostbuild/vcs.py @@ -69,3 +69,7 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): if tmp_dest != dest: os.rename(tmp_dest, dest) return dest + +def clean(keytype, checkoutdir): + assert keytype == 'git' + run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)