shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-19 22:50:35 +03:00
Code Issues Projects Releases Wiki Activity

ostbuild: Tighten build process significantly

Browse Source 
First, "resolve" now just picks git commits.  We don't expand
config-opts and patches, nor do we generate tree contents.  This makes
the generated files *much* more human editable.

Next, fold "build-components" and "compose" into just "build".  One
never really wants to just build components.  This lets us eliminate
binary snapshots as a concept; instead we always have a combination of
source snapshot and component/ refs.
This commit is contained in:
Colin Walters 2012-05-13 15:53:55 -04:00
parent 92244c80cc
commit f4aba2a339
10 changed files with 212 additions and 472 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
Makefile-ostbuild.am
View File

@ -24,20 +24,18 @@ pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_branch_prefix.py \
src/ostbuild/pyostbuild/builtin_build_components.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
src/ostbuild/pyostbuild/builtin_import_tree.py \
src/ostbuild/pyostbuild/builtin_pull_components.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
src/ostbuild/pyostbuild/builtin_tree_to_src.py \
src/ostbuild/pyostbuild/builtin_init.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \

277
src/ostbuild/pyostbuild/builtin_build_components.py → src/ostbuild/pyostbuild/builtin_build.py
View File

@ -37,9 +37,9 @@ from . import vcs
class BuildOptions(object):
pass
class OstbuildBuildComponents(builtins.Builtin):
name = "build-components"
short_description = "Build multiple components from given source snapshot"
class OstbuildBuild(builtins.Builtin):
name = "build"
short_description = "Build multiple components and generate trees"
def __init__(self):
builtins.Builtin.__init__(self)
@ -62,19 +62,20 @@ class OstbuildBuildComponents(builtins.Builtin):
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
def _build_one_component(self, basename, component, architecture):
def _build_one_component(self, component, architecture):
basename = component['name']
branch = component['branch']
name = '%s/%s' % (basename, architecture)
buildname = 'components/%s' % (name, )
archname = '%s/%s' % (basename, architecture)
buildname = 'components/%s' % (archname, )
current_vcs_version = component.get('revision')
expanded_component = self.expand_component(component)
# TODO - deduplicate this with chroot_compile_one
current_meta_io = StringIO()
meta_copy = dict(component)
meta_copy['name'] = basename # Note we have to match the name here
json.dump(meta_copy, current_meta_io, indent=4, sort_keys=True)
json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True)
current_metadata_text = current_meta_io.getvalue()
sha = hashlib.sha256()
sha.update(current_metadata_text)
@ -90,7 +91,7 @@ class OstbuildBuildComponents(builtins.Builtin):
none_on_error=True)
if (current_vcs_version is not None
and previous_build_version is not None):
log("Previous build of '%s' is %s" % (name, previous_build_version))
log("Previous build of '%s' is %s" % (archname, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
@ -102,33 +103,33 @@ class OstbuildBuildComponents(builtins.Builtin):
if current_meta_digest == previous_meta_digest:
log("Metadata is unchanged from previous")
return False
return previous_build_version
else:
previous_metadata = json.loads(previous_metadata_text)
previous_vcs_version = previous_metadata['revision']
if current_vcs_version == previous_vcs_version:
log("Metadata differs; VCS version unchanged")
if self.buildopts.skip_vcs_matches:
return False
for k,v in meta_copy.iteritems():
return previous_build_version
for k,v in expanded_component.iteritems():
previous_v = previous_metadata.get(k)
if v != previous_v:
log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
else:
log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
log("No previous build for '%s' found" % (name, ))
log("No previous build for '%s' found" % (archname, ))
checkoutdir = os.path.join(self.workdir, 'checkouts')
fileutil.ensure_dir(checkoutdir)
component_src = os.path.join(checkoutdir, basename)
component_src = os.path.join(checkoutdir, archname)
fileutil.ensure_parent_dir(component_src)
run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--checkoutdir=' + component_src,
'--clean', '--overwrite', basename])
artifact_meta = dict(component)
logdir = os.path.join(self.workdir, 'logs', name)
logdir = os.path.join(self.workdir, 'logs', archname)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
if os.path.isfile(log_path):
@ -139,7 +140,7 @@ class OstbuildBuildComponents(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
chroot_args.extend(['--name=' + basename, '--arch=' + architecture])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@ -162,76 +163,87 @@ class OstbuildBuildComponents(builtins.Builtin):
f.close()
args.append('--statoverride=' + statoverride_path)
component_resultdir = os.path.join(self.workdir, 'results', name)
component_resultdir = os.path.join(self.workdir, 'results', archname)
run_sync(args, cwd=component_resultdir)
if statoverride_path is not None:
os.unlink(statoverride_path)
return True
def _resolve_refs(self, refs):
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
return run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', buildname])
def _save_bin_snapshot(self, components, component_architectures):
bin_snapshot = dict(self.snapshot)
def _compose_one_target(self, target, component_build_revs):
base = target['base']
base_name = 'bases/%s' % (base['name'], )
del bin_snapshot['00ostree-src-snapshot-version']
bin_snapshot['00ostree-bin-snapshot-version'] = 1
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
for target in bin_snapshot['targets']:
base = target['base']
base_name = 'bases/%s' % (base['name'], )
if 'ostree-revision' not in target:
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
base['ostree-revision'] = base_revision
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
if 'architecture-buildroots2' in bin_snapshot:
for arch,buildroot in bin_snapshot['architecture-buildroots2'].iteritems():
name = buildroot['name']
if 'ostree-revision' not in buildroot:
rev = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', name])
buildroot['ostree-revision'] = rev
compose_contents = [(base_name, base_revision, '/')]
for tree_content in target['contents']:
name = tree_content['name']
rev = component_build_revs[name]
subtrees = tree_content['trees']
for subpath in subtrees:
compose_contents.append((name, rev, subpath))
component_revisions = bin_snapshot.get('component-revisions', {})
(related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
related_f = os.fdopen(related_fd, 'w')
resolved_refs = {}
for (name, branch, subpath) in compose_contents:
resolved_refs[name] = branch
for (name, rev) in resolved_refs.iteritems():
related_f.write(name)
related_f.write(' ')
related_f.write(rev)
related_f.write('\n')
related_f.close()
component_refs_to_resolve = []
for name in components.iterkeys():
for architecture in component_architectures[name]:
archname = '%s/%s' % (name, architecture)
if archname not in component_revisions:
component_refs_to_resolve.append('components/' + archname)
(contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
contents_f = os.fdopen(contents_fd, 'w')
for (name, branch, subpath) in compose_contents:
contents_f.write(branch)
contents_f.write('\0')
contents_f.write(subpath)
contents_f.write('\0')
contents_f.close()
if len(component_refs_to_resolve) > 0:
resolved_refs = self._resolve_refs(component_refs_to_resolve)
for name,rev in zip(component_refs_to_resolve, resolved_refs):
assert name.startswith('components/')
archname = name[len('components/'):]
component_revisions[archname] = rev
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--no-triggers', '--union',
'--from-file=' + contents_tmppath, compose_rootdir])
os.unlink(contents_tmppath)
bin_snapshot['component-revisions'] = component_revisions
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(self.snapshot, f, indent=4, sort_keys=True)
f.close()
path = self.get_bin_snapshot_db().store(bin_snapshot)
log("Binary snapshot: %s" % (path, ))
treename = 'trees/%s' % (target['name'], )
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', treename, '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--related-objects-file=' + related_tmppath,
'--skip-if-unchanged'], cwd=compose_rootdir)
os.unlink(related_tmppath)
shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--force-rebuild', action='store_true')
parser.add_argument('--skip-vcs-matches', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('--compose', action='store_true')
parser.add_argument('--force-rebuild', action='store_true')
parser.add_argument('--skip-vcs-matches', action='store_true')
parser.add_argument('--no-compose', action='store_true')
parser.add_argument('--compose-only', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.args = args
@ -239,15 +251,7 @@ class OstbuildBuildComponents(builtins.Builtin):
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
component_revisions = self.snapshot.get('component-revisions', {})
if component_revisions is not None:
snapshot_type = "source+binary"
else:
snapshot_type = "source"
log("Using %s snapshot: %s" % (snapshot_type,
os.path.basename(self.snapshot_path), ))
log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
@ -256,68 +260,87 @@ class OstbuildBuildComponents(builtins.Builtin):
self.force_build_components = set()
required_components = {}
component_architectures = {}
for target in self.snapshot['targets']:
for tree_content in target['contents']:
(name, arch) = tree_content['name'].rsplit('/', 1)
required_components[name] = self.snapshot['components'][name]
if name not in component_architectures:
component_architectures[name] = set([arch])
else:
component_architectures[name].add(arch)
components = self.snapshot['components']
build_component_order = []
if len(args.components) == 0:
tsorted = buildutil.tsort_components(required_components, 'build-depends')
tsorted.reverse()
build_component_order = tsorted
else:
if args.start_at is not None:
fatal("Can't specify --start-at with component list")
for name in args.components:
found = False
component = self.snapshot['components'].get(name)
if component is None:
fatal("Unknown component %r" % (name, ))
build_component_order.append(name)
self.force_build_components.add(name)
prefix = self.snapshot['prefix']
base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix)
start_at_index = -1
if args.start_at is not None:
for i,component_name in enumerate(build_component_order):
if component_name == args.start_at:
start_at_index = i
break
if start_at_index == -1:
fatal("Unknown component %r specified for --start-at" % (args.start_at, ))
else:
start_at_index = 0
architectures = self.snapshot['architectures']
component_to_arches = {}
runtime_components = []
devel_components = []
for component in components:
name = component['name']
is_runtime = component.get('component', 'runtime') == 'runtime'
if is_runtime:
runtime_components.append(component)
devel_components.append(component)
is_noarch = component.get('noarch', False)
if is_noarch:
# Just use the first specified architecture
component_arches = [architectures[0]]
else:
component_arches = component.get('architectures', architectures)
component_to_arches[name] = component_arches
for name in args.components:
component = components.get(name)
if component is None:
fatal("Unknown component %r" % (name, ))
self.force_build_components.add(name)
components_to_build = []
component_skipped_count = 0
component_build_revs = {}
if not args.compose_only:
for component_name in build_component_order[start_at_index:]:
component = required_components[component_name]
architectures = component_architectures[component_name]
for component in components:
for architecture in architectures:
archname = '%s/%s' % (component_name, architecture)
if (component_revisions is not None and
archname in component_revisions):
component_skipped_count += 1
else:
components_to_build.append((component_name, component, architecture))
components_to_build.append((component, architecture))
log("%d components to build" % (len(components_to_build), ))
if component_skipped_count > 0:
log("%d components skipped due to existing component-revisions" % (component_skipped_count, ))
for (component_name, component, architecture) in components_to_build:
self._build_one_component(component_name, component, architecture)
for (component, architecture) in components_to_build:
archname = '%s/%s' % (component['name'], architecture)
build_rev = self._build_one_component(component, architecture)
component_build_revs[archname] = build_rev
self._save_bin_snapshot(required_components, component_architectures)
targets_list = []
for target_component_type in ['runtime', 'devel']:
for architecture in architectures:
target = {}
targets_list.append(target)
target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
if args.compose or args.compose_only:
run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
builtins.register(OstbuildBuildComponents)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
target_components = runtime_components
else:
target_components = devel_components
contents = []
for component in target_components:
builds_for_component = component_to_arches[component['name']]
if architecture not in builds_for_component:
continue
binary_name = '%s/%s' % (component['name'], architecture)
component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
component_ref['trees'] = ['/runtime', '/devel', '/doc']
contents.append(component_ref)
target['contents'] = contents
for target in targets_list:
self._compose_one_target(target, component_build_revs)
builtins.register(OstbuildBuild)

8
src/ostbuild/pyostbuild/builtin_checkout.py
View File

@ -60,7 +60,7 @@ class OstbuildCheckout(builtins.Builtin):
component_name = args.component
found = False
component = self.get_component_meta(component_name)
component = self.get_expanded_component(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
is_local = (keytype == 'local')
@ -82,9 +82,9 @@ class OstbuildCheckout(builtins.Builtin):
else:
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
if args.clean:
if is_local:

48
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
View File

@ -34,6 +34,8 @@ class OstbuildChrootCompileOne(builtins.Builtin):
short_description = "Build artifacts from the current source directory in a chroot"
def _resolve_refs(self, refs):
if len(refs) == 0:
return []
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
@ -52,32 +54,27 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
dependencies = buildutil.build_depends(component_name, components)
component = components.get(component_name)
component = None
build_dependencies = []
for component in components:
if component['name'] == component_name:
break
build_dependencies.append(component)
ref_to_rev = {}
arch_buildroot_name = None
arch_buildroot_rev = None
if 'architecture-buildroots2' in self.snapshot:
buildroots = self.snapshot['architecture-buildroots2']
arch_buildroot = buildroots[architecture]
arch_buildroot_name = arch_buildroot['name']
arch_buildroot_rev = arch_buildroot.get('ostree-revision')
else:
buildroots = self.snapshot['architecture-buildroots']
arch_rev_suffix = buildroots[architecture]
arch_buildroot_name = 'bases/' + arch_rev_suffix
arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
self.snapshot['prefix'],
architecture)
if arch_buildroot_rev is None:
arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
arch_buildroot_name]).strip()
arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
arch_buildroot_name]).strip()
ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
checkout_trees = [(arch_buildroot_name, '/')]
refs_to_resolve = []
for dependency_name in dependencies:
buildname = 'components/%s/%s' % (dependency_name, architecture)
for dependency in build_dependencies:
buildname = 'components/%s/%s' % (dependency['name'], architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
@ -152,7 +149,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--pristine', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
@ -169,15 +165,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
else:
component_name = self.get_component_from_cwd()
components = self.snapshot['components']
component = components.get(component_name)
if component is None:
fatal("Couldn't find component '%s' in manifest" % (component_name, ))
self.metadata = dict(component)
self.metadata['name'] = component_name
if not args.pristine:
self.metadata['src'] = 'dirty:worktree'
self.metadata['revision'] = 'dirty-worktree'
component = self.get_expanded_component(component_name)
workdir = self.workdir
@ -202,7 +190,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
json.dump(self.metadata, output_metadata, indent=4, sort_keys=True)
json.dump(component, output_metadata, indent=4, sort_keys=True)
output_metadata.close()
chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
@ -229,7 +217,7 @@ class OstbuildChrootCompileOne(builtins.Builtin):
recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
recorded_meta_f = open(recorded_meta_path, 'w')
json.dump(self.metadata, recorded_meta_f, indent=4, sort_keys=True)
json.dump(component, recorded_meta_f, indent=4, sort_keys=True)
recorded_meta_f.close()
builtins.register(OstbuildChrootCompileOne)

113
src/ostbuild/pyostbuild/builtin_compose.py
View File

@ -1,113 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
from . import fileutil
from . import kvfile
from . import odict
from . import vcs
class OstbuildCompose(builtins.Builtin):
name = "compose"
short_description = "Build complete trees from components"
def __init__(self):
builtins.Builtin.__init__(self)
def _compose_one_target(self, bin_snapshot, target):
components = bin_snapshot['component-revisions']
base = target['base']
base_name = 'bases/%s' % (base['name'], )
base_revision = target['base']['ostree-revision']
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
compose_contents = [(base_revision, '/')]
for tree_content in target['contents']:
name = tree_content['name']
rev = components[name]
subtrees = tree_content['trees']
for subpath in subtrees:
compose_contents.append((rev, subpath))
(related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
(contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
related_f = os.fdopen(related_fd, 'w')
contents_f = os.fdopen(contents_fd, 'w')
for (branch, subpath) in compose_contents:
related_f.write(' ')
related_f.write(branch)
related_f.write('\n')
contents_f.write(branch)
contents_f.write('\0')
contents_f.write(subpath)
contents_f.write('\0')
related_f.close()
contents_f.close()
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--no-triggers', '--union',
'--from-file=' + contents_tmppath, compose_rootdir])
os.unlink(contents_tmppath)
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(bin_snapshot, f, indent=4, sort_keys=True)
f.close()
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--related-objects-file=' + related_tmppath,
'--skip-if-unchanged'], cwd=compose_rootdir)
os.unlink(related_tmppath)
shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
for target in self.bin_snapshot['targets']:
log("Composing target %r from %u components" % (target['name'],
len(target['contents'])))
self._compose_one_target(self.bin_snapshot, target)
builtins.register(OstbuildCompose)

8
src/ostbuild/pyostbuild/builtin_tree_to_src.py → src/ostbuild/pyostbuild/builtin_import_tree.py
View File

@ -28,9 +28,9 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildTreeToSrc(builtins.Builtin):
name = "tree-to-src"
short_description = "Turn a tree into a source snapshot"
class OstbuildImportTree(builtins.Builtin):
name = "import-tree"
short_description = "Extract source data from tree"
def __init__(self):
builtins.Builtin.__init__(self)
@ -63,4 +63,4 @@ class OstbuildTreeToSrc(builtins.Builtin):
path = db.store(snapshot)
log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildTreeToSrc)
builtins.register(OstbuildImportTree)

63
src/ostbuild/pyostbuild/builtin_modify_snapshot.py
View File

@ -1,63 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
import select,time
import argparse
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
class OstbuildModifySnapshot(builtins.Builtin):
name = "modify-snapshot"
short_description = "Change the current source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
component_name = self.get_component_from_cwd()
current_meta = self.get_component_meta(component_name)
new_snapshot = dict(self.snapshot)
new_meta = dict(current_meta)
if 'patches' in new_meta:
del new_meta['patches']
new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
new_snapshot['components'][component_name] = new_meta
db = self.get_src_snapshot_db()
path = db.store(new_snapshot)
log("Replaced %s with %s %s" % (component_name, new_meta['src'],
new_meta['revision']))
log("New source snapshot: %s" % (path, ))
builtins.register(OstbuildModifySnapshot)

117
src/ostbuild/pyostbuild/builtin_resolve.py
View File

@ -35,7 +35,7 @@ from . import odict
class OstbuildResolve(builtins.Builtin):
name = "resolve"
short_description = "Download the source code for a given manifest"
short_description = "Expand git revisions in source to exact targets"
def __init__(self):
builtins.Builtin.__init__(self)
@ -88,8 +88,8 @@ class OstbuildResolve(builtins.Builtin):
self.prefix = self.manifest['prefix']
snapshot = copy.deepcopy(self.manifest)
component_source_list = map(self._resolve_component_meta, self.manifest['components'])
del snapshot['components']
components = map(self._resolve_component_meta, self.manifest['components'])
snapshot['components'] = components
if args.fetch:
if len(args.components) == 0:
@ -98,7 +98,7 @@ class OstbuildResolve(builtins.Builtin):
fetch_components = args.components
for component_name in fetch_components:
found = False
for component in component_source_list:
for component in components:
if component['name'] == component_name:
found = True
break
@ -112,6 +112,7 @@ class OstbuildResolve(builtins.Builtin):
fetch_components = []
global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
snapshot['patches'] = global_patches_meta
(keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
if args.fetch_patches:
@ -120,7 +121,7 @@ class OstbuildResolve(builtins.Builtin):
global_patches_meta['revision'] = revision
unique_component_names = set()
for component in component_source_list:
for component in components:
(keytype, uri) = vcs.parse_src_key(component['src'])
name = component['name']
@ -133,112 +134,6 @@ class OstbuildResolve(builtins.Builtin):
component['branch'])
component['revision'] = revision
config_opts = list(self.manifest['config-opts'])
config_opts.extend(component.get('config-opts', []))
component['config-opts'] = config_opts
patch_files = component.get('patches')
if patch_files is not None:
component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
manifest_architectures = snapshot['architectures']
ostree_prefix = snapshot['prefix']
base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
snapshot['architecture-buildroots'] = {}
for architecture in manifest_architectures:
snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
# Lame bit neeeded because I didn't have enough foresight to use an object
# for this in the first place, and I don't want to break backwards compatibility
# right now.
snapshot['architecture-buildroots2'] = {}
for architecture in manifest_architectures:
snapshot['architecture-buildroots2'][architecture] = {'name': 'bases/%s-%s-devel' % (base_prefix, architecture)}
components_by_name = {}
component_ordering = []
build_prev_component = None
runtime_prev_component = None
runtime_components = []
devel_components = []
builds = {}
for component in component_source_list:
base_name = component['name']
name = '%s/%s' % (ostree_prefix, base_name)
component['name'] = name
components_by_name[name] = component
if build_prev_component is not None:
component['build-depends'] = [build_prev_component['name']]
build_prev_component = component
is_runtime = component.get('component', 'runtime') == 'runtime'
if runtime_prev_component is not None:
component['runtime-depends'] = [runtime_prev_component['name']]
if is_runtime:
runtime_prev_component = component
runtime_components.append(component)
devel_components.append(component)
is_noarch = component.get('noarch', False)
if is_noarch:
# Just use the first specified architecture
component_arches = [manifest_architectures[0]]
else:
component_arches = component.get('architectures', manifest_architectures)
builds[name] = component_arches
# We expanded these keys
del snapshot['config-opts']
del snapshot['vcsconfig']
del snapshot['patches']
del snapshot['architectures']
targets_list = []
snapshot['targets'] = targets_list
for target_component_type in ['runtime', 'devel']:
for architecture in manifest_architectures:
target = {}
targets_list.append(target)
target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
target_components = runtime_components
else:
target_components = devel_components
contents = []
for component in target_components:
builds_for_component = builds[component['name']]
if architecture not in builds_for_component:
continue
binary_name = '%s/%s' % (component['name'], architecture)
component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
component_ref['trees'] = ['/runtime', '/devel', '/doc']
contents.append(component_ref)
target['contents'] = contents
for component in components_by_name.itervalues():
del component['name']
snapshot['components'] = components_by_name
snapshot['00ostree-src-snapshot-version'] = 0
current_time = time.time()
src_db = self.get_src_snapshot_db()
path = src_db.store(snapshot)
log("Source snapshot: %s" % (path, ))

39
src/ostbuild/pyostbuild/builtins.py
View File

@ -129,19 +129,31 @@ class Builtin(object):
'/_ostbuild-meta.json'])
return json.loads(text)
def get_component_meta(self, name):
assert self.repo is not None
if self.snapshot is not None:
return self.snapshot['components'][name]
meta = self._meta_cache.get(name)
if meta is None:
content = self.get_component_snapshot(name)
meta = self.get_component_meta_from_revision(content['ostree-revision'])
self._meta_cache[name] = meta
def expand_component(self, component):
meta = dict(component)
global_patchmeta = self.snapshot.get('patches')
if global_patchmeta is not None:
component_patch_files = component.get('patches', [])
if len(component_patch_files) > 0:
patches = dict(global_patchmeta)
patches['files'] = component_patch_files
meta['patches'] = patches
config_opts = self.snapshot.get('config-opts', [])
config_opts.extend(component.get('config-opts', []))
meta['config-opts'] = config_opts
return meta
def get_component(self, name):
assert self.repo is not None
assert self.snapshot is not None
for component in self.snapshot['components']:
if component['name'] == name:
return component
fatal("Couldn't find component '%s' in manifest" % (component_name, ))
def get_expanded_component(self, name):
return self.expand_component(self.get_component(name))
def get_prefix(self):
if self.prefix is None:
path = os.path.expanduser('~/.config/ostbuild-prefix')
@ -197,9 +209,10 @@ class Builtin(object):
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
src_ver = self.snapshot['00ostree-src-snapshot-version']
key = '00ostbuild-manifest-version'
src_ver = self.snapshot[key]
if src_ver != 0:
fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0" % (src_ver, ))
fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
def parse_bin_snapshot(self, prefix, path):
self.parse_prefix(prefix)

5
src/ostbuild/pyostbuild/main.py
View File

@ -22,21 +22,20 @@ import sys
import argparse
from . import builtins
from . import builtin_build_components
from . import builtin_build
from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_compose
from . import builtin_compile_one
from . import builtin_deploy_root
from . import builtin_deploy_qemu
from . import builtin_import_tree
from . import builtin_git_mirror
from . import builtin_pull_components
from . import builtin_privhelper_deploy_qemu
from . import builtin_prefix
from . import builtin_resolve
from . import builtin_modify_snapshot
from . import builtin_tree_to_src
from . import builtin_init
from . import builtin_status