shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-11 20:58:45 +03:00
Code Issues Projects Releases Wiki Activity

ostbuild: Move to gnome-ostree module

Browse Source 
Since we want to keep "ostree" buildsystem-independent, let's move
ostbuild out into the gnome-ostree module.

http://git.gnome.org/browse/gnome-ostree/commit/?id=04ba1926e33b442af1e357460e4415d4c0311e87
This commit is contained in:
Colin Walters 2012-06-08 19:01:20 -04:00
parent 6aaa2089f0
commit 270dc05176
35 changed files with 0 additions and 3447 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
Makefile-ostbuild.am
View File

@ -1,57 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
ostbuild: src/ostbuild/ostbuild.in Makefile
sed -e s,@libdir\@,$(libdir), -e s,@datarootdir\@,$(datarootdir), -e s,@PYTHON\@,$(PYTHON), $< > $@.tmp && mv $@.tmp $@
bin_SCRIPTS += ostbuild
EXTRA_DIST += src/ostbuild/ostbuild.in
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_deploy_root.py \
src/ostbuild/pyostbuild/builtin_run_qemu.py \
src/ostbuild/pyostbuild/builtin_import_tree.py \
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \
src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py \
src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
src/ostbuild/pyostbuild/builtin_init.py \
src/ostbuild/pyostbuild/builtin_source_diff.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/jsondb.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/mainloop.py \
src/ostbuild/pyostbuild/odict.py \
src/ostbuild/pyostbuild/ostbuildlog.py \
src/ostbuild/pyostbuild/ostbuildrc.py \
src/ostbuild/pyostbuild/privileged_subproc.py \
src/ostbuild/pyostbuild/warningfilter.py \
src/ostbuild/pyostbuild/subprocess_helpers.py \
src/ostbuild/pyostbuild/vcs.py \
$(NULL)

1
Makefile.am
View File

@ -40,7 +40,6 @@ include Makefile-libostree.am
include Makefile-ostree.am
include Makefile-ostadmin.am
include Makefile-switchroot.am
include Makefile-ostbuild.am
include Makefile-triggers.am
# Docbook generation copied from systemd/Makefile.am

2
configure.ac
View File

@ -84,8 +84,6 @@ if test x$with_libarchive != xno; then
fi
AM_CONDITIONAL(USE_LIBARCHIVE, test $with_libarchive != no)
AM_PATH_PYTHON([2.7])
AC_CONFIG_FILES([
Makefile
])

32
src/ostbuild/ostbuild.in
View File

@ -1,32 +0,0 @@
#!/usr/bin/python
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
import __builtin__
__builtin__.__dict__['DATADIR'] = '@datarootdir@'
# This is a private directory, we don't want to pollute the global
# namespace.
path = os.path.join('@libdir@', 'ostbuild')
sys.path.insert(0, path)
from pyostbuild.main import main
sys.exit(main(sys.argv[1:]))

0
src/ostbuild/pyostbuild/__init__.py
View File

187
src/ostbuild/pyostbuild/buildutil.py
View File

@ -1,187 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import re
import urlparse
import tempfile
import StringIO
from . import ostbuildrc
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync_get_output
BUILD_ENV = {
'HOME' : '/',
'HOSTNAME' : 'ostbuild',
'LANG': 'C',
'PATH' : '/usr/bin:/bin:/usr/sbin:/sbin',
'SHELL' : '/bin/bash',
'TERM' : 'vt100',
'TMPDIR' : '/tmp',
'TZ': 'EST5EDT'
}
def parse_src_key(srckey):
idx = srckey.find(':')
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git', 'local']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
def get_mirrordir(mirrordir, keytype, uri, prefix=''):
if keytype != 'git':
fatal("Unhandled keytype '%s' for uri '%s'" % (keytype, uri))
parsed = urlparse.urlsplit(uri)
return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
def find_user_chroot_path():
# We need to search PATH here manually so we correctly pick up an
# ostree install in e.g. ~/bin even though we're going to set PATH
# below for our children inside the chroot.
ostbuild_user_chroot_path = None
for dirname in os.environ['PATH'].split(':'):
path = os.path.join(dirname, 'linux-user-chroot')
if os.access(path, os.X_OK):
ostbuild_user_chroot_path = path
break
if ostbuild_user_chroot_path is None:
ostbuild_user_chroot_path = 'linux-user-chroot'
return ostbuild_user_chroot_path
def branch_name_for_artifact(a):
return 'artifacts/%s/%s/%s' % (a['buildroot'],
a['name'],
a['branch'])
def get_git_version_describe(dirpath, commit=None):
args = ['git', 'describe', '--long', '--abbrev=42', '--always']
if commit is not None:
args.append(commit)
version = run_sync_get_output(args, cwd=dirpath)
return version.strip()
def ref_to_unix_name(ref):
return ref.replace('/', '.')
def tsort_components(components, key):
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-')
f = os.fdopen(fd, 'w')
for name,component in components.iteritems():
build_prev = component.get(key)
if (build_prev is not None and len(build_prev) > 0):
for dep_name in build_prev:
f.write('%s %s\n' % (name, dep_name))
f.close()
output = run_sync_get_output(['tsort', path])
os.unlink(path)
output_stream = StringIO.StringIO(output)
result = []
for line in output_stream:
result.append(line.strip())
return result
def _recurse_depends(depkey, component_name, components, dep_names):
component = components[component_name]
depends = component.get(depkey)
if (depends is None or len(depends) == 0):
return
for depname in depends:
dep_names.add(depname)
_recurse_depends(depkey, depname, components, dep_names)
def _sorted_depends(deptype, component_name, components):
dep_names = set()
_recurse_depends(deptype, component_name, components, dep_names)
dep_components = {}
for component_name in dep_names:
dep_components[component_name] = components[component_name]
result = tsort_components(dep_components, deptype)
result.reverse()
return result
def build_depends(component_name, components):
return _sorted_depends('build-depends', component_name, components)
def runtime_depends(component_name, components):
return _sorted_depends('runtime-depends', component_name, components)
def find_component_in_manifest(manifest, component_name):
for component in manifest['components']:
if component['name'] == component_name:
return component
return None
def compose(repo, target, artifacts):
child_args = ['ostree', '--repo=' + repo, 'compose',
'-b', target, '-s', 'Compose']
(fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for artifact in artifacts:
f.write(artifact)
f.write('\n')
f.close()
child_args.extend(['-F', path])
revision = run_sync_get_output(child_args, log_initiation=True).strip()
os.unlink(path)
return revision
def get_base_user_chroot_args():
path = find_user_chroot_path()
args = [path, '--unshare-pid', '--unshare-ipc']
if not ostbuildrc.get_key('preserve_net', default=False):
args.append('--unshare-net')
return args
def resolve_component_meta(snapshot, component_meta):
result = dict(component_meta)
orig_src = component_meta['src']
did_expand = False
for (vcsprefix, expansion) in snapshot['vcsconfig'].iteritems():
prefix = vcsprefix + ':'
if orig_src.startswith(prefix):
result['src'] = expansion + orig_src[len(prefix):]
did_expand = True
break
name = component_meta.get('name')
if name is None:
if did_expand:
src = orig_src
idx = src.rindex(':')
name = src[idx+1:]
else:
src = result['src']
idx = src.rindex('/')
name = src[idx+1:]
if name.endswith('.git'):
name = name[:-4]
name = name.replace('/', '-')
result['name'] = name
branch_or_tag = result.get('branch') or result.get('tag')
if branch_or_tag is None:
result['branch'] = 'master'
return result

382
src/ostbuild/pyostbuild/builtin_build.py
View File

@ -1,382 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import hashlib
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
from . import fileutil
from . import kvfile
from . import odict
from . import vcs
class BuildOptions(object):
pass
class OstbuildBuild(builtins.Builtin):
name = "build"
short_description = "Build multiple components and generate trees"
def __init__(self):
builtins.Builtin.__init__(self)
def _get_ostbuild_chroot_args(self, architecture, component, component_resultdir):
basename = component['name']
current_machine = os.uname()[4]
if current_machine != architecture:
args = ['setarch', architecture]
else:
args = []
args.extend(['ostbuild', 'chroot-compile-one',
'--snapshot=' + self.snapshot_path,
'--name=' + basename, '--arch=' + architecture,
'--resultdir=' + component_resultdir])
return args
def _launch_debug_shell(self, architecture, component, component_resultdir, cwd=None):
args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir)
args.append('--debug-shell')
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
def _build_one_component(self, component, architecture):
basename = component['name']
buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture)
build_ref = 'components/%s' % (buildname, )
current_vcs_version = component.get('revision')
expanded_component = self.expand_component(component)
# TODO - deduplicate this with chroot_compile_one
current_meta_io = StringIO()
json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True)
current_metadata_text = current_meta_io.getvalue()
sha = hashlib.sha256()
sha.update(current_metadata_text)
current_meta_digest = sha.hexdigest()
if (self.buildopts.force_rebuild or
basename in self.force_build_components):
previous_build_version = None
else:
previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', build_ref],
stderr=open('/dev/null', 'w'),
none_on_error=True)
if (current_vcs_version is not None
and previous_build_version is not None):
log("Previous build of '%s' is %s" % (buildname, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
'/_ostbuild-meta.json'],
log_initiation=True)
sha = hashlib.sha256()
sha.update(previous_metadata_text)
previous_meta_digest = sha.hexdigest()
if current_meta_digest == previous_meta_digest:
log("Metadata is unchanged from previous")
return previous_build_version
else:
previous_metadata = json.loads(previous_metadata_text)
previous_vcs_version = previous_metadata.get('revision')
if current_vcs_version == previous_vcs_version:
log("Metadata differs; VCS version unchanged")
if self.buildopts.skip_vcs_matches:
return previous_build_version
for k,v in expanded_component.iteritems():
previous_v = previous_metadata.get(k)
if v != previous_v:
log("Key %r differs: old: %r new: %r" % (k, previous_v, v))
else:
log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
log("No previous build for '%s' found" % (buildname, ))
checkoutdir = os.path.join(self.workdir, 'checkouts')
component_src = os.path.join(checkoutdir, buildname)
fileutil.ensure_parent_dir(component_src)
child_args = ['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--checkoutdir=' + component_src,
'--clean', '--overwrite', basename]
if self.args.patches_path:
child_args.append('--patches-path=' + self.args.patches_path)
run_sync(child_args)
artifact_meta = dict(component)
logdir = os.path.join(self.workdir, 'logs', buildname)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
if os.path.isfile(log_path):
curtime = int(time.time())
saved_name = os.path.join(logdir, 'compile-prev.log')
os.rename(log_path, saved_name)
component_resultdir = os.path.join(self.workdir, 'results', buildname)
if os.path.isdir(component_resultdir):
shutil.rmtree(component_resultdir)
fileutil.ensure_dir(component_resultdir)
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir)
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
self._launch_debug_shell(architecture, component, component_resultdir, cwd=component_src)
else:
run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src)
args = ['ostree', '--repo=' + self.repo,
'commit', '-b', build_ref, '-s', 'Build',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged']
setuid_files = artifact_meta.get('setuid', [])
statoverride_path = None
if len(setuid_files) > 0:
(fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-')
f = os.fdopen(fd, 'w')
for path in setuid_files:
f.write('+2048 ' + path)
f.write('\n')
f.close()
args.append('--statoverride=' + statoverride_path)
run_sync(args, cwd=component_resultdir)
if statoverride_path is not None:
os.unlink(statoverride_path)
if os.path.islink(component_src):
os.unlink(component_src)
else:
shutil.rmtree(component_src)
shutil.rmtree(component_resultdir)
return run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', build_ref])
def _compose_one_target(self, target, component_build_revs):
base = target['base']
base_name = 'bases/%s' % (base['name'], )
runtime_name = 'bases/%s' % (base['runtime'], )
devel_name = 'bases/%s' % (base['devel'], )
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
fileutil.ensure_parent_dir(compose_rootdir)
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
related_refs = {}
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
runtime_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', runtime_name])
related_refs[runtime_name] = runtime_revision
devel_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', devel_name])
related_refs[devel_name] = devel_revision
for name,rev in component_build_revs.iteritems():
build_ref = 'components/%s/%s' % (self.snapshot['prefix'], name)
related_refs[build_ref] = rev
(related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
related_f = os.fdopen(related_fd, 'w')
for (name, rev) in related_refs.iteritems():
related_f.write(name)
related_f.write(' ')
related_f.write(rev)
related_f.write('\n')
related_f.close()
compose_contents = [(base_revision, '/')]
for tree_content in target['contents']:
name = tree_content['name']
rev = component_build_revs[name]
subtrees = tree_content['trees']
for subpath in subtrees:
compose_contents.append((rev, subpath))
(contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
contents_f = os.fdopen(contents_fd, 'w')
for (branch, subpath) in compose_contents:
contents_f.write(branch)
contents_f.write('\0')
contents_f.write(subpath)
contents_f.write('\0')
contents_f.close()
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--no-triggers', '--union',
'--from-file=' + contents_tmppath, compose_rootdir])
os.unlink(contents_tmppath)
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(self.snapshot, f, indent=4, sort_keys=True)
f.close()
treename = 'trees/%s' % (target['name'], )
child_args = ['ostree', '--repo=' + self.repo,
'commit', '-b', treename, '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--related-objects-file=' + related_tmppath,
]
if not self.buildopts.no_skip_if_unchanged:
child_args.append('--skip-if-unchanged')
run_sync(child_args, cwd=compose_rootdir)
os.unlink(related_tmppath)
shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('--patches-path')
parser.add_argument('--force-rebuild', action='store_true')
parser.add_argument('--skip-vcs-matches', action='store_true')
parser.add_argument('--no-compose', action='store_true')
parser.add_argument('--no-skip-if-unchanged', action='store_true')
parser.add_argument('--compose-only', action='store_true')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.force_rebuild = args.force_rebuild
self.buildopts.skip_vcs_matches = args.skip_vcs_matches
self.buildopts.no_skip_if_unchanged = args.no_skip_if_unchanged
self.force_build_components = set()
components = self.snapshot['components']
prefix = self.snapshot['prefix']
base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix)
architectures = self.snapshot['architectures']
component_to_arches = {}
runtime_components = []
devel_components = []
for component in components:
name = component['name']
is_runtime = component.get('component', 'runtime') == 'runtime'
if is_runtime:
runtime_components.append(component)
devel_components.append(component)
is_noarch = component.get('noarch', False)
if is_noarch:
# Just use the first specified architecture
component_arches = [architectures[0]]
else:
component_arches = component.get('architectures', architectures)
component_to_arches[name] = component_arches
for name in args.components:
component = self.get_component(name)
self.force_build_components.add(component['name'])
components_to_build = []
component_skipped_count = 0
component_build_revs = {}
if not args.compose_only:
for component in components:
for architecture in architectures:
components_to_build.append((component, architecture))
log("%d components to build" % (len(components_to_build), ))
for (component, architecture) in components_to_build:
archname = '%s/%s' % (component['name'], architecture)
build_rev = self._build_one_component(component, architecture)
component_build_revs[archname] = build_rev
targets_list = []
for target_component_type in ['runtime', 'devel']:
for architecture in architectures:
target = {}
targets_list.append(target)
target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type)
runtime_ref = '%s-%s-runtime' % (base_prefix, architecture)
buildroot_ref = '%s-%s-devel' % (base_prefix, architecture)
if target_component_type == 'runtime':
base_ref = runtime_ref
else:
base_ref = buildroot_ref
target['base'] = {'name': base_ref,
'runtime': runtime_ref,
'devel': buildroot_ref}
if target_component_type == 'runtime':
target_components = runtime_components
else:
target_components = devel_components
contents = []
for component in target_components:
builds_for_component = component_to_arches[component['name']]
if architecture not in builds_for_component:
continue
binary_name = '%s/%s' % (component['name'], architecture)
component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
component_ref['trees'] = ['/runtime', '/devel', '/doc']
contents.append(component_ref)
target['contents'] = contents
for target in targets_list:
self._compose_one_target(target, component_build_revs)
builtins.register(OstbuildBuild)

127
src/ostbuild/pyostbuild/builtin_checkout.py
View File

@ -1,127 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import json
import urlparse
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import buildutil
from . import fileutil
from . import odict
from . import vcs
class OstbuildCheckout(builtins.Builtin):
name = "checkout"
short_description = "Check out specified modules"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--overwrite', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--patches-path')
parser.add_argument('--snapshot')
parser.add_argument('--checkoutdir')
parser.add_argument('-a', '--active-tree', action='store_true')
parser.add_argument('--clean', action='store_true')
parser.add_argument('component')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
if args.active_tree:
self.parse_active_branch()
else:
self.parse_snapshot(args.prefix, args.snapshot)
component_name = args.component
found = False
component = self.get_expanded_component(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
is_local = (keytype == 'local')
if is_local:
if args.checkoutdir:
checkoutdir = args.checkoutdir
# Kind of a hack, but...
if os.path.islink(checkoutdir):
os.unlink(checkoutdir)
if args.overwrite and os.path.isdir(checkoutdir):
shutil.rmtree(checkoutdir)
os.symlink(uri, checkoutdir)
else:
checkoutdir = uri
else:
if args.checkoutdir:
checkoutdir = args.checkoutdir
else:
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
if args.clean:
if is_local:
log("note: ignoring --clean argument due to \"local:\" specification")
else:
vcs.clean(keytype, checkoutdir)
patches = component.get('patches')
if patches is not None:
if self.args.patches_path:
(patches_keytype, patches_uri) = ('local', self.args.patches_path)
else:
(patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src'])
if patches_keytype == 'git':
patches_mirror = buildutil.get_mirrordir(self.mirrordir, patches_keytype, patches_uri)
vcs.get_vcs_checkout(self.mirrordir, patches_keytype, patches_uri,
self.patchdir, patches['branch'],
overwrite=True)
patchdir = self.patchdir
else:
patchdir = patches_uri
patch_subdir = patches.get('subdir', None)
if patch_subdir is not None:
patchdir = os.path.join(patchdir, patch_subdir)
else:
patchdir = self.patchdir
for patch in patches['files']:
patch_path = os.path.join(patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(component, f, indent=4, sort_keys=True)
f.close()
log("Checked out: %r" % (checkoutdir, ))
builtins.register(OstbuildCheckout)

223
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
View File

@ -1,223 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import time
import json
import hashlib
from . import builtins
from . import buildutil
from . import fileutil
from . import ostbuildrc
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
class OstbuildChrootCompileOne(builtins.Builtin):
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
def _resolve_refs(self, refs):
if len(refs) == 0:
return []
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
def _compose_buildroot(self, component_name, architecture):
starttime = time.time()
rootdir_prefix = os.path.join(self.workdir, 'roots')
rootdir = os.path.join(rootdir_prefix, component_name)
fileutil.ensure_parent_dir(rootdir)
# Clean up any leftover root dir
rootdir_tmp = rootdir + '.tmp'
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
component = None
build_dependencies = []
for component in components:
if component['name'] == component_name:
break
build_dependencies.append(component)
ref_to_rev = {}
prefix = self.snapshot['prefix']
arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'],
prefix,
architecture)
arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse',
arch_buildroot_name]).strip()
ref_to_rev[arch_buildroot_name] = arch_buildroot_rev
checkout_trees = [(arch_buildroot_name, '/')]
refs_to_resolve = []
for dependency in build_dependencies:
buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
resolved_refs = self._resolve_refs(refs_to_resolve)
for ref,rev in zip(refs_to_resolve, resolved_refs):
ref_to_rev[ref] = rev
sha = hashlib.sha256()
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
f = os.fdopen(fd, 'w')
for (branch, subpath) in checkout_trees:
f.write(ref_to_rev[branch])
f.write('\0')
f.write(subpath)
f.write('\0')
f.close()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
sha.update(buf)
buf = f.read(8192)
f.close()
new_root_cacheid = sha.hexdigest()
rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
if os.path.isdir(rootdir):
if os.path.isfile(rootdir_cache_path):
f = open(rootdir_cache_path)
prev_cache_id = f.read().strip()
f.close()
if prev_cache_id == new_root_cacheid:
log("Reusing previous buildroot")
os.unlink(tmppath)
return rootdir
else:
log("New buildroot differs from previous")
shutil.rmtree(rootdir)
os.mkdir(rootdir_tmp)
if len(checkout_trees) > 0:
log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
checkout_trees[-1][0]))
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--union',
'--from-file=' + tmppath, rootdir_tmp])
os.unlink(tmppath);
builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
os.mkdir(builddir_tmp)
os.mkdir(os.path.join(builddir_tmp, 'source'))
os.mkdir(os.path.join(builddir_tmp, 'results'))
os.rename(rootdir_tmp, rootdir)
f = open(rootdir_cache_path, 'w')
f.write(new_root_cacheid)
f.write('\n')
f.close()
endtime = time.time()
log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
return rootdir
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
parser.add_argument('--resultdir')
parser.add_argument('--arch', required=True)
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot(args.prefix, args.snapshot)
if args.name:
component_name = args.name
else:
component_name = self.get_component_from_cwd()
component = self.get_expanded_component(component_name)
workdir = self.workdir
log("Using working directory: %s" % (workdir, ))
child_tmpdir=os.path.join(workdir, 'tmp')
if os.path.isdir(child_tmpdir):
log("Cleaning up previous tmpdir: %r" % (child_tmpdir, ))
shutil.rmtree(child_tmpdir)
fileutil.ensure_dir(child_tmpdir)
resultdir = args.resultdir
rootdir = self._compose_buildroot(component_name, args.arch)
log("Checked out buildroot: %s" % (rootdir, ))
sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
json.dump(component, output_metadata, indent=4, sort_keys=True)
output_metadata.close()
chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name)
child_args = buildutil.get_base_user_chroot_args()
child_args.extend([
'--mount-readonly', '/',
'--mount-proc', '/proc',
'--mount-bind', '/dev', '/dev',
'--mount-bind', child_tmpdir, '/tmp',
'--mount-bind', os.getcwd(), chroot_sourcedir,
'--mount-bind', resultdir, '/ostbuild/results',
'--chdir', chroot_sourcedir])
if args.debug_shell:
child_args.extend([rootdir, '/bin/sh'])
else:
child_args.extend([rootdir, '/usr/bin/ostbuild',
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta.json'])
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)
recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json')
recorded_meta_f = open(recorded_meta_path, 'w')
json.dump(component, recorded_meta_f, indent=4, sort_keys=True)
recorded_meta_f.close()
builtins.register(OstbuildChrootCompileOne)

273
src/ostbuild/pyostbuild/builtin_compile_one.py
View File

@ -1,273 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
from multiprocessing import cpu_count
import select,time
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
PREFIX = '/usr'
_DOC_DIRS = ['usr/share/doc',
'usr/share/gtk-doc',
'usr/share/man',
'usr/share/info']
_DEVEL_DIRS = ['usr/include',
'usr/share/aclocal',
'usr/share/pkgconfig',
'usr/lib/pkgconfig']
class OstbuildCompileOne(builtins.Builtin):
name = "compile-one"
short_description = "Build artifacts from the current source directory"
def __init__(self):
builtins.Builtin.__init__(self)
self.tempfiles = []
def _has_buildapi_configure_variable(self, name):
var = '#buildapi-variable-%s' % (name, )
for line in open('configure'):
if line.find(var) >= 0:
return True
return False
def execute(self, args):
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
starttime = time.time()
uname=os.uname()
kernel=uname[0].lower()
machine=uname[4]
self.build_target='%s-%s' % (machine, kernel)
self.configargs = ['--build=' + self.build_target,
'--prefix=' + PREFIX,
'--libdir=' + os.path.join(PREFIX, 'lib'),
'--sysconfdir=/etc',
'--localstatedir=/var',
'--bindir=' + os.path.join(PREFIX, 'bin'),
'--sbindir=' + os.path.join(PREFIX, 'sbin'),
'--datadir=' + os.path.join(PREFIX, 'share'),
'--includedir=' + os.path.join(PREFIX, 'include'),
'--libexecdir=' + os.path.join(PREFIX, 'libexec'),
'--mandir=' + os.path.join(PREFIX, 'share', 'man'),
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
self.makeargs = ['make']
self.ostbuild_resultdir='_ostbuild-results'
self.ostbuild_meta_path='_ostbuild-meta.json'
chdir = None
opt_install = False
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
elif arg.startswith('--ostbuild-meta='):
self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
elif arg.startswith('--chdir='):
os.chdir(arg[len('--chdir='):])
else:
self.makeargs.append(arg)
f = open(self.ostbuild_meta_path)
self.metadata = json.load(f)
f.close()
self.configargs.extend(self.metadata.get('config-opts', []))
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
os.unlink(configure_path)
autogen_script = None
if not os.path.exists('configure'):
log("No 'configure' script found, looking for autogen/bootstrap")
for name in ['autogen', 'autogen.sh', 'bootstrap']:
if os.path.exists(name):
log("Using bootstrap script '%s'" % (name, ))
autogen_script = name
if autogen_script is None:
fatal("No configure or autogen script detected; unknown buildsystem")
if autogen_script is not None:
env = dict(os.environ)
env['NOCONFIGURE'] = '1'
run_sync(['./' + autogen_script], env=env)
else:
log("Using existing 'configure' script")
builddir = '_build'
use_builddir = True
doesnot_support_builddir = self._has_buildapi_configure_variable('no-builddir')
if doesnot_support_builddir:
log("Found no-builddir Build API variable; copying source tree to _build")
if os.path.isdir('_build'):
shutil.rmtree('_build')
shutil.copytree('.', '_build', symlinks=True,
ignore=shutil.ignore_patterns('_build'))
use_builddir = False
if use_builddir:
log("Using build directory %r" % (builddir, ))
if not os.path.isdir(builddir):
os.mkdir(builddir)
if use_builddir:
args = ['../configure']
else:
args = ['./configure']
args.extend(self.configargs)
run_sync(args, cwd=builddir)
makefile_path = None
for name in ['Makefile', 'makefile', 'GNUmakefile']:
makefile_path = os.path.join(builddir, name)
if os.path.exists(makefile_path):
break
if makefile_path is None:
fatal("No Makefile found")
args = list(self.makeargs)
user_specified_jobs = False
for arg in args:
if arg == '-j':
user_specified_jobs = True
if not user_specified_jobs:
has_notparallel = False
for line in open(makefile_path):
if line.startswith('.NOTPARALLEL'):
has_notparallel = True
log("Found .NOTPARALLEL")
if not has_notparallel:
log("Didn't find NOTPARALLEL, using parallel make by default")
args.extend(self.default_buildapi_jobs)
run_sync(args, cwd=builddir)
tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), ))
self.tempfiles.append(tempdir)
args = ['make', 'install', 'DESTDIR=' + tempdir]
run_sync(args, cwd=builddir)
runtime_path = os.path.join(self.ostbuild_resultdir, 'runtime')
devel_path = os.path.join(self.ostbuild_resultdir, 'devel')
doc_path = os.path.join(self.ostbuild_resultdir, 'doc')
for artifact_type in ['runtime', 'devel', 'doc']:
resultdir = os.path.join(self.ostbuild_resultdir, artifact_type)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
os.makedirs(resultdir)
# Remove /var from the install - components are required to
# auto-create these directories on demand.
varpath = os.path.join(tempdir, 'var')
if os.path.isdir(varpath):
shutil.rmtree(varpath)
# Move symbolic links for shared libraries as well
# as static libraries. And delete all .la files.
for libdirname in ['lib', 'usr/lib']:
path = os.path.join(tempdir, libdirname)
if not os.path.isdir(path):
continue
for filename in os.listdir(path):
subpath = os.path.join(path, filename)
if filename.endswith('.la'):
os.unlink(subpath)
continue
if not ((filename.endswith('.so')
and os.path.islink(filename))
or filename.endswith('.a')):
continue
dest = os.path.join(devel_path, libdirname, filename)
self._install_and_unlink(subpath, dest)
for dirname in _DEVEL_DIRS:
dirpath = os.path.join(tempdir, dirname)
if os.path.isdir(dirpath):
dest = os.path.join(devel_path, dirname)
self._install_and_unlink(dirpath, dest)
for dirname in _DOC_DIRS:
dirpath = os.path.join(tempdir, dirname)
if os.path.isdir(dirpath):
dest = os.path.join(doc_path, dirname)
self._install_and_unlink(dirpath, dest)
for filename in os.listdir(tempdir):
src_path = os.path.join(tempdir, filename)
dest_path = os.path.join(runtime_path, filename)
self._install_and_unlink(src_path, dest_path)
for tmpname in self.tempfiles:
assert os.path.isabs(tmpname)
if os.path.isdir(tmpname):
shutil.rmtree(tmpname)
else:
try:
os.unlink(tmpname)
except OSError, e:
pass
endtime = time.time()
log("Compilation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
log("Results placed in %s" % (self.ostbuild_resultdir, ))
def _install_and_unlink(self, src, dest):
statsrc = os.lstat(src)
dirname = os.path.dirname(dest)
if not os.path.isdir(dirname):
os.makedirs(dirname)
if stat.S_ISDIR(statsrc.st_mode):
if not os.path.isdir(dest):
os.mkdir(dest)
for filename in os.listdir(src):
src_child = os.path.join(src, filename)
dest_child = os.path.join(dest, filename)
self._install_and_unlink(src_child, dest_child)
os.rmdir(src)
else:
try:
os.rename(src, dest)
except OSError, e:
if stat.S_ISLNK(statsrc.st_mode):
linkto = os.readlink(src)
os.symlink(linkto, dest)
else:
shutil.copy2(src, dest)
os.unlink(src)
builtins.register(OstbuildCompileOne)

65
src/ostbuild/pyostbuild/builtin_deploy_qemu.py
View File

@ -1,65 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from . import ostbuildrc
from . import privileged_subproc
class OstbuildDeployQemu(builtins.Builtin):
name = "deploy-qemu"
short_description = "Extract data from shadow repository to qemu"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--snapshot')
parser.add_argument('targets', nargs='*')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_snapshot(args.prefix, args.snapshot)
if len(args.targets) > 0:
targets = args.targets
else:
targets = []
prefix = self.snapshot['prefix']
for target_component_type in ['runtime', 'devel']:
for architecture in self.snapshot['architectures']:
name = '%s-%s-%s' % (prefix, architecture, target_component_type)
targets.append(name)
helper = privileged_subproc.PrivilegedSubprocess()
shadow_path = os.path.join(self.workdir, 'shadow-repo')
child_args = ['ostbuild', 'privhelper-deploy-qemu', shadow_path]
child_args.extend(targets)
helper.spawn_sync(child_args)
builtins.register(OstbuildDeployQemu)

67
src/ostbuild/pyostbuild/builtin_deploy_root.py
View File

@ -1,67 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from . import ostbuildrc
from . import privileged_subproc
class OstbuildDeployRoot(builtins.Builtin):
name = "deploy-root"
short_description = "Extract data from shadow repository to system repository"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--snapshot')
parser.add_argument('targets', nargs='*')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_snapshot(args.prefix, args.snapshot)
if len(args.targets) > 0:
targets = args.targets
else:
targets = []
prefix = self.snapshot['prefix']
for target_component_type in ['runtime', 'devel']:
for architecture in self.snapshot['architectures']:
name = '%s-%s-%s' % (prefix, architecture, target_component_type)
targets.append(name)
helper = privileged_subproc.PrivilegedSubprocess()
sys_repo = os.path.join(self.ostree_dir, 'repo')
shadow_path = os.path.join(self.workdir, 'shadow-repo')
child_args = ['ostree', '--repo=' + sys_repo,
'pull-local', shadow_path]
child_args.extend(['trees/' + x for x in targets])
helper.spawn_sync(child_args)
builtins.register(OstbuildDeployRoot)

109
src/ostbuild/pyostbuild/builtin_git_mirror.py
View File

@ -1,109 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil,time
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from . import vcs
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildGitMirror(builtins.Builtin):
name = "git-mirror"
short_description = "Update internal git mirror for one or more components"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--manifest')
parser.add_argument('--src-snapshot')
parser.add_argument('--start-at',
help="Start at the given component")
parser.add_argument('--fetch-skip-secs', type=int, default=0,
help="Don't perform a fetch if we have done so in the last N seconds")
parser.add_argument('--fetch', action='store_true',
help="Also do a git fetch for components")
parser.add_argument('-k', '--keep-going', action='store_true',
help="Don't exit on fetch failures")
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.parse_config()
if args.manifest:
self.snapshot = json.load(open(args.manifest))
components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
self.snapshot['components'] = components
self.snapshot['patches'] = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches'])
else:
self.parse_snapshot(args.prefix, args.src_snapshot)
if len(args.components) == 0:
components = []
for component in self.snapshot['components']:
components.append(component['name'])
if 'patches' in self.snapshot:
components.append(self.snapshot['patches']['name'])
if args.start_at:
idx = components.index(args.start_at)
components = components[idx:]
else:
components = args.components
for name in components:
component = self.get_component(name)
src = component['src']
(keytype, uri) = vcs.parse_src_key(src)
branch = component.get('branch')
tag = component.get('tag')
branch_or_tag = branch or tag
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
if not args.fetch:
continue
if tag is not None:
log("Skipping fetch for %s at tag %s" % (name, tag))
continue
curtime = time.time()
if args.fetch_skip_secs > 0:
last_fetch_path = vcs.get_lastfetch_path(self.mirrordir, keytype, uri, branch_or_tag)
try:
stbuf = os.stat(last_fetch_path)
except OSError, e:
stbuf = None
if stbuf is not None:
mtime = stbuf.st_mtime
delta = curtime - mtime
if delta < args.fetch_skip_secs:
log("Skipping fetch for %s updated in last %d seconds" % (name, delta))
continue
log("Running git fetch for %s" % (name, ))
vcs.fetch(self.mirrordir, keytype, uri, branch_or_tag, keep_going=args.keep_going)
builtins.register(OstbuildGitMirror)

93
src/ostbuild/pyostbuild/builtin_import_tree.py
View File

@ -1,93 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildImportTree(builtins.Builtin):
name = "import-tree"
short_description = "Extract source data from tree into new prefix"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--tree')
parser.add_argument('--prefix')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot_from_current()
log("Loading source from tree %r" % (self.snapshot_path, ))
related_objects = run_sync_get_output(['ostree', '--repo='+ self.repo,
'show', '--print-related',
self.active_branch_checksum])
ref_to_revision = {}
for line in StringIO(related_objects):
line = line.strip()
(ref, revision) = line.split(' ', 1)
ref_to_revision[ref] = revision
if args.prefix:
target_prefix = args.prefix
else:
target_prefix = self.snapshot['prefix']
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-import-tree-')
f = os.fdopen(fd, 'w')
for (ref, rev) in ref_to_revision.iteritems():
if ref.startswith('components/'):
ref = ref[len('components/'):]
(prefix, subref) = ref.split('/', 1)
newref = 'components/%s/%s' % (target_prefix, subref)
elif ref.startswith('bases/'):
# hack
base_key = '/' + self.snapshot['prefix'] + '-'
replace_key = '/' + target_prefix + '-'
newref = ref.replace(base_key, replace_key)
else:
fatal("Unhandled ref %r; expected components/ or bases/" % (ref, ))
f.write('%s %s\n' % (newref, rev))
f.close()
run_sync(['ostree', '--repo=' + self.repo,
'write-refs'], stdin=open(tmppath))
self.snapshot['prefix'] = target_prefix
run_sync(['ostbuild', 'prefix', target_prefix])
self.prefix = target_prefix
db = self.get_src_snapshot_db()
path = db.store(self.snapshot)
log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildImportTree)

58
src/ostbuild/pyostbuild/builtin_init.py
View File

@ -1,58 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
import select,time
import argparse
from . import builtins
from . import ostbuildrc
from .ostbuildlog import log, fatal
from . import fileutil
from .subprocess_helpers import run_sync, run_sync_get_output
class OstbuildInit(builtins.Builtin):
name = "init"
short_description = "Initialize working state"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
args = parser.parse_args(argv)
mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
fileutil.ensure_dir(mirrordir)
workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
fileutil.ensure_dir(workdir)
self.parse_config()
path = os.path.join(self.workdir, 'shadow-repo')
fileutil.ensure_dir(path)
if os.path.isdir(os.path.join(path, 'objects')):
log("note: shadow repository '%s' already exists" % (path, ))
else:
run_sync(['ostree', '--repo=' + path, 'init', '--archive'])
run_sync(['ostree', '--repo=' + path, 'config', 'set', 'core.parent', '/ostree/repo'])
log("Created shadow repository: %s" % (path, ))
builtins.register(OstbuildInit)

73
src/ostbuild/pyostbuild/builtin_prefix.py
View File

@ -1,73 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildPrefix(builtins.Builtin):
name = "prefix"
short_description = "Display or modify \"prefix\" (build target)"
def __init__(self):
builtins.Builtin.__init__(self)
def _set_prefix(self, prefix):
f = open(self.path, 'w')
f.write(prefix)
f.write('\n')
f.close()
log("Prefix is now %r" % (prefix, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('-a', '--active', action='store_true')
parser.add_argument('prefix', nargs='?', default=None)
args = parser.parse_args(argv)
self.path = os.path.expanduser('~/.config/ostbuild-prefix')
if args.prefix is None and not args.active:
if os.path.exists(self.path):
f = open(self.path)
print "%s" % (f.read().strip(), )
f.close()
else:
log("No currently active prefix")
elif args.prefix is not None and args.active:
fatal("Can't specify -a with prefix")
elif args.prefix is not None:
self._set_prefix(args.prefix)
else:
assert args.active
self.parse_active_branch()
active_prefix = self.active_branch_contents['prefix']
self._set_prefix(active_prefix)
builtins.register(OstbuildPrefix)

115
src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py
View File

@ -1,115 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync
from . import ostbuildrc
from . import fileutil
class OstbuildPrivhelperDeployQemu(builtins.Builtin):
name = "privhelper-deploy-qemu"
short_description = "Helper for deploy-qemu"
def __init__(self):
builtins.Builtin.__init__(self)
def _create_qemu_disk(self):
log("%s not found, creating" % (self.qemu_path, ))
success = False
tmppath = self.qemu_path + '.tmp'
if os.path.exists(tmppath):
os.unlink(tmppath)
subprocess.check_call(['qemu-img', 'create', tmppath, '6G'])
subprocess.check_call(['mkfs.ext4', '-q', '-F', tmppath])
subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w'))
try:
subprocess.check_call(['mount', '-o', 'loop', tmppath, self.mountpoint])
for topdir in ['mnt', 'sys', 'root', 'home', 'opt', 'tmp', 'run',
'ostree']:
path = os.path.join(self.mountpoint, topdir)
fileutil.ensure_dir(path)
os.chmod(os.path.join(self.mountpoint, 'root'), 0700)
os.chmod(os.path.join(self.mountpoint, 'tmp'), 01777)
varpath = os.path.join(self.mountpoint, 'ostree', 'var')
fileutil.ensure_dir(varpath)
modulespath = os.path.join(self.mountpoint, 'ostree', 'modules')
fileutil.ensure_dir(modulespath)
repo_path = os.path.join(self.mountpoint, 'ostree', 'repo')
fileutil.ensure_dir(repo_path)
subprocess.check_call(['ostree', '--repo=' + repo_path, 'init'])
success = True
finally:
subprocess.call(['umount', self.mountpoint])
if success:
os.rename(tmppath, self.qemu_path)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--rootdir',
help="Directory containing OSTree data (default: /ostree)")
parser.add_argument('srcrepo')
parser.add_argument('targets', nargs='+')
args = parser.parse_args(argv)
if os.geteuid() != 0:
fatal("This helper can only be run as root")
if args.rootdir:
self.ostree_dir = args.rootdir
else:
self.ostree_dir = self.find_ostree_dir()
self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img")
self.mountpoint = os.path.join(self.ostree_dir, 'ostree-qemu-mnt')
fileutil.ensure_dir(self.mountpoint)
if not os.path.exists(self.qemu_path):
self._create_qemu_disk()
subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w'))
repo_path = os.path.join(self.mountpoint, 'ostree', 'repo')
try:
subprocess.check_call(['mount', '-o', 'loop', self.qemu_path, self.mountpoint])
child_args = ['ostree', '--repo=' + repo_path, 'pull-local', args.srcrepo]
child_args.extend(['trees/' + x for x in args.targets])
run_sync(child_args)
first_target = args.targets[0]
for target in args.targets:
run_sync(['ostree', '--repo=' + repo_path, 'checkout', '--atomic-retarget', 'trees/'+ target, target],
cwd=os.path.join(self.mountpoint, 'ostree'))
current_link_path = os.path.join(self.mountpoint, 'ostree', 'current')
os.symlink(first_target, current_link_path + '.tmp')
os.rename(current_link_path + '.tmp', current_link_path)
finally:
subprocess.call(['umount', self.mountpoint])
builtins.register(OstbuildPrivhelperDeployQemu)

63
src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py
View File

@ -1,63 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync
from . import ostbuildrc
from . import fileutil
class OstbuildPrivhelperRunQemu(builtins.Builtin):
name = "privhelper-run-qemu"
short_description = "Helper for run-qemu"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('target')
args = parser.parse_args(argv)
if os.geteuid() != 0:
fatal("This helper can only be run as root")
self.ostree_dir = self.find_ostree_dir()
self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img")
release = os.uname()[2]
qemu = 'qemu-kvm'
kernel = '/boot/vmlinuz-%s' % (release, )
initramfs = '/boot/initramfs-ostree-%s.img' % (release, )
memory = '512M'
extra_args = 'root=/dev/sda rd.pymouth=0 ostree=%s' % (args.target, )
args = [qemu, '-kernel', kernel, '-initrd', initramfs,
'-hda', self.qemu_path, '-m', memory, '-append', extra_args]
log("Running: %s" % (subprocess.list2cmdline(args), ))
os.execvp(qemu, args)
builtins.register(OstbuildPrivhelperRunQemu)

106
src/ostbuild/pyostbuild/builtin_resolve.py
View File

@ -1,106 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
import time
import urlparse
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import vcs
from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
class OstbuildResolve(builtins.Builtin):
name = "resolve"
short_description = "Expand git revisions in source to exact targets"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--manifest', required=True,
help="Path to manifest file")
parser.add_argument('--fetch-patches', action='store_true',
help="Git fetch the patches")
parser.add_argument('--fetch', action='store_true',
help="Also perform a git fetch")
parser.add_argument('components', nargs='*',
help="List of component names to git fetch")
args = parser.parse_args(argv)
self.args = args
if len(args.components) > 0 and not args.fetch:
fatal("Can't specify components without --fetch")
self.parse_config()
self.snapshot = json.load(open(args.manifest))
self.prefix = self.snapshot['prefix']
components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components'])
self.snapshot['components'] = components
unique_component_names = set()
for component in components:
name = component['name']
if name in unique_component_names:
fatal("Duplicate component name '%s'" % (name, ))
unique_component_names.add(name)
global_patches_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches'])
self.snapshot['patches'] = global_patches_meta
(keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
if args.fetch_patches:
run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
git_mirror_args = ['ostbuild', 'git-mirror', '--manifest=' + args.manifest]
if args.fetch:
git_mirror_args.append('--fetch')
git_mirror_args.extend(args.components)
run_sync(git_mirror_args)
patch_revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
global_patches_meta['revision'] = patch_revision
for component in components:
src = component['src']
(keytype, uri) = vcs.parse_src_key(src)
branch = component.get('branch')
tag = component.get('tag')
branch_or_tag = branch or tag
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
revision = buildutil.get_git_version_describe(mirrordir, branch_or_tag)
component['revision'] = revision
src_db = self.get_src_snapshot_db()
path = src_db.store(self.snapshot)
log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildResolve)

49
src/ostbuild/pyostbuild/builtin_run_qemu.py
View File

@ -1,49 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from . import ostbuildrc
from . import privileged_subproc
class OstbuildRunQemu(builtins.Builtin):
name = "run-qemu"
short_description = "Run QEMU image"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('target')
args = parser.parse_args(argv)
self.parse_config()
helper = privileged_subproc.PrivilegedSubprocess()
child_args = ['ostbuild', 'privhelper-run-qemu', args.target]
helper.spawn_sync(child_args)
builtins.register(OstbuildRunQemu)

162
src/ostbuild/pyostbuild/builtin_source_diff.py
View File

@ -1,162 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from . import vcs
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildSourceDiff(builtins.Builtin):
name = "source-diff"
short_description = "Show differences in source code between builds"
def __init__(self):
builtins.Builtin.__init__(self)
def _diff(self, name, mirrordir, from_revision, to_revision,
diffstat=False):
diff_replace_re = re.compile(' [ab]')
env = dict(os.environ)
env['LANG'] = 'C'
spacename = ' ' + name
sys.stdout.write('diff of %s revision range %s..%s\n' % (name, from_revision, to_revision))
sys.stdout.flush()
diff_proc = subprocess.Popen(['git', 'diff', from_revision, to_revision],
env=env, cwd=mirrordir, stdout=subprocess.PIPE)
if diffstat:
diffstat_proc = subprocess.Popen(['diffstat', '-p0'],
stdin=subprocess.PIPE,
stdout=sys.stdout)
diff_pipe = diffstat_proc.stdin
else:
diffstat_proc = None
diff_pipe = sys.stdout
for line in diff_proc.stdout:
if (line.startswith('diff --git ')
or line.startswith('--- a/')
or line.startswith('+++ b/')
or line.startswith('Binary files /dev/null and b/')):
line = diff_replace_re.sub(spacename, line)
diff_pipe.write(line)
else:
diff_pipe.write(line)
diff_proc.wait()
if diffstat_proc is not None:
diffstat_proc.stdin.close()
diffstat_proc.wait()
def _log(self, opts, name, mirrordir, from_revision, to_revision):
env = dict(os.environ)
env['LANG'] = 'C'
spacename = ' ' + name
args = ['git', 'log']
args.extend(opts)
args.append(from_revision + '...' + to_revision)
proc = subprocess.Popen(args, env=env, cwd=mirrordir, stdout=subprocess.PIPE)
for line in proc.stdout:
sys.stdout.write(line)
proc.wait()
def _snapshot_from_rev(self, rev):
self.init_repo()
text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', rev, '/contents.json'],
log_initiation=False)
return json.loads(text)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--log', action='store_true')
parser.add_argument('--logp', action='store_true')
parser.add_argument('--diffstat', action='store_true')
parser.add_argument('--rev-from')
parser.add_argument('--rev-to')
parser.add_argument('--snapshot-from')
parser.add_argument('--snapshot-to')
args = parser.parse_args(argv)
self.parse_config()
to_snap = None
from_snap = None
if args.rev_to:
to_snap = self._snapshot_from_rev(args.rev_to)
if args.rev_from:
from_snap = self._snapshot_from_rev(args.rev_from)
if args.snapshot_from:
from_snap = json.load(open(args.snapshot_from))
if args.snapshot_to:
to_snap = json.load(open(args.snapshot_to))
if to_snap is None:
fatal("One of --rev-to/--snapshot-to must be given")
if from_snap is None:
if args.rev_to:
from_snap = self._snapshot_from_rev(args.rev_to + '^')
else:
fatal("One of --rev-from/--snapshot-from must be given")
for from_component in from_snap['components']:
name = from_component['name']
src = from_component['src']
(keytype, uri) = vcs.parse_src_key(src)
if keytype == 'local':
log("Component %r has local URI" % (name, ))
continue
branch_or_tag = from_component.get('branch') or from_component.get('tag')
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
to_component = self.find_component_in_snapshot(name, to_snap)
if to_component is None:
log("DELETED COMPONENT: %s" % (name, ))
continue
from_revision = from_component.get('revision')
to_revision = to_component.get('revision')
if from_revision is None:
log("From component %s missing revision" % (name, ))
continue
if to_revision is None:
log("From component %s missing revision" % (name, ))
continue
if from_revision != to_revision:
if args.log:
self._log([], name, mirrordir, from_revision, to_revision)
elif args.logp:
self._log(['-p'], name, mirrordir, from_revision, to_revision)
else:
self._diff(name, mirrordir, from_revision, to_revision,
diffstat=args.diffstat)
builtins.register(OstbuildSourceDiff)

237
src/ostbuild/pyostbuild/builtins.py
View File

@ -1,237 +0,0 @@
#!/usr/bin/python
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
import stat
import argparse
import json
from . import ostbuildrc
from . import fileutil
from . import jsondb
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
_all_builtins = {}
class Builtin(object):
name = None
short_description = None
def __init__(self):
self._meta_cache = {}
self.prefix = None
self.manifest = None
self.snapshot = None
self.bin_snapshot = None
self.repo = None
self.ostree_dir = self.find_ostree_dir()
(self.active_branch, self.active_branch_checksum) = self._find_active_branch()
self._src_snapshots = None
self._bin_snapshots = None
def find_ostree_dir(self):
for path in ['/ostree', '/sysroot/ostree']:
if os.path.isdir(path):
return path
return None
def _find_active_branch(self):
if self.ostree_dir is None:
return (None, None)
current_path = os.path.join(self.ostree_dir, 'current')
while True:
try:
target = os.path.join(self.ostree_dir, current_path)
stbuf = os.lstat(target)
except OSError, e:
current_path = None
break
if not stat.S_ISLNK(stbuf.st_mode):
break
current_path = os.readlink(target)
if current_path is not None:
basename = os.path.basename(current_path)
return basename.rsplit('-', 1)
else:
return (None, None)
def get_component_from_cwd(self):
cwd = os.getcwd()
parent = os.path.dirname(cwd)
parentparent = os.path.dirname(parent)
return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
def parse_config(self):
self.ostbuildrc = ostbuildrc
self.mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir'))
if not os.path.isdir(self.mirrordir):
fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = os.path.expanduser(ostbuildrc.get_key('workdir'))
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
self.patchdir = os.path.join(self.workdir, 'patches')
def get_component_snapshot(self, name):
found = False
for content in self.active_branch_contents['contents']:
if content['name'] == name:
found = True
break
if not found:
fatal("Unknown component '%s'" % (name, ))
return content
def get_component_meta_from_revision(self, revision):
text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', revision,
'/_ostbuild-meta.json'])
return json.loads(text)
def expand_component(self, component):
meta = dict(component)
global_patchmeta = self.snapshot.get('patches')
if global_patchmeta is not None:
component_patch_files = component.get('patches', [])
if len(component_patch_files) > 0:
patches = dict(global_patchmeta)
patches['files'] = component_patch_files
meta['patches'] = patches
config_opts = list(self.snapshot.get('config-opts', []))
config_opts.extend(component.get('config-opts', []))
meta['config-opts'] = config_opts
return meta
def find_component_in_snapshot(self, name, snapshot):
for component in snapshot['components']:
if component['name'] == name:
return component
return None
def get_component(self, name, in_snapshot=None):
if in_snapshot is None:
assert self.snapshot is not None
target_snapshot = self.snapshot
else:
target_snapshot = in_snapshot
component = self.find_component_in_snapshot(name, target_snapshot)
if (component is None and
'patches' in self.snapshot and
self.snapshot['patches']['name'] == name):
return self.snapshot['patches']
if component is None:
fatal("Couldn't find component '%s' in manifest" % (name, ))
return component
def get_expanded_component(self, name):
return self.expand_component(self.get_component(name))
def get_prefix(self):
if self.prefix is None:
path = os.path.expanduser('~/.config/ostbuild-prefix')
if not os.path.exists(path):
fatal("No prefix set; use \"ostbuild prefix\" to set one")
f = open(path)
self.prefix = f.read().strip()
f.close()
return self.prefix
def create_db(self, dbsuffix, prefix=None):
if prefix is None:
target_prefix = self.get_prefix()
else:
target_prefix = prefix
name = '%s-%s' % (target_prefix, dbsuffix)
fileutil.ensure_dir(self.snapshot_dir)
return jsondb.JsonDB(self.snapshot_dir, prefix=name)
def get_src_snapshot_db(self):
if self._src_snapshots is None:
self._src_snapshots = self.create_db('src-snapshot')
return self._src_snapshots
def get_bin_snapshot_db(self):
if self._bin_snapshots is None:
self._bin_snapshots = self.create_db('bin-snapshot')
return self._bin_snapshots
def init_repo(self):
if self.repo is not None:
return self.repo
repo = ostbuildrc.get_key('repo', default=None)
if repo is not None:
self.repo = repo
else:
shadow_path = os.path.join(self.workdir, 'shadow-repo')
if os.path.isdir(shadow_path):
self.repo = shadow_path
else:
fatal("No repository configured, and shadow-repo not found. Use \"ostbuild init\" to make one")
def parse_prefix(self, prefix):
if prefix is not None:
self.prefix = prefix
def parse_snapshot(self, prefix, path):
self.parse_prefix(prefix)
self.init_repo()
if path is None:
latest_path = self.get_src_snapshot_db().get_latest_path()
if latest_path is None:
raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
self.snapshot_path = latest_path
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
key = '00ostbuild-manifest-version'
src_ver = self.snapshot[key]
if src_ver != 0:
fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, ))
if self.prefix is None:
self.prefix = self.snapshot['prefix']
def parse_snapshot_from_current(self):
if self.ostree_dir is None:
fatal("/ostree directory not found")
repo_path = os.path.join(self.ostree_dir, 'repo')
if not os.path.isdir(repo_path):
fatal("Repository '%s' doesn't exist" % (repo_path, ))
if self.active_branch is None:
fatal("No \"current\" link found")
tree_path = os.path.join(self.ostree_dir, self.active_branch)
self.parse_snapshot(None, os.path.join(tree_path, 'contents.json'))
def execute(self, args):
raise NotImplementedError()
def register(builtin):
_all_builtins[builtin.name] = builtin
def get(name):
builtin = _all_builtins.get(name)
if builtin is not None:
return builtin()
return None
def get_all():
return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))

64
src/ostbuild/pyostbuild/filemonitor.py
View File

@ -1,64 +0,0 @@
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import stat
from . import mainloop
_global_filemon = None
class FileMonitor(object):
def __init__(self):
self._paths = {}
self._path_modtimes = {}
self._timeout = 1000
self._timeout_installed = False
self._loop = mainloop.Mainloop.get(None)
@classmethod
def get(cls):
global _global_filemon
if _global_filemon is None:
_global_filemon = cls()
return _global_filemon
def _stat(self, path):
try:
st = os.stat(path)
return st[stat.ST_MTIME]
except OSError, e:
return None
def add(self, path, callback):
if path not in self._paths:
self._paths[path] = []
self._path_modtimes[path] = self._stat(path)
self._paths[path].append(callback)
if not self._timeout_installed:
self._timeout_installed = True
self._loop.timeout_add(self._timeout, self._check_files)
def _check_files(self):
for (path,callbacks) in self._paths.iteritems():
mtime = self._stat(path)
orig_mtime = self._path_modtimes[path]
if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)):
self._path_modtimes[path] = mtime
for cb in callbacks:
cb()

26
src/ostbuild/pyostbuild/fileutil.py
View File

@ -1,26 +0,0 @@
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
def ensure_dir(path):
if not os.path.isdir(path):
os.makedirs(path)
def ensure_parent_dir(path):
ensure_dir(os.path.dirname(path))

112
src/ostbuild/pyostbuild/jsondb.py
View File

@ -1,112 +0,0 @@
#
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import stat
import time
import tempfile
import re
import shutil
import hashlib
import json
class JsonDB(object):
def __init__(self, dirpath, prefix):
self._dirpath = dirpath
self._prefix = prefix
self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
def _cmp_match_by_version(self, a, b):
# Note this is a reversed comparison; bigger is earlier
a_major = a[0]
a_minor = a[1]
b_major = b[0]
b_minor = b[1]
c = cmp(b_major, a_major)
if c == 0:
return cmp(b_minor, a_minor)
return 0
def _get_all(self):
result = []
for fname in os.listdir(self._dirpath):
if not (fname.startswith(self._prefix) and fname.endswith('.json')):
continue
path = os.path.join(self._dirpath, fname)
match = self._version_csum_re.search(fname)
if not match:
raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
path)
result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
result.sort(self._cmp_match_by_version)
return result
def get_latest(self):
path = self.get_latest_path()
if path is None:
return None
return json.load(open(path))
def get_latest_path(self):
files = self._get_all()
if len(files) == 0:
return None
return os.path.join(self._dirpath, files[0][3])
def store(self, obj):
files = self._get_all()
if len(files) == 0:
latest = None
else:
latest = files[0]
current_time = time.gmtime()
(fd, tmppath) = tempfile.mkstemp(suffix='.tmp',
prefix='tmp-jsondb-', dir=self._dirpath)
os.close(fd)
f = open(tmppath, 'w')
json.dump(obj, f, indent=4, sort_keys=True)
f.close()
csum = hashlib.sha256()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
csum.update(buf)
buf = f.read(8192)
f.close()
digest = csum.hexdigest()
if latest is not None:
if digest == latest[2]:
os.unlink(tmppath)
return latest[3]
latest_version = (latest[0], latest[1])
else:
latest_version = (current_time.tm_year, 0)
target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
latest_version[1] + 1, digest)
target_path = os.path.join(self._dirpath, target_name)
os.rename(tmppath, target_path)
return target_path

23
src/ostbuild/pyostbuild/kvfile.py
View File

@ -1,23 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
def parse(stream):
ret = {}
for line in stream:
(k,v) = line.split('=', 1)
ret[k.strip()] = v.strip()
return ret

61
src/ostbuild/pyostbuild/main.py
View File

@ -1,61 +0,0 @@
#!/usr/bin/python
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
import argparse
from . import builtins
from . import builtin_build
from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_compile_one
from . import builtin_deploy_root
from . import builtin_deploy_qemu
from . import builtin_git_mirror
from . import builtin_import_tree
from . import builtin_init
from . import builtin_run_qemu
from . import builtin_prefix
from . import builtin_privhelper_deploy_qemu
from . import builtin_privhelper_run_qemu
from . import builtin_resolve
from . import builtin_source_diff
def usage(ecode):
print "Builtins:"
for builtin in builtins.get_all():
if builtin.name.startswith('privhelper'):
continue
print " %s - %s" % (builtin.name, builtin.short_description)
return ecode
def main(args):
if len(args) < 1:
return usage(1)
elif args[0] in ('-h', '--help'):
return usage(0)
else:
builtin = builtins.get(args[0])
if builtin is None:
print "error: Unknown builtin '%s'" % (args[0], )
return usage(1)
return builtin.execute(args[1:])

96
src/ostbuild/pyostbuild/mainloop.py
View File

@ -1,96 +0,0 @@
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
import select
import time
class Mainloop(object):
DEFAULT = None
def __init__(self):
self._running = True
self.poll = None
self._timeouts = []
self._pid_watches = {}
self._fd_callbacks = {}
@classmethod
def get(cls, context):
if context is None:
if cls.DEFAULT is None:
cls.DEFAULT = cls()
return cls.DEFAULT
raise NotImplementedError("Unknown context %r" % (context, ))
def _ensure_poll(self):
if self.poll is None:
self.poll = select.poll()
def watch_fd(self, fd, callback):
self._ensure_poll()
self.poll.register(fd)
self._fd_callbacks[fd] = callback
def unwatch_fd(self, fd):
self.poll.unregister(fd)
del self._fd_callbacks[fd]
def watch_pid(self, pid, callback):
self._pid_watches[pid] = callback
def timeout_add(self, ms, callback):
self._timeouts.append((ms, callback))
def quit(self):
self._running = False
def run_once(self):
min_timeout = None
if len(self._pid_watches) > 0:
min_timeout = 500
for (ms, callback) in self._timeouts:
if (min_timeout is None) or (ms < min_timeout):
min_timeout = ms
origtime = time.time() * 1000
self._ensure_poll()
fds = self.poll.poll(min_timeout)
for fd in fds:
self._fd_callbacks[fd]()
to_delete_pids = []
for pid in self._pid_watches:
(opid, status) = os.waitpid(pid, os.WNOHANG)
if opid == pid:
to_delete_pids.append(pid)
self._pid_watches[pid](pid, status)
for pid in to_delete_pids:
del self._pid_watches[pid]
newtime = time.time() * 1000
diff = int(newtime - origtime)
if diff < 0: diff = 0
for i,(ms, callback) in enumerate(self._timeouts):
remaining_ms = ms - diff
if remaining_ms <= 0:
callback()
else:
self._timeouts[i] = (remaining_ms, callback)
def run(self):
self._running = True
while self._running:
self.run_once()

45
src/ostbuild/pyostbuild/odict.py
View File

@ -1,45 +0,0 @@
# -*- Mode: Python -*-
# GObject-Introspection - a framework for introspecting GObject libraries
# Copyright (C) 2008 Johan Dahlin
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
#
"""odict - an ordered dictionary"""
from UserDict import DictMixin
class odict(DictMixin):
def __init__(self):
self._items = {}
self._keys = []
def __setitem__(self, key, value):
if key not in self._items:
self._keys.append(key)
self._items[key] = value
def __getitem__(self, key):
return self._items[key]
def __delitem__(self, key):
del self._items[key]
self._keys.remove(key)
def keys(self):
return self._keys[:]

35
src/ostbuild/pyostbuild/ostbuildlog.py
View File

@ -1,35 +0,0 @@
#!/usr/bin/python
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
def log(msg, prefix=None):
if prefix is None:
prefix_target = ''
else:
prefix_target = prefix
fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
sys.stdout.write(fullmsg)
sys.stdout.flush()
def fatal(msg):
log(msg, prefix="FATAL: ")
sys.exit(1)

52
src/ostbuild/pyostbuild/ostbuildrc.py
View File

@ -1,52 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,ConfigParser
_config = None
def get():
global _config
if _config is None:
configpath = os.path.expanduser('~/.config/ostbuild.cfg')
parser = ConfigParser.RawConfigParser()
parser.read([configpath])
_config = {}
for (k, v) in parser.items('global'):
_config[k.strip()] = v.strip()
return _config
# This hack is because we want people to be able to pass None
# for "default", but still distinguish default=None from default
# not passed.
_default_not_supplied = object()
def get_key(name, provided_args=None, default=_default_not_supplied):
global _default_not_supplied
config = get()
if provided_args:
v = provided_args.get(name)
if v is not None:
return v
if default is _default_not_supplied:
# Possibly throw a KeyError
return config[name]
value = config.get(name, _default_not_supplied)
if value is _default_not_supplied:
return default
return value

39
src/ostbuild/pyostbuild/privileged_subproc.py
View File

@ -1,39 +0,0 @@
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess
from .ostbuildlog import log, fatal
from . import ostbuildrc
from .subprocess_helpers import run_sync
class PrivilegedSubprocess(object):
def spawn_sync(self, argv):
helper = ostbuildrc.get_key('privileged_exec', default='pkexec')
handlers = {'pkexec': self._pkexec_spawn_sync}
handler = handlers.get(helper)
if handler is None:
fatal("Unrecognized privileged_exec; valid values=%r" % (handlers.keys(),))
else:
handler(argv)
def _pkexec_spawn_sync(self, argv):
pkexec_argv = ['pkexec'] + argv
run_sync(pkexec_argv)

145
src/ostbuild/pyostbuild/subprocess_helpers.py
View File

@ -1,145 +0,0 @@
#!/usr/bin/python
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import sys
import subprocess
from .ostbuildlog import log, fatal
from .warningfilter import WarningFilter
from .mainloop import Mainloop
def _get_env_for_cwd(cwd=None, env=None):
# This dance is necessary because we want to keep the PWD
# environment variable up to date. Not doing so is a recipie
# for triggering edge conditions in pwd lookup.
if (cwd is not None) and (env is None or ('PWD' in env)):
if env is None:
env_copy = os.environ.copy()
else:
env_copy = env.copy()
if ('PWD' in env_copy) and (not cwd.startswith('/')):
env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd)
else:
env_copy['PWD'] = cwd
else:
env_copy = env
return env_copy
def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none_on_error=False,
log_success=False, log_initiation=False):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
f = open('/dev/null', 'r')
if stderr is None:
stderr_target = sys.stderr
else:
stderr_target = stderr
proc = subprocess.Popen(args, stdin=f, stdout=subprocess.PIPE, stderr=stderr_target,
close_fds=True, cwd=cwd, env=env_copy)
f.close()
output = proc.communicate()[0].strip()
if proc.returncode != 0 and not none_on_error:
logfn = fatal
elif log_success:
logfn = log
else:
logfn = None
if logfn is not None:
logfn("cmd '%s' (cwd=%s) exited with code %d, %d bytes of output" % (subprocess.list2cmdline(args),
cwd, proc.returncode, len(output)))
if proc.returncode == 0:
return output
return None
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
log_success=True, log_initiation=True, stdin=None, stdout=None,
stderr=None):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
if stdin is not None:
stdin_target = stdin
elif keep_stdin:
stdin_target = sys.stdin
else:
stdin_target = open('/dev/null', 'r')
if stdout is None:
stdout_target = sys.stdout
else:
stdout_target = stdout
if stderr is None:
stderr_target = sys.stderr
else:
stderr_target = stderr
proc = subprocess.Popen(args, stdin=stdin_target, stdout=stdout_target, stderr=stderr_target,
close_fds=True, cwd=cwd, env=env_copy)
if not keep_stdin:
stdin_target.close()
returncode = proc.wait()
if fatal_on_error and returncode != 0:
logfn = fatal
elif log_success:
logfn = log
else:
logfn = None
if logfn is not None:
logfn("pid %d exited with code %d" % (proc.pid, returncode))
return returncode
def run_sync_monitor_log_file(args, logfile, cwd=None, env=None,
fatal_on_error=True, log_initiation=True):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
logfile_f = open(logfile, 'w')
proc = subprocess.Popen(args, stdin=open('/dev/null', 'r'),
stdout=logfile_f,
stderr=subprocess.STDOUT,
close_fds=True, cwd=cwd, env=env_copy)
warnfilter = WarningFilter(logfile, sys.stdout)
warnfilter.start()
loop = Mainloop.get(None)
proc_estatus = None
def _on_pid_exited(pid, estatus):
global proc_estatus
proc_estatus = estatus
failed = estatus != 0
warnfilter.finish(not failed)
if fatal_on_error and failed:
logfn = fatal
else:
logfn = log
logfn("pid %d exited with code %d" % (pid, estatus))
loop.quit()
loop.watch_pid(proc.pid, _on_pid_exited)
loop.run()
return proc_estatus

155
src/ostbuild/pyostbuild/vcs.py
View File

@ -1,155 +0,0 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import re
import urlparse
import shutil
from .subprocess_helpers import run_sync_get_output, run_sync
from . import buildutil
from .ostbuildlog import log, fatal
def get_mirrordir(mirrordir, keytype, uri, prefix=''):
assert keytype == 'git'
parsed = urlparse.urlsplit(uri)
return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:])
def _fixup_submodule_references(mirrordir, cwd):
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd)
submodule_status_lines = submodules_status_text.split('\n')
have_submodules = False
for line in submodule_status_lines:
if line == '': continue
have_submodules = True
line = line[1:]
(sub_checksum, sub_name) = line.split(' ', 1)
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
'submodule.%s.url' % (sub_name, )], cwd=cwd)
mirrordir = get_mirrordir(mirrordir, 'git', sub_url)
run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd)
return have_submodules
def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
module_mirror = get_mirrordir(mirrordir, keytype, uri)
assert keytype == 'git'
checkoutdir_parent=os.path.dirname(dest)
if not os.path.isdir(checkoutdir_parent):
os.makedirs(checkoutdir_parent)
tmp_dest = dest + '.tmp'
if os.path.isdir(tmp_dest):
shutil.rmtree(tmp_dest)
if os.path.islink(dest):
os.unlink(dest)
if os.path.isdir(dest):
if overwrite:
shutil.rmtree(dest)
else:
tmp_dest = dest
if not os.path.isdir(tmp_dest):
run_sync(['git', 'clone', '-q', '--origin', 'localmirror',
'--no-checkout', module_mirror, tmp_dest])
run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
else:
run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest)
run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
have_submodules = _fixup_submodule_references(mirrordir, tmp_dest)
if have_submodules:
run_sync(['git', 'submodule', 'update'], cwd=tmp_dest)
if tmp_dest != dest:
os.rename(tmp_dest, dest)
return dest
def clean(keytype, checkoutdir):
assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
def parse_src_key(srckey):
idx = srckey.find(':')
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git', 'local']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
def get_lastfetch_path(mirrordir, keytype, uri, branch):
mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
branch_safename = branch.replace('/','_').replace('.', '_')
return mirror + '.lastfetch-%s' % (branch_safename, )
def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
shutil.rmtree(tmp_mirror)
if not os.path.isdir(mirror):
run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
os.rename(tmp_mirror, mirror)
if branch is None:
return mirror
last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
if os.path.exists(last_fetch_path):
f = open(last_fetch_path)
last_fetch_contents = f.read()
f.close()
last_fetch_contents = last_fetch_contents.strip()
else:
last_fetch_contents = None
current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
current_vcs_version = current_vcs_version.strip()
if current_vcs_version != last_fetch_contents:
log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
if os.path.isdir(tmp_checkout):
shutil.rmtree(tmp_checkout)
parent = os.path.dirname(tmp_checkout)
if not os.path.isdir(parent):
os.makedirs(parent)
run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
submodules = []
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
submodule_status_lines = submodules_status_text.split('\n')
for line in submodule_status_lines:
if line == '': continue
line = line[1:]
(sub_checksum, sub_name) = line.split(' ', 1)
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum)
shutil.rmtree(tmp_checkout)
f = open(last_fetch_path, 'w')
f.write(current_vcs_version + '\n')
f.close()
return mirror
def fetch(mirrordir, keytype, uri, branch, keep_going=False):
mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False,
fatal_on_error=not keep_going)
current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
if current_vcs_version is not None:
current_vcs_version = current_vcs_version.strip()
f = open(last_fetch_path, 'w')
f.write(current_vcs_version + '\n')
f.close()

113
src/ostbuild/pyostbuild/warningfilter.py
View File

@ -1,113 +0,0 @@
#
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import re
import stat
import fcntl
import subprocess
from . import filemonitor
from . import mainloop
warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)')
output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostbuild:)')
_bold_sequence = None
_normal_sequence = None
if os.isatty(1):
_bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
_normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0]
def _bold(text):
if _bold_sequence is not None:
return '%s%s%s' % (_bold_sequence, text, _normal_sequence)
else:
return text
class WarningFilter(object):
def __init__(self, filename, output):
self.filename = filename
self.output = output
# inherit globals
self._warning_re = warning_re
self._nonfilter_re = output_whitelist_re
self._buf = ''
self._warning_count = 0
self._filtered_line_count = 0
filemon = filemonitor.FileMonitor.get()
filemon.add(filename, self._on_changed)
self._fd = os.open(filename, os.O_RDONLY)
fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK)
def _do_read(self):
while True:
buf = os.read(self._fd, 4096)
if buf == '':
break
self._buf += buf
self._flush()
def _write_last_log_lines(self):
_last_line_limit = 100
f = open(self.filename)
lines = []
for line in f:
if line.startswith('ostbuild '):
continue
lines.append(line)
if len(lines) > _last_line_limit:
lines.pop(0)
f.close()
for line in lines:
self.output.write('| ')
self.output.write(line)
def _flush(self):
while True:
p = self._buf.find('\n')
if p < 0:
break
line = self._buf[0:p]
self._buf = self._buf[p+1:]
match = self._warning_re.search(line)
if match:
self._warning_count += 1
self.output.write(line + '\n')
else:
match = self._nonfilter_re.search(line)
if match:
self.output.write(line + '\n')
else:
self._filtered_line_count += 1
def _on_changed(self):
self._do_read()
def start(self):
self._do_read()
def finish(self, successful):
self._do_read()
if not successful:
self._write_last_log_lines()
pass
self.output.write("ostbuild %s: %d warnings\n" % ('success' if successful else _bold('failed'),
self._warning_count, ))
self.output.write("ostbuild: full log path: %s\n" % (self.filename, ))