From 270dc05176661c601696022b26a4fda7435866b1 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Fri, 8 Jun 2012 19:01:20 -0400 Subject: [PATCH] ostbuild: Move to gnome-ostree module Since we want to keep "ostree" buildsystem-independent, let's move ostbuild out into the gnome-ostree module. http://git.gnome.org/browse/gnome-ostree/commit/?id=04ba1926e33b442af1e357460e4415d4c0311e87 --- Makefile-ostbuild.am | 57 --- Makefile.am | 1 - configure.ac | 2 - src/ostbuild/ostbuild.in | 32 -- src/ostbuild/pyostbuild/__init__.py | 0 src/ostbuild/pyostbuild/buildutil.py | 187 --------- src/ostbuild/pyostbuild/builtin_build.py | 382 ------------------ src/ostbuild/pyostbuild/builtin_checkout.py | 127 ------ .../pyostbuild/builtin_chroot_compile_one.py | 223 ---------- .../pyostbuild/builtin_compile_one.py | 273 ------------- .../pyostbuild/builtin_deploy_qemu.py | 65 --- .../pyostbuild/builtin_deploy_root.py | 67 --- src/ostbuild/pyostbuild/builtin_git_mirror.py | 109 ----- .../pyostbuild/builtin_import_tree.py | 93 ----- src/ostbuild/pyostbuild/builtin_init.py | 58 --- src/ostbuild/pyostbuild/builtin_prefix.py | 73 ---- .../builtin_privhelper_deploy_qemu.py | 115 ------ .../pyostbuild/builtin_privhelper_run_qemu.py | 63 --- src/ostbuild/pyostbuild/builtin_resolve.py | 106 ----- src/ostbuild/pyostbuild/builtin_run_qemu.py | 49 --- .../pyostbuild/builtin_source_diff.py | 162 -------- src/ostbuild/pyostbuild/builtins.py | 237 ----------- src/ostbuild/pyostbuild/filemonitor.py | 64 --- src/ostbuild/pyostbuild/fileutil.py | 26 -- src/ostbuild/pyostbuild/jsondb.py | 112 ----- src/ostbuild/pyostbuild/kvfile.py | 23 -- src/ostbuild/pyostbuild/main.py | 61 --- src/ostbuild/pyostbuild/mainloop.py | 96 ----- src/ostbuild/pyostbuild/odict.py | 45 --- src/ostbuild/pyostbuild/ostbuildlog.py | 35 -- src/ostbuild/pyostbuild/ostbuildrc.py | 52 --- src/ostbuild/pyostbuild/privileged_subproc.py | 39 -- src/ostbuild/pyostbuild/subprocess_helpers.py | 145 ------- src/ostbuild/pyostbuild/vcs.py | 155 ------- src/ostbuild/pyostbuild/warningfilter.py | 113 ------ 35 files changed, 3447 deletions(-) delete mode 100644 Makefile-ostbuild.am delete mode 100755 src/ostbuild/ostbuild.in delete mode 100644 src/ostbuild/pyostbuild/__init__.py delete mode 100755 src/ostbuild/pyostbuild/buildutil.py delete mode 100755 src/ostbuild/pyostbuild/builtin_build.py delete mode 100755 src/ostbuild/pyostbuild/builtin_checkout.py delete mode 100755 src/ostbuild/pyostbuild/builtin_chroot_compile_one.py delete mode 100755 src/ostbuild/pyostbuild/builtin_compile_one.py delete mode 100755 src/ostbuild/pyostbuild/builtin_deploy_qemu.py delete mode 100755 src/ostbuild/pyostbuild/builtin_deploy_root.py delete mode 100755 src/ostbuild/pyostbuild/builtin_git_mirror.py delete mode 100755 src/ostbuild/pyostbuild/builtin_import_tree.py delete mode 100755 src/ostbuild/pyostbuild/builtin_init.py delete mode 100755 src/ostbuild/pyostbuild/builtin_prefix.py delete mode 100755 src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py delete mode 100755 src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py delete mode 100755 src/ostbuild/pyostbuild/builtin_resolve.py delete mode 100755 src/ostbuild/pyostbuild/builtin_run_qemu.py delete mode 100755 src/ostbuild/pyostbuild/builtin_source_diff.py delete mode 100755 src/ostbuild/pyostbuild/builtins.py delete mode 100644 src/ostbuild/pyostbuild/filemonitor.py delete mode 100644 src/ostbuild/pyostbuild/fileutil.py delete mode 100644 src/ostbuild/pyostbuild/jsondb.py delete mode 100755 src/ostbuild/pyostbuild/kvfile.py delete mode 100755 src/ostbuild/pyostbuild/main.py delete mode 100644 src/ostbuild/pyostbuild/mainloop.py delete mode 100644 src/ostbuild/pyostbuild/odict.py delete mode 100755 src/ostbuild/pyostbuild/ostbuildlog.py delete mode 100755 src/ostbuild/pyostbuild/ostbuildrc.py delete mode 100755 src/ostbuild/pyostbuild/privileged_subproc.py delete mode 100755 src/ostbuild/pyostbuild/subprocess_helpers.py delete mode 100755 src/ostbuild/pyostbuild/vcs.py delete mode 100644 src/ostbuild/pyostbuild/warningfilter.py diff --git a/Makefile-ostbuild.am b/Makefile-ostbuild.am deleted file mode 100644 index f0385694..00000000 --- a/Makefile-ostbuild.am +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -ostbuild: src/ostbuild/ostbuild.in Makefile - sed -e s,@libdir\@,$(libdir), -e s,@datarootdir\@,$(datarootdir), -e s,@PYTHON\@,$(PYTHON), $< > $@.tmp && mv $@.tmp $@ -bin_SCRIPTS += ostbuild -EXTRA_DIST += src/ostbuild/ostbuild.in - -pyostbuilddir=$(libdir)/ostbuild/pyostbuild -pyostbuild_PYTHON = \ - src/ostbuild/pyostbuild/buildutil.py \ - src/ostbuild/pyostbuild/builtin_build.py \ - src/ostbuild/pyostbuild/builtin_checkout.py \ - src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \ - src/ostbuild/pyostbuild/builtin_compile_one.py \ - src/ostbuild/pyostbuild/builtin_deploy_qemu.py \ - src/ostbuild/pyostbuild/builtin_deploy_root.py \ - src/ostbuild/pyostbuild/builtin_run_qemu.py \ - src/ostbuild/pyostbuild/builtin_import_tree.py \ - src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py \ - src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py \ - src/ostbuild/pyostbuild/builtin_git_mirror.py \ - src/ostbuild/pyostbuild/builtin_prefix.py \ - src/ostbuild/pyostbuild/builtin_resolve.py \ - src/ostbuild/pyostbuild/builtin_init.py \ - src/ostbuild/pyostbuild/builtin_source_diff.py \ - src/ostbuild/pyostbuild/builtins.py \ - src/ostbuild/pyostbuild/filemonitor.py \ - src/ostbuild/pyostbuild/fileutil.py \ - src/ostbuild/pyostbuild/__init__.py \ - src/ostbuild/pyostbuild/jsondb.py \ - src/ostbuild/pyostbuild/kvfile.py \ - src/ostbuild/pyostbuild/main.py \ - src/ostbuild/pyostbuild/mainloop.py \ - src/ostbuild/pyostbuild/odict.py \ - src/ostbuild/pyostbuild/ostbuildlog.py \ - src/ostbuild/pyostbuild/ostbuildrc.py \ - src/ostbuild/pyostbuild/privileged_subproc.py \ - src/ostbuild/pyostbuild/warningfilter.py \ - src/ostbuild/pyostbuild/subprocess_helpers.py \ - src/ostbuild/pyostbuild/vcs.py \ - $(NULL) - diff --git a/Makefile.am b/Makefile.am index fba107fd..ed10d5cb 100644 --- a/Makefile.am +++ b/Makefile.am @@ -40,7 +40,6 @@ include Makefile-libostree.am include Makefile-ostree.am include Makefile-ostadmin.am include Makefile-switchroot.am -include Makefile-ostbuild.am include Makefile-triggers.am # Docbook generation copied from systemd/Makefile.am diff --git a/configure.ac b/configure.ac index 4b36ad37..dd75a078 100644 --- a/configure.ac +++ b/configure.ac @@ -84,8 +84,6 @@ if test x$with_libarchive != xno; then fi AM_CONDITIONAL(USE_LIBARCHIVE, test $with_libarchive != no) -AM_PATH_PYTHON([2.7]) - AC_CONFIG_FILES([ Makefile ]) diff --git a/src/ostbuild/ostbuild.in b/src/ostbuild/ostbuild.in deleted file mode 100755 index a2002351..00000000 --- a/src/ostbuild/ostbuild.in +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys -import __builtin__ - -__builtin__.__dict__['DATADIR'] = '@datarootdir@' -# This is a private directory, we don't want to pollute the global -# namespace. -path = os.path.join('@libdir@', 'ostbuild') -sys.path.insert(0, path) - -from pyostbuild.main import main - -sys.exit(main(sys.argv[1:])) diff --git a/src/ostbuild/pyostbuild/__init__.py b/src/ostbuild/pyostbuild/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/src/ostbuild/pyostbuild/buildutil.py b/src/ostbuild/pyostbuild/buildutil.py deleted file mode 100755 index 94ad63bd..00000000 --- a/src/ostbuild/pyostbuild/buildutil.py +++ /dev/null @@ -1,187 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import re -import urlparse -import tempfile -import StringIO - -from . import ostbuildrc -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync_get_output - -BUILD_ENV = { - 'HOME' : '/', - 'HOSTNAME' : 'ostbuild', - 'LANG': 'C', - 'PATH' : '/usr/bin:/bin:/usr/sbin:/sbin', - 'SHELL' : '/bin/bash', - 'TERM' : 'vt100', - 'TMPDIR' : '/tmp', - 'TZ': 'EST5EDT' - } - -def parse_src_key(srckey): - idx = srckey.find(':') - if idx < 0: - raise ValueError("Invalid SRC uri=%s" % (srckey, )) - keytype = srckey[:idx] - if keytype not in ['git', 'local']: - raise ValueError("Unsupported SRC uri=%s" % (srckey, )) - uri = srckey[idx+1:] - return (keytype, uri) - - -def get_mirrordir(mirrordir, keytype, uri, prefix=''): - if keytype != 'git': - fatal("Unhandled keytype '%s' for uri '%s'" % (keytype, uri)) - parsed = urlparse.urlsplit(uri) - return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:]) - -def find_user_chroot_path(): - # We need to search PATH here manually so we correctly pick up an - # ostree install in e.g. ~/bin even though we're going to set PATH - # below for our children inside the chroot. - ostbuild_user_chroot_path = None - for dirname in os.environ['PATH'].split(':'): - path = os.path.join(dirname, 'linux-user-chroot') - if os.access(path, os.X_OK): - ostbuild_user_chroot_path = path - break - if ostbuild_user_chroot_path is None: - ostbuild_user_chroot_path = 'linux-user-chroot' - return ostbuild_user_chroot_path - -def branch_name_for_artifact(a): - return 'artifacts/%s/%s/%s' % (a['buildroot'], - a['name'], - a['branch']) - -def get_git_version_describe(dirpath, commit=None): - args = ['git', 'describe', '--long', '--abbrev=42', '--always'] - if commit is not None: - args.append(commit) - version = run_sync_get_output(args, cwd=dirpath) - return version.strip() - -def ref_to_unix_name(ref): - return ref.replace('/', '.') - -def tsort_components(components, key): - (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-tsort-') - f = os.fdopen(fd, 'w') - for name,component in components.iteritems(): - build_prev = component.get(key) - if (build_prev is not None and len(build_prev) > 0): - for dep_name in build_prev: - f.write('%s %s\n' % (name, dep_name)) - f.close() - - output = run_sync_get_output(['tsort', path]) - os.unlink(path) - output_stream = StringIO.StringIO(output) - result = [] - for line in output_stream: - result.append(line.strip()) - return result - -def _recurse_depends(depkey, component_name, components, dep_names): - component = components[component_name] - depends = component.get(depkey) - if (depends is None or len(depends) == 0): - return - for depname in depends: - dep_names.add(depname) - _recurse_depends(depkey, depname, components, dep_names) - -def _sorted_depends(deptype, component_name, components): - dep_names = set() - _recurse_depends(deptype, component_name, components, dep_names) - dep_components = {} - for component_name in dep_names: - dep_components[component_name] = components[component_name] - result = tsort_components(dep_components, deptype) - result.reverse() - return result - -def build_depends(component_name, components): - return _sorted_depends('build-depends', component_name, components) - -def runtime_depends(component_name, components): - return _sorted_depends('runtime-depends', component_name, components) - -def find_component_in_manifest(manifest, component_name): - for component in manifest['components']: - if component['name'] == component_name: - return component - return None - -def compose(repo, target, artifacts): - child_args = ['ostree', '--repo=' + repo, 'compose', - '-b', target, '-s', 'Compose'] - (fd, path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - f = os.fdopen(fd, 'w') - for artifact in artifacts: - f.write(artifact) - f.write('\n') - f.close() - child_args.extend(['-F', path]) - revision = run_sync_get_output(child_args, log_initiation=True).strip() - os.unlink(path) - return revision - -def get_base_user_chroot_args(): - path = find_user_chroot_path() - args = [path, '--unshare-pid', '--unshare-ipc'] - if not ostbuildrc.get_key('preserve_net', default=False): - args.append('--unshare-net') - return args - - -def resolve_component_meta(snapshot, component_meta): - result = dict(component_meta) - orig_src = component_meta['src'] - - did_expand = False - for (vcsprefix, expansion) in snapshot['vcsconfig'].iteritems(): - prefix = vcsprefix + ':' - if orig_src.startswith(prefix): - result['src'] = expansion + orig_src[len(prefix):] - did_expand = True - break - - name = component_meta.get('name') - if name is None: - if did_expand: - src = orig_src - idx = src.rindex(':') - name = src[idx+1:] - else: - src = result['src'] - idx = src.rindex('/') - name = src[idx+1:] - if name.endswith('.git'): - name = name[:-4] - name = name.replace('/', '-') - result['name'] = name - - branch_or_tag = result.get('branch') or result.get('tag') - if branch_or_tag is None: - result['branch'] = 'master' - - return result diff --git a/src/ostbuild/pyostbuild/builtin_build.py b/src/ostbuild/pyostbuild/builtin_build.py deleted file mode 100755 index 225584ba..00000000 --- a/src/ostbuild/pyostbuild/builtin_build.py +++ /dev/null @@ -1,382 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import hashlib -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from .subprocess_helpers import run_sync_monitor_log_file -from . import ostbuildrc -from . import buildutil -from . import fileutil -from . import kvfile -from . import odict -from . import vcs - -class BuildOptions(object): - pass - -class OstbuildBuild(builtins.Builtin): - name = "build" - short_description = "Build multiple components and generate trees" - - def __init__(self): - builtins.Builtin.__init__(self) - - def _get_ostbuild_chroot_args(self, architecture, component, component_resultdir): - basename = component['name'] - current_machine = os.uname()[4] - if current_machine != architecture: - args = ['setarch', architecture] - else: - args = [] - args.extend(['ostbuild', 'chroot-compile-one', - '--snapshot=' + self.snapshot_path, - '--name=' + basename, '--arch=' + architecture, - '--resultdir=' + component_resultdir]) - return args - - def _launch_debug_shell(self, architecture, component, component_resultdir, cwd=None): - args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir) - args.append('--debug-shell') - run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True) - fatal("Exiting after debug shell") - - def _build_one_component(self, component, architecture): - basename = component['name'] - - buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture) - build_ref = 'components/%s' % (buildname, ) - - current_vcs_version = component.get('revision') - - expanded_component = self.expand_component(component) - - # TODO - deduplicate this with chroot_compile_one - current_meta_io = StringIO() - json.dump(expanded_component, current_meta_io, indent=4, sort_keys=True) - current_metadata_text = current_meta_io.getvalue() - sha = hashlib.sha256() - sha.update(current_metadata_text) - current_meta_digest = sha.hexdigest() - - if (self.buildopts.force_rebuild or - basename in self.force_build_components): - previous_build_version = None - else: - previous_build_version = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', build_ref], - stderr=open('/dev/null', 'w'), - none_on_error=True) - if (current_vcs_version is not None - and previous_build_version is not None): - log("Previous build of '%s' is %s" % (buildname, previous_build_version)) - - previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'cat', previous_build_version, - '/_ostbuild-meta.json'], - log_initiation=True) - sha = hashlib.sha256() - sha.update(previous_metadata_text) - previous_meta_digest = sha.hexdigest() - - if current_meta_digest == previous_meta_digest: - log("Metadata is unchanged from previous") - return previous_build_version - else: - previous_metadata = json.loads(previous_metadata_text) - previous_vcs_version = previous_metadata.get('revision') - if current_vcs_version == previous_vcs_version: - log("Metadata differs; VCS version unchanged") - if self.buildopts.skip_vcs_matches: - return previous_build_version - for k,v in expanded_component.iteritems(): - previous_v = previous_metadata.get(k) - if v != previous_v: - log("Key %r differs: old: %r new: %r" % (k, previous_v, v)) - else: - log("Metadata differs; note vcs version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version)) - else: - log("No previous build for '%s' found" % (buildname, )) - - checkoutdir = os.path.join(self.workdir, 'checkouts') - component_src = os.path.join(checkoutdir, buildname) - fileutil.ensure_parent_dir(component_src) - child_args = ['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path, - '--checkoutdir=' + component_src, - '--clean', '--overwrite', basename] - if self.args.patches_path: - child_args.append('--patches-path=' + self.args.patches_path) - run_sync(child_args) - - artifact_meta = dict(component) - - logdir = os.path.join(self.workdir, 'logs', buildname) - fileutil.ensure_dir(logdir) - log_path = os.path.join(logdir, 'compile.log') - if os.path.isfile(log_path): - curtime = int(time.time()) - saved_name = os.path.join(logdir, 'compile-prev.log') - os.rename(log_path, saved_name) - - component_resultdir = os.path.join(self.workdir, 'results', buildname) - if os.path.isdir(component_resultdir): - shutil.rmtree(component_resultdir) - fileutil.ensure_dir(component_resultdir) - - log("Logging to %s" % (log_path, )) - f = open(log_path, 'w') - chroot_args = self._get_ostbuild_chroot_args(architecture, component, component_resultdir) - if self.buildopts.shell_on_failure: - ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False) - if ecode != 0: - self._launch_debug_shell(architecture, component, component_resultdir, cwd=component_src) - else: - run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src) - - args = ['ostree', '--repo=' + self.repo, - 'commit', '-b', build_ref, '-s', 'Build', - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--skip-if-unchanged'] - - setuid_files = artifact_meta.get('setuid', []) - statoverride_path = None - if len(setuid_files) > 0: - (fd, statoverride_path) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-statoverride-') - f = os.fdopen(fd, 'w') - for path in setuid_files: - f.write('+2048 ' + path) - f.write('\n') - f.close() - args.append('--statoverride=' + statoverride_path) - - run_sync(args, cwd=component_resultdir) - if statoverride_path is not None: - os.unlink(statoverride_path) - - if os.path.islink(component_src): - os.unlink(component_src) - else: - shutil.rmtree(component_src) - shutil.rmtree(component_resultdir) - - return run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', build_ref]) - - def _compose_one_target(self, target, component_build_revs): - base = target['base'] - base_name = 'bases/%s' % (base['name'], ) - runtime_name = 'bases/%s' % (base['runtime'], ) - devel_name = 'bases/%s' % (base['devel'], ) - - compose_rootdir = os.path.join(self.workdir, 'roots', target['name']) - fileutil.ensure_parent_dir(compose_rootdir) - if os.path.isdir(compose_rootdir): - shutil.rmtree(compose_rootdir) - os.mkdir(compose_rootdir) - - related_refs = {} - - base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', base_name]) - - runtime_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', runtime_name]) - related_refs[runtime_name] = runtime_revision - devel_revision = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'rev-parse', devel_name]) - related_refs[devel_name] = devel_revision - - for name,rev in component_build_revs.iteritems(): - build_ref = 'components/%s/%s' % (self.snapshot['prefix'], name) - related_refs[build_ref] = rev - - (related_fd, related_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - related_f = os.fdopen(related_fd, 'w') - for (name, rev) in related_refs.iteritems(): - related_f.write(name) - related_f.write(' ') - related_f.write(rev) - related_f.write('\n') - related_f.close() - - compose_contents = [(base_revision, '/')] - for tree_content in target['contents']: - name = tree_content['name'] - rev = component_build_revs[name] - subtrees = tree_content['trees'] - for subpath in subtrees: - compose_contents.append((rev, subpath)) - - (contents_fd, contents_tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-') - contents_f = os.fdopen(contents_fd, 'w') - for (branch, subpath) in compose_contents: - contents_f.write(branch) - contents_f.write('\0') - contents_f.write(subpath) - contents_f.write('\0') - contents_f.close() - - run_sync(['ostree', '--repo=' + self.repo, - 'checkout', '--user-mode', '--no-triggers', '--union', - '--from-file=' + contents_tmppath, compose_rootdir]) - os.unlink(contents_tmppath) - - contents_path = os.path.join(compose_rootdir, 'contents.json') - f = open(contents_path, 'w') - json.dump(self.snapshot, f, indent=4, sort_keys=True) - f.close() - - treename = 'trees/%s' % (target['name'], ) - - child_args = ['ostree', '--repo=' + self.repo, - 'commit', '-b', treename, '-s', 'Compose', - '--owner-uid=0', '--owner-gid=0', '--no-xattrs', - '--related-objects-file=' + related_tmppath, - ] - if not self.buildopts.no_skip_if_unchanged: - child_args.append('--skip-if-unchanged') - run_sync(child_args, cwd=compose_rootdir) - os.unlink(related_tmppath) - shutil.rmtree(compose_rootdir) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--src-snapshot') - parser.add_argument('--patches-path') - parser.add_argument('--force-rebuild', action='store_true') - parser.add_argument('--skip-vcs-matches', action='store_true') - parser.add_argument('--no-compose', action='store_true') - parser.add_argument('--no-skip-if-unchanged', action='store_true') - parser.add_argument('--compose-only', action='store_true') - parser.add_argument('--shell-on-failure', action='store_true') - parser.add_argument('--debug-shell', action='store_true') - parser.add_argument('components', nargs='*') - - args = parser.parse_args(argv) - self.args = args - - self.parse_config() - self.parse_snapshot(args.prefix, args.src_snapshot) - - log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), )) - - self.buildopts = BuildOptions() - self.buildopts.shell_on_failure = args.shell_on_failure - self.buildopts.force_rebuild = args.force_rebuild - self.buildopts.skip_vcs_matches = args.skip_vcs_matches - self.buildopts.no_skip_if_unchanged = args.no_skip_if_unchanged - - self.force_build_components = set() - - components = self.snapshot['components'] - - prefix = self.snapshot['prefix'] - base_prefix = '%s/%s' % (self.snapshot['base']['name'], prefix) - - architectures = self.snapshot['architectures'] - - component_to_arches = {} - - runtime_components = [] - devel_components = [] - - for component in components: - name = component['name'] - - is_runtime = component.get('component', 'runtime') == 'runtime' - - if is_runtime: - runtime_components.append(component) - devel_components.append(component) - - is_noarch = component.get('noarch', False) - if is_noarch: - # Just use the first specified architecture - component_arches = [architectures[0]] - else: - component_arches = component.get('architectures', architectures) - component_to_arches[name] = component_arches - - for name in args.components: - component = self.get_component(name) - self.force_build_components.add(component['name']) - - components_to_build = [] - component_skipped_count = 0 - - component_build_revs = {} - - if not args.compose_only: - for component in components: - for architecture in architectures: - components_to_build.append((component, architecture)) - - log("%d components to build" % (len(components_to_build), )) - for (component, architecture) in components_to_build: - archname = '%s/%s' % (component['name'], architecture) - build_rev = self._build_one_component(component, architecture) - component_build_revs[archname] = build_rev - - targets_list = [] - for target_component_type in ['runtime', 'devel']: - for architecture in architectures: - target = {} - targets_list.append(target) - target['name'] = '%s-%s-%s' % (prefix, architecture, target_component_type) - - runtime_ref = '%s-%s-runtime' % (base_prefix, architecture) - buildroot_ref = '%s-%s-devel' % (base_prefix, architecture) - if target_component_type == 'runtime': - base_ref = runtime_ref - else: - base_ref = buildroot_ref - target['base'] = {'name': base_ref, - 'runtime': runtime_ref, - 'devel': buildroot_ref} - - if target_component_type == 'runtime': - target_components = runtime_components - else: - target_components = devel_components - - contents = [] - for component in target_components: - builds_for_component = component_to_arches[component['name']] - if architecture not in builds_for_component: - continue - binary_name = '%s/%s' % (component['name'], architecture) - component_ref = {'name': binary_name} - if target_component_type == 'runtime': - component_ref['trees'] = ['/runtime'] - else: - component_ref['trees'] = ['/runtime', '/devel', '/doc'] - contents.append(component_ref) - target['contents'] = contents - - for target in targets_list: - self._compose_one_target(target, component_build_revs) - -builtins.register(OstbuildBuild) diff --git a/src/ostbuild/pyostbuild/builtin_checkout.py b/src/ostbuild/pyostbuild/builtin_checkout.py deleted file mode 100755 index 153ea8a9..00000000 --- a/src/ostbuild/pyostbuild/builtin_checkout.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import json -import urlparse -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from . import ostbuildrc -from . import buildutil -from . import fileutil -from . import odict -from . import vcs - -class OstbuildCheckout(builtins.Builtin): - name = "checkout" - short_description = "Check out specified modules" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--overwrite', action='store_true') - parser.add_argument('--prefix') - parser.add_argument('--patches-path') - parser.add_argument('--snapshot') - parser.add_argument('--checkoutdir') - parser.add_argument('-a', '--active-tree', action='store_true') - parser.add_argument('--clean', action='store_true') - parser.add_argument('component') - - args = parser.parse_args(argv) - self.args = args - - self.parse_config() - - if args.active_tree: - self.parse_active_branch() - else: - self.parse_snapshot(args.prefix, args.snapshot) - - component_name = args.component - - found = False - component = self.get_expanded_component(component_name) - (keytype, uri) = buildutil.parse_src_key(component['src']) - - is_local = (keytype == 'local') - - if is_local: - if args.checkoutdir: - checkoutdir = args.checkoutdir - # Kind of a hack, but... - if os.path.islink(checkoutdir): - os.unlink(checkoutdir) - if args.overwrite and os.path.isdir(checkoutdir): - shutil.rmtree(checkoutdir) - os.symlink(uri, checkoutdir) - else: - checkoutdir = uri - else: - if args.checkoutdir: - checkoutdir = args.checkoutdir - else: - checkoutdir = os.path.join(os.getcwd(), component_name) - fileutil.ensure_parent_dir(checkoutdir) - vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir, - component['revision'], - overwrite=args.overwrite) - - if args.clean: - if is_local: - log("note: ignoring --clean argument due to \"local:\" specification") - else: - vcs.clean(keytype, checkoutdir) - - patches = component.get('patches') - if patches is not None: - if self.args.patches_path: - (patches_keytype, patches_uri) = ('local', self.args.patches_path) - else: - (patches_keytype, patches_uri) = buildutil.parse_src_key(patches['src']) - if patches_keytype == 'git': - patches_mirror = buildutil.get_mirrordir(self.mirrordir, patches_keytype, patches_uri) - vcs.get_vcs_checkout(self.mirrordir, patches_keytype, patches_uri, - self.patchdir, patches['branch'], - overwrite=True) - patchdir = self.patchdir - else: - patchdir = patches_uri - - patch_subdir = patches.get('subdir', None) - if patch_subdir is not None: - patchdir = os.path.join(patchdir, patch_subdir) - else: - patchdir = self.patchdir - for patch in patches['files']: - patch_path = os.path.join(patchdir, patch) - run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir) - - metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json') - f = open(metadata_path, 'w') - json.dump(component, f, indent=4, sort_keys=True) - f.close() - - log("Checked out: %r" % (checkoutdir, )) - -builtins.register(OstbuildCheckout) diff --git a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py b/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py deleted file mode 100755 index 9ff1122c..00000000 --- a/src/ostbuild/pyostbuild/builtin_chroot_compile_one.py +++ /dev/null @@ -1,223 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,re,subprocess,tempfile,shutil -from StringIO import StringIO -import argparse -import time -import json -import hashlib - -from . import builtins -from . import buildutil -from . import fileutil -from . import ostbuildrc -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output - -class OstbuildChrootCompileOne(builtins.Builtin): - name = "chroot-compile-one" - short_description = "Build artifacts from the current source directory in a chroot" - - def _resolve_refs(self, refs): - if len(refs) == 0: - return [] - args = ['ostree', '--repo=' + self.repo, 'rev-parse'] - args.extend(refs) - output = run_sync_get_output(args) - return output.split('\n') - - def _compose_buildroot(self, component_name, architecture): - starttime = time.time() - - rootdir_prefix = os.path.join(self.workdir, 'roots') - rootdir = os.path.join(rootdir_prefix, component_name) - fileutil.ensure_parent_dir(rootdir) - - # Clean up any leftover root dir - rootdir_tmp = rootdir + '.tmp' - if os.path.isdir(rootdir_tmp): - shutil.rmtree(rootdir_tmp) - - components = self.snapshot['components'] - component = None - build_dependencies = [] - for component in components: - if component['name'] == component_name: - break - build_dependencies.append(component) - - ref_to_rev = {} - - prefix = self.snapshot['prefix'] - - arch_buildroot_name = 'bases/%s/%s-%s-devel' % (self.snapshot['base']['name'], - prefix, - architecture) - - arch_buildroot_rev = run_sync_get_output(['ostree', '--repo=' + self.repo, 'rev-parse', - arch_buildroot_name]).strip() - - ref_to_rev[arch_buildroot_name] = arch_buildroot_rev - checkout_trees = [(arch_buildroot_name, '/')] - refs_to_resolve = [] - for dependency in build_dependencies: - buildname = 'components/%s/%s/%s' % (prefix, dependency['name'], architecture) - refs_to_resolve.append(buildname) - checkout_trees.append((buildname, '/runtime')) - checkout_trees.append((buildname, '/devel')) - - resolved_refs = self._resolve_refs(refs_to_resolve) - for ref,rev in zip(refs_to_resolve, resolved_refs): - ref_to_rev[ref] = rev - - sha = hashlib.sha256() - - (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-') - f = os.fdopen(fd, 'w') - for (branch, subpath) in checkout_trees: - f.write(ref_to_rev[branch]) - f.write('\0') - f.write(subpath) - f.write('\0') - f.close() - - f = open(tmppath) - buf = f.read(8192) - while buf != '': - sha.update(buf) - buf = f.read(8192) - f.close() - - new_root_cacheid = sha.hexdigest() - - rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid') - - if os.path.isdir(rootdir): - if os.path.isfile(rootdir_cache_path): - f = open(rootdir_cache_path) - prev_cache_id = f.read().strip() - f.close() - if prev_cache_id == new_root_cacheid: - log("Reusing previous buildroot") - os.unlink(tmppath) - return rootdir - else: - log("New buildroot differs from previous") - - shutil.rmtree(rootdir) - - os.mkdir(rootdir_tmp) - - if len(checkout_trees) > 0: - log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees), - checkout_trees[-1][0])) - - run_sync(['ostree', '--repo=' + self.repo, - 'checkout', '--user-mode', '--union', - '--from-file=' + tmppath, rootdir_tmp]) - - os.unlink(tmppath); - - builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild') - os.mkdir(builddir_tmp) - os.mkdir(os.path.join(builddir_tmp, 'source')) - os.mkdir(os.path.join(builddir_tmp, 'results')) - os.rename(rootdir_tmp, rootdir) - - f = open(rootdir_cache_path, 'w') - f.write(new_root_cacheid) - f.write('\n') - f.close() - - endtime = time.time() - log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),)) - - return rootdir - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--snapshot', required=True) - parser.add_argument('--name') - parser.add_argument('--resultdir') - parser.add_argument('--arch', required=True) - parser.add_argument('--debug-shell', action='store_true') - - args = parser.parse_args(argv) - - self.parse_config() - self.parse_snapshot(args.prefix, args.snapshot) - - if args.name: - component_name = args.name - else: - component_name = self.get_component_from_cwd() - - component = self.get_expanded_component(component_name) - - workdir = self.workdir - - log("Using working directory: %s" % (workdir, )) - - child_tmpdir=os.path.join(workdir, 'tmp') - if os.path.isdir(child_tmpdir): - log("Cleaning up previous tmpdir: %r" % (child_tmpdir, )) - shutil.rmtree(child_tmpdir) - fileutil.ensure_dir(child_tmpdir) - - resultdir = args.resultdir - - rootdir = self._compose_buildroot(component_name, args.arch) - - log("Checked out buildroot: %s" % (rootdir, )) - - sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name) - fileutil.ensure_dir(sourcedir) - - output_metadata = open('_ostbuild-meta.json', 'w') - json.dump(component, output_metadata, indent=4, sort_keys=True) - output_metadata.close() - - chroot_sourcedir = os.path.join('/ostbuild', 'source', component_name) - - child_args = buildutil.get_base_user_chroot_args() - child_args.extend([ - '--mount-readonly', '/', - '--mount-proc', '/proc', - '--mount-bind', '/dev', '/dev', - '--mount-bind', child_tmpdir, '/tmp', - '--mount-bind', os.getcwd(), chroot_sourcedir, - '--mount-bind', resultdir, '/ostbuild/results', - '--chdir', chroot_sourcedir]) - if args.debug_shell: - child_args.extend([rootdir, '/bin/sh']) - else: - child_args.extend([rootdir, '/usr/bin/ostbuild', - 'compile-one', - '--ostbuild-resultdir=/ostbuild/results', - '--ostbuild-meta=_ostbuild-meta.json']) - env_copy = dict(buildutil.BUILD_ENV) - env_copy['PWD'] = chroot_sourcedir - run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell) - - recorded_meta_path = os.path.join(resultdir, '_ostbuild-meta.json') - recorded_meta_f = open(recorded_meta_path, 'w') - json.dump(component, recorded_meta_f, indent=4, sort_keys=True) - recorded_meta_f.close() - -builtins.register(OstbuildChrootCompileOne) diff --git a/src/ostbuild/pyostbuild/builtin_compile_one.py b/src/ostbuild/pyostbuild/builtin_compile_one.py deleted file mode 100755 index aa5fc84c..00000000 --- a/src/ostbuild/pyostbuild/builtin_compile_one.py +++ /dev/null @@ -1,273 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# ostbuild-compile-one-make wraps systems that implement the GNOME build API: -# http://people.gnome.org/~walters/docs/build-api.txt - -import os,sys,stat,subprocess,tempfile,re,shutil -from StringIO import StringIO -import json -from multiprocessing import cpu_count -import select,time - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output - -PREFIX = '/usr' - -_DOC_DIRS = ['usr/share/doc', - 'usr/share/gtk-doc', - 'usr/share/man', - 'usr/share/info'] - -_DEVEL_DIRS = ['usr/include', - 'usr/share/aclocal', - 'usr/share/pkgconfig', - 'usr/lib/pkgconfig'] - -class OstbuildCompileOne(builtins.Builtin): - name = "compile-one" - short_description = "Build artifacts from the current source directory" - - def __init__(self): - builtins.Builtin.__init__(self) - self.tempfiles = [] - - def _has_buildapi_configure_variable(self, name): - var = '#buildapi-variable-%s' % (name, ) - for line in open('configure'): - if line.find(var) >= 0: - return True - return False - - def execute(self, args): - self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )] - - starttime = time.time() - - uname=os.uname() - kernel=uname[0].lower() - machine=uname[4] - self.build_target='%s-%s' % (machine, kernel) - - self.configargs = ['--build=' + self.build_target, - '--prefix=' + PREFIX, - '--libdir=' + os.path.join(PREFIX, 'lib'), - '--sysconfdir=/etc', - '--localstatedir=/var', - '--bindir=' + os.path.join(PREFIX, 'bin'), - '--sbindir=' + os.path.join(PREFIX, 'sbin'), - '--datadir=' + os.path.join(PREFIX, 'share'), - '--includedir=' + os.path.join(PREFIX, 'include'), - '--libexecdir=' + os.path.join(PREFIX, 'libexec'), - '--mandir=' + os.path.join(PREFIX, 'share', 'man'), - '--infodir=' + os.path.join(PREFIX, 'share', 'info')] - self.makeargs = ['make'] - - self.ostbuild_resultdir='_ostbuild-results' - self.ostbuild_meta_path='_ostbuild-meta.json' - - chdir = None - opt_install = False - - for arg in args: - if arg.startswith('--ostbuild-resultdir='): - self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):] - elif arg.startswith('--ostbuild-meta='): - self.ostbuild_meta_path=arg[len('--ostbuild-meta='):] - elif arg.startswith('--chdir='): - os.chdir(arg[len('--chdir='):]) - else: - self.makeargs.append(arg) - - f = open(self.ostbuild_meta_path) - self.metadata = json.load(f) - f.close() - - self.configargs.extend(self.metadata.get('config-opts', [])) - - if self.metadata.get('rm-configure', False): - configure_path = 'configure' - if os.path.exists(configure_path): - os.unlink(configure_path) - - autogen_script = None - if not os.path.exists('configure'): - log("No 'configure' script found, looking for autogen/bootstrap") - for name in ['autogen', 'autogen.sh', 'bootstrap']: - if os.path.exists(name): - log("Using bootstrap script '%s'" % (name, )) - autogen_script = name - if autogen_script is None: - fatal("No configure or autogen script detected; unknown buildsystem") - - if autogen_script is not None: - env = dict(os.environ) - env['NOCONFIGURE'] = '1' - run_sync(['./' + autogen_script], env=env) - else: - log("Using existing 'configure' script") - - builddir = '_build' - - use_builddir = True - doesnot_support_builddir = self._has_buildapi_configure_variable('no-builddir') - if doesnot_support_builddir: - log("Found no-builddir Build API variable; copying source tree to _build") - if os.path.isdir('_build'): - shutil.rmtree('_build') - shutil.copytree('.', '_build', symlinks=True, - ignore=shutil.ignore_patterns('_build')) - use_builddir = False - - if use_builddir: - log("Using build directory %r" % (builddir, )) - if not os.path.isdir(builddir): - os.mkdir(builddir) - - if use_builddir: - args = ['../configure'] - else: - args = ['./configure'] - args.extend(self.configargs) - run_sync(args, cwd=builddir) - - makefile_path = None - for name in ['Makefile', 'makefile', 'GNUmakefile']: - makefile_path = os.path.join(builddir, name) - if os.path.exists(makefile_path): - break - if makefile_path is None: - fatal("No Makefile found") - - args = list(self.makeargs) - user_specified_jobs = False - for arg in args: - if arg == '-j': - user_specified_jobs = True - - if not user_specified_jobs: - has_notparallel = False - for line in open(makefile_path): - if line.startswith('.NOTPARALLEL'): - has_notparallel = True - log("Found .NOTPARALLEL") - - if not has_notparallel: - log("Didn't find NOTPARALLEL, using parallel make by default") - args.extend(self.default_buildapi_jobs) - - run_sync(args, cwd=builddir) - - tempdir = tempfile.mkdtemp(prefix='ostbuild-destdir-%s' % (self.metadata['name'].replace('/', '_'), )) - self.tempfiles.append(tempdir) - args = ['make', 'install', 'DESTDIR=' + tempdir] - run_sync(args, cwd=builddir) - - runtime_path = os.path.join(self.ostbuild_resultdir, 'runtime') - devel_path = os.path.join(self.ostbuild_resultdir, 'devel') - doc_path = os.path.join(self.ostbuild_resultdir, 'doc') - for artifact_type in ['runtime', 'devel', 'doc']: - resultdir = os.path.join(self.ostbuild_resultdir, artifact_type) - if os.path.isdir(resultdir): - shutil.rmtree(resultdir) - os.makedirs(resultdir) - - # Remove /var from the install - components are required to - # auto-create these directories on demand. - varpath = os.path.join(tempdir, 'var') - if os.path.isdir(varpath): - shutil.rmtree(varpath) - - # Move symbolic links for shared libraries as well - # as static libraries. And delete all .la files. - for libdirname in ['lib', 'usr/lib']: - path = os.path.join(tempdir, libdirname) - if not os.path.isdir(path): - continue - for filename in os.listdir(path): - subpath = os.path.join(path, filename) - if filename.endswith('.la'): - os.unlink(subpath) - continue - if not ((filename.endswith('.so') - and os.path.islink(filename)) - or filename.endswith('.a')): - continue - dest = os.path.join(devel_path, libdirname, filename) - self._install_and_unlink(subpath, dest) - - for dirname in _DEVEL_DIRS: - dirpath = os.path.join(tempdir, dirname) - if os.path.isdir(dirpath): - dest = os.path.join(devel_path, dirname) - self._install_and_unlink(dirpath, dest) - - for dirname in _DOC_DIRS: - dirpath = os.path.join(tempdir, dirname) - if os.path.isdir(dirpath): - dest = os.path.join(doc_path, dirname) - self._install_and_unlink(dirpath, dest) - - for filename in os.listdir(tempdir): - src_path = os.path.join(tempdir, filename) - dest_path = os.path.join(runtime_path, filename) - self._install_and_unlink(src_path, dest_path) - - for tmpname in self.tempfiles: - assert os.path.isabs(tmpname) - if os.path.isdir(tmpname): - shutil.rmtree(tmpname) - else: - try: - os.unlink(tmpname) - except OSError, e: - pass - - endtime = time.time() - - log("Compilation succeeded; %d seconds elapsed" % (int(endtime - starttime),)) - log("Results placed in %s" % (self.ostbuild_resultdir, )) - - def _install_and_unlink(self, src, dest): - statsrc = os.lstat(src) - dirname = os.path.dirname(dest) - if not os.path.isdir(dirname): - os.makedirs(dirname) - - if stat.S_ISDIR(statsrc.st_mode): - if not os.path.isdir(dest): - os.mkdir(dest) - for filename in os.listdir(src): - src_child = os.path.join(src, filename) - dest_child = os.path.join(dest, filename) - - self._install_and_unlink(src_child, dest_child) - os.rmdir(src) - else: - try: - os.rename(src, dest) - except OSError, e: - if stat.S_ISLNK(statsrc.st_mode): - linkto = os.readlink(src) - os.symlink(linkto, dest) - else: - shutil.copy2(src, dest) - os.unlink(src) - -builtins.register(OstbuildCompileOne) diff --git a/src/ostbuild/pyostbuild/builtin_deploy_qemu.py b/src/ostbuild/pyostbuild/builtin_deploy_qemu.py deleted file mode 100755 index f06b669a..00000000 --- a/src/ostbuild/pyostbuild/builtin_deploy_qemu.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from . import ostbuildrc -from . import privileged_subproc - -class OstbuildDeployQemu(builtins.Builtin): - name = "deploy-qemu" - short_description = "Extract data from shadow repository to qemu" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--snapshot') - parser.add_argument('targets', nargs='*') - - args = parser.parse_args(argv) - self.args = args - - self.parse_config() - self.parse_snapshot(args.prefix, args.snapshot) - - if len(args.targets) > 0: - targets = args.targets - else: - targets = [] - prefix = self.snapshot['prefix'] - for target_component_type in ['runtime', 'devel']: - for architecture in self.snapshot['architectures']: - name = '%s-%s-%s' % (prefix, architecture, target_component_type) - targets.append(name) - - helper = privileged_subproc.PrivilegedSubprocess() - shadow_path = os.path.join(self.workdir, 'shadow-repo') - child_args = ['ostbuild', 'privhelper-deploy-qemu', shadow_path] - child_args.extend(targets) - helper.spawn_sync(child_args) - -builtins.register(OstbuildDeployQemu) diff --git a/src/ostbuild/pyostbuild/builtin_deploy_root.py b/src/ostbuild/pyostbuild/builtin_deploy_root.py deleted file mode 100755 index f56d8f8f..00000000 --- a/src/ostbuild/pyostbuild/builtin_deploy_root.py +++ /dev/null @@ -1,67 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from . import ostbuildrc -from . import privileged_subproc - -class OstbuildDeployRoot(builtins.Builtin): - name = "deploy-root" - short_description = "Extract data from shadow repository to system repository" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--snapshot') - parser.add_argument('targets', nargs='*') - - args = parser.parse_args(argv) - self.args = args - - self.parse_config() - self.parse_snapshot(args.prefix, args.snapshot) - - if len(args.targets) > 0: - targets = args.targets - else: - targets = [] - prefix = self.snapshot['prefix'] - for target_component_type in ['runtime', 'devel']: - for architecture in self.snapshot['architectures']: - name = '%s-%s-%s' % (prefix, architecture, target_component_type) - targets.append(name) - - helper = privileged_subproc.PrivilegedSubprocess() - sys_repo = os.path.join(self.ostree_dir, 'repo') - shadow_path = os.path.join(self.workdir, 'shadow-repo') - child_args = ['ostree', '--repo=' + sys_repo, - 'pull-local', shadow_path] - child_args.extend(['trees/' + x for x in targets]) - helper.spawn_sync(child_args) - -builtins.register(OstbuildDeployRoot) diff --git a/src/ostbuild/pyostbuild/builtin_git_mirror.py b/src/ostbuild/pyostbuild/builtin_git_mirror.py deleted file mode 100755 index 6ecdafac..00000000 --- a/src/ostbuild/pyostbuild/builtin_git_mirror.py +++ /dev/null @@ -1,109 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# ostbuild-compile-one-make wraps systems that implement the GNOME build API: -# http://people.gnome.org/~walters/docs/build-api.txt - -import os,sys,stat,subprocess,tempfile,re,shutil,time -import argparse -from StringIO import StringIO -import json - -from . import builtins -from .ostbuildlog import log, fatal -from . import vcs -from .subprocess_helpers import run_sync, run_sync_get_output -from . import buildutil - -class OstbuildGitMirror(builtins.Builtin): - name = "git-mirror" - short_description = "Update internal git mirror for one or more components" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--prefix') - parser.add_argument('--manifest') - parser.add_argument('--src-snapshot') - parser.add_argument('--start-at', - help="Start at the given component") - parser.add_argument('--fetch-skip-secs', type=int, default=0, - help="Don't perform a fetch if we have done so in the last N seconds") - parser.add_argument('--fetch', action='store_true', - help="Also do a git fetch for components") - parser.add_argument('-k', '--keep-going', action='store_true', - help="Don't exit on fetch failures") - parser.add_argument('components', nargs='*') - - args = parser.parse_args(argv) - self.parse_config() - if args.manifest: - self.snapshot = json.load(open(args.manifest)) - components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components']) - self.snapshot['components'] = components - self.snapshot['patches'] = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches']) - else: - self.parse_snapshot(args.prefix, args.src_snapshot) - - if len(args.components) == 0: - components = [] - for component in self.snapshot['components']: - components.append(component['name']) - if 'patches' in self.snapshot: - components.append(self.snapshot['patches']['name']) - if args.start_at: - idx = components.index(args.start_at) - components = components[idx:] - else: - components = args.components - - for name in components: - component = self.get_component(name) - src = component['src'] - (keytype, uri) = vcs.parse_src_key(src) - branch = component.get('branch') - tag = component.get('tag') - branch_or_tag = branch or tag - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) - - if not args.fetch: - continue - - if tag is not None: - log("Skipping fetch for %s at tag %s" % (name, tag)) - continue - - curtime = time.time() - if args.fetch_skip_secs > 0: - last_fetch_path = vcs.get_lastfetch_path(self.mirrordir, keytype, uri, branch_or_tag) - try: - stbuf = os.stat(last_fetch_path) - except OSError, e: - stbuf = None - if stbuf is not None: - mtime = stbuf.st_mtime - delta = curtime - mtime - if delta < args.fetch_skip_secs: - log("Skipping fetch for %s updated in last %d seconds" % (name, delta)) - continue - - log("Running git fetch for %s" % (name, )) - vcs.fetch(self.mirrordir, keytype, uri, branch_or_tag, keep_going=args.keep_going) - -builtins.register(OstbuildGitMirror) diff --git a/src/ostbuild/pyostbuild/builtin_import_tree.py b/src/ostbuild/pyostbuild/builtin_import_tree.py deleted file mode 100755 index 9fed209a..00000000 --- a/src/ostbuild/pyostbuild/builtin_import_tree.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# ostbuild-compile-one-make wraps systems that implement the GNOME build API: -# http://people.gnome.org/~walters/docs/build-api.txt - -import os,sys,stat,subprocess,tempfile,re,shutil -import argparse -from StringIO import StringIO -import json - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from . import buildutil - -class OstbuildImportTree(builtins.Builtin): - name = "import-tree" - short_description = "Extract source data from tree into new prefix" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--tree') - parser.add_argument('--prefix') - - args = parser.parse_args(argv) - self.parse_config() - self.parse_snapshot_from_current() - - log("Loading source from tree %r" % (self.snapshot_path, )) - - related_objects = run_sync_get_output(['ostree', '--repo='+ self.repo, - 'show', '--print-related', - self.active_branch_checksum]) - ref_to_revision = {} - for line in StringIO(related_objects): - line = line.strip() - (ref, revision) = line.split(' ', 1) - ref_to_revision[ref] = revision - - if args.prefix: - target_prefix = args.prefix - else: - target_prefix = self.snapshot['prefix'] - - (fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-import-tree-') - f = os.fdopen(fd, 'w') - for (ref, rev) in ref_to_revision.iteritems(): - if ref.startswith('components/'): - ref = ref[len('components/'):] - (prefix, subref) = ref.split('/', 1) - newref = 'components/%s/%s' % (target_prefix, subref) - elif ref.startswith('bases/'): - # hack - base_key = '/' + self.snapshot['prefix'] + '-' - replace_key = '/' + target_prefix + '-' - newref = ref.replace(base_key, replace_key) - else: - fatal("Unhandled ref %r; expected components/ or bases/" % (ref, )) - - f.write('%s %s\n' % (newref, rev)) - f.close() - - run_sync(['ostree', '--repo=' + self.repo, - 'write-refs'], stdin=open(tmppath)) - - self.snapshot['prefix'] = target_prefix - - run_sync(['ostbuild', 'prefix', target_prefix]) - self.prefix = target_prefix - - db = self.get_src_snapshot_db() - path = db.store(self.snapshot) - log("Source snapshot: %s" % (path, )) - -builtins.register(OstbuildImportTree) diff --git a/src/ostbuild/pyostbuild/builtin_init.py b/src/ostbuild/pyostbuild/builtin_init.py deleted file mode 100755 index de45eb44..00000000 --- a/src/ostbuild/pyostbuild/builtin_init.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,stat,subprocess,tempfile,re,shutil -from StringIO import StringIO -import json -import select,time -import argparse - -from . import builtins -from . import ostbuildrc -from .ostbuildlog import log, fatal -from . import fileutil -from .subprocess_helpers import run_sync, run_sync_get_output - -class OstbuildInit(builtins.Builtin): - name = "init" - short_description = "Initialize working state" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - - args = parser.parse_args(argv) - - mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir')) - fileutil.ensure_dir(mirrordir) - workdir = os.path.expanduser(ostbuildrc.get_key('workdir')) - fileutil.ensure_dir(workdir) - - self.parse_config() - - path = os.path.join(self.workdir, 'shadow-repo') - fileutil.ensure_dir(path) - if os.path.isdir(os.path.join(path, 'objects')): - log("note: shadow repository '%s' already exists" % (path, )) - else: - run_sync(['ostree', '--repo=' + path, 'init', '--archive']) - run_sync(['ostree', '--repo=' + path, 'config', 'set', 'core.parent', '/ostree/repo']) - log("Created shadow repository: %s" % (path, )) - -builtins.register(OstbuildInit) diff --git a/src/ostbuild/pyostbuild/builtin_prefix.py b/src/ostbuild/pyostbuild/builtin_prefix.py deleted file mode 100755 index 4ffa9814..00000000 --- a/src/ostbuild/pyostbuild/builtin_prefix.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# ostbuild-compile-one-make wraps systems that implement the GNOME build API: -# http://people.gnome.org/~walters/docs/build-api.txt - -import os,sys,stat,subprocess,tempfile,re,shutil -import argparse -from StringIO import StringIO -import json - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from . import buildutil - -class OstbuildPrefix(builtins.Builtin): - name = "prefix" - short_description = "Display or modify \"prefix\" (build target)" - - def __init__(self): - builtins.Builtin.__init__(self) - - def _set_prefix(self, prefix): - f = open(self.path, 'w') - f.write(prefix) - f.write('\n') - f.close() - log("Prefix is now %r" % (prefix, )) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('-a', '--active', action='store_true') - parser.add_argument('prefix', nargs='?', default=None) - - args = parser.parse_args(argv) - - self.path = os.path.expanduser('~/.config/ostbuild-prefix') - if args.prefix is None and not args.active: - if os.path.exists(self.path): - f = open(self.path) - print "%s" % (f.read().strip(), ) - f.close() - else: - log("No currently active prefix") - elif args.prefix is not None and args.active: - fatal("Can't specify -a with prefix") - elif args.prefix is not None: - self._set_prefix(args.prefix) - else: - assert args.active - - self.parse_active_branch() - - active_prefix = self.active_branch_contents['prefix'] - - self._set_prefix(active_prefix) - -builtins.register(OstbuildPrefix) diff --git a/src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py b/src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py deleted file mode 100755 index 9380c462..00000000 --- a/src/ostbuild/pyostbuild/builtin_privhelper_deploy_qemu.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync -from . import ostbuildrc -from . import fileutil - -class OstbuildPrivhelperDeployQemu(builtins.Builtin): - name = "privhelper-deploy-qemu" - short_description = "Helper for deploy-qemu" - - def __init__(self): - builtins.Builtin.__init__(self) - - def _create_qemu_disk(self): - log("%s not found, creating" % (self.qemu_path, )) - success = False - tmppath = self.qemu_path + '.tmp' - if os.path.exists(tmppath): - os.unlink(tmppath) - subprocess.check_call(['qemu-img', 'create', tmppath, '6G']) - subprocess.check_call(['mkfs.ext4', '-q', '-F', tmppath]) - - subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w')) - try: - subprocess.check_call(['mount', '-o', 'loop', tmppath, self.mountpoint]) - - for topdir in ['mnt', 'sys', 'root', 'home', 'opt', 'tmp', 'run', - 'ostree']: - path = os.path.join(self.mountpoint, topdir) - fileutil.ensure_dir(path) - os.chmod(os.path.join(self.mountpoint, 'root'), 0700) - os.chmod(os.path.join(self.mountpoint, 'tmp'), 01777) - - varpath = os.path.join(self.mountpoint, 'ostree', 'var') - fileutil.ensure_dir(varpath) - modulespath = os.path.join(self.mountpoint, 'ostree', 'modules') - fileutil.ensure_dir(modulespath) - - repo_path = os.path.join(self.mountpoint, 'ostree', 'repo') - fileutil.ensure_dir(repo_path) - subprocess.check_call(['ostree', '--repo=' + repo_path, 'init']) - success = True - finally: - subprocess.call(['umount', self.mountpoint]) - if success: - os.rename(tmppath, self.qemu_path) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--rootdir', - help="Directory containing OSTree data (default: /ostree)") - parser.add_argument('srcrepo') - parser.add_argument('targets', nargs='+') - - args = parser.parse_args(argv) - - if os.geteuid() != 0: - fatal("This helper can only be run as root") - - if args.rootdir: - self.ostree_dir = args.rootdir - else: - self.ostree_dir = self.find_ostree_dir() - self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img") - - self.mountpoint = os.path.join(self.ostree_dir, 'ostree-qemu-mnt') - fileutil.ensure_dir(self.mountpoint) - - if not os.path.exists(self.qemu_path): - self._create_qemu_disk() - - subprocess.call(['umount', self.mountpoint], stderr=open('/dev/null', 'w')) - repo_path = os.path.join(self.mountpoint, 'ostree', 'repo') - try: - subprocess.check_call(['mount', '-o', 'loop', self.qemu_path, self.mountpoint]) - child_args = ['ostree', '--repo=' + repo_path, 'pull-local', args.srcrepo] - child_args.extend(['trees/' + x for x in args.targets]) - run_sync(child_args) - - first_target = args.targets[0] - for target in args.targets: - run_sync(['ostree', '--repo=' + repo_path, 'checkout', '--atomic-retarget', 'trees/'+ target, target], - cwd=os.path.join(self.mountpoint, 'ostree')) - current_link_path = os.path.join(self.mountpoint, 'ostree', 'current') - os.symlink(first_target, current_link_path + '.tmp') - os.rename(current_link_path + '.tmp', current_link_path) - finally: - subprocess.call(['umount', self.mountpoint]) - - -builtins.register(OstbuildPrivhelperDeployQemu) diff --git a/src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py b/src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py deleted file mode 100755 index a3e556e5..00000000 --- a/src/ostbuild/pyostbuild/builtin_privhelper_run_qemu.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync -from . import ostbuildrc -from . import fileutil - -class OstbuildPrivhelperRunQemu(builtins.Builtin): - name = "privhelper-run-qemu" - short_description = "Helper for run-qemu" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('target') - - args = parser.parse_args(argv) - - if os.geteuid() != 0: - fatal("This helper can only be run as root") - - self.ostree_dir = self.find_ostree_dir() - self.qemu_path = os.path.join(self.ostree_dir, "ostree-qemu.img") - - release = os.uname()[2] - - qemu = 'qemu-kvm' - kernel = '/boot/vmlinuz-%s' % (release, ) - initramfs = '/boot/initramfs-ostree-%s.img' % (release, ) - memory = '512M' - extra_args = 'root=/dev/sda rd.pymouth=0 ostree=%s' % (args.target, ) - - args = [qemu, '-kernel', kernel, '-initrd', initramfs, - '-hda', self.qemu_path, '-m', memory, '-append', extra_args] - log("Running: %s" % (subprocess.list2cmdline(args), )) - os.execvp(qemu, args) - -builtins.register(OstbuildPrivhelperRunQemu) diff --git a/src/ostbuild/pyostbuild/builtin_resolve.py b/src/ostbuild/pyostbuild/builtin_resolve.py deleted file mode 100755 index 5109fd14..00000000 --- a/src/ostbuild/pyostbuild/builtin_resolve.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import copy -import argparse -import json -import time -import urlparse -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output -from . import ostbuildrc -from . import vcs -from . import jsondb -from . import buildutil -from . import kvfile -from . import odict - -class OstbuildResolve(builtins.Builtin): - name = "resolve" - short_description = "Expand git revisions in source to exact targets" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--manifest', required=True, - help="Path to manifest file") - parser.add_argument('--fetch-patches', action='store_true', - help="Git fetch the patches") - parser.add_argument('--fetch', action='store_true', - help="Also perform a git fetch") - parser.add_argument('components', nargs='*', - help="List of component names to git fetch") - - args = parser.parse_args(argv) - self.args = args - - if len(args.components) > 0 and not args.fetch: - fatal("Can't specify components without --fetch") - - self.parse_config() - - self.snapshot = json.load(open(args.manifest)) - self.prefix = self.snapshot['prefix'] - - components = map(lambda x: buildutil.resolve_component_meta(self.snapshot, x), self.snapshot['components']) - self.snapshot['components'] = components - - unique_component_names = set() - for component in components: - name = component['name'] - - if name in unique_component_names: - fatal("Duplicate component name '%s'" % (name, )) - unique_component_names.add(name) - - global_patches_meta = buildutil.resolve_component_meta(self.snapshot, self.snapshot['patches']) - self.snapshot['patches'] = global_patches_meta - (keytype, uri) = vcs.parse_src_key(global_patches_meta['src']) - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch']) - if args.fetch_patches: - run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False) - - git_mirror_args = ['ostbuild', 'git-mirror', '--manifest=' + args.manifest] - if args.fetch: - git_mirror_args.append('--fetch') - git_mirror_args.extend(args.components) - run_sync(git_mirror_args) - - patch_revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch']) - global_patches_meta['revision'] = patch_revision - - for component in components: - src = component['src'] - (keytype, uri) = vcs.parse_src_key(src) - branch = component.get('branch') - tag = component.get('tag') - branch_or_tag = branch or tag - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) - revision = buildutil.get_git_version_describe(mirrordir, branch_or_tag) - component['revision'] = revision - - src_db = self.get_src_snapshot_db() - path = src_db.store(self.snapshot) - log("Source snapshot: %s" % (path, )) - -builtins.register(OstbuildResolve) diff --git a/src/ostbuild/pyostbuild/builtin_run_qemu.py b/src/ostbuild/pyostbuild/builtin_run_qemu.py deleted file mode 100755 index ca33f4f3..00000000 --- a/src/ostbuild/pyostbuild/builtin_run_qemu.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess,tempfile,re,shutil -import argparse -import time -import urlparse -import json -from StringIO import StringIO - -from . import builtins -from .ostbuildlog import log, fatal -from . import ostbuildrc -from . import privileged_subproc - -class OstbuildRunQemu(builtins.Builtin): - name = "run-qemu" - short_description = "Run QEMU image" - - def __init__(self): - builtins.Builtin.__init__(self) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('target') - - args = parser.parse_args(argv) - - self.parse_config() - - helper = privileged_subproc.PrivilegedSubprocess() - child_args = ['ostbuild', 'privhelper-run-qemu', args.target] - helper.spawn_sync(child_args) - -builtins.register(OstbuildRunQemu) diff --git a/src/ostbuild/pyostbuild/builtin_source_diff.py b/src/ostbuild/pyostbuild/builtin_source_diff.py deleted file mode 100755 index 0e3ff07b..00000000 --- a/src/ostbuild/pyostbuild/builtin_source_diff.py +++ /dev/null @@ -1,162 +0,0 @@ -# Copyright (C) 2011,2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -# ostbuild-compile-one-make wraps systems that implement the GNOME build API: -# http://people.gnome.org/~walters/docs/build-api.txt - -import os,sys,stat,subprocess,tempfile,re,shutil -import argparse -from StringIO import StringIO -import json - -from . import builtins -from .ostbuildlog import log, fatal -from . import vcs -from .subprocess_helpers import run_sync, run_sync_get_output -from . import buildutil - -class OstbuildSourceDiff(builtins.Builtin): - name = "source-diff" - short_description = "Show differences in source code between builds" - - def __init__(self): - builtins.Builtin.__init__(self) - - def _diff(self, name, mirrordir, from_revision, to_revision, - diffstat=False): - diff_replace_re = re.compile(' [ab]') - - env = dict(os.environ) - env['LANG'] = 'C' - - spacename = ' ' + name - - sys.stdout.write('diff of %s revision range %s..%s\n' % (name, from_revision, to_revision)) - sys.stdout.flush() - - diff_proc = subprocess.Popen(['git', 'diff', from_revision, to_revision], - env=env, cwd=mirrordir, stdout=subprocess.PIPE) - if diffstat: - diffstat_proc = subprocess.Popen(['diffstat', '-p0'], - stdin=subprocess.PIPE, - stdout=sys.stdout) - diff_pipe = diffstat_proc.stdin - else: - diffstat_proc = None - diff_pipe = sys.stdout - for line in diff_proc.stdout: - if (line.startswith('diff --git ') - or line.startswith('--- a/') - or line.startswith('+++ b/') - or line.startswith('Binary files /dev/null and b/')): - line = diff_replace_re.sub(spacename, line) - diff_pipe.write(line) - else: - diff_pipe.write(line) - diff_proc.wait() - if diffstat_proc is not None: - diffstat_proc.stdin.close() - diffstat_proc.wait() - - def _log(self, opts, name, mirrordir, from_revision, to_revision): - env = dict(os.environ) - env['LANG'] = 'C' - - spacename = ' ' + name - - args = ['git', 'log'] - args.extend(opts) - args.append(from_revision + '...' + to_revision) - proc = subprocess.Popen(args, env=env, cwd=mirrordir, stdout=subprocess.PIPE) - for line in proc.stdout: - sys.stdout.write(line) - proc.wait() - - def _snapshot_from_rev(self, rev): - self.init_repo() - text = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'cat', rev, '/contents.json'], - log_initiation=False) - return json.loads(text) - - def execute(self, argv): - parser = argparse.ArgumentParser(description=self.short_description) - parser.add_argument('--log', action='store_true') - parser.add_argument('--logp', action='store_true') - parser.add_argument('--diffstat', action='store_true') - parser.add_argument('--rev-from') - parser.add_argument('--rev-to') - parser.add_argument('--snapshot-from') - parser.add_argument('--snapshot-to') - - args = parser.parse_args(argv) - self.parse_config() - - to_snap = None - from_snap = None - - if args.rev_to: - to_snap = self._snapshot_from_rev(args.rev_to) - if args.rev_from: - from_snap = self._snapshot_from_rev(args.rev_from) - if args.snapshot_from: - from_snap = json.load(open(args.snapshot_from)) - if args.snapshot_to: - to_snap = json.load(open(args.snapshot_to)) - - if to_snap is None: - fatal("One of --rev-to/--snapshot-to must be given") - if from_snap is None: - if args.rev_to: - from_snap = self._snapshot_from_rev(args.rev_to + '^') - else: - fatal("One of --rev-from/--snapshot-from must be given") - - for from_component in from_snap['components']: - name = from_component['name'] - src = from_component['src'] - (keytype, uri) = vcs.parse_src_key(src) - if keytype == 'local': - log("Component %r has local URI" % (name, )) - continue - branch_or_tag = from_component.get('branch') or from_component.get('tag') - mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag) - - to_component = self.find_component_in_snapshot(name, to_snap) - if to_component is None: - log("DELETED COMPONENT: %s" % (name, )) - continue - - from_revision = from_component.get('revision') - to_revision = to_component.get('revision') - if from_revision is None: - log("From component %s missing revision" % (name, )) - continue - if to_revision is None: - log("From component %s missing revision" % (name, )) - continue - - if from_revision != to_revision: - if args.log: - self._log([], name, mirrordir, from_revision, to_revision) - elif args.logp: - self._log(['-p'], name, mirrordir, from_revision, to_revision) - else: - self._diff(name, mirrordir, from_revision, to_revision, - diffstat=args.diffstat) - -builtins.register(OstbuildSourceDiff) diff --git a/src/ostbuild/pyostbuild/builtins.py b/src/ostbuild/pyostbuild/builtins.py deleted file mode 100755 index b5fd1043..00000000 --- a/src/ostbuild/pyostbuild/builtins.py +++ /dev/null @@ -1,237 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys -import stat -import argparse -import json - -from . import ostbuildrc -from . import fileutil -from . import jsondb -from .ostbuildlog import log, fatal -from .subprocess_helpers import run_sync, run_sync_get_output - -_all_builtins = {} - -class Builtin(object): - name = None - short_description = None - - def __init__(self): - self._meta_cache = {} - self.prefix = None - self.manifest = None - self.snapshot = None - self.bin_snapshot = None - self.repo = None - self.ostree_dir = self.find_ostree_dir() - (self.active_branch, self.active_branch_checksum) = self._find_active_branch() - self._src_snapshots = None - self._bin_snapshots = None - - def find_ostree_dir(self): - for path in ['/ostree', '/sysroot/ostree']: - if os.path.isdir(path): - return path - return None - - def _find_active_branch(self): - if self.ostree_dir is None: - return (None, None) - current_path = os.path.join(self.ostree_dir, 'current') - while True: - try: - target = os.path.join(self.ostree_dir, current_path) - stbuf = os.lstat(target) - except OSError, e: - current_path = None - break - if not stat.S_ISLNK(stbuf.st_mode): - break - current_path = os.readlink(target) - if current_path is not None: - basename = os.path.basename(current_path) - return basename.rsplit('-', 1) - else: - return (None, None) - - def get_component_from_cwd(self): - cwd = os.getcwd() - parent = os.path.dirname(cwd) - parentparent = os.path.dirname(parent) - return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd])) - - def parse_config(self): - self.ostbuildrc = ostbuildrc - - self.mirrordir = os.path.expanduser(ostbuildrc.get_key('mirrordir')) - if not os.path.isdir(self.mirrordir): - fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, )) - self.workdir = os.path.expanduser(ostbuildrc.get_key('workdir')) - if not os.path.isdir(self.workdir): - fatal("Specified workdir '%s' is not a directory" % (self.workdir, )) - - self.snapshot_dir = os.path.join(self.workdir, 'snapshots') - self.patchdir = os.path.join(self.workdir, 'patches') - - def get_component_snapshot(self, name): - found = False - for content in self.active_branch_contents['contents']: - if content['name'] == name: - found = True - break - if not found: - fatal("Unknown component '%s'" % (name, )) - return content - - def get_component_meta_from_revision(self, revision): - text = run_sync_get_output(['ostree', '--repo=' + self.repo, - 'cat', revision, - '/_ostbuild-meta.json']) - return json.loads(text) - - def expand_component(self, component): - meta = dict(component) - global_patchmeta = self.snapshot.get('patches') - if global_patchmeta is not None: - component_patch_files = component.get('patches', []) - if len(component_patch_files) > 0: - patches = dict(global_patchmeta) - patches['files'] = component_patch_files - meta['patches'] = patches - config_opts = list(self.snapshot.get('config-opts', [])) - config_opts.extend(component.get('config-opts', [])) - meta['config-opts'] = config_opts - return meta - - def find_component_in_snapshot(self, name, snapshot): - for component in snapshot['components']: - if component['name'] == name: - return component - return None - - def get_component(self, name, in_snapshot=None): - if in_snapshot is None: - assert self.snapshot is not None - target_snapshot = self.snapshot - else: - target_snapshot = in_snapshot - component = self.find_component_in_snapshot(name, target_snapshot) - if (component is None and - 'patches' in self.snapshot and - self.snapshot['patches']['name'] == name): - return self.snapshot['patches'] - if component is None: - fatal("Couldn't find component '%s' in manifest" % (name, )) - return component - - def get_expanded_component(self, name): - return self.expand_component(self.get_component(name)) - - def get_prefix(self): - if self.prefix is None: - path = os.path.expanduser('~/.config/ostbuild-prefix') - if not os.path.exists(path): - fatal("No prefix set; use \"ostbuild prefix\" to set one") - f = open(path) - self.prefix = f.read().strip() - f.close() - return self.prefix - - def create_db(self, dbsuffix, prefix=None): - if prefix is None: - target_prefix = self.get_prefix() - else: - target_prefix = prefix - name = '%s-%s' % (target_prefix, dbsuffix) - fileutil.ensure_dir(self.snapshot_dir) - return jsondb.JsonDB(self.snapshot_dir, prefix=name) - - def get_src_snapshot_db(self): - if self._src_snapshots is None: - self._src_snapshots = self.create_db('src-snapshot') - return self._src_snapshots - - def get_bin_snapshot_db(self): - if self._bin_snapshots is None: - self._bin_snapshots = self.create_db('bin-snapshot') - return self._bin_snapshots - - def init_repo(self): - if self.repo is not None: - return self.repo - repo = ostbuildrc.get_key('repo', default=None) - if repo is not None: - self.repo = repo - else: - shadow_path = os.path.join(self.workdir, 'shadow-repo') - if os.path.isdir(shadow_path): - self.repo = shadow_path - else: - fatal("No repository configured, and shadow-repo not found. Use \"ostbuild init\" to make one") - - def parse_prefix(self, prefix): - if prefix is not None: - self.prefix = prefix - - def parse_snapshot(self, prefix, path): - self.parse_prefix(prefix) - self.init_repo() - if path is None: - latest_path = self.get_src_snapshot_db().get_latest_path() - if latest_path is None: - raise Exception("No source snapshot found for prefix %r" % (self.prefix, )) - self.snapshot_path = latest_path - else: - self.snapshot_path = path - self.snapshot = json.load(open(self.snapshot_path)) - key = '00ostbuild-manifest-version' - src_ver = self.snapshot[key] - if src_ver != 0: - fatal("Unhandled %s version \"%d\", expected 0" % (key, src_ver, )) - if self.prefix is None: - self.prefix = self.snapshot['prefix'] - - def parse_snapshot_from_current(self): - if self.ostree_dir is None: - fatal("/ostree directory not found") - repo_path = os.path.join(self.ostree_dir, 'repo') - if not os.path.isdir(repo_path): - fatal("Repository '%s' doesn't exist" % (repo_path, )) - if self.active_branch is None: - fatal("No \"current\" link found") - tree_path = os.path.join(self.ostree_dir, self.active_branch) - self.parse_snapshot(None, os.path.join(tree_path, 'contents.json')) - - def execute(self, args): - raise NotImplementedError() - -def register(builtin): - _all_builtins[builtin.name] = builtin - -def get(name): - builtin = _all_builtins.get(name) - if builtin is not None: - return builtin() - return None - -def get_all(): - return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name)) diff --git a/src/ostbuild/pyostbuild/filemonitor.py b/src/ostbuild/pyostbuild/filemonitor.py deleted file mode 100644 index 0eec07b7..00000000 --- a/src/ostbuild/pyostbuild/filemonitor.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import stat - -from . import mainloop - -_global_filemon = None - -class FileMonitor(object): - def __init__(self): - self._paths = {} - self._path_modtimes = {} - self._timeout = 1000 - self._timeout_installed = False - self._loop = mainloop.Mainloop.get(None) - - @classmethod - def get(cls): - global _global_filemon - if _global_filemon is None: - _global_filemon = cls() - return _global_filemon - - def _stat(self, path): - try: - st = os.stat(path) - return st[stat.ST_MTIME] - except OSError, e: - return None - - def add(self, path, callback): - if path not in self._paths: - self._paths[path] = [] - self._path_modtimes[path] = self._stat(path) - self._paths[path].append(callback) - if not self._timeout_installed: - self._timeout_installed = True - self._loop.timeout_add(self._timeout, self._check_files) - - def _check_files(self): - for (path,callbacks) in self._paths.iteritems(): - mtime = self._stat(path) - orig_mtime = self._path_modtimes[path] - if (mtime is not None) and (orig_mtime is None or (mtime > orig_mtime)): - self._path_modtimes[path] = mtime - for cb in callbacks: - cb() diff --git a/src/ostbuild/pyostbuild/fileutil.py b/src/ostbuild/pyostbuild/fileutil.py deleted file mode 100644 index d9ae91fa..00000000 --- a/src/ostbuild/pyostbuild/fileutil.py +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os - -def ensure_dir(path): - if not os.path.isdir(path): - os.makedirs(path) - -def ensure_parent_dir(path): - ensure_dir(os.path.dirname(path)) diff --git a/src/ostbuild/pyostbuild/jsondb.py b/src/ostbuild/pyostbuild/jsondb.py deleted file mode 100644 index 810103df..00000000 --- a/src/ostbuild/pyostbuild/jsondb.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import stat -import time -import tempfile -import re -import shutil -import hashlib -import json - -class JsonDB(object): - def __init__(self, dirpath, prefix): - self._dirpath = dirpath - self._prefix = prefix - self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$') - - def _cmp_match_by_version(self, a, b): - # Note this is a reversed comparison; bigger is earlier - a_major = a[0] - a_minor = a[1] - b_major = b[0] - b_minor = b[1] - - c = cmp(b_major, a_major) - if c == 0: - return cmp(b_minor, a_minor) - return 0 - - def _get_all(self): - result = [] - for fname in os.listdir(self._dirpath): - if not (fname.startswith(self._prefix) and fname.endswith('.json')): - continue - - path = os.path.join(self._dirpath, fname) - match = self._version_csum_re.search(fname) - if not match: - raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum", - path) - result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname)) - result.sort(self._cmp_match_by_version) - return result - - def get_latest(self): - path = self.get_latest_path() - if path is None: - return None - return json.load(open(path)) - - def get_latest_path(self): - files = self._get_all() - if len(files) == 0: - return None - return os.path.join(self._dirpath, files[0][3]) - - def store(self, obj): - files = self._get_all() - if len(files) == 0: - latest = None - else: - latest = files[0] - - current_time = time.gmtime() - - (fd, tmppath) = tempfile.mkstemp(suffix='.tmp', - prefix='tmp-jsondb-', dir=self._dirpath) - os.close(fd) - f = open(tmppath, 'w') - json.dump(obj, f, indent=4, sort_keys=True) - f.close() - - csum = hashlib.sha256() - f = open(tmppath) - buf = f.read(8192) - while buf != '': - csum.update(buf) - buf = f.read(8192) - f.close() - digest = csum.hexdigest() - - if latest is not None: - if digest == latest[2]: - os.unlink(tmppath) - return latest[3] - latest_version = (latest[0], latest[1]) - else: - latest_version = (current_time.tm_year, 0) - target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year, - latest_version[1] + 1, digest) - target_path = os.path.join(self._dirpath, target_name) - os.rename(tmppath, target_path) - return target_path - - - diff --git a/src/ostbuild/pyostbuild/kvfile.py b/src/ostbuild/pyostbuild/kvfile.py deleted file mode 100755 index 075d4b09..00000000 --- a/src/ostbuild/pyostbuild/kvfile.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -def parse(stream): - ret = {} - for line in stream: - (k,v) = line.split('=', 1) - ret[k.strip()] = v.strip() - return ret diff --git a/src/ostbuild/pyostbuild/main.py b/src/ostbuild/pyostbuild/main.py deleted file mode 100755 index 0adccd68..00000000 --- a/src/ostbuild/pyostbuild/main.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys -import argparse - -from . import builtins -from . import builtin_build -from . import builtin_checkout -from . import builtin_chroot_compile_one -from . import builtin_compile_one -from . import builtin_deploy_root -from . import builtin_deploy_qemu -from . import builtin_git_mirror -from . import builtin_import_tree -from . import builtin_init -from . import builtin_run_qemu -from . import builtin_prefix -from . import builtin_privhelper_deploy_qemu -from . import builtin_privhelper_run_qemu -from . import builtin_resolve -from . import builtin_source_diff - -def usage(ecode): - print "Builtins:" - for builtin in builtins.get_all(): - if builtin.name.startswith('privhelper'): - continue - print " %s - %s" % (builtin.name, builtin.short_description) - return ecode - -def main(args): - if len(args) < 1: - return usage(1) - elif args[0] in ('-h', '--help'): - return usage(0) - else: - builtin = builtins.get(args[0]) - if builtin is None: - print "error: Unknown builtin '%s'" % (args[0], ) - return usage(1) - return builtin.execute(args[1:]) - - diff --git a/src/ostbuild/pyostbuild/mainloop.py b/src/ostbuild/pyostbuild/mainloop.py deleted file mode 100644 index 40a67be8..00000000 --- a/src/ostbuild/pyostbuild/mainloop.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys -import select -import time - -class Mainloop(object): - DEFAULT = None - def __init__(self): - self._running = True - self.poll = None - self._timeouts = [] - self._pid_watches = {} - self._fd_callbacks = {} - - @classmethod - def get(cls, context): - if context is None: - if cls.DEFAULT is None: - cls.DEFAULT = cls() - return cls.DEFAULT - raise NotImplementedError("Unknown context %r" % (context, )) - - def _ensure_poll(self): - if self.poll is None: - self.poll = select.poll() - - def watch_fd(self, fd, callback): - self._ensure_poll() - self.poll.register(fd) - self._fd_callbacks[fd] = callback - - def unwatch_fd(self, fd): - self.poll.unregister(fd) - del self._fd_callbacks[fd] - - def watch_pid(self, pid, callback): - self._pid_watches[pid] = callback - - def timeout_add(self, ms, callback): - self._timeouts.append((ms, callback)) - - def quit(self): - self._running = False - - def run_once(self): - min_timeout = None - if len(self._pid_watches) > 0: - min_timeout = 500 - for (ms, callback) in self._timeouts: - if (min_timeout is None) or (ms < min_timeout): - min_timeout = ms - origtime = time.time() * 1000 - self._ensure_poll() - fds = self.poll.poll(min_timeout) - for fd in fds: - self._fd_callbacks[fd]() - to_delete_pids = [] - for pid in self._pid_watches: - (opid, status) = os.waitpid(pid, os.WNOHANG) - if opid == pid: - to_delete_pids.append(pid) - self._pid_watches[pid](pid, status) - for pid in to_delete_pids: - del self._pid_watches[pid] - newtime = time.time() * 1000 - diff = int(newtime - origtime) - if diff < 0: diff = 0 - for i,(ms, callback) in enumerate(self._timeouts): - remaining_ms = ms - diff - if remaining_ms <= 0: - callback() - else: - self._timeouts[i] = (remaining_ms, callback) - - def run(self): - self._running = True - while self._running: - self.run_once() diff --git a/src/ostbuild/pyostbuild/odict.py b/src/ostbuild/pyostbuild/odict.py deleted file mode 100644 index df703cbb..00000000 --- a/src/ostbuild/pyostbuild/odict.py +++ /dev/null @@ -1,45 +0,0 @@ -# -*- Mode: Python -*- -# GObject-Introspection - a framework for introspecting GObject libraries -# Copyright (C) 2008 Johan Dahlin -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. -# - -"""odict - an ordered dictionary""" - -from UserDict import DictMixin - - -class odict(DictMixin): - - def __init__(self): - self._items = {} - self._keys = [] - - def __setitem__(self, key, value): - if key not in self._items: - self._keys.append(key) - self._items[key] = value - - def __getitem__(self, key): - return self._items[key] - - def __delitem__(self, key): - del self._items[key] - self._keys.remove(key) - - def keys(self): - return self._keys[:] diff --git a/src/ostbuild/pyostbuild/ostbuildlog.py b/src/ostbuild/pyostbuild/ostbuildlog.py deleted file mode 100755 index 0f852298..00000000 --- a/src/ostbuild/pyostbuild/ostbuildlog.py +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys - -def log(msg, prefix=None): - if prefix is None: - prefix_target = '' - else: - prefix_target = prefix - fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg) - sys.stdout.write(fullmsg) - sys.stdout.flush() - -def fatal(msg): - log(msg, prefix="FATAL: ") - sys.exit(1) - diff --git a/src/ostbuild/pyostbuild/ostbuildrc.py b/src/ostbuild/pyostbuild/ostbuildrc.py deleted file mode 100755 index a36febb7..00000000 --- a/src/ostbuild/pyostbuild/ostbuildrc.py +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,ConfigParser - -_config = None - -def get(): - global _config - if _config is None: - configpath = os.path.expanduser('~/.config/ostbuild.cfg') - parser = ConfigParser.RawConfigParser() - parser.read([configpath]) - - _config = {} - for (k, v) in parser.items('global'): - _config[k.strip()] = v.strip() - return _config - -# This hack is because we want people to be able to pass None -# for "default", but still distinguish default=None from default -# not passed. -_default_not_supplied = object() -def get_key(name, provided_args=None, default=_default_not_supplied): - global _default_not_supplied - config = get() - if provided_args: - v = provided_args.get(name) - if v is not None: - return v - if default is _default_not_supplied: - # Possibly throw a KeyError - return config[name] - value = config.get(name, _default_not_supplied) - if value is _default_not_supplied: - return default - return value - diff --git a/src/ostbuild/pyostbuild/privileged_subproc.py b/src/ostbuild/pyostbuild/privileged_subproc.py deleted file mode 100755 index 0116cef2..00000000 --- a/src/ostbuild/pyostbuild/privileged_subproc.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (C) 2012 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os,sys,subprocess - -from .ostbuildlog import log, fatal -from . import ostbuildrc -from .subprocess_helpers import run_sync - -class PrivilegedSubprocess(object): - - def spawn_sync(self, argv): - helper = ostbuildrc.get_key('privileged_exec', default='pkexec') - - handlers = {'pkexec': self._pkexec_spawn_sync} - - handler = handlers.get(helper) - if handler is None: - fatal("Unrecognized privileged_exec; valid values=%r" % (handlers.keys(),)) - else: - handler(argv) - - def _pkexec_spawn_sync(self, argv): - pkexec_argv = ['pkexec'] + argv - run_sync(pkexec_argv) diff --git a/src/ostbuild/pyostbuild/subprocess_helpers.py b/src/ostbuild/pyostbuild/subprocess_helpers.py deleted file mode 100755 index 37549007..00000000 --- a/src/ostbuild/pyostbuild/subprocess_helpers.py +++ /dev/null @@ -1,145 +0,0 @@ -#!/usr/bin/python -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import sys -import subprocess - -from .ostbuildlog import log, fatal -from .warningfilter import WarningFilter -from .mainloop import Mainloop - -def _get_env_for_cwd(cwd=None, env=None): - # This dance is necessary because we want to keep the PWD - # environment variable up to date. Not doing so is a recipie - # for triggering edge conditions in pwd lookup. - if (cwd is not None) and (env is None or ('PWD' in env)): - if env is None: - env_copy = os.environ.copy() - else: - env_copy = env.copy() - if ('PWD' in env_copy) and (not cwd.startswith('/')): - env_copy['PWD'] = os.path.join(env_copy['PWD'], cwd) - else: - env_copy['PWD'] = cwd - else: - env_copy = env - return env_copy - -def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none_on_error=False, - log_success=False, log_initiation=False): - if log_initiation: - log("running: %s" % (subprocess.list2cmdline(args),)) - env_copy = _get_env_for_cwd(cwd, env) - f = open('/dev/null', 'r') - if stderr is None: - stderr_target = sys.stderr - else: - stderr_target = stderr - proc = subprocess.Popen(args, stdin=f, stdout=subprocess.PIPE, stderr=stderr_target, - close_fds=True, cwd=cwd, env=env_copy) - f.close() - output = proc.communicate()[0].strip() - if proc.returncode != 0 and not none_on_error: - logfn = fatal - elif log_success: - logfn = log - else: - logfn = None - if logfn is not None: - logfn("cmd '%s' (cwd=%s) exited with code %d, %d bytes of output" % (subprocess.list2cmdline(args), - cwd, proc.returncode, len(output))) - if proc.returncode == 0: - return output - return None - -def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False, - log_success=True, log_initiation=True, stdin=None, stdout=None, - stderr=None): - if log_initiation: - log("running: %s" % (subprocess.list2cmdline(args),)) - - env_copy = _get_env_for_cwd(cwd, env) - - if stdin is not None: - stdin_target = stdin - elif keep_stdin: - stdin_target = sys.stdin - else: - stdin_target = open('/dev/null', 'r') - - if stdout is None: - stdout_target = sys.stdout - else: - stdout_target = stdout - - if stderr is None: - stderr_target = sys.stderr - else: - stderr_target = stderr - - proc = subprocess.Popen(args, stdin=stdin_target, stdout=stdout_target, stderr=stderr_target, - close_fds=True, cwd=cwd, env=env_copy) - if not keep_stdin: - stdin_target.close() - returncode = proc.wait() - if fatal_on_error and returncode != 0: - logfn = fatal - elif log_success: - logfn = log - else: - logfn = None - if logfn is not None: - logfn("pid %d exited with code %d" % (proc.pid, returncode)) - return returncode - -def run_sync_monitor_log_file(args, logfile, cwd=None, env=None, - fatal_on_error=True, log_initiation=True): - if log_initiation: - log("running: %s" % (subprocess.list2cmdline(args),)) - - env_copy = _get_env_for_cwd(cwd, env) - - logfile_f = open(logfile, 'w') - - proc = subprocess.Popen(args, stdin=open('/dev/null', 'r'), - stdout=logfile_f, - stderr=subprocess.STDOUT, - close_fds=True, cwd=cwd, env=env_copy) - warnfilter = WarningFilter(logfile, sys.stdout) - - warnfilter.start() - - loop = Mainloop.get(None) - - proc_estatus = None - def _on_pid_exited(pid, estatus): - global proc_estatus - proc_estatus = estatus - failed = estatus != 0 - warnfilter.finish(not failed) - if fatal_on_error and failed: - logfn = fatal - else: - logfn = log - logfn("pid %d exited with code %d" % (pid, estatus)) - loop.quit() - loop.watch_pid(proc.pid, _on_pid_exited) - loop.run() - return proc_estatus diff --git a/src/ostbuild/pyostbuild/vcs.py b/src/ostbuild/pyostbuild/vcs.py deleted file mode 100755 index 85f2f85c..00000000 --- a/src/ostbuild/pyostbuild/vcs.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import re -import urlparse -import shutil - -from .subprocess_helpers import run_sync_get_output, run_sync -from . import buildutil -from .ostbuildlog import log, fatal - -def get_mirrordir(mirrordir, keytype, uri, prefix=''): - assert keytype == 'git' - parsed = urlparse.urlsplit(uri) - return os.path.join(mirrordir, prefix, keytype, parsed.scheme, parsed.netloc, parsed.path[1:]) - -def _fixup_submodule_references(mirrordir, cwd): - submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=cwd) - submodule_status_lines = submodules_status_text.split('\n') - have_submodules = False - for line in submodule_status_lines: - if line == '': continue - have_submodules = True - line = line[1:] - (sub_checksum, sub_name) = line.split(' ', 1) - sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', - 'submodule.%s.url' % (sub_name, )], cwd=cwd) - mirrordir = get_mirrordir(mirrordir, 'git', sub_url) - run_sync(['git', 'config', 'submodule.%s.url' % (sub_name, ), 'file://' + mirrordir], cwd=cwd) - return have_submodules - -def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True): - module_mirror = get_mirrordir(mirrordir, keytype, uri) - assert keytype == 'git' - checkoutdir_parent=os.path.dirname(dest) - if not os.path.isdir(checkoutdir_parent): - os.makedirs(checkoutdir_parent) - tmp_dest = dest + '.tmp' - if os.path.isdir(tmp_dest): - shutil.rmtree(tmp_dest) - if os.path.islink(dest): - os.unlink(dest) - if os.path.isdir(dest): - if overwrite: - shutil.rmtree(dest) - else: - tmp_dest = dest - if not os.path.isdir(tmp_dest): - run_sync(['git', 'clone', '-q', '--origin', 'localmirror', - '--no-checkout', module_mirror, tmp_dest]) - run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest) - else: - run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest) - run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest) - run_sync(['git', 'submodule', 'init'], cwd=tmp_dest) - have_submodules = _fixup_submodule_references(mirrordir, tmp_dest) - if have_submodules: - run_sync(['git', 'submodule', 'update'], cwd=tmp_dest) - if tmp_dest != dest: - os.rename(tmp_dest, dest) - return dest - -def clean(keytype, checkoutdir): - assert keytype in ('git', 'dirty-git') - run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir) - -def parse_src_key(srckey): - idx = srckey.find(':') - if idx < 0: - raise ValueError("Invalid SRC uri=%s" % (srckey, )) - keytype = srckey[:idx] - if keytype not in ['git', 'local']: - raise ValueError("Unsupported SRC uri=%s" % (srckey, )) - uri = srckey[idx+1:] - return (keytype, uri) - -def get_lastfetch_path(mirrordir, keytype, uri, branch): - mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) - branch_safename = branch.replace('/','_').replace('.', '_') - return mirror + '.lastfetch-%s' % (branch_safename, ) - -def ensure_vcs_mirror(mirrordir, keytype, uri, branch): - mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) - tmp_mirror = mirror + '.tmp' - if os.path.isdir(tmp_mirror): - shutil.rmtree(tmp_mirror) - if not os.path.isdir(mirror): - run_sync(['git', 'clone', '--mirror', uri, tmp_mirror]) - run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror) - os.rename(tmp_mirror, mirror) - if branch is None: - return mirror - last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch) - if os.path.exists(last_fetch_path): - f = open(last_fetch_path) - last_fetch_contents = f.read() - f.close() - last_fetch_contents = last_fetch_contents.strip() - else: - last_fetch_contents = None - current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) - current_vcs_version = current_vcs_version.strip() - if current_vcs_version != last_fetch_contents: - log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version)) - tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts') - if os.path.isdir(tmp_checkout): - shutil.rmtree(tmp_checkout) - parent = os.path.dirname(tmp_checkout) - if not os.path.isdir(parent): - os.makedirs(parent) - run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout]) - run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout) - submodules = [] - submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout) - submodule_status_lines = submodules_status_text.split('\n') - for line in submodule_status_lines: - if line == '': continue - line = line[1:] - (sub_checksum, sub_name) = line.split(' ', 1) - sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules', - 'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout) - ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum) - shutil.rmtree(tmp_checkout) - f = open(last_fetch_path, 'w') - f.write(current_vcs_version + '\n') - f.close() - return mirror - -def fetch(mirrordir, keytype, uri, branch, keep_going=False): - mirror = buildutil.get_mirrordir(mirrordir, keytype, uri) - last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch) - run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False, - fatal_on_error=not keep_going) - current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror) - if current_vcs_version is not None: - current_vcs_version = current_vcs_version.strip() - f = open(last_fetch_path, 'w') - f.write(current_vcs_version + '\n') - f.close() - diff --git a/src/ostbuild/pyostbuild/warningfilter.py b/src/ostbuild/pyostbuild/warningfilter.py deleted file mode 100644 index 24619533..00000000 --- a/src/ostbuild/pyostbuild/warningfilter.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (C) 2011 Colin Walters -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place - Suite 330, -# Boston, MA 02111-1307, USA. - -import os -import re -import stat -import fcntl -import subprocess - -from . import filemonitor -from . import mainloop - -warning_re = re.compile(r'(: ((warning)|(error)|(fatal error)): )|(make(\[[0-9]+\])?: \*\*\*)') -output_whitelist_re = re.compile(r'^(make(\[[0-9]+\])?: Entering directory)|(ostbuild:)') - -_bold_sequence = None -_normal_sequence = None -if os.isatty(1): - _bold_sequence = subprocess.Popen(['tput', 'bold'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0] - _normal_sequence = subprocess.Popen(['tput', 'sgr0'], stdout=subprocess.PIPE, stderr=open('/dev/null', 'w')).communicate()[0] -def _bold(text): - if _bold_sequence is not None: - return '%s%s%s' % (_bold_sequence, text, _normal_sequence) - else: - return text - -class WarningFilter(object): - def __init__(self, filename, output): - self.filename = filename - self.output = output - - # inherit globals - self._warning_re = warning_re - self._nonfilter_re = output_whitelist_re - - self._buf = '' - self._warning_count = 0 - self._filtered_line_count = 0 - filemon = filemonitor.FileMonitor.get() - filemon.add(filename, self._on_changed) - self._fd = os.open(filename, os.O_RDONLY) - fcntl.fcntl(self._fd, fcntl.F_SETFL, os.O_NONBLOCK) - - def _do_read(self): - while True: - buf = os.read(self._fd, 4096) - if buf == '': - break - self._buf += buf - self._flush() - - def _write_last_log_lines(self): - _last_line_limit = 100 - f = open(self.filename) - lines = [] - for line in f: - if line.startswith('ostbuild '): - continue - lines.append(line) - if len(lines) > _last_line_limit: - lines.pop(0) - f.close() - for line in lines: - self.output.write('| ') - self.output.write(line) - - def _flush(self): - while True: - p = self._buf.find('\n') - if p < 0: - break - line = self._buf[0:p] - self._buf = self._buf[p+1:] - match = self._warning_re.search(line) - if match: - self._warning_count += 1 - self.output.write(line + '\n') - else: - match = self._nonfilter_re.search(line) - if match: - self.output.write(line + '\n') - else: - self._filtered_line_count += 1 - - def _on_changed(self): - self._do_read() - - def start(self): - self._do_read() - - def finish(self, successful): - self._do_read() - if not successful: - self._write_last_log_lines() - pass - self.output.write("ostbuild %s: %d warnings\n" % ('success' if successful else _bold('failed'), - self._warning_count, )) - self.output.write("ostbuild: full log path: %s\n" % (self.filename, ))