shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-21 02:50:37 +03:00
Code Issues Projects Releases Wiki Activity

ostbuild: Major rework

Browse Source
This commit is contained in:
Colin Walters 2012-04-06 15:12:57 -04:00
parent b052758652
commit 717cec46c3
27 changed files with 1205 additions and 353 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
Makefile-ostbuild.am
View File

@ -22,18 +22,23 @@ bin_SCRIPTS += ostbuild
pyostbuilddir=$(libdir)/ostbuild/pyostbuild
pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/buildutil.py \
src/ostbuild/pyostbuild/builtin_build.py \
src/ostbuild/pyostbuild/builtin_bin_to_src.py \
src/ostbuild/pyostbuild/builtin_branch_prefix.py \
src/ostbuild/pyostbuild/builtin_build_components.py \
src/ostbuild/pyostbuild/builtin_checkout.py \
src/ostbuild/pyostbuild/builtin_compose.py \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_query_content.py \
src/ostbuild/pyostbuild/builtin_pull_components.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
src/ostbuild/pyostbuild/builtin_modify_snapshot.py \
src/ostbuild/pyostbuild/builtin_status.py \
src/ostbuild/pyostbuild/builtins.py \
src/ostbuild/pyostbuild/filemonitor.py \
src/ostbuild/pyostbuild/fileutil.py \
src/ostbuild/pyostbuild/__init__.py \
src/ostbuild/pyostbuild/jsondb.py \
src/ostbuild/pyostbuild/kvfile.py \
src/ostbuild/pyostbuild/main.py \
src/ostbuild/pyostbuild/mainloop.py \

15
gnomeos/3.4/gnomeos-3.4-src.json
View File

@ -1,7 +1,10 @@
{
"name-prefix": "gnomeos-3.4",
"prefix": "gnomeos-3.4",
"architectures": ["i686"],
"base-prefix": "yocto/gnomeos-3.4",
"base": {
"name": "yocto",
"src": "cgwalters:poky"
},
"config-opts": ["--disable-static", "--disable-silent-rules"],
@ -25,13 +28,15 @@
"patches": {"src": "git:/src/ostree",
"branch": "wip/ostbuild-v2",
"prefix": "gnomeos/3.4"},
"subdir": "gnomeos/3.4"},
"components": [
{"src": "cgwalters:ginitscripts"},
{"src": "cgwalters:ginitscripts",
"noarch": true},
{"src": "gnome:gtk-doc-stub",
"component": "devel"},
"component": "devel",
"noarch": true},
{"src": "git:git://github.com/atgreen/libffi.git"},

42
gnomeos/README-build.md
View File

@ -16,6 +16,9 @@ base "runtime", and one "devel" with all of the development tools like
gcc. We then import that into an OSTree branch
e.g. "bases/yocto/gnomeos-3.4-i686-devel".
At present, it's still (mostly) possible to put this data on an ext4
filesystem and boot into it.
We also have a Yocto recipe "ostree-native" which generates (as you
might guess) a native binary of ostree. That binary is used to import
into an "archive mode" OSTree repository. You can see it in
@ -26,26 +29,59 @@ can use "ostbuild" which uses "linux-user-chroot" to chroot inside,
run a build on a source tree, and outputs binaries, which we then add
to the build tree for the next module, and so on.
The final result of all of this is that the OSTree repository gains
new commits (which can be downloaded by clients), while still
retaining old build history.
Yocto details
-------------
I have a branch of Yocto here:
https://github.com/cgwalters/poky
It has a collection of patches on top of the "Edison" release of
Yocto, some of which are hacky, others upstreamable. The most
important part though are the modifications to commit the generated
root filesystem into OSTree.
ostbuild details
----------------
The simple goal of ostbuild is that it only takes as input a
"manifest" which is basically just a list of components to build. A
component is a pure metadata file which includes the git repository
"manifest" which is basically just a list of components to build. You
can see this here:
http://git.gnome.org/browse/ostree/tree/gnomeos/3.4/gnomeos-3.4-src.json
A component is a pure metadata file which includes the git repository
URL and branch name, as well as ./configure flags (--enable-foo).
There is no support for building from "tarballs" - I want the ability
to review all of the code that goes in, and to efficiently store
source code updates.
source code updates. It's also just significantly easier from an
implementation perspective, versus having to maintain a version
control abstraction layer.
The result of a build of a component is an OSTree branch like
"artifacts/gnomeos-3.4-i686-devel/libxslt/master". Then, a "compose"
process merges together the individual filesystem trees into the final
branches (e.g. gnomeos-3.4-i686-devel).
Doing local builds
------------------
This is where you want to modify one (or a few) components on top of
what comes from the ostree.gnome.org server, and test the result
locally. I'm working on this.
Doing a full build on your system
---------------------------------
Following this process is equivalent to what we have set up on the
ostree.gnome.org build server. It will generate a completely new
repository.
srcdir=/src
builddir=/src/build

20
gnomeos/gnomeos-install.sh
View File

@ -46,17 +46,19 @@ if ! test -d /ostree/repo/objects; then
mkdir -p /ostree
$SRCDIR/gnomeos-setup.sh /ostree
cd /ostree
ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
ostree-pull --repo=repo gnome
for branch in runtime devel; do
ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
done
ln -sf ${BRANCH_PREFIX}runtime current
cd ${WORKDIR}
fi
cd /ostree
ostree --repo=repo remote add gnome http://ostree.gnome.org/repo ${BRANCH_PREFIX}{runtime,devel}
ostree-pull --repo=repo gnome
for branch in runtime devel; do
ostree --repo=repo checkout --atomic-retarget ${BRANCH_PREFIX}${branch}
done
ln -sf ${BRANCH_PREFIX}runtime current
uname=$(uname -r)
if test -d /etc/grub.d; then

74
gnomeos/gnomeos-qemu-create.sh Executable file
View File

@ -0,0 +1,74 @@
#!/bin/bash
# -*- indent-tabs-mode: nil; -*-
# Create ostree-qemu.img file in the current directory, suitable
# for booting via qemu.
#
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -e
set -x
SRCDIR=`dirname $0`
WORKDIR=`pwd`
if test $(id -u) != 0; then
cat <<EOF
This script should be run as root.
EOF
exit 1
fi
usage () {
echo "$0"
exit 1
}
OBJ=ostree-qemu.img
if ! test -f ${OBJ}; then
# Hardcoded 6 gigabyte filesystem size here; 6 gigabytes should be
# enough for everybody.
qemu-img create $OBJ 6G
mkfs.ext4 -q -F $OBJ
fi
mkdir -p fs
umount fs || true
sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
mount -o loop ostree-qemu.img fs
cd fs
if ! test -d ./ostree/repo/objects; then
mkdir -p ./ostree
$SRCDIR/gnomeos-setup.sh $(pwd)/ostree
fi
mkdir -p ./run ./home ./root ./sys
mkdir -p ./tmp
chmod 01777 ./tmp
mkdir -p $(pwd)/ostree/modules
rsync -a -H -v --delete /ostree/modules/ ./ostree/modules/
cd ..
umount fs
cat << EOF
Next, run gnomeos-qemu-pull.sh to copy data.
EOF

70
gnomeos/gnomeos-qemu-pull.sh Executable file
View File

@ -0,0 +1,70 @@
#!/bin/bash
# -*- indent-tabs-mode: nil; -*-
# Run built image in QEMU
#
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
set -e
set -x
SRCDIR=`dirname $0`
WORKDIR=`pwd`
if test $(id -u) != 0; then
cat <<EOF
This script should be run as root.
EOF
exit 1
fi
usage () {
cat <<EOF
usage: $0 SRC_REPO_PATH CURRENT_REF [REFS...]
EOF
exit 1
}
SRC_REPO_PATH=$1
test -n "$SRC_REPO_PATH" || usage
shift
CURRENT_REF=$1
test -n "$CURRENT_REF" || usage
shift
if ! test -f ostree-qemu.img; then
cat <<EOF
ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first
EOF
fi
mkdir -p fs
umount fs || true
sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
mount -o loop ostree-qemu.img fs
cd fs
ostree --repo=${SRC_REPO_PATH} local-clone ./ostree/repo ${CURRENT_REF} "$@"
cd ostree
ostree --repo=./repo checkout --atomic-retarget ${CURRENT_REF}
ln -sf ${CURRENT_REF} ${CURRENT_REF}.tmplink
mv -T ${CURRENT_REF}.tmplink current
cd ${WORKDIR}
umount fs

54
gnomeos/gnomeos-clone-qemu.sh → gnomeos/gnomeos-qemu-run.sh
View File

@ -33,62 +33,16 @@ EOF
fi
usage () {
echo "$0 OSTREE_REPO_PATH"
echo "$0"
exit 1
}
ARCH=i686
BRANCH_PREFIX="gnomeos-3.4-${ARCH}-"
OBJ=gnomeos-fs.img
if ! test -f ${OBJ}; then
cat <<EOF
Create gnomeos-fs.img like this:
$ qemu-img create gnomeos-fs.img 6G
$ mkfs.ext4 -q -F gnomeos-fs.img
EOF
exit 1
if ! test -f ostree-qemu.img; then
echo "ostree-qemu.img not found; You must run gnomeos-qemu-create.sh first"
fi
if ! test -d ${WORKDIR}/repo/objects; then
cat <<EOF
No ./repo/objects found
EOF
exit 1
fi
mkdir -p fs
umount fs || true
sleep 1 # Avoid Linux kernel bug, pretty sure it's the new RCU pathname lookup
mount -o loop gnomeos-fs.img fs
cd fs
TOPROOT_BIND_MOUNTS="home root tmp"
for d in $TOPROOT_BIND_MOUNTS; do
if ! test -d $d; then
mkdir -m 0755 $d
fi
done
chmod a=rwxt tmp
if ! test -d ostree; then
mkdir -p ostree
$SRCDIR/gnomeos-setup.sh $(pwd)/ostree
fi
rsync -a -H -v ${WORKDIR}/repo ${WORKDIR}/current ${WORKDIR}/modules ${WORKDIR}/var ${WORKDIR}/gnomeos-3.4-* ./ostree
current_uname=$(uname -r)
cd ${WORKDIR}
sync
umount fs
rmdir fs
ARGS="$@"
if ! echo $ARGS | grep -q 'ostree='; then
ARGS="ostree=current $ARGS"
@ -97,4 +51,4 @@ ARGS="rd.plymouth=0 root=/dev/sda $ARGS"
KERNEL=/boot/vmlinuz-${current_uname}
INITRD_ARG="-initrd /boot/initramfs-ostree-${current_uname}.img"
exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda gnomeos-fs.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio
exec qemu-kvm -kernel ${KERNEL} ${INITRD_ARG} -hda ostree-qemu.img -net user -net nic,model=virtio -m 512M -append "$ARGS" -monitor stdio

5
gnomeos/gnomeos-setup.sh
View File

@ -58,6 +58,11 @@ fi
mkdir -p ./var/lib/gdm
chown 2:2 ./var/lib/gdm
mkdir -p ./var/log/gdm
chown 2:2 ./var/log/gdm
chmod 01770 ./var/log/gdm
mkdir -p ./var/lib/AccountsService
touch ./var/shadow
chmod 0600 ./var/shadow

2
src/ostbuild/pyostbuild/buildutil.py
View File

@ -40,7 +40,7 @@ def parse_src_key(srckey):
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git']:
if keytype not in ['git', 'dirty-git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)

76
src/ostbuild/pyostbuild/builtin_bin_to_src.py Executable file
View File

@ -0,0 +1,76 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildBinToSrc(builtins.Builtin):
name = "bin-to-src"
short_description = "Turn a binary snapshot into a source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
def bin_snapshot_to_src(self, bin_snapshot):
del bin_snapshot['00ostree-bin-snapshot-version']
src_snapshot = dict(bin_snapshot)
src_snapshot['00ostree-src-snapshot-version'] = 0
all_architectures = src_snapshot['architecture-buildroots'].keys()
# Arbitrarily take first architecture
first_arch = all_architectures[0]
bin_components = src_snapshot['components']
src_components = {}
src_snapshot['components'] = src_components
for archname,rev in bin_components.iteritems():
(name, arch) = archname.rsplit('/', 1)
if arch != first_arch:
continue
meta = dict(self.get_component_meta_from_revision(rev))
del meta['name']
src_components[name] = meta
for target in src_snapshot['targets']:
del target['base']['ostree-revision']
return src_snapshot
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
self.parse_config()
self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
snapshot = self.bin_snapshot_to_src(self.bin_snapshot)
json.dump(snapshot, sys.stdout, indent=4, sort_keys=True)
builtins.register(OstbuildBinToSrc)

54
src/ostbuild/pyostbuild/builtin_query_content.py → src/ostbuild/pyostbuild/builtin_branch_prefix.py
View File

@ -28,40 +28,44 @@ from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildQueryContent(builtins.Builtin):
name = "query-content"
short_description = "Output metadata from a component"
class OstbuildBranchPrefix(builtins.Builtin):
name = "prefix-branch"
short_description = "Copy current source snapshot to new prefix"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--branch', required=True)
parser.add_argument('--component')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('newprefix')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
contents_json_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', args.branch, 'contents.json'])
if args.newprefix == self.prefix:
fatal("Specified prefix %r matches active prefix" % (args.newprefix, ))
db = self.create_db('src-snapshot', prefix=args.newprefix)
log("Branching from source snapshot %r" % (self.snapshot_path, ))
orig_prefix = self.prefix
forked_snapshot = dict(self.snapshot)
forked_snapshot['prefix'] = args.newprefix
for target in forked_snapshot['targets']:
name = target['name']
if not name.startswith(orig_prefix):
fatal("Mismatched name %r in snapshot" % (name, ))
target['name'] = name.replace(orig_prefix, args.newprefix)
if args.component is None:
sys.stdout.write(contents_json_text)
else:
contents = json.loads(contents_json_text)
contents_list = contents['contents']
found = False
for content in contents_list:
if content['name'] != args.component:
found = True
break
if not found:
fatal("Unknown component '%s'" % (args.component, ))
ostbuildmeta_json = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', content['ostree-revision'],
'/_ostbuild-meta.json'])
sys.stdout.write(ostbuildmeta_json)
db.store(forked_snapshot)
builtins.register(OstbuildQueryContent)
run_sync(['ostbuild', 'prefix', args.newprefix],
log_initiation=False, log_success=False)
builtins.register(OstbuildBranchPrefix)

150
src/ostbuild/pyostbuild/builtin_build.py → src/ostbuild/pyostbuild/builtin_build_components.py
View File

@ -36,9 +36,9 @@ from . import vcs
class BuildOptions(object):
pass
class OstbuildBuild(builtins.Builtin):
name = "build"
short_description = "Rebuild all artifacts from the given manifest"
class OstbuildBuildComponents(builtins.Builtin):
name = "build-components"
short_description = "Build multiple components from given source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
@ -49,7 +49,8 @@ class OstbuildBuild(builtins.Builtin):
args = ['setarch', architecture]
else:
args = []
args.extend(['ostbuild', 'chroot-compile-one'])
args.extend(['ostbuild', 'chroot-compile-one',
'--snapshot=' + self.snapshot_path])
return args
def _launch_debug_shell(self, architecture, buildroot, cwd=None):
@ -60,10 +61,10 @@ class OstbuildBuild(builtins.Builtin):
run_sync(args, cwd=cwd, fatal_on_error=False, keep_stdin=True)
fatal("Exiting after debug shell")
def _build_one_component(self, name, component):
def _build_one_component(self, basename, component, architecture):
branch = component['branch']
architecture = component['architecture']
name = '%s/%s' % (basename, architecture)
buildname = 'components/%s' % (name, )
current_vcs_version = component['revision']
@ -73,7 +74,7 @@ class OstbuildBuild(builtins.Builtin):
stderr=open('/dev/null', 'w'),
none_on_error=True)
if previous_build_version is not None:
log("Previous build of '%s' is %s" % (buildname, previous_build_version))
log("Previous build of '%s' is %s" % (name, previous_build_version))
previous_metadata_text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', previous_build_version,
@ -92,19 +93,15 @@ class OstbuildBuild(builtins.Builtin):
else:
log("VCS version is now '%s', was '%s'" % (current_vcs_version, previous_vcs_version))
else:
log("No previous build for '%s' found" % (buildname, ))
log("No previous build for '%s' found" % (name, ))
checkoutdir = os.path.join(self.workdir, 'src')
component_src = os.path.join(checkoutdir, name)
run_sync(['ostbuild', 'checkout', '--clean', '--overwrite', name], cwd=checkoutdir)
component_src = os.path.join(checkoutdir, basename)
run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--clean', '--overwrite', basename], cwd=checkoutdir)
artifact_meta = dict(component)
metadata_path = os.path.join(component_src, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(artifact_meta, f, indent=4, sort_keys=True)
f.close()
logdir = os.path.join(self.workdir, 'logs', name)
fileutil.ensure_dir(logdir)
log_path = os.path.join(logdir, 'compile.log')
@ -116,7 +113,7 @@ class OstbuildBuild(builtins.Builtin):
log("Logging to %s" % (log_path, ))
f = open(log_path, 'w')
chroot_args = self._get_ostbuild_chroot_args(architecture)
chroot_args.extend(['--pristine', '--name=' + name])
chroot_args.extend(['--pristine', '--name=' + basename, '--arch=' + architecture])
if self.buildopts.shell_on_failure:
ecode = run_sync_monitor_log_file(chroot_args, log_path, cwd=component_src, fatal_on_error=False)
if ecode != 0:
@ -146,69 +143,48 @@ class OstbuildBuild(builtins.Builtin):
os.unlink(statoverride_path)
return True
def _compose(self, target):
base_name = 'bases/%s' % (target['base']['name'], )
branch_to_rev = {}
branch_to_subtrees = {}
contents = [base_name]
branch_to_subtrees[base_name] = ['/']
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
branch_to_rev[base_name] = base_revision
def _resolve_refs(self, refs):
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
for component in target['contents']:
name = component['name']
contents.append(name)
args.append('components/%s' % (name, ))
branch_to_subtrees[name] = component['trees']
branch_revs_text = run_sync_get_output(args)
branch_revs = branch_revs_text.split('\n')
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
for (content, rev) in zip(target['contents'], branch_revs):
name = content['name']
branch_to_rev[name] = rev
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
def _save_bin_snapshot(self, components, component_architectures):
bin_snapshot = dict(self.snapshot)
resolved_base = dict(target['base'])
resolved_base['ostree-revision'] = base_revision
resolved_contents = list(target['contents'])
for component in resolved_contents:
component['ostree-revision'] = branch_to_rev[component['name']]
metadata = {'source': 'ostbuild compose v0',
'base': resolved_base,
'contents': resolved_contents}
del bin_snapshot['00ostree-src-snapshot-version']
bin_snapshot['00ostree-bin-snapshot-version'] = 0
for branch in contents:
branch_rev = branch_to_rev[branch]
subtrees = branch_to_subtrees[branch]
for subtree in subtrees:
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode',
'--union', '--subpath=' + subtree,
branch_rev, compose_rootdir])
for target in bin_snapshot['targets']:
base = target['base']
base_name = 'bases/%s' % (base['name'], )
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', base_name])
base['ostree-revision'] = base_revision
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(metadata, f, indent=4, sort_keys=True)
f.close()
component_refs = []
for name in components.iterkeys():
for architecture in component_architectures[name]:
component_refs.append('components/%s/%s' % (name, architecture))
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'], cwd=compose_rootdir)
new_components = {}
resolved_refs = self._resolve_refs(component_refs)
for name,rev in zip(components.iterkeys(), resolved_refs):
for architecture in component_architectures[name]:
archname = '%s/%s' % (name, architecture)
new_components[archname] = rev
bin_snapshot['components'] = new_components
path = self.get_bin_snapshot_db().store(bin_snapshot)
log("Binary snapshot: %s" % (path, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--skip-built', action='store_true')
parser.add_argument('--recompose', action='store_true')
parser.add_argument('--skip-compose', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('--compose', action='store_true')
parser.add_argument('--start-at')
parser.add_argument('--shell-on-failure', action='store_true')
parser.add_argument('--debug-shell', action='store_true')
@ -218,17 +194,28 @@ class OstbuildBuild(builtins.Builtin):
self.args = args
self.parse_config()
self.parse_snapshot()
self.parse_snapshot(args.prefix, args.src_snapshot)
log("Using source snapshot: %s" % (os.path.basename(self.snapshot_path), ))
self.buildopts = BuildOptions()
self.buildopts.shell_on_failure = args.shell_on_failure
self.buildopts.skip_built = args.skip_built
required_components = {}
component_architectures = {}
for target in self.snapshot['targets']:
for tree_content in target['contents']:
(name, arch) = tree_content['name'].rsplit('/', 1)
required_components[name] = self.snapshot['components'][name]
if name not in component_architectures:
component_architectures[name] = set([arch])
else:
component_architectures[name].add(arch)
build_component_order = []
if args.recompose:
pass
elif len(args.components) == 0:
tsorted = buildutil.tsort_components(self.snapshot['components'], 'build-depends')
if len(args.components) == 0:
tsorted = buildutil.tsort_components(required_components, 'build-depends')
tsorted.reverse()
build_component_order = tsorted
else:
@ -253,11 +240,14 @@ class OstbuildBuild(builtins.Builtin):
start_at_index = 0
for component_name in build_component_order[start_at_index:]:
component = self.snapshot['components'].get(component_name)
self._build_one_component(component_name, component)
component = required_components[component_name]
architectures = component_architectures[component_name]
for architecture in architectures:
self._build_one_component(component_name, component, architecture)
if not args.skip_compose:
for target in self.snapshot['targets']:
self._compose(target)
self._save_bin_snapshot(required_components, component_architectures)
if args.compose:
run_sync(['ostbuild', 'compose', '--prefix=' + self.prefix])
builtins.register(OstbuildBuild)
builtins.register(OstbuildBuildComponents)

45
src/ostbuild/pyostbuild/builtin_checkout.py
View File

@ -40,6 +40,9 @@ class OstbuildCheckout(builtins.Builtin):
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--overwrite', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot')
parser.add_argument('-a', '--active-tree', action='store_true')
parser.add_argument('--clean', action='store_true')
parser.add_argument('components', nargs='*')
@ -47,28 +50,41 @@ class OstbuildCheckout(builtins.Builtin):
self.args = args
self.parse_config()
self.parse_snapshot()
if len(args.components) > 0:
checkout_components = args.components
else:
checkout_components = [os.path.basename(os.getcwd())]
if args.active_tree:
self.parse_active_branch()
else:
self.parse_snapshot(args.prefix, args.snapshot)
for component_name in checkout_components:
found = False
component = self.snapshot['components'].get(component_name)
if component is None:
fatal("Unknown component %r" % (component_name, ))
component = self.get_component_meta(component_name)
(keytype, uri) = buildutil.parse_src_key(component['src'])
checkoutdir = os.path.join(os.getcwd(), component_name)
fileutil.ensure_parent_dir(checkoutdir)
component_src = vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
is_dirty = (keytype == 'dirty-git')
if is_dirty:
# Kind of a hack, but...
if os.path.lexists(checkoutdir):
os.unlink(checkoutdir)
os.symlink(uri, checkoutdir)
else:
vcs.get_vcs_checkout(self.mirrordir, keytype, uri, checkoutdir,
component['revision'],
overwrite=args.overwrite)
if args.clean:
vcs.clean(keytype, checkoutdir)
if is_dirty:
log("note: ignoring --clean argument due to \"dirty-git:\" specification")
else:
vcs.clean(keytype, checkoutdir)
patches = component.get('patches')
if patches is not None:
@ -78,15 +94,20 @@ class OstbuildCheckout(builtins.Builtin):
self.patchdir, patches['branch'],
overwrite=True)
patch_prefix = patches.get('prefix', None)
if patch_prefix is not None:
patchdir = os.path.join(self.patchdir, patch_prefix)
patch_subdir = patches.get('subdir', None)
if patch_subdir is not None:
patchdir = os.path.join(self.patchdir, patch_subdir)
else:
patchdir = self.patchdir
for patch in patches['files']:
patch_path = os.path.join(patchdir, patch)
run_sync(['git', 'am', '--ignore-date', '-3', patch_path], cwd=checkoutdir)
metadata_path = os.path.join(checkoutdir, '_ostbuild-meta.json')
f = open(metadata_path, 'w')
json.dump(component, f, indent=4, sort_keys=True)
f.close()
print "Checked out: %r" % (component_src, )
log("Checked out: %r" % (checkoutdir, ))
builtins.register(OstbuildCheckout)

146
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py
View File

@ -18,7 +18,9 @@
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import time
import json
import hashlib
from . import builtins
from . import buildutil
@ -31,47 +33,133 @@ class OstbuildChrootCompileOne(builtins.Builtin):
name = "chroot-compile-one"
short_description = "Build artifacts from the current source directory in a chroot"
def _compose_buildroot(self, component_name, dirpath):
def _resolve_refs(self, refs):
args = ['ostree', '--repo=' + self.repo, 'rev-parse']
args.extend(refs)
output = run_sync_get_output(args)
return output.split('\n')
def _compose_buildroot(self, component_name, architecture):
starttime = time.time()
rootdir_prefix = os.path.join(self.workdir, 'roots')
rootdir = os.path.join(rootdir_prefix, component_name)
fileutil.ensure_parent_dir(rootdir)
# Clean up any leftover root dir
rootdir_tmp = rootdir + '.tmp'
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
components = self.snapshot['components']
dependencies = buildutil.build_depends(component_name, components)
component = components.get(component_name)
base_devel_name = 'bases/%s-%s-%s' % (self.snapshot['base-prefix'],
component['architecture'],
'devel')
buildroots = self.snapshot['architecture-buildroots']
base_devel_name = 'bases/' + buildroots[architecture]
refs_to_resolve = [base_devel_name]
checkout_trees = [(base_devel_name, '/')]
for dependency_name in dependencies:
buildname = 'components/%s' % (dependency_name, )
buildname = 'components/%s/%s' % (dependency_name, architecture)
refs_to_resolve.append(buildname)
checkout_trees.append((buildname, '/runtime'))
checkout_trees.append((buildname, '/devel'))
resolved_refs = self._resolve_refs(refs_to_resolve)
ref_to_rev = {}
for ref,rev in zip(refs_to_resolve, resolved_refs):
ref_to_rev[ref] = rev
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
for (branch, rootpath) in checkout_trees:
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--user-mode', '--link-cache=' + link_cache_dir,
'--union', '--subpath=' + rootpath,
branch, dirpath])
sha = hashlib.sha256()
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-buildroot-')
f = os.fdopen(fd, 'w')
for (branch, subpath) in checkout_trees:
f.write(ref_to_rev[branch])
f.write('\0')
f.write(subpath)
f.write('\0')
f.close()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
sha.update(buf)
buf = f.read(8192)
f.close()
new_root_cacheid = sha.hexdigest()
rootdir_cache_path = os.path.join(rootdir_prefix, component_name + '.cacheid')
if os.path.isdir(rootdir):
if os.path.isfile(rootdir_cache_path):
f = open(rootdir_cache_path)
prev_cache_id = f.read().strip()
f.close()
if prev_cache_id == new_root_cacheid:
log("Reusing previous buildroot")
os.unlink(tmppath)
return rootdir
else:
log("New buildroot differs from previous")
shutil.rmtree(rootdir)
os.mkdir(rootdir_tmp)
if len(checkout_trees) > 0:
log("composing buildroot from %d parents (last: %r)" % (len(checkout_trees),
checkout_trees[-1][0]))
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--link-cache=' + link_cache_dir,
'--user-mode', '--union', '--from-stdin', rootdir_tmp],
stdin=open(tmppath))
os.unlink(tmppath);
builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
os.mkdir(builddir_tmp)
os.mkdir(os.path.join(builddir_tmp, 'source'))
os.mkdir(os.path.join(builddir_tmp, 'results'))
os.rename(rootdir_tmp, rootdir)
f = open(rootdir_cache_path, 'w')
f.write(new_root_cacheid)
f.write('\n')
f.close()
endtime = time.time()
log("Composed buildroot; %d seconds elapsed" % (int(endtime - starttime),))
return rootdir
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--pristine', action='store_true')
parser.add_argument('--prefix')
parser.add_argument('--snapshot', required=True)
parser.add_argument('--name')
parser.add_argument('--arch', required=True)
parser.add_argument('--debug-shell', action='store_true')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot()
self.parse_snapshot(args.prefix, args.snapshot)
if args.name:
component_name = args.name
else:
cwd = os.getcwd()
parent = os.path.dirname(cwd)
parentparent = os.path.dirname(parent)
component_name = '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
component_name = self.get_component_from_cwd()
components = self.snapshot['components']
component = components.get(component_name)
@ -93,37 +181,16 @@ class OstbuildChrootCompileOne(builtins.Builtin):
shutil.rmtree(child_tmpdir)
fileutil.ensure_dir(child_tmpdir)
resultdir = os.path.join(self.workdir, 'results', component_name)
resultdir = os.path.join(self.workdir, 'results', component_name, args.arch)
if os.path.isdir(resultdir):
shutil.rmtree(resultdir)
fileutil.ensure_dir(resultdir)
rootdir_prefix = os.path.join(workdir, 'roots')
fileutil.ensure_dir(rootdir_prefix)
rootdir = os.path.join(rootdir_prefix, component_name)
fileutil.ensure_parent_dir(rootdir)
if os.path.isdir(rootdir):
shutil.rmtree(rootdir)
rootdir_tmp = rootdir + '.tmp'
builddir = os.path.join(rootdir, 'ostbuild');
if os.path.isdir(rootdir_tmp):
shutil.rmtree(rootdir_tmp)
os.mkdir(rootdir_tmp)
self._compose_buildroot(component_name, rootdir_tmp)
rootdir = self._compose_buildroot(component_name, args.arch)
child_args = ['ostbuild', 'chroot-run-triggers', rootdir_tmp]
run_sync(child_args)
builddir_tmp = os.path.join(rootdir_tmp, 'ostbuild')
os.mkdir(builddir_tmp)
os.mkdir(os.path.join(builddir_tmp, 'source'))
os.mkdir(os.path.join(builddir_tmp, 'results'))
os.rename(rootdir_tmp, rootdir)
log("Checked out buildroot: %s" % (rootdir, ))
sourcedir=os.path.join(builddir, 'source', component_name)
sourcedir=os.path.join(rootdir, 'ostbuild', 'source', component_name)
fileutil.ensure_dir(sourcedir)
output_metadata = open('_ostbuild-meta.json', 'w')
@ -148,7 +215,6 @@ class OstbuildChrootCompileOne(builtins.Builtin):
'compile-one',
'--ostbuild-resultdir=/ostbuild/results',
'--ostbuild-meta=_ostbuild-meta.json'])
child_args.extend(self.metadata.get('config-opts', []))
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = chroot_sourcedir
run_sync(child_args, env=env_copy, keep_stdin=args.debug_shell)

47
src/ostbuild/pyostbuild/builtin_chroot_run_triggers.py
View File

@ -1,47 +0,0 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,re,subprocess,tempfile,shutil
from StringIO import StringIO
import argparse
import json
from . import builtins
from . import buildutil
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync
class OstbuildChrootRunTriggers(builtins.Builtin):
name = "chroot-run-triggers"
short_description = "Run ostree-run-triggers inside a chroot"
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('root')
args = parser.parse_args(argv)
child_args = buildutil.get_base_user_chroot_args()
child_args.extend(['--mount-proc', '/proc',
'--mount-bind', '/dev', '/dev',
args.root,
'/usr/bin/ostree-run-triggers'])
env_copy = dict(buildutil.BUILD_ENV)
env_copy['PWD'] = '/'
run_sync(child_args, env=env_copy)
builtins.register(OstbuildChrootRunTriggers)

25
src/ostbuild/pyostbuild/builtin_compile_one.py
View File

@ -57,6 +57,8 @@ class OstbuildCompileOne(builtins.Builtin):
def execute(self, args):
self.default_buildapi_jobs = ['-j', '%d' % (cpu_count() * 2, )]
starttime = time.time()
uname=os.uname()
kernel=uname[0].lower()
@ -77,32 +79,28 @@ class OstbuildCompileOne(builtins.Builtin):
'--infodir=' + os.path.join(PREFIX, 'share', 'info')]
self.makeargs = ['make']
self.ostbuild_resultdir=None
self.ostbuild_meta=None
self.ostbuild_resultdir='_ostbuild-results'
self.ostbuild_meta_path='_ostbuild-meta.json'
chdir = None
opt_install = False
for arg in args:
if arg.startswith('--ostbuild-resultdir='):
self.ostbuild_resultdir=arg[len('--ostbuild-resultdir='):]
elif arg.startswith('--ostbuild-meta='):
self.ostbuild_meta=arg[len('--ostbuild-meta='):]
self.ostbuild_meta_path=arg[len('--ostbuild-meta='):]
elif arg.startswith('--chdir='):
os.chdir(arg[len('--chdir='):])
elif arg.startswith('--'):
self.configargs.append(arg)
else:
self.makeargs.append(arg)
if self.ostbuild_resultdir is None:
fatal("Must specify --ostbuild-resultdir=")
if self.ostbuild_meta is None:
fatal("Must specify --ostbuild-meta=")
f = open(self.ostbuild_meta)
f = open(self.ostbuild_meta_path)
self.metadata = json.load(f)
f.close()
self.configargs.extend(self.metadata.get('config-opts', []))
if self.metadata.get('rm-configure', False):
configure_path = 'configure'
if os.path.exists(configure_path):
@ -235,6 +233,11 @@ class OstbuildCompileOne(builtins.Builtin):
except OSError, e:
pass
endtime = time.time()
log("Compliation succeeded; %d seconds elapsed" % (int(endtime - starttime),))
log("Results placed in %s" % (self.ostbuild_resultdir, ))
def _install_and_unlink(self, src, dest):
statsrc = os.lstat(src)
dirname = os.path.dirname(dest)

110
src/ostbuild/pyostbuild/builtin_compose.py Executable file
View File

@ -0,0 +1,110 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import argparse
import time
import urlparse
import json
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from .subprocess_helpers import run_sync_monitor_log_file
from . import ostbuildrc
from . import buildutil
from . import fileutil
from . import kvfile
from . import odict
from . import vcs
class OstbuildCompose(builtins.Builtin):
name = "compose"
short_description = "Build complete trees from components"
def __init__(self):
builtins.Builtin.__init__(self)
def _compose_one_target(self, bin_snapshot, target):
components = bin_snapshot['components']
base = target['base']
base_name = 'bases/%s' % (base['name'], )
base_revision = target['base']['ostree-revision']
compose_rootdir = os.path.join(self.workdir, 'roots', target['name'])
if os.path.isdir(compose_rootdir):
shutil.rmtree(compose_rootdir)
os.mkdir(compose_rootdir)
compose_contents = [(base_revision, '/')]
for tree_content in target['contents']:
name = tree_content['name']
rev = components[name]
subtrees = tree_content['trees']
for subpath in subtrees:
compose_contents.append((rev, subpath))
(fd, tmppath) = tempfile.mkstemp(suffix='.txt', prefix='ostbuild-compose-')
f = os.fdopen(fd, 'w')
for (branch, subpath) in compose_contents:
f.write(branch)
f.write('\0')
f.write(subpath)
f.write('\0')
f.close()
link_cache_dir = os.path.join(self.workdir, 'link-cache')
fileutil.ensure_dir(link_cache_dir)
run_sync(['ostree', '--repo=' + self.repo,
'checkout', '--link-cache=' + link_cache_dir,
'--user-mode', '--no-triggers',
'--union', '--from-stdin', compose_rootdir],
stdin=open(tmppath))
os.unlink(tmppath)
contents_path = os.path.join(compose_rootdir, 'contents.json')
f = open(contents_path, 'w')
json.dump(bin_snapshot, f, indent=4, sort_keys=True)
f.close()
run_sync(['ostree', '--repo=' + self.repo,
'commit', '-b', target['name'], '-s', 'Compose',
'--owner-uid=0', '--owner-gid=0', '--no-xattrs',
'--skip-if-unchanged'], cwd=compose_rootdir)
shutil.rmtree(compose_rootdir)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--bin-snapshot')
args = parser.parse_args(argv)
self.args = args
self.parse_config()
self.parse_bin_snapshot(args.prefix, args.bin_snapshot)
log("Using binary snapshot: %s" % (os.path.basename(self.bin_snapshot_path), ))
for target in self.bin_snapshot['targets']:
log("Composing target %r from %u components" % (target['name'],
len(target['contents'])))
self._compose_one_target(self.bin_snapshot, target)
builtins.register(OstbuildCompose)

63
src/ostbuild/pyostbuild/builtin_modify_snapshot.py Executable file
View File

@ -0,0 +1,63 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,stat,subprocess,tempfile,re,shutil
from StringIO import StringIO
import json
import select,time
import argparse
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
class OstbuildModifySnapshot(builtins.Builtin):
name = "modify-snapshot"
short_description = "Change the current source snapshot"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
component_name = self.get_component_from_cwd()
current_meta = self.get_component_meta(component_name)
new_snapshot = dict(self.snapshot)
new_meta = dict(current_meta)
if 'patches' in new_meta:
del new_meta['patches']
new_meta['src'] = "dirty-git:%s" % (os.getcwd(), )
new_meta['revision'] = run_sync_get_output(['git', 'rev-parse', 'HEAD'])
new_snapshot['components'][component_name] = new_meta
db = self.get_src_snapshot_db()
path = db.store(new_snapshot)
log("Replaced %s with %s %s" % (component_name, new_meta['src'],
new_meta['revision']))
log("New source snapshot: %s" % (path, ))
builtins.register(OstbuildModifySnapshot)

73
src/ostbuild/pyostbuild/builtin_prefix.py Executable file
View File

@ -0,0 +1,73 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildPrefix(builtins.Builtin):
name = "prefix"
short_description = "Display or modify \"prefix\" (build target)"
def __init__(self):
builtins.Builtin.__init__(self)
def _set_prefix(self, prefix):
f = open(self.path, 'w')
f.write(prefix)
f.write('\n')
f.close()
log("Prefix is now %r" % (prefix, ))
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('-a', '--active', action='store_true')
parser.add_argument('prefix', nargs='?', default=None)
args = parser.parse_args(argv)
self.path = os.path.expanduser('~/.config/ostbuild-prefix')
if args.prefix is None and not args.active:
if os.path.exists(self.path):
f = open(self.path)
print "%s" % (f.read().strip(), )
f.close()
else:
log("No currently active prefix")
elif args.prefix is not None and args.active:
fatal("Can't specify -a with prefix")
elif args.prefix is not None:
self._set_prefix(args.prefix)
else:
assert args.active
self.parse_active_branch()
active_prefix = self.active_branch_contents['prefix']
self._set_prefix(active_prefix)
builtins.register(OstbuildPrefix)

70
src/ostbuild/pyostbuild/builtin_pull_components.py Executable file
View File

@ -0,0 +1,70 @@
# Copyright (C) 2011 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
import time
import urlparse
from StringIO import StringIO
from . import builtins
from .ostbuildlog import log, fatal
from . import ostbuildrc
from . import buildutil
from .subprocess_helpers import run_sync, run_sync_get_output
from . import kvfile
from . import odict
class OstbuildPullComponents(builtins.Builtin):
name = "pull-components"
short_description = "Download the component data for active branch"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('targets', nargs='*')
args = parser.parse_args(argv)
self.parse_active_branch()
if len(args.targets) == 0:
targets = [self.active_branch]
else:
targets = args.targets
tree_contents_list = []
for target in targets:
tree_contents_path = os.path.join(self.ostree_dir, target, 'contents.json')
tree_contents = json.load(open(tree_contents_path))
tree_contents_list.append(tree_contents)
revisions = set()
for tree_contents in tree_contents_list:
for content_item in tree_contents['contents']:
revisions.add(content_item['ostree-revision'])
args = ['ostree-pull', '--repo=' + self.repo]
# FIXME FIXME - don't hardcode origin here
args.append('gnome')
for revision in revisions:
args.append(revision)
run_sync(args)
builtins.register(OstbuildPullComponents)

103
src/ostbuild/pyostbuild/builtin_resolve.py
View File

@ -19,6 +19,7 @@ import os,sys,subprocess,tempfile,re,shutil
import copy
import argparse
import json
import time
import urlparse
from StringIO import StringIO
@ -26,6 +27,7 @@ from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import jsondb
from . import buildutil
from . import kvfile
from . import odict
@ -38,6 +40,7 @@ class OstbuildResolve(builtins.Builtin):
builtins.Builtin.__init__(self)
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
# FIXME - remove "name" from parameter list here - hash uri?
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
@ -135,6 +138,7 @@ class OstbuildResolve(builtins.Builtin):
self.args = args
self.parse_config()
self.repo = ostbuildrc.get_key('repo')
manifest_path = self.ostbuildrc.get_key('manifest')
self.manifest = json.load(open(manifest_path))
@ -192,52 +196,52 @@ class OstbuildResolve(builtins.Builtin):
component['patches'] = copy.deepcopy(global_patches_meta)
component['patches']['files'] = patch_files
name_prefix = snapshot['name-prefix']
base_prefix = snapshot['base-prefix']
manifest_architectures = snapshot['architectures']
ostree_prefix = snapshot['prefix']
base_prefix = '%s/%s' % (snapshot['base']['name'], ostree_prefix)
snapshot['architecture-buildroots'] = {}
for architecture in manifest_architectures:
snapshot['architecture-buildroots'][architecture] = '%s-%s-devel' % (base_prefix, architecture)
components_by_name = {}
component_ordering = []
build_prev_component_by_arch = {}
runtime_prev_component_by_arch = {}
runtime_components_by_arch = {}
devel_components_by_arch = {}
for architecture in manifest_architectures:
runtime_components_by_arch[architecture] = []
devel_components_by_arch[architecture] = []
build_prev_component = None
runtime_prev_component = None
runtime_components = []
devel_components = []
builds = {}
for component in component_source_list:
component_architectures = component.get('architectures', manifest_architectures)
for architecture in component_architectures:
component_binary = copy.deepcopy(component)
source_name = component['name']
binary_name = '%s/%s/%s' % (name_prefix, architecture, source_name)
component_binary['name'] = binary_name
component_binary['architecture'] = architecture
base_name = component['name']
name = '%s/%s' % (ostree_prefix, base_name)
component['name'] = name
components_by_name[binary_name] = component_binary
components_by_name[name] = component
prev_component = build_prev_component_by_arch.get(architecture)
if prev_component is not None:
component_binary['build-depends'] = [prev_component['name']]
build_prev_component_by_arch[architecture] = component_binary
if build_prev_component is not None:
component['build-depends'] = [build_prev_component['name']]
build_prev_component = component
is_runtime = component.get('component', 'runtime') == 'runtime'
is_runtime = component.get('component', 'runtime') == 'runtime'
prev_component = runtime_prev_component_by_arch.get(architecture)
if prev_component is not None:
component_binary['runtime-depends'] = [prev_component['name']]
if runtime_prev_component is not None:
component['runtime-depends'] = [runtime_prev_component['name']]
if is_runtime:
runtime_prev_component_by_arch[architecture] = component_binary
if is_runtime:
runtime_prev_component = component
runtime_components.append(component)
devel_components.append(component)
if is_runtime:
runtime_components_by_arch[architecture].append(component_binary)
devel_components_by_arch[architecture].append(component_binary)
if 'architectures' in component_binary:
del component_binary['architectures']
is_noarch = component.get('noarch', False)
if is_noarch:
# Just use the first specified architecture
component_arches = [manifest_architectures[0]]
else:
component_arches = component.get('architectures', manifest_architectures)
builds[name] = component_arches
# We expanded these keys
del snapshot['config-opts']
@ -247,26 +251,27 @@ class OstbuildResolve(builtins.Builtin):
targets_list = []
snapshot['targets'] = targets_list
for architecture in manifest_architectures:
for target_component_type in ['runtime', 'devel']:
for target_component_type in ['runtime', 'devel']:
for architecture in manifest_architectures:
target = {}
targets_list.append(target)
target['name'] = '%s-%s-%s' % (name_prefix, architecture, target_component_type)
target['name'] = '%s-%s-%s' % (ostree_prefix, architecture, target_component_type)
base_ref = '%s-%s-%s' % (base_prefix, architecture, target_component_type)
base_revision = run_sync_get_output(['ostree', '--repo=' + self.repo,
'rev-parse', 'bases/%s' % (base_ref, )])
target['base'] = {'name': base_ref}
if target_component_type == 'runtime':
target_components = runtime_components_by_arch[architecture]
target_components = runtime_components
else:
target_components = devel_components_by_arch[architecture]
target_components = devel_components
contents = []
for component in target_components:
name = component['name']
component_ref = {'name': name}
builds_for_component = builds[component['name']]
if architecture not in builds_for_component:
continue
binary_name = '%s/%s' % (component['name'], architecture)
component_ref = {'name': binary_name}
if target_component_type == 'runtime':
component_ref['trees'] = ['/runtime']
else:
@ -278,10 +283,12 @@ class OstbuildResolve(builtins.Builtin):
del component['name']
snapshot['components'] = components_by_name
out_snapshot = os.path.join(self.workdir, '%s-snapshot.json' % (name_prefix, ))
f = open(out_snapshot, 'w')
json.dump(snapshot, f, indent=4, sort_keys=True)
f.close()
print "Created: %s" % (out_snapshot, )
snapshot['00ostree-src-snapshot-version'] = 0
current_time = time.time()
src_db = self.get_src_snapshot_db()
path = src_db.store(snapshot)
log("Source snapshot: %s" % (path, ))
builtins.register(OstbuildResolve)

165
src/ostbuild/pyostbuild/builtins.py
View File

@ -19,11 +19,15 @@
import os
import sys
import stat
import argparse
import json
from . import ostbuildrc
from . import fileutil
from . import jsondb
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
_all_builtins = {}
@ -31,26 +35,167 @@ class Builtin(object):
name = None
short_description = None
def __init__(self):
self._meta_cache = {}
self.prefix = None
self.manifest = None
self.snapshot = None
self.bin_snapshot = None
self.repo = None
self.ostree_dir = self._find_ostree_dir()
(self.active_branch, self.active_branch_checksum) = self._find_active_branch()
self._src_snapshots = None
self._bin_snapshots = None
def _find_ostree_dir(self):
for path in ['/ostree', '/sysroot/ostree']:
if os.path.isdir(path):
return path
return None
def _find_active_branch(self):
if self.ostree_dir is None:
return (None, None)
current_path = os.path.join(self.ostree_dir, 'current')
while True:
try:
target = os.path.join(self.ostree_dir, current_path)
stbuf = os.lstat(target)
except OSError, e:
current_path = None
break
if not stat.S_ISLNK(stbuf.st_mode):
break
current_path = os.readlink(target)
if current_path is not None:
basename = os.path.basename(current_path)
return basename.rsplit('-', 1)
else:
return (None, None)
def get_component_from_cwd(self):
cwd = os.getcwd()
parent = os.path.dirname(cwd)
parentparent = os.path.dirname(parent)
return '%s/%s/%s' % tuple(map(os.path.basename, [parentparent, parent, cwd]))
def parse_config(self):
self.ostbuildrc = ostbuildrc
self.repo = ostbuildrc.get_key('repo')
self.mirrordir = ostbuildrc.get_key('mirrordir')
if not os.path.isdir(self.mirrordir):
fatal("Specified mirrordir '%s' is not a directory" % (self.mirrordir, ))
self.workdir = ostbuildrc.get_key('workdir')
if not os.path.isdir(self.workdir):
fatal("Specified workdir '%s' is not a directory" % (self.workdir, ))
self.snapshot_dir = os.path.join(self.workdir, 'snapshots')
self.patchdir = os.path.join(self.workdir, 'patches')
def parse_manifest(self):
self.manifest_path = ostbuildrc.get_key('manifest')
self.manifest = json.load(open(self.manifest_path))
self.name_prefix = self.manifest['name-prefix']
def parse_active_branch(self):
if self.ostree_dir is None:
fatal("/ostree directory not found")
repo_path = os.path.join(self.ostree_dir, 'repo')
if not os.path.isdir(repo_path):
fatal("Repository '%s' doesn't exist" % (repo_path, ))
self.repo = repo_path
if self.active_branch is None:
fatal("No \"current\" link found")
branch_path = os.path.join(self.ostree_dir, self.active_branch)
contents_path = os.path.join(branch_path, 'contents.json')
f = open(contents_path)
self.active_branch_contents = json.load(f)
f.close()
def parse_snapshot(self):
self.parse_manifest()
snapshot_path = os.path.join(self.workdir, '%s-snapshot.json' % (self.name_prefix, ))
self.snapshot = json.load(open(snapshot_path))
def get_component_snapshot(self, name):
found = False
for content in self.active_branch_contents['contents']:
if content['name'] == name:
found = True
break
if not found:
fatal("Unknown component '%s'" % (name, ))
return content
def get_component_meta_from_revision(self, revision):
text = run_sync_get_output(['ostree', '--repo=' + self.repo,
'cat', revision,
'/_ostbuild-meta.json'])
return json.loads(text)
def get_component_meta(self, name):
assert self.repo is not None
if self.snapshot is not None:
return self.snapshot['components'][name]
meta = self._meta_cache.get(name)
if meta is None:
content = self.get_component_snapshot(name)
meta = self.get_component_meta_from_revision(content['ostree-revision'])
self._meta_cache[name] = meta
return meta
def get_prefix(self):
if self.prefix is None:
path = os.path.expanduser('~/.config/ostbuild-prefix')
if not os.path.exists(path):
fatal("No prefix set; use \"ostbuild prefix\" to set one")
f = open(path)
self.prefix = f.read().strip()
f.close()
return self.prefix
def create_db(self, dbsuffix, prefix=None):
if prefix is None:
target_prefix = self.get_prefix()
else:
target_prefix = prefix
name = '%s-%s' % (target_prefix, dbsuffix)
fileutil.ensure_dir(self.snapshot_dir)
return jsondb.JsonDB(self.snapshot_dir, prefix=name)
def get_src_snapshot_db(self):
if self._src_snapshots is None:
self._src_snapshots = self.create_db('src-snapshot')
return self._src_snapshots
def get_bin_snapshot_db(self):
if self._bin_snapshots is None:
self._bin_snapshots = self.create_db('bin-snapshot')
return self._bin_snapshots
def parse_snapshot(self, prefix, path):
if prefix is not None:
self.prefix = prefix
self.repo = ostbuildrc.get_key('repo')
if path is None:
latest_path = self.get_src_snapshot_db().get_latest_path()
if latest_path is None:
raise Exception("No source snapshot found for prefix %r" % (self.prefix, ))
self.snapshot_path = latest_path
else:
self.snapshot_path = path
self.snapshot = json.load(open(self.snapshot_path))
src_ver = self.snapshot['00ostree-src-snapshot-version']
if src_ver != 0:
fatal("Unhandled 00ostree-src-snapshot-version \"%d\", expected 0", src_ver)
def parse_bin_snapshot(self, prefix, path):
if prefix is not None:
self.prefix = prefix
self.repo = ostbuildrc.get_key('repo')
if path is None:
latest_path = self.get_bin_snapshot_db().get_latest_path()
if latest_path is None:
raise Exception("No binary snapshot found for prefix %r" % (self.prefix, ))
self.bin_snapshot_path = latest_path
else:
self.bin_snapshot_path = path
self.bin_snapshot = json.load(open(self.bin_snapshot_path))
bin_ver = self.bin_snapshot['00ostree-bin-snapshot-version']
if bin_ver != 0:
fatal("Unhandled 00ostree-bin-snapshot-version \"%d\", expected 0", bin_ver)
def execute(self, args):
raise NotImplementedError()
@ -65,4 +210,4 @@ def get(name):
return None
def get_all():
return _all_builtins.itervalues()
return sorted(_all_builtins.itervalues(), lambda a, b: cmp(a.name, b.name))

110
src/ostbuild/pyostbuild/jsondb.py Normal file
View File

@ -0,0 +1,110 @@
#
# Copyright (C) 2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
import os
import stat
import time
import tempfile
import re
import shutil
import hashlib
import json
class JsonDB(object):
def __init__(self, dirpath, prefix):
self._dirpath = dirpath
self._prefix = prefix
self._version_csum_re = re.compile(r'-(\d+)\.(\d+)-([0-9a-f]+).json$')
def _cmp_match_by_version(self, a, b):
# Note this is a reversed comparison; bigger is earlier
a_major = a[0]
a_minor = a[1]
b_major = b[0]
b_minor = b[1]
c = cmp(b_major, a_major)
if c == 0:
return cmp(b_minor, a_minor)
return 0
def _get_all(self):
result = []
for fname in os.listdir(self._dirpath):
if not (fname.startswith(self._prefix) and fname.endswith('.json')):
continue
path = os.path.join(self._dirpath, fname)
match = self._version_csum_re.search(fname)
if not match:
raise Exception("Invalid file '%s' in JsonDB; doesn't contain version+checksum",
path)
result.append((int(match.group(1)), int(match.group(2)), match.group(3), fname))
result.sort(self._cmp_match_by_version)
return result
def get_latest(self):
path = self.get_latest_path()
if path is None:
return None
return json.load(open(path))
def get_latest_path(self):
files = self._get_all()
if len(files) == 0:
return None
return os.path.join(self._dirpath, files[0][3])
def store(self, obj):
files = self._get_all()
if len(files) == 0:
latest = None
else:
latest = files[0]
current_time = time.gmtime()
(fd, tmppath) = tempfile.mkstemp(suffix='.tmp', prefix='tmp-jsondb-')
os.close(fd)
f = open(tmppath, 'w')
json.dump(obj, f, indent=4, sort_keys=True)
f.close()
csum = hashlib.sha256()
f = open(tmppath)
buf = f.read(8192)
while buf != '':
csum.update(buf)
buf = f.read(8192)
f.close()
digest = csum.hexdigest()
if latest is not None:
if digest == latest[2]:
return latest[3]
latest_version = (latest[0], latest[1])
else:
latest_version = (current_time.tm_year, 0)
target_name = '%s-%d.%d-%s.json' % (self._prefix, current_time.tm_year,
latest_version[1] + 1, digest)
target_path = os.path.join(self._dirpath, target_name)
os.rename(tmppath, target_path)
return target_path

10
src/ostbuild/pyostbuild/main.py
View File

@ -22,13 +22,17 @@ import sys
import argparse
from . import builtins
from . import builtin_build
from . import builtin_bin_to_src
from . import builtin_build_components
from . import builtin_branch_prefix
from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_chroot_run_triggers
from . import builtin_compose
from . import builtin_compile_one
from . import builtin_query_content
from . import builtin_pull_components
from . import builtin_prefix
from . import builtin_resolve
from . import builtin_modify_snapshot
from . import builtin_status
def usage(ecode):

10
src/ostbuild/pyostbuild/ostbuildlog.py
View File

@ -20,12 +20,16 @@
import os
import sys
def log(msg):
fullmsg = '%s: %s\n' % (os.path.basename(sys.argv[0]), msg)
def log(msg, prefix=None):
if prefix is None:
prefix_target = ''
else:
prefix_target = prefix
fullmsg = '%s: %s%s\n' % (os.path.basename(sys.argv[0]), prefix_target, msg)
sys.stdout.write(fullmsg)
sys.stdout.flush()
def fatal(msg):
log(msg)
log(msg, prefix="FATAL: ")
sys.exit(1)

6
src/ostbuild/pyostbuild/subprocess_helpers.py
View File

@ -70,14 +70,16 @@ def run_sync_get_output(args, cwd=None, env=None, stdout=None, stderr=None, none
return None
def run_sync(args, cwd=None, env=None, fatal_on_error=True, keep_stdin=False,
log_success=True, log_initiation=True, stdout=None,
log_success=True, log_initiation=True, stdin=None, stdout=None,
stderr=None):
if log_initiation:
log("running: %s" % (subprocess.list2cmdline(args),))
env_copy = _get_env_for_cwd(cwd, env)
if keep_stdin:
if stdin is not None:
stdin_target = stdin
elif keep_stdin:
stdin_target = sys.stdin
else:
stdin_target = open('/dev/null', 'r')

2
src/ostbuild/pyostbuild/vcs.py
View File

@ -71,5 +71,5 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
return dest
def clean(keytype, checkoutdir):
assert keytype == 'git'
assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)