shaba/ostree
1
0
Fork 0
mirror of https://github.com/ostreedev/ostree.git synced 2025-03-19 22:50:35 +03:00
Code Issues Projects Releases Wiki Activity

ostbuild: Break out git-mirror from resolve

Browse Source 
Other bits
This commit is contained in:
Colin Walters 2012-05-01 20:51:29 -04:00
parent 986cd2172b
commit 29b230107e
6 changed files with 134 additions and 66 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile-ostbuild.am
View File

@ -30,6 +30,7 @@ pyostbuild_PYTHON = \
src/ostbuild/pyostbuild/builtin_chroot_compile_one.py \
src/ostbuild/pyostbuild/builtin_compile_one.py \
src/ostbuild/pyostbuild/builtin_pull_components.py \
src/ostbuild/pyostbuild/builtin_git_mirror.py \
src/ostbuild/pyostbuild/builtin_prefix.py \
src/ostbuild/pyostbuild/builtin_resolve.py \
src/ostbuild/pyostbuild/builtin_modify_snapshot.py \

1
src/ostbuild/pyostbuild/builtin_build_components.py
View File

@ -96,6 +96,7 @@ class OstbuildBuildComponents(builtins.Builtin):
log("No previous build for '%s' found" % (name, ))
checkoutdir = os.path.join(self.workdir, 'checkouts')
fileutil.ensure_dir(checkoutdir)
component_src = os.path.join(checkoutdir, basename)
run_sync(['ostbuild', 'checkout', '--snapshot=' + self.snapshot_path,
'--clean', '--overwrite', basename], cwd=checkoutdir)

63
src/ostbuild/pyostbuild/builtin_git_mirror.py Executable file
View File

@ -0,0 +1,63 @@
# Copyright (C) 2011,2012 Colin Walters <walters@verbum.org>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330,
# Boston, MA 02111-1307, USA.
# ostbuild-compile-one-make wraps systems that implement the GNOME build API:
# http://people.gnome.org/~walters/docs/build-api.txt
import os,sys,stat,subprocess,tempfile,re,shutil
import argparse
from StringIO import StringIO
import json
from . import builtins
from .ostbuildlog import log, fatal
from . import vcs
from .subprocess_helpers import run_sync, run_sync_get_output
from . import buildutil
class OstbuildGitMirror(builtins.Builtin):
name = "git-mirror"
short_description = "Update internal git mirror for one or more components"
def __init__(self):
builtins.Builtin.__init__(self)
def execute(self, argv):
parser = argparse.ArgumentParser(description=self.short_description)
parser.add_argument('--prefix')
parser.add_argument('--src-snapshot')
parser.add_argument('--fetch', action='store_true')
parser.add_argument('components', nargs='*')
args = parser.parse_args(argv)
self.parse_config()
self.parse_snapshot(args.prefix, args.src_snapshot)
if len(args.components) == 0:
components = self.snapshot['components'].keys()
else:
components = args.components
for name in components:
component = self.snapshot['components'][name]
src = component['src']
(keytype, uri) = vcs.parse_src_key(src)
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
if args.fetch:
run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
builtins.register(OstbuildGitMirror)

71
src/ostbuild/pyostbuild/builtin_resolve.py
View File

@ -27,6 +27,7 @@ from . import builtins
from .ostbuildlog import log, fatal
from .subprocess_helpers import run_sync, run_sync_get_output
from . import ostbuildrc
from . import vcs
from . import jsondb
from . import buildutil
from . import kvfile
@ -39,64 +40,6 @@ class OstbuildResolve(builtins.Builtin):
def __init__(self):
builtins.Builtin.__init__(self)
def _ensure_vcs_mirror(self, name, keytype, uri, branch):
# FIXME - remove "name" from parameter list here - hash uri?
mirror = buildutil.get_mirrordir(self.mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
shutil.rmtree(tmp_mirror)
if not os.path.isdir(mirror):
run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
os.rename(tmp_mirror, mirror)
if branch is None:
return mirror
last_fetch_path = mirror + '.%s-lastfetch' % (name, )
if os.path.exists(last_fetch_path):
f = open(last_fetch_path)
last_fetch_contents = f.read()
f.close()
last_fetch_contents = last_fetch_contents.strip()
else:
last_fetch_contents = None
current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
current_vcs_version = current_vcs_version.strip()
if current_vcs_version != last_fetch_contents:
log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
tmp_checkout = buildutil.get_mirrordir(self.mirrordir, keytype, uri, prefix='_tmp-checkouts')
if os.path.isdir(tmp_checkout):
shutil.rmtree(tmp_checkout)
parent = os.path.dirname(tmp_checkout)
if not os.path.isdir(parent):
os.makedirs(parent)
run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
submodules = []
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
submodule_status_lines = submodules_status_text.split('\n')
for line in submodule_status_lines:
if line == '': continue
line = line[1:]
(sub_checksum, sub_name) = line.split(' ', 1)
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
self._ensure_vcs_mirror(name + '-' + sub_name, keytype, sub_url, sub_checksum)
shutil.rmtree(tmp_checkout)
f = open(last_fetch_path, 'w')
f.write(current_vcs_version + '\n')
f.close()
return mirror
def _parse_src_key(self, srckey):
idx = srckey.find(':')
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
def _resolve_component_meta(self, component_meta):
result = dict(component_meta)
orig_src = component_meta['src']
@ -160,29 +103,29 @@ class OstbuildResolve(builtins.Builtin):
break
if not found:
fatal("Unknown component %r" % (component_name, ))
(keytype, uri) = self._parse_src_key(component['src'])
mirrordir = self._ensure_vcs_mirror(component_name, keytype, uri, None)
(keytype, uri) = vcs.parse_src_key(component['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, None)
log("Running git fetch for %s" % (component['name'], ))
run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
else:
fetch_components = []
global_patches_meta = self._resolve_component_meta(self.manifest['patches'])
(keytype, uri) = self._parse_src_key(global_patches_meta['src'])
mirrordir = self._ensure_vcs_mirror(global_patches_meta['name'], keytype, uri, global_patches_meta['branch'])
(keytype, uri) = vcs.parse_src_key(global_patches_meta['src'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, global_patches_meta['branch'])
revision = buildutil.get_git_version_describe(mirrordir, global_patches_meta['branch'])
global_patches_meta['revision'] = revision
unique_component_names = set()
for component in component_source_list:
(keytype, uri) = self._parse_src_key(component['src'])
(keytype, uri) = vcs.parse_src_key(component['src'])
name = component['name']
if name in unique_component_names:
fatal("Duplicate component name '%s'" % (name, ))
unique_component_names.add(name)
mirrordir = self._ensure_vcs_mirror(name, keytype, uri, component['branch'])
mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
revision = buildutil.get_git_version_describe(mirrordir,
component['branch'])
component['revision'] = revision

1
src/ostbuild/pyostbuild/main.py
View File

@ -29,6 +29,7 @@ from . import builtin_checkout
from . import builtin_chroot_compile_one
from . import builtin_compose
from . import builtin_compile_one
from . import builtin_git_mirror
from . import builtin_pull_components
from . import builtin_prefix
from . import builtin_resolve

63
src/ostbuild/pyostbuild/vcs.py
View File

@ -21,6 +21,8 @@ import urlparse
import shutil
from .subprocess_helpers import run_sync_get_output, run_sync
from . import buildutil
from .ostbuildlog import log, fatal
def get_mirrordir(mirrordir, keytype, uri, prefix=''):
assert keytype == 'git'
@ -61,10 +63,10 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
if not os.path.isdir(tmp_dest):
run_sync(['git', 'clone', '-q', '--origin', 'localmirror',
'--no-checkout', module_mirror, tmp_dest])
run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
else:
run_sync(['git', 'fetch'], cwd=tmp_dest)
run_sync(['git', 'fetch', 'localmirror'], cwd=tmp_dest)
run_sync(['git', 'checkout', '-q', branch], cwd=tmp_dest)
run_sync(['git', 'remote', 'add', 'upstream', uri], cwd=tmp_dest)
run_sync(['git', 'submodule', 'init'], cwd=tmp_dest)
have_submodules = _fixup_submodule_references(mirrordir, tmp_dest)
if have_submodules:
@ -76,3 +78,60 @@ def get_vcs_checkout(mirrordir, keytype, uri, dest, branch, overwrite=True):
def clean(keytype, checkoutdir):
assert keytype in ('git', 'dirty-git')
run_sync(['git', 'clean', '-d', '-f', '-x'], cwd=checkoutdir)
def parse_src_key(srckey):
idx = srckey.find(':')
if idx < 0:
raise ValueError("Invalid SRC uri=%s" % (srckey, ))
keytype = srckey[:idx]
if keytype not in ['git']:
raise ValueError("Unsupported SRC uri=%s" % (srckey, ))
uri = srckey[idx+1:]
return (keytype, uri)
def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
tmp_mirror = mirror + '.tmp'
if os.path.isdir(tmp_mirror):
shutil.rmtree(tmp_mirror)
if not os.path.isdir(mirror):
run_sync(['git', 'clone', '--mirror', uri, tmp_mirror])
run_sync(['git', 'config', 'gc.auto', '0'], cwd=tmp_mirror)
os.rename(tmp_mirror, mirror)
if branch is None:
return mirror
last_fetch_path = mirror + '.lastfetch'
if os.path.exists(last_fetch_path):
f = open(last_fetch_path)
last_fetch_contents = f.read()
f.close()
last_fetch_contents = last_fetch_contents.strip()
else:
last_fetch_contents = None
current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
current_vcs_version = current_vcs_version.strip()
if current_vcs_version != last_fetch_contents:
log("last fetch %r differs from branch %r" % (last_fetch_contents, current_vcs_version))
tmp_checkout = buildutil.get_mirrordir(mirrordir, keytype, uri, prefix='_tmp-checkouts')
if os.path.isdir(tmp_checkout):
shutil.rmtree(tmp_checkout)
parent = os.path.dirname(tmp_checkout)
if not os.path.isdir(parent):
os.makedirs(parent)
run_sync(['git', 'clone', '-q', '--no-checkout', mirror, tmp_checkout])
run_sync(['git', 'checkout', '-q', '-f', current_vcs_version], cwd=tmp_checkout)
submodules = []
submodules_status_text = run_sync_get_output(['git', 'submodule', 'status'], cwd=tmp_checkout)
submodule_status_lines = submodules_status_text.split('\n')
for line in submodule_status_lines:
if line == '': continue
line = line[1:]
(sub_checksum, sub_name) = line.split(' ', 1)
sub_url = run_sync_get_output(['git', 'config', '-f', '.gitmodules',
'submodule.%s.url' % (sub_name, )], cwd=tmp_checkout)
ensure_vcs_mirror(mirrordir, keytype, sub_url, sub_checksum)
shutil.rmtree(tmp_checkout)
f = open(last_fetch_path, 'w')
f.write(current_vcs_version + '\n')
f.close()
return mirror