ostbuild: Add 'tag' concept to manifest, improve git mirroring

We should explicitly distinguish between the case where we have a git
branch we're following, versus an immutable tag.  In the latter case,
we can entirely avoid running 'git fetch' for it once we have it.
This is a noticeable speedup in our current scenario of pinning WebKit
to a tag.

The git mirroring code now has a --fetch-skip-secs=X option which
allows us to basically run it in a loop, without hitting remote git
repositories too often.

src/ostbuild/pyostbuild/builtin_build.py

@@ -65,7 +65,6 @@ class OstbuildBuild(builtins.Builtin):
 
     def _build_one_component(self, component, architecture):
         basename = component['name']
-        branch = component['branch']
 
         buildname = '%s/%s/%s' % (self.snapshot['prefix'], basename, architecture)
         build_ref = 'components/%s' % (buildname, )

src/ostbuild/pyostbuild/builtin_git_mirror.py

@@ -18,7 +18,7 @@
 # ostbuild-compile-one-make wraps systems that implement the GNOME build API:
 # http://people.gnome.org/~walters/docs/build-api.txt
 
-import os,sys,stat,subprocess,tempfile,re,shutil
+import os,sys,stat,subprocess,tempfile,re,shutil,time
 import argparse
 from StringIO import StringIO
 import json
@@ -40,8 +40,12 @@ class OstbuildGitMirror(builtins.Builtin):
         parser = argparse.ArgumentParser(description=self.short_description)
         parser.add_argument('--prefix')
         parser.add_argument('--src-snapshot')
-        parser.add_argument('--start-at')
-        parser.add_argument('--fetch', action='store_true')
+        parser.add_argument('--start-at',
+                            help="Start at the given component")
+        parser.add_argument('--fetch-skip-secs', type=int, default=0,
+                            help="Don't perform a fetch if we have done so in the last N seconds")
+        parser.add_argument('--fetch', action='store_true',
+                            help="Also do a git fetch for components")
         parser.add_argument('components', nargs='*')
 
         args = parser.parse_args(argv)
@@ -64,10 +68,33 @@ class OstbuildGitMirror(builtins.Builtin):
             component = self.get_component(name)
             src = component['src']
             (keytype, uri) = vcs.parse_src_key(src)
-            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
+            branch = component.get('branch')
+            tag = component.get('tag')
+            branch_or_tag = branch or tag
+            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
 
-            if args.fetch:
-                log("Running git fetch for %s" % (name, ))
-                run_sync(['git', 'fetch'], cwd=mirrordir, log_initiation=False)
+            if not args.fetch:
+                continue
+
+            if tag is not None:
+                log("Skipping fetch for %s at tag %s" % (name, tag))
+                continue
+
+            curtime = time.time()
+            if args.fetch_skip_secs > 0:
+                last_fetch_path = vcs.get_lastfetch_path(self.mirrordir, keytype, uri, branch_or_tag)
+                try:
+                    stbuf = os.stat(last_fetch_path)
+                except OSError, e:
+                    stbuf = None
+                if stbuf is not None:
+                    mtime = stbuf.st_mtime
+                    delta = curtime - mtime
+                    if delta < args.fetch_skip_secs:
+                        log("Skipping fetch for %s updated in last %d seconds" % (name, delta))
+                        continue
+
+            log("Running git fetch for %s" % (name, ))
+            vcs.fetch(self.mirrordir, keytype, uri, branch_or_tag)
 
 builtins.register(OstbuildGitMirror)

src/ostbuild/pyostbuild/builtin_resolve.py

@@ -67,7 +67,8 @@ class OstbuildResolve(builtins.Builtin):
             name = name.replace('/', '-')
             result['name'] = name
 
-        if 'branch' not in result:
+        branch_or_tag = result.get('branch') or result.get('tag')
+        if branch_or_tag is None:
             result['branch'] = 'master'
 
         return result
@@ -107,9 +108,9 @@ class OstbuildResolve(builtins.Builtin):
                 fatal("Duplicate component name '%s'" % (name, ))
             unique_component_names.add(name)
 
-            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, component['branch'])
-            revision = buildutil.get_git_version_describe(mirrordir,
-                                                          component['branch'])
+            branch_or_tag = component.get('branch') or component.get('tag')
+            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
+            revision = buildutil.get_git_version_describe(mirrordir, branch_or_tag)
             component['revision'] = revision
 
         src_db = self.get_src_snapshot_db()

src/ostbuild/pyostbuild/builtin_source_diff.py

@@ -117,7 +117,8 @@ class OstbuildSourceDiff(builtins.Builtin):
             if keytype == 'local':
                 log("Component %r has local URI" % (name, ))
                 continue
-            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, from_component['branch'])
+            branch_or_tag = from_component.get('branch') or from_component.get('tag')
+            mirrordir = vcs.ensure_vcs_mirror(self.mirrordir, keytype, uri, branch_or_tag)
 
             to_component = self.find_component_in_snapshot(name, to_snap)
             if to_component is None:

src/ostbuild/pyostbuild/vcs.py

@@ -89,6 +89,11 @@ def parse_src_key(srckey):
     uri = srckey[idx+1:]
     return (keytype, uri)
 
+def get_lastfetch_path(mirrordir, keytype, uri, branch):
+    mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+    branch_safename = branch.replace('/','_').replace('.', '_')
+    return mirror + '.lastfetch-%s' % (branch_safename, )
+
 def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
     mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
     tmp_mirror = mirror + '.tmp'
@@ -100,8 +105,7 @@ def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
         os.rename(tmp_mirror, mirror)
     if branch is None:
         return mirror
-    branch_safename = branch.replace('/','_').replace('.', '_')
-    last_fetch_path = mirror + '.lastfetch-%s' % (branch_safename, )
+    last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
     if os.path.exists(last_fetch_path):
         f = open(last_fetch_path)
         last_fetch_contents = f.read()
@@ -136,3 +140,14 @@ def ensure_vcs_mirror(mirrordir, keytype, uri, branch):
         f.write(current_vcs_version + '\n')
         f.close()
     return mirror
+
+def fetch(mirrordir, keytype, uri, branch):
+    mirror = buildutil.get_mirrordir(mirrordir, keytype, uri)
+    last_fetch_path = get_lastfetch_path(mirrordir, keytype, uri, branch)
+    run_sync(['git', 'fetch'], cwd=mirror, log_initiation=False) 
+    current_vcs_version = run_sync_get_output(['git', 'rev-parse', branch], cwd=mirror)
+    current_vcs_version = current_vcs_version.strip()
+    f = open(last_fetch_path, 'w')
+    f.write(current_vcs_version + '\n')
+    f.close()
+