Add some code

Python code copyed from port-compare commit
153130a18f2ce80d8f9646ab94b0f9eeb0e994de
with minor modifications.

Runner script added

Makefile

@@ -0,0 +1,9 @@
+
+
+all: flake8; @:
+
+flake8:
+	flake8 repos_cmp/
+
+run:
+	scripts/run.sh

config/sm-titan.json

@@ -0,0 +1,20 @@
+{
+    "description": "Configuration for local machine of iv@",
+    "repos": {
+        "sisyphus": {
+            "path": "/srv/mirrors/alt/Sisyphus/",
+            "arch": ["x86_64", "noarch"],
+            "bits": 64
+        },
+        "sisyphus_mipsel": {
+            "path": "/srv/mirrors/alt/ports/sisyphus_mipsel/",
+            "arch": ["mipsel", "noarch"],
+            "bits": 32
+        },
+        "sisyphus_riscv64": {
+            "path": "/srv/mirrors/alt/ports/riscv64/Sisyphus/",
+            "arch": ["riscv64", "noarch"],
+            "bits": 64
+        }
+    }
+}

config/titan.json

@@ -0,0 +1,35 @@
+{
+    "description": "Configuration for local machine of iv@",
+    "repos": {
+        "sisyphus-i586": {
+            "path": "/srv/mirrors/alt/Sisyphus/",
+            "arch": ["i586", "noarch"]
+            "bits": 32
+        },
+        "sisyphus": {
+            "path": "/srv/mirrors/alt/Sisyphus/",
+            "arch": ["x86_64", "noarch"],
+            "bits": 64
+        },
+        "sisyphus_mipsel": {
+            "path": "/srv/mirrors/alt/ports/sisyphus_mipsel/",
+            "arch": ["mipsel", "noarch"],
+            "bits": 32
+        },
+        "p9_mipsel": {
+            "path": "/srv/mirrors/alt/ports/p9_mipsel/",
+            "arch": ["mipsel", "noarch"]
+            "bits": 32
+        },
+        "sisyphus_riscv64": {
+            "path": "/srv/mirrors/alt/ports/riscv64/Sisyphus/",
+            "arch": ["riscv64", "noarch"]
+            "bits": 64
+        },
+        "sisyphus_mips64el": {
+            "path": "/srv/mirrors/alt/ports/sisyphus_mips64el/",
+            "arch": ["mips64el", "noarch"]
+            "bits": 64
+        }
+    }
+}

repos_cmp/lists.py

@@ -0,0 +1,155 @@
+
+import collections
+import logging
+import os
+import shutil
+import subprocess
+
+try:
+    import rpm
+except ImportError:
+    rpm = None
+
+from repos_cmp import rpm_ffi
+
+LOG = logging.getLogger(__name__)
+
+
+class NEVR(collections.namedtuple('NEVR', ['name', 'epoch',
+                                           'version', 'release'])):
+
+    if rpm:
+        @classmethod
+        def from_header(cls, header):
+            return cls(header[rpm.RPMTAG_NAME],
+                       header[rpm.RPMTAG_EPOCH],
+                       header[rpm.RPMTAG_VERSION],
+                       header[rpm.RPMTAG_RELEASE])
+
+    @classmethod
+    def from_tsv_line(cls, line):
+        """Creates a NEVR object from a tab-separated line.
+
+        The line should have the following format:
+            name\tepoch\tversion\trelease
+        """
+        try:
+            n, e, v, r = line.split(b'\t')
+        except Exception:
+            LOG.error("Failed to parse line: %s", line, exc_info=True)
+            return None
+
+        if e in ('', '(none)', 'None'):
+            e = None
+        else:
+            try:
+                e = int(e)
+            except Exception:
+                LOG.error("Failed to parse epoch from line: %s",
+                          line, exc_info=True)
+                return None
+        return cls(n, e, v, r)
+
+    def format_evr(self):
+        if self.epoch is None:
+            return '%s-%s' % (self.version, self.release)
+        else:
+            return '%s:%s-%s' % (self.epoch, self.version, self.release)
+
+    @property
+    def evr(self):
+        return self[1:]
+
+
+def format_evr(nevr):
+    return nevr.format_evr() if nevr else 'MISSING'
+
+
+def _uncompressed(path):
+    if not path.endswith('.xz'):
+        return open(path, 'rb')
+    pixz = shutil.which('pixz')
+    if pixz:
+        return subprocess.Popen([pixz, '-d', '-i', path, '-o', '/dev/stdout'],
+                                stdout=subprocess.PIPE).stdout
+    return subprocess.Popen(['xz', '-dc', path], stdout=subprocess.PIPE).stdout
+
+
+def read_pkglist_headers_rpm(path):
+    LOG.info("Reading %s using python-module-rpm", path)
+    if not rpm:
+        raise RuntimeError('rpm module is not avalable')
+    with _uncompressed(path) as input_file:
+        return rpm.readHeaderListFromFD(input_file.fileno())
+
+
+def read_pkglist_heders_for_repo(repo_path, arches, components=None):
+    bin_headers = []
+    src_headers = []
+    for arch in arches:
+        basedir = os.path.join(repo_path, arch, 'base')
+        for pkglist in os.listdir(basedir):
+            parts = pkglist.split('.', 3)
+            if parts[0] not in ('pkglist', 'srclist'):
+                continue
+            if components is not None and parts[1] not in components:
+                continue
+            (src_headers if parts[0] == 'srclist' else bin_headers).extend(
+                read_pkglist_headers_rpm(os.path.join(basedir, pkglist)))
+    return src_headers, bin_headers
+
+
+def _read_pkglist_rpm(path):
+    return (NEVR.from_header(h) for h in read_pkglist_headers_rpm(path))
+
+
+_PKGLIST_QUERY_FORMAT = '%{NAME}\t%{EPOCH}\t%{VERSION}\t%{RELEASE}\n'
+
+
+def _read_pkglist_pkglist_query(path):
+    LOG.info("Reading %s using pkglist-query", path)
+    with _uncompressed(path) as input_file:
+        query = subprocess.Popen(
+            ["pkglist-query", _PKGLIST_QUERY_FORMAT, '-'],
+            stdin=input_file, stdout=subprocess.PIPE)
+    return (NEVR.from_tsv_line(line)
+            for line in query.communicate()[0].splitlines())
+
+
+def read_pkglist(path):
+    if rpm:
+        result = _read_pkglist_rpm(path)
+    else:
+        result = _read_pkglist_pkglist_query(path)
+    return [r for r in result if r]
+
+
+def read_src_dot_list(repo_path):
+    path = os.path.join(repo_path, 'files/list/src.list.xz')
+    LOG.info("Reading src.list %s", path)
+
+    result = []
+    with _uncompressed(path) as input_file:
+        for line in input_file:
+            try:
+                name, evr = line.split(b'\t', 2)[:2]
+                e, v, r = rpm_ffi.parse_evr(evr)
+                result.append(NEVR(name, e, v, r))
+            except Exception:
+                LOG.warning('Failed to parse line %r', line, exc_info=True)
+    return frozenset(result)
+
+
+def read_srclists(prefix, arches):
+    result = frozenset()
+    for arch in arches:
+        srclist = os.path.join(prefix, arch, 'base', 'srclist.classic.xz')
+        result = result.union(read_pkglist(srclist))
+    if not result:
+        raise RuntimeError('Empty lists at %s' % prefix)
+    return result
+
+
+def read_all_srclists(repos):
+    return dict((name, read_src_dot_list(v['path']))
+                for name, v in repos.items())

repos_cmp/repos.py

@@ -0,0 +1,531 @@
+
+# This module does not use str: use bytes everywhere.
+
+import collections
+import graphlib
+import itertools
+import logging
+import re
+
+import rpm
+
+from repos_cmp import lists
+from repos_cmp import rpm_ffi
+
+LOG = logging.getLogger(__name__)
+
+
+class Dependency(collections.namedtuple(
+        'Dependency', ['name', 'flags', 'version'])):
+
+    HEADER_TRIPLETS = {
+        'require': (rpm.RPMTAG_REQUIRENAME,
+                    rpm.RPMTAG_REQUIREFLAGS,
+                    rpm.RPMTAG_REQUIREVERSION),
+        'provide': (rpm.RPMTAG_PROVIDENAME,
+                    rpm.RPMTAG_PROVIDEFLAGS,
+                    rpm.RPMTAG_PROVIDEVERSION)
+    }
+
+    RPMSENSE_COMPARE = (rpm.RPMSENSE_EQUAL
+                        | rpm.RPMSENSE_GREATER
+                        | rpm.RPMSENSE_LESS)
+
+    @classmethod
+    def from_header(cls, header, kind):
+        tp, tf, tv = cls.HEADER_TRIPLETS[kind]
+        triplets = zip(header[tp], header[tf], header[tv])
+        for name, flags, version in triplets:
+            if not name.startswith(b'rpmlib('):
+                yield cls(name, flags, version or None)
+        if kind == 'provide':
+            for name in header[rpm.RPMTAG_FILENAMES]:
+                yield cls(name, 0, None)
+            yield cls(header[rpm.RPMTAG_NAME],
+                      rpm.RPMSENSE_EQUAL,
+                      header.format('%{EVR}:%{DISTTAG}').encode('utf-8'))
+
+    @classmethod
+    def wrap(cls, other):
+        if isinstance(other, cls):
+            return other
+        if isinstance(other, bytes):
+            return cls(other, 0, None)
+        raise ValueError("Don't know how to make %s from %s"
+                         % (cls, repr(other)))
+
+    def is_provide_for(self, other):
+        if self.name != other.name:
+            return False
+        if self.version is None or other.version is None:
+            return True
+        return bool(rpm_ffi.ranges_overlap(
+            self.name, self.version, self.flags,
+            other.name, other.version, other.flags))
+
+    def is_setversion(self):
+        return self.version and self.version.startswith(b'set:')
+
+    def pretty_str(self):
+        if not self.flags and not self.version:
+            return '{}[ANY]'.format(self.name.decode())
+        if self.is_setversion():
+            return '{}[set:<>]'.format(self.name.decode())
+        return '{}[{} {}]'.format(
+            self.name.decode(), self.flags,
+            self.version.decode() if self.version else None)
+
+
+class Source:
+
+    def __init__(self, name, epoch, version, release):
+        self.name = name
+        self.epoch = epoch
+        self.version = version
+        self.release = release
+        self.requires = set()
+        self.bin_names = set()
+
+    @classmethod
+    def from_header(cls, header):
+        '''Initialize Source from rpm.header corresponding to SRPM'''
+        src = cls(header[rpm.RPMTAG_NAME],
+                  header[rpm.RPMTAG_EPOCH],
+                  header[rpm.RPMTAG_VERSION],
+                  header[rpm.RPMTAG_RELEASE])
+        src.requires.update(Dependency.from_header(header, 'require'))
+        return src
+
+    @property
+    def source_rpm(self):
+        return b'%s-%s-%s.src.rpm' % (
+            self.name, self.version, self.release)
+
+    def __repr__(self):
+        return 'Source[{} {} {} {}]'.format(
+            self.name, self.epoch, self.version, self.release)
+
+
+class Binary:
+
+    def __init__(self, name, epoch, version, release, source_rpm):
+        self.name = name
+        self.epoch = epoch
+        self.version = version
+        self.release = release
+        self.source_rpm = source_rpm
+        self.source_name = source_rpm.rsplit(b'-', 2)[0]
+        self.requires = set()
+        self.provides = set()
+
+    @classmethod
+    def from_header(cls, header):
+        '''Initialize Source from rpm.header corresponding to SRPM'''
+        pkg = cls(header[rpm.RPMTAG_NAME],
+                  header[rpm.RPMTAG_EPOCH],
+                  header[rpm.RPMTAG_VERSION],
+                  header[rpm.RPMTAG_RELEASE],
+                  header[rpm.RPMTAG_SOURCERPM])
+        pkg.requires.update(Dependency.from_header(header, 'require'))
+        pkg.provides.update(Dependency.from_header(header, 'provide'))
+        return pkg
+
+    def __repr__(self):
+        return 'Binary[{} {} {} {}]'.format(
+            self.name, self.epoch, self.version, self.release)
+
+
+class Repository:
+
+    def __init__(self, repo_name, sources, binaries, bits=None):
+        self.name = repo_name
+        self.sources = sources
+        self.binaries = binaries
+        if bits is not None:
+            self.bits = bits
+        else:
+            self.bits = 64 if ('64' in self.name) else 32
+        self.reverse_prov = {}  # name -> [(provide, binary)]
+        self.update_indexes()
+
+    def copy(self, new_name):
+        return Repository(new_name,
+                          dict(self.sources),
+                          dict(self.binaries),
+                          self.bits)
+
+    def update_indexes(self):
+        rprov = collections.defaultdict(list)
+        for b in self.binaries.values():
+            for p in b.provides:
+                rprov[p.name].append((p, b))
+        self.reverse_prov = dict(rprov)
+
+    @classmethod
+    def load(cls, repo_name, path, arch, components=('classic',)):
+        src_list, bin_list = lists.read_pkglist_heders_for_repo(
+            path, arch, components)
+        sources = {}
+        for header in src_list:
+            name = header[rpm.RPMTAG_NAME]
+            if name not in sources:
+                sources[name] = Source.from_header(header)
+        binaries = {}
+        for header in bin_list:
+            name = header[rpm.RPMTAG_NAME]
+            found = binaries.get(name)
+            if not found:
+                binaries[name] = Binary.from_header(header)
+            else:
+                LOG.warning('Duplicate binaries: %s %s', found,
+                            header.format('%{NAME}-%{EVR}:%{DISTTAG})'))
+        return cls(repo_name, sources, binaries)
+
+    @classmethod
+    def load_from_config(cls, repo_name, config):
+        r = config['repos'][repo_name]
+        return cls.load(repo_name, r['path'], r['arch'])
+
+    def binaries_from(self, *source_names):
+        '''Return binary packages build from this source'''
+        sources = set(source_names)
+        for b in self.binaries.values():
+            if b.source_name in sources:
+                yield b
+
+    def providers(self, dependency):
+        for item in self.reverse_prov.get(dependency.name, ()):
+            if item[0].is_provide_for(dependency):
+                yield item
+
+    def _unmets(self, packages):
+        result = []
+        for pkg in packages:
+            for dep in pkg.requires:
+                if not any(self.providers(dep)):
+                    result.append((pkg, dep))
+        return result
+
+    def unmets(self):
+        return self._unmets(self.binaries.values())
+
+    def build_unmets(self):
+        return self._unmets(self.sources.values())
+
+    def delete_sources(self, *source_names):
+        bin_names = [b.name for b in self.binaries_from(*source_names)]
+        for source in source_names:
+            del self.sources[source]
+        for name in bin_names:
+            del self.binaries[name]
+        self.update_indexes()
+
+    def chroot_for(self, requires):
+        stack = list(reversed(list(requires)))
+        chroot_provides = collections.defaultdict(set)
+        chroot_binaries = []
+        unmets = []
+
+        while stack:
+            dep = stack.pop()
+            if any(p.is_provide_for(dep) for p in chroot_provides[dep.name]):
+                continue
+            LOG.debug("looking up %s", dep)
+            providers = set(item[1] for item in self.providers(dep))
+            if not providers:
+                unmets.append(dep)
+                continue
+            if len(providers) > 1:
+                LOG.warning('Ambiguous provide: %s (%s)', dep, providers)
+            # that's almost random if ambigous:
+            p = providers.pop()
+            LOG.debug("  installing %s", p)
+            chroot_binaries.append(p)
+            for prov in p.provides:
+                chroot_provides[prov.name].add(prov)
+            stack.extend(p.requires)
+
+        return unmets, chroot_binaries
+
+
+_SPECIAL_DEPS = (
+    # ABI parts:
+    b'(GCC_', b'(CXXABI_', b'(GLIBC_',
+    # elf loaders:
+    b'/lib64/ld', b'/lib/ld', b'ld-linux')
+
+
+_GHC_HASHDEP = re.compile(rb'ghc[0-9\.]+\(.+-[^-]+-[^-]+\)')
+_GHC_HASHLIB = re.compile(rb'^lib.*-ghc[0-9.]+.so')
+
+
+def _from_64bit_dep(dep):
+    if dep.name.endswith(b'()(64bit)'):
+        new_name = dep.name[:-9]
+    elif dep.name.endswith(b'(64bit)'):
+        new_name = dep.name[:-7]
+    elif b'/lib64/' in dep.name:
+        new_name = dep.name.replace(b'/lib64/', b'/lib/')
+    else:
+        return dep
+    return Dependency(new_name, dep.flags, dep.version)
+
+
+def have_same_source(repoA, repoB, source_name):
+    sourceA = repoA.sources.get(source_name)
+    sourceB = repoB.sources.get(source_name)
+    return (sourceA and sourceB
+            and sourceA.epoch == sourceB.epoch
+            and sourceA.version == sourceB.version
+            and sourceA.release == sourceB.release)
+
+
+_BUILDREQ = '0000-BR'
+
+
+def _missing_requires(from_repo, to_repo, requires, kind, ignore):
+    missing_buidreqs = (dep for dep in requires
+                        if not any(to_repo.providers(dep)))
+    for dep in missing_buidreqs:
+        for _dep, provider in from_repo.providers(dep):
+            if provider.source_name not in ignore:
+                yield kind, dep, provider
+
+
+def _raw_build_report(from_repo, to_repo, source_name, ignore=()):
+    """Build report for one source, by name
+
+    Returns an iterable over tuples (kind, dep, provider).
+    """
+    # XXX: assumes from_repo is x86_64
+    assert from_repo.bits == 64
+    translate = (to_repo.bits != 64)
+
+    result = set(_missing_requires(
+        from_repo, to_repo, from_repo.sources[source_name].requires,
+        _BUILDREQ, ignore))
+
+    for b in from_repo.binaries_from(source_name):
+        for dep in b.requires:
+            # skip some platform-specific stuff
+            if any(x in dep.name for x in _SPECIAL_DEPS):
+                continue
+            # if needed, try to translate from 64 to 32 bits
+            the_dep = _from_64bit_dep(dep) if translate else dep
+            # skip dependencies already present in to_repo
+            if any(to_repo.providers(the_dep)):
+                continue
+            # skip inter-sub-package dependencies
+            if any(p.source_name == source_name
+                   for _d, p in from_repo.providers(dep)):
+                continue
+            # set-versions may be platform-dependent.
+            # double-check that if have the same source
+            if (dep.is_setversion()
+                    or _GHC_HASHLIB.match(dep.name)
+                    or _GHC_HASHDEP.fullmatch(dep.name)):
+                if any(have_same_source(from_repo, to_repo, p.source_name)
+                       for _d, p in from_repo.providers(dep)):
+                    continue
+            # ok, it's true missing dependency
+            for _dep, provider in from_repo.providers(dep):
+                if provider.source_name not in ignore:
+                    result.add((b.name.decode(), dep, provider))
+    return result
+
+
+def format_triplet_report(report, title=None):
+    """Format a report
+
+    Arguments:
+      report: iterable over triplets (kind, dep, binary)
+      title: the title of report, if any
+    """
+    lines = []
+    if title:
+        lines.append('\n== %s ==\n' % title)
+
+    by_source = collections.defaultdict(list)
+    for item in report:
+        by_source[item[2].source_rpm].append(item)
+    for that_source in sorted(by_source):
+        lines.append(that_source.decode())
+        by_knp = collections.defaultdict(list)
+        for kind, dep, provider in by_source[that_source]:
+            by_knp[(kind, provider.name.decode())].append(dep.pretty_str())
+        for (kind, pname), pdep in sorted(by_knp.items()):
+            lines.append('\t {} {}  {}'.format(kind, pname, '  '.join(pdep)))
+    return '\n'.join(lines)
+
+
+def build_report(from_repo, to_repo, source_name):
+    report = _raw_build_report(from_repo, to_repo, source_name)
+    return format_triplet_report(report, source_name.decode())
+
+
+def _src_name(source):
+    if source.endswith(b'.src.rpm'):
+        return source.rsplit(b'-', 2)[0]
+    return source
+
+
+def recursive_build_report(from_repo, to_repo, *source_names,
+                           ignore=(), ignore_sort=()):
+    reports = []
+    build_source_deps = {}
+    pkg_source_deps = {}
+    requested_source_names = set(_src_name(s) for s in source_names)
+    stack = list(requested_source_names)
+    seen = set(stack)
+
+    while stack:
+        cur = stack.pop()
+        report = _raw_build_report(from_repo, to_repo, cur, ignore)
+        cur_in_to = to_repo.binaries.get(cur)
+        cur_source_srpm = cur_in_to.source_rpm if cur_in_to else b'NONE'
+        reports.append('\n== %s ==\n%s has %s\n' % (
+            cur.decode(), to_repo.name, cur_source_srpm.decode()))
+        reports.append(format_triplet_report(report))
+
+        # find all sources in the report
+        bld_deps = set(p.source_name for k, _d, p in report if k == _BUILDREQ)
+        pkg_deps = set(p.source_name for k, _d, p in report if k != _BUILDREQ)
+        pkg_deps -= bld_deps
+
+        build_source_deps[cur] = sorted(bld_deps)
+        pkg_source_deps[cur] = sorted(pkg_deps)
+
+        unseen = bld_deps.union(pkg_deps) - seen
+        seen.update(unseen)
+        stack.extend(unseen)
+
+    # expand the build requires with the pkg reqs of dependencies:
+    full_req = {}
+    for source in build_source_deps:
+        _u, bins = from_repo.chroot_for(from_repo.sources[source].requires)
+        dep_sources = set()
+        for b in bins:
+            if b.source_name not in build_source_deps:
+                continue
+            if (b.source_name, source) in ignore_sort:
+                continue
+            dep_sources.add(b.source_name)
+        full_req[source] = dep_sources
+
+    cycles = []
+    while True:
+        try:
+            order = list(graphlib.TopologicalSorter(full_req).static_order())
+        except graphlib.CycleError as ex:
+            LOG.warning("Cycle detected: %s", ex)
+            cycle = ex.args[1]
+            cycles.append(cycle)
+            # break the cycle and retry
+            full_req[cycle[1]].remove(cycle[0])
+        else:
+            break
+
+    reports.append('\n== SUMMARY ==\n')
+    for source in order:
+        reports.append('{}\t{:50} {} :: {}'.format(
+            '' if source in requested_source_names else 'N',
+            from_repo.sources[source].source_rpm.decode(),
+            b' '.join(build_source_deps[source]).decode(),
+            b' '.join(pkg_source_deps[source]).decode()))
+
+    if cycles:
+        reports.append('\nCycles:')
+        reports.extend('\t%s' % ' '.join(source.decode() for source in c)
+                       for c in cycles)
+
+    special = set()
+    for source_name in order:
+        source = to_repo.sources.get(source_name)
+        if not source:
+            continue
+        if b'.mips' in source.release or b'.rv64' in source.release:
+            special.add(source.source_rpm)
+
+    if special:
+        reports.append('\nThe followng packages are special:')
+        reports.extend('\t%s' % source.decode()
+                       for source in sorted(special))
+
+    return '\n'.join(reports)
+
+
+def requires_build_report(from_repo, to_repo, *requires,
+                          ignore=(), ignore_sort=()):
+    missing_reqs = list(_missing_requires(
+        from_repo, to_repo,
+        [Dependency.wrap(r) for r in requires],
+        ignore, ignore_sort))
+    reports = [format_triplet_report(missing_reqs, 'Requested')]
+    source_names = set(p.source_name for _k, _d, p in missing_reqs)
+    reports.append(recursive_build_report(
+        from_repo, to_repo, *source_names,
+        ignore=ignore, ignore_sort=ignore_sort))
+    return '\n'.join(reports)
+
+
+def _unmets(repo, old_repo=None):
+    old_unmets = set(old_repo.unmets()) if old_repo else set()
+    unmets = (u for u in repo.unmets() if u not in old_unmets)
+
+    old_build_unmets = set(old_repo.build_unmets()) if old_repo else set()
+    build_unmets = (u for u in repo.build_unmets()
+                    if u not in old_build_unmets)
+
+    return itertools.chain(
+        (('unmet', d, b) for b, d in unmets),
+        (('BUILD', d, b) for b, d in build_unmets))
+
+
+def unmets_report(repo):
+    return format_triplet_report(_unmets(repo))
+
+
+def who_cares(repo, *source_names):
+    '''What new unmets deletion of the pacakge will produce?'''
+    next_repo = repo.copy('next_' + repo.name)
+    next_repo.delete_sources(*source_names)
+    return format_triplet_report(_unmets(next_repo, repo))
+
+
+_GOOD_EXTRAS_PREFIXES = (b'kernel-', b'u-boot', b'riscv', b'fu540', b'opensbi')
+
+
+def extras_to_remove(base_repo, repo, ignore=()):
+    extras = set(repo.sources) - set(base_repo.sources) - set(ignore)
+    # filter out certain packages that must be left alone
+    filtered_extras = set(
+        name for name in extras
+        if (not any(name.startswith(p) for p in _GOOD_EXTRAS_PREFIXES)
+            and b'jpp' not in repo.sources[name].release))
+    while True:
+        LOG.info('Checking %d filtered extras', len(filtered_extras))
+        next_repo = repo.copy('removal_test_' + repo.name)
+        next_repo.delete_sources(*filtered_extras)
+        leave_alone = set()
+        for kind, req, pkg in _unmets(next_repo, repo):
+            for b in repo.binaries_from(*filtered_extras):
+                if b.source_name in leave_alone:
+                    continue
+                if any(p.is_provide_for(req) for p in b.provides):
+                    leave_alone.add(b.source_name)
+        if leave_alone:
+            filtered_extras -= leave_alone
+        else:
+            break
+    for name in sorted(filtered_extras):
+        print(name.decode())
+
+
+if __name__ == '__main__':
+    from repos_cmp.utils import interactive_setup
+    from pydoc import pager  # noqa
+    CONFIG = interactive_setup()
+    riscv64 = Repository.load_from_config('sisyphus_riscv64', CONFIG)
+    x86_64 = Repository.load_from_config('sisyphus', CONFIG)
+    x86_64.bits = 64
+    mipsel = Repository.load_from_config('sisyphus_mipsel', CONFIG)

repos_cmp/rpm_ffi.py

@@ -0,0 +1,101 @@
+
+import cffi
+
+
+_CDEF = """
+void
+parseEVR(const char* evr,
+const char **e, const char **v, const char **r);
+
+int
+rpmEVRcmp(const char * const aE, const char * const aV, const char * const aR,
+          const char * const aDepend,
+          const char * const bE, const char * const bV, const char * const bR,
+          const char * const bDepend);
+
+int rpmRangesOverlap(const char * AName, const char * AEVR, int AFlags,
+                     const char * BName, const char * BEVR, int BFlags);
+"""
+
+
+_FFI = cffi.FFI()
+_FFI.cdef(_CDEF)
+_LIBRPM = _FFI.dlopen('librpm-4.0.4.so')
+
+_EMPTY = _FFI.new('char[]', b'')
+
+
+def _pp_str(p_str):
+    """Convert FFI's char** to Python string.
+
+    Assumes that pointer points to a zero-terminated C-string.
+    Returns None if the pointer is NULL.
+    """
+    if p_str and p_str[0]:
+        return _FFI.string(p_str[0])
+    return None
+
+
+def parse_evr(evr):
+    """Returns 3-tuple (epoch, version, release)"""
+    p_evr = _FFI.new('char[]', evr)
+    e = _FFI.new('char**')
+    v = _FFI.new('char**')
+    r = _FFI.new('char**')
+    _LIBRPM.parseEVR(p_evr, e, v, r)
+    epoch = _pp_str(e)
+    epoch = int(epoch) if epoch else None
+    return epoch, _pp_str(v), _pp_str(r)
+
+
+def ranges_overlap(aname, aevr, aflags, bname, bevr, bflags):
+    return _LIBRPM.rpmRangesOverlap(
+        _FFI.new('char[]', aname), _FFI.new('char[]', aevr), aflags,
+        _FFI.new('char[]', bname), _FFI.new('char[]', bevr), bflags)
+
+
+def _to_pchar(part):
+    if not part:
+        return _FFI.NULL
+    if not isinstance(part, bytes):
+        part = part.encode()
+    return _FFI.new('char[]', part) if part else _FFI.NULL
+
+
+def evr_cmp(evr1, evr2):
+    # Each EVR is 3-tuple 'epoch, version, release'.
+    # If epoch is None we replace it with '0'.
+    return _LIBRPM.rpmEVRcmp(
+        _to_pchar(str(evr1[0]).encode() if evr1[0] else b'0'),
+        _to_pchar(evr1[1]), _to_pchar(evr1[2]), _EMPTY,
+        _to_pchar(str(evr2[0]).encode() if evr2[0] else b'0'),
+        _to_pchar(evr2[1]), _to_pchar(evr2[2]), _EMPTY)
+
+
+def ver_cmp(ver1, ver2):
+    return evr_cmp((None, ver1, b'1'), (None, ver2, b'1'))
+
+
+def _tests():
+    assert parse_evr(b'3:42.8.4-alt1.mips0') == (3, b'42.8.4', b'alt1.mips0')
+    assert parse_evr(b'0:42.8.4-alt1.mips0') == (0, b'42.8.4', b'alt1.mips0')
+    assert parse_evr(b'42.8.4-alt1.mips0') == (None, b'42.8.4', b'alt1.mips0')
+
+    assert parse_evr(b'1.20.0-alt1_1') == (None, b'1.20.0', b'alt1_1')
+    assert parse_evr(b'1:1.20.1-alt1') == (1, b'1.20.1', b'alt1')
+
+    assert evr_cmp((3, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt2')) < 0
+    assert evr_cmp((3, b'42.9.4', b'alt1'), (3, b'42.8.4', b'alt2')) > 0
+    assert evr_cmp((3, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt1')) == 0
+    assert evr_cmp((5, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt1')) > 0
+    assert evr_cmp((3, b'42.9.4', b'alt1'), (5, b'42.8.4', b'alt1')) < 0
+
+    # unicode test
+    assert evr_cmp((3, u'42.8.4', u'alt1'), (3, b'42.8.4', b'alt1')) == 0
+
+    assert evr_cmp((1, b'1.2.0', b'alt1'), (None, b'1.2.0', b'alt1_1')) > 0
+    assert evr_cmp((None, b'1.2.0', b'alt1_1'), (1, b'1.2.0', b'alt1')) < 0
+
+
+if __name__ == '__main__':
+    _tests()

repos_cmp/utils.py

@@ -0,0 +1,39 @@
+
+import json
+import logging
+import os
+import sys
+
+LOG = logging.getLogger(__name__)
+
+
+def readline_setup():
+    import atexit
+    import readline
+
+    readline.parse_and_bind('tab:complete')
+    histfile = os.path.join(os.environ['HOME'], '.pythonhistory')
+    try:
+        readline.read_history_file(histfile)
+    except IOError:
+        pass
+    atexit.register(readline.write_history_file, histfile)
+
+
+def logging_setup(loglevel=logging.INFO):
+    logging.basicConfig(
+        format='%(asctime)s %(levelname)-5s %(name)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+        stream=sys.stderr, level=loglevel)
+
+
+def interactive_setup(readline=True, loglevel=logging.INFO):
+    logging_setup()
+
+    if readline:
+        readline_setup()
+
+    config = sys.argv[1]
+    LOG.info("Loading configuraition file: %s", config)
+    with open(config, 'r') as f:
+        return json.load(f)

scripts/run.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+set -eux
+
+BASE_BASES="$(pwd)/bases"
+
+function get_base() {
+  local name="$1" remote="$2" arch="$3"
+  local dst="$BASE_BASES/$name/$arch"
+
+  mkdir -p "$dst"
+  rsync -yavP "$remote/$arch/base" "$dst/"
+}
+
+get_base x86_64 xrs:/space/ALT/Sisyphus x86_64
+get_base x86_64 xrs:/space/ALT/Sisyphus noarch
+
+get_base riscv64 xrs:/riscv64/repo_depot/repo/sisyphus_riscv64/release/latest riscv64
+get_base riscv64 xrs:/riscv64/repo_depot/repo/sisyphus_riscv64/release/latest noarch
+
+get_base mipsel xrs:/repo_depot/repo/sisyphus_mipsel/release/latest mipsel
+get_base mipsel xrs:/repo_depot/repo/sisyphus_mipsel/release/latest noarch
+
+cat > "$BASE_BASES/config.json" << EOF
+{
+  "description": "Configuration for reports-secondary.office.basealt.ru",
+  "repos": {
+    "sisyphus": {
+      "path": "$BASE_BASES/x86_64",
+      "arch": ["x86_64", "noarch"],
+      "bits": 64
+    },
+    "sisyphus_riscv64": {
+      "path": "$BASE_BASES/riscv64",
+      "arch": ["riscv64", "noarch"],
+      "bits": 64
+    },
+    "sisyphus_mipsel": {
+      "path": "$BASE_BASES/mipsel",
+      "arch": ["mipsel", "noarch"],
+      "bits": 32
+    }
+  }
+}
+EOF
+
+python3 -im repos_cmp.repos "$BASE_BASES/config.json"