diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..3214e67 --- /dev/null +++ b/Makefile @@ -0,0 +1,9 @@ + + +all: flake8; @: + +flake8: + flake8 repos_cmp/ + +run: + scripts/run.sh diff --git a/config/sm-titan.json b/config/sm-titan.json new file mode 100644 index 0000000..88d29d4 --- /dev/null +++ b/config/sm-titan.json @@ -0,0 +1,20 @@ +{ + "description": "Configuration for local machine of iv@", + "repos": { + "sisyphus": { + "path": "/srv/mirrors/alt/Sisyphus/", + "arch": ["x86_64", "noarch"], + "bits": 64 + }, + "sisyphus_mipsel": { + "path": "/srv/mirrors/alt/ports/sisyphus_mipsel/", + "arch": ["mipsel", "noarch"], + "bits": 32 + }, + "sisyphus_riscv64": { + "path": "/srv/mirrors/alt/ports/riscv64/Sisyphus/", + "arch": ["riscv64", "noarch"], + "bits": 64 + } + } +} diff --git a/config/titan.json b/config/titan.json new file mode 100644 index 0000000..4cd939a --- /dev/null +++ b/config/titan.json @@ -0,0 +1,35 @@ +{ + "description": "Configuration for local machine of iv@", + "repos": { + "sisyphus-i586": { + "path": "/srv/mirrors/alt/Sisyphus/", + "arch": ["i586", "noarch"] + "bits": 32 + }, + "sisyphus": { + "path": "/srv/mirrors/alt/Sisyphus/", + "arch": ["x86_64", "noarch"], + "bits": 64 + }, + "sisyphus_mipsel": { + "path": "/srv/mirrors/alt/ports/sisyphus_mipsel/", + "arch": ["mipsel", "noarch"], + "bits": 32 + }, + "p9_mipsel": { + "path": "/srv/mirrors/alt/ports/p9_mipsel/", + "arch": ["mipsel", "noarch"] + "bits": 32 + }, + "sisyphus_riscv64": { + "path": "/srv/mirrors/alt/ports/riscv64/Sisyphus/", + "arch": ["riscv64", "noarch"] + "bits": 64 + }, + "sisyphus_mips64el": { + "path": "/srv/mirrors/alt/ports/sisyphus_mips64el/", + "arch": ["mips64el", "noarch"] + "bits": 64 + } + } +} diff --git a/repos_cmp/__init__.py b/repos_cmp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/repos_cmp/lists.py b/repos_cmp/lists.py new file mode 100644 index 0000000..d3c471f --- /dev/null +++ b/repos_cmp/lists.py @@ -0,0 +1,155 @@ + +import collections +import logging +import os +import shutil +import subprocess + +try: + import rpm +except ImportError: + rpm = None + +from repos_cmp import rpm_ffi + +LOG = logging.getLogger(__name__) + + +class NEVR(collections.namedtuple('NEVR', ['name', 'epoch', + 'version', 'release'])): + + if rpm: + @classmethod + def from_header(cls, header): + return cls(header[rpm.RPMTAG_NAME], + header[rpm.RPMTAG_EPOCH], + header[rpm.RPMTAG_VERSION], + header[rpm.RPMTAG_RELEASE]) + + @classmethod + def from_tsv_line(cls, line): + """Creates a NEVR object from a tab-separated line. + + The line should have the following format: + name\tepoch\tversion\trelease + """ + try: + n, e, v, r = line.split(b'\t') + except Exception: + LOG.error("Failed to parse line: %s", line, exc_info=True) + return None + + if e in ('', '(none)', 'None'): + e = None + else: + try: + e = int(e) + except Exception: + LOG.error("Failed to parse epoch from line: %s", + line, exc_info=True) + return None + return cls(n, e, v, r) + + def format_evr(self): + if self.epoch is None: + return '%s-%s' % (self.version, self.release) + else: + return '%s:%s-%s' % (self.epoch, self.version, self.release) + + @property + def evr(self): + return self[1:] + + +def format_evr(nevr): + return nevr.format_evr() if nevr else 'MISSING' + + +def _uncompressed(path): + if not path.endswith('.xz'): + return open(path, 'rb') + pixz = shutil.which('pixz') + if pixz: + return subprocess.Popen([pixz, '-d', '-i', path, '-o', '/dev/stdout'], + stdout=subprocess.PIPE).stdout + return subprocess.Popen(['xz', '-dc', path], stdout=subprocess.PIPE).stdout + + +def read_pkglist_headers_rpm(path): + LOG.info("Reading %s using python-module-rpm", path) + if not rpm: + raise RuntimeError('rpm module is not avalable') + with _uncompressed(path) as input_file: + return rpm.readHeaderListFromFD(input_file.fileno()) + + +def read_pkglist_heders_for_repo(repo_path, arches, components=None): + bin_headers = [] + src_headers = [] + for arch in arches: + basedir = os.path.join(repo_path, arch, 'base') + for pkglist in os.listdir(basedir): + parts = pkglist.split('.', 3) + if parts[0] not in ('pkglist', 'srclist'): + continue + if components is not None and parts[1] not in components: + continue + (src_headers if parts[0] == 'srclist' else bin_headers).extend( + read_pkglist_headers_rpm(os.path.join(basedir, pkglist))) + return src_headers, bin_headers + + +def _read_pkglist_rpm(path): + return (NEVR.from_header(h) for h in read_pkglist_headers_rpm(path)) + + +_PKGLIST_QUERY_FORMAT = '%{NAME}\t%{EPOCH}\t%{VERSION}\t%{RELEASE}\n' + + +def _read_pkglist_pkglist_query(path): + LOG.info("Reading %s using pkglist-query", path) + with _uncompressed(path) as input_file: + query = subprocess.Popen( + ["pkglist-query", _PKGLIST_QUERY_FORMAT, '-'], + stdin=input_file, stdout=subprocess.PIPE) + return (NEVR.from_tsv_line(line) + for line in query.communicate()[0].splitlines()) + + +def read_pkglist(path): + if rpm: + result = _read_pkglist_rpm(path) + else: + result = _read_pkglist_pkglist_query(path) + return [r for r in result if r] + + +def read_src_dot_list(repo_path): + path = os.path.join(repo_path, 'files/list/src.list.xz') + LOG.info("Reading src.list %s", path) + + result = [] + with _uncompressed(path) as input_file: + for line in input_file: + try: + name, evr = line.split(b'\t', 2)[:2] + e, v, r = rpm_ffi.parse_evr(evr) + result.append(NEVR(name, e, v, r)) + except Exception: + LOG.warning('Failed to parse line %r', line, exc_info=True) + return frozenset(result) + + +def read_srclists(prefix, arches): + result = frozenset() + for arch in arches: + srclist = os.path.join(prefix, arch, 'base', 'srclist.classic.xz') + result = result.union(read_pkglist(srclist)) + if not result: + raise RuntimeError('Empty lists at %s' % prefix) + return result + + +def read_all_srclists(repos): + return dict((name, read_src_dot_list(v['path'])) + for name, v in repos.items()) diff --git a/repos_cmp/repos.py b/repos_cmp/repos.py new file mode 100644 index 0000000..d100e25 --- /dev/null +++ b/repos_cmp/repos.py @@ -0,0 +1,531 @@ + +# This module does not use str: use bytes everywhere. + +import collections +import graphlib +import itertools +import logging +import re + +import rpm + +from repos_cmp import lists +from repos_cmp import rpm_ffi + +LOG = logging.getLogger(__name__) + + +class Dependency(collections.namedtuple( + 'Dependency', ['name', 'flags', 'version'])): + + HEADER_TRIPLETS = { + 'require': (rpm.RPMTAG_REQUIRENAME, + rpm.RPMTAG_REQUIREFLAGS, + rpm.RPMTAG_REQUIREVERSION), + 'provide': (rpm.RPMTAG_PROVIDENAME, + rpm.RPMTAG_PROVIDEFLAGS, + rpm.RPMTAG_PROVIDEVERSION) + } + + RPMSENSE_COMPARE = (rpm.RPMSENSE_EQUAL + | rpm.RPMSENSE_GREATER + | rpm.RPMSENSE_LESS) + + @classmethod + def from_header(cls, header, kind): + tp, tf, tv = cls.HEADER_TRIPLETS[kind] + triplets = zip(header[tp], header[tf], header[tv]) + for name, flags, version in triplets: + if not name.startswith(b'rpmlib('): + yield cls(name, flags, version or None) + if kind == 'provide': + for name in header[rpm.RPMTAG_FILENAMES]: + yield cls(name, 0, None) + yield cls(header[rpm.RPMTAG_NAME], + rpm.RPMSENSE_EQUAL, + header.format('%{EVR}:%{DISTTAG}').encode('utf-8')) + + @classmethod + def wrap(cls, other): + if isinstance(other, cls): + return other + if isinstance(other, bytes): + return cls(other, 0, None) + raise ValueError("Don't know how to make %s from %s" + % (cls, repr(other))) + + def is_provide_for(self, other): + if self.name != other.name: + return False + if self.version is None or other.version is None: + return True + return bool(rpm_ffi.ranges_overlap( + self.name, self.version, self.flags, + other.name, other.version, other.flags)) + + def is_setversion(self): + return self.version and self.version.startswith(b'set:') + + def pretty_str(self): + if not self.flags and not self.version: + return '{}[ANY]'.format(self.name.decode()) + if self.is_setversion(): + return '{}[set:<>]'.format(self.name.decode()) + return '{}[{} {}]'.format( + self.name.decode(), self.flags, + self.version.decode() if self.version else None) + + +class Source: + + def __init__(self, name, epoch, version, release): + self.name = name + self.epoch = epoch + self.version = version + self.release = release + self.requires = set() + self.bin_names = set() + + @classmethod + def from_header(cls, header): + '''Initialize Source from rpm.header corresponding to SRPM''' + src = cls(header[rpm.RPMTAG_NAME], + header[rpm.RPMTAG_EPOCH], + header[rpm.RPMTAG_VERSION], + header[rpm.RPMTAG_RELEASE]) + src.requires.update(Dependency.from_header(header, 'require')) + return src + + @property + def source_rpm(self): + return b'%s-%s-%s.src.rpm' % ( + self.name, self.version, self.release) + + def __repr__(self): + return 'Source[{} {} {} {}]'.format( + self.name, self.epoch, self.version, self.release) + + +class Binary: + + def __init__(self, name, epoch, version, release, source_rpm): + self.name = name + self.epoch = epoch + self.version = version + self.release = release + self.source_rpm = source_rpm + self.source_name = source_rpm.rsplit(b'-', 2)[0] + self.requires = set() + self.provides = set() + + @classmethod + def from_header(cls, header): + '''Initialize Source from rpm.header corresponding to SRPM''' + pkg = cls(header[rpm.RPMTAG_NAME], + header[rpm.RPMTAG_EPOCH], + header[rpm.RPMTAG_VERSION], + header[rpm.RPMTAG_RELEASE], + header[rpm.RPMTAG_SOURCERPM]) + pkg.requires.update(Dependency.from_header(header, 'require')) + pkg.provides.update(Dependency.from_header(header, 'provide')) + return pkg + + def __repr__(self): + return 'Binary[{} {} {} {}]'.format( + self.name, self.epoch, self.version, self.release) + + +class Repository: + + def __init__(self, repo_name, sources, binaries, bits=None): + self.name = repo_name + self.sources = sources + self.binaries = binaries + if bits is not None: + self.bits = bits + else: + self.bits = 64 if ('64' in self.name) else 32 + self.reverse_prov = {} # name -> [(provide, binary)] + self.update_indexes() + + def copy(self, new_name): + return Repository(new_name, + dict(self.sources), + dict(self.binaries), + self.bits) + + def update_indexes(self): + rprov = collections.defaultdict(list) + for b in self.binaries.values(): + for p in b.provides: + rprov[p.name].append((p, b)) + self.reverse_prov = dict(rprov) + + @classmethod + def load(cls, repo_name, path, arch, components=('classic',)): + src_list, bin_list = lists.read_pkglist_heders_for_repo( + path, arch, components) + sources = {} + for header in src_list: + name = header[rpm.RPMTAG_NAME] + if name not in sources: + sources[name] = Source.from_header(header) + binaries = {} + for header in bin_list: + name = header[rpm.RPMTAG_NAME] + found = binaries.get(name) + if not found: + binaries[name] = Binary.from_header(header) + else: + LOG.warning('Duplicate binaries: %s %s', found, + header.format('%{NAME}-%{EVR}:%{DISTTAG})')) + return cls(repo_name, sources, binaries) + + @classmethod + def load_from_config(cls, repo_name, config): + r = config['repos'][repo_name] + return cls.load(repo_name, r['path'], r['arch']) + + def binaries_from(self, *source_names): + '''Return binary packages build from this source''' + sources = set(source_names) + for b in self.binaries.values(): + if b.source_name in sources: + yield b + + def providers(self, dependency): + for item in self.reverse_prov.get(dependency.name, ()): + if item[0].is_provide_for(dependency): + yield item + + def _unmets(self, packages): + result = [] + for pkg in packages: + for dep in pkg.requires: + if not any(self.providers(dep)): + result.append((pkg, dep)) + return result + + def unmets(self): + return self._unmets(self.binaries.values()) + + def build_unmets(self): + return self._unmets(self.sources.values()) + + def delete_sources(self, *source_names): + bin_names = [b.name for b in self.binaries_from(*source_names)] + for source in source_names: + del self.sources[source] + for name in bin_names: + del self.binaries[name] + self.update_indexes() + + def chroot_for(self, requires): + stack = list(reversed(list(requires))) + chroot_provides = collections.defaultdict(set) + chroot_binaries = [] + unmets = [] + + while stack: + dep = stack.pop() + if any(p.is_provide_for(dep) for p in chroot_provides[dep.name]): + continue + LOG.debug("looking up %s", dep) + providers = set(item[1] for item in self.providers(dep)) + if not providers: + unmets.append(dep) + continue + if len(providers) > 1: + LOG.warning('Ambiguous provide: %s (%s)', dep, providers) + # that's almost random if ambigous: + p = providers.pop() + LOG.debug(" installing %s", p) + chroot_binaries.append(p) + for prov in p.provides: + chroot_provides[prov.name].add(prov) + stack.extend(p.requires) + + return unmets, chroot_binaries + + +_SPECIAL_DEPS = ( + # ABI parts: + b'(GCC_', b'(CXXABI_', b'(GLIBC_', + # elf loaders: + b'/lib64/ld', b'/lib/ld', b'ld-linux') + + +_GHC_HASHDEP = re.compile(rb'ghc[0-9\.]+\(.+-[^-]+-[^-]+\)') +_GHC_HASHLIB = re.compile(rb'^lib.*-ghc[0-9.]+.so') + + +def _from_64bit_dep(dep): + if dep.name.endswith(b'()(64bit)'): + new_name = dep.name[:-9] + elif dep.name.endswith(b'(64bit)'): + new_name = dep.name[:-7] + elif b'/lib64/' in dep.name: + new_name = dep.name.replace(b'/lib64/', b'/lib/') + else: + return dep + return Dependency(new_name, dep.flags, dep.version) + + +def have_same_source(repoA, repoB, source_name): + sourceA = repoA.sources.get(source_name) + sourceB = repoB.sources.get(source_name) + return (sourceA and sourceB + and sourceA.epoch == sourceB.epoch + and sourceA.version == sourceB.version + and sourceA.release == sourceB.release) + + +_BUILDREQ = '0000-BR' + + +def _missing_requires(from_repo, to_repo, requires, kind, ignore): + missing_buidreqs = (dep for dep in requires + if not any(to_repo.providers(dep))) + for dep in missing_buidreqs: + for _dep, provider in from_repo.providers(dep): + if provider.source_name not in ignore: + yield kind, dep, provider + + +def _raw_build_report(from_repo, to_repo, source_name, ignore=()): + """Build report for one source, by name + + Returns an iterable over tuples (kind, dep, provider). + """ + # XXX: assumes from_repo is x86_64 + assert from_repo.bits == 64 + translate = (to_repo.bits != 64) + + result = set(_missing_requires( + from_repo, to_repo, from_repo.sources[source_name].requires, + _BUILDREQ, ignore)) + + for b in from_repo.binaries_from(source_name): + for dep in b.requires: + # skip some platform-specific stuff + if any(x in dep.name for x in _SPECIAL_DEPS): + continue + # if needed, try to translate from 64 to 32 bits + the_dep = _from_64bit_dep(dep) if translate else dep + # skip dependencies already present in to_repo + if any(to_repo.providers(the_dep)): + continue + # skip inter-sub-package dependencies + if any(p.source_name == source_name + for _d, p in from_repo.providers(dep)): + continue + # set-versions may be platform-dependent. + # double-check that if have the same source + if (dep.is_setversion() + or _GHC_HASHLIB.match(dep.name) + or _GHC_HASHDEP.fullmatch(dep.name)): + if any(have_same_source(from_repo, to_repo, p.source_name) + for _d, p in from_repo.providers(dep)): + continue + # ok, it's true missing dependency + for _dep, provider in from_repo.providers(dep): + if provider.source_name not in ignore: + result.add((b.name.decode(), dep, provider)) + return result + + +def format_triplet_report(report, title=None): + """Format a report + + Arguments: + report: iterable over triplets (kind, dep, binary) + title: the title of report, if any + """ + lines = [] + if title: + lines.append('\n== %s ==\n' % title) + + by_source = collections.defaultdict(list) + for item in report: + by_source[item[2].source_rpm].append(item) + for that_source in sorted(by_source): + lines.append(that_source.decode()) + by_knp = collections.defaultdict(list) + for kind, dep, provider in by_source[that_source]: + by_knp[(kind, provider.name.decode())].append(dep.pretty_str()) + for (kind, pname), pdep in sorted(by_knp.items()): + lines.append('\t {} {} {}'.format(kind, pname, ' '.join(pdep))) + return '\n'.join(lines) + + +def build_report(from_repo, to_repo, source_name): + report = _raw_build_report(from_repo, to_repo, source_name) + return format_triplet_report(report, source_name.decode()) + + +def _src_name(source): + if source.endswith(b'.src.rpm'): + return source.rsplit(b'-', 2)[0] + return source + + +def recursive_build_report(from_repo, to_repo, *source_names, + ignore=(), ignore_sort=()): + reports = [] + build_source_deps = {} + pkg_source_deps = {} + requested_source_names = set(_src_name(s) for s in source_names) + stack = list(requested_source_names) + seen = set(stack) + + while stack: + cur = stack.pop() + report = _raw_build_report(from_repo, to_repo, cur, ignore) + cur_in_to = to_repo.binaries.get(cur) + cur_source_srpm = cur_in_to.source_rpm if cur_in_to else b'NONE' + reports.append('\n== %s ==\n%s has %s\n' % ( + cur.decode(), to_repo.name, cur_source_srpm.decode())) + reports.append(format_triplet_report(report)) + + # find all sources in the report + bld_deps = set(p.source_name for k, _d, p in report if k == _BUILDREQ) + pkg_deps = set(p.source_name for k, _d, p in report if k != _BUILDREQ) + pkg_deps -= bld_deps + + build_source_deps[cur] = sorted(bld_deps) + pkg_source_deps[cur] = sorted(pkg_deps) + + unseen = bld_deps.union(pkg_deps) - seen + seen.update(unseen) + stack.extend(unseen) + + # expand the build requires with the pkg reqs of dependencies: + full_req = {} + for source in build_source_deps: + _u, bins = from_repo.chroot_for(from_repo.sources[source].requires) + dep_sources = set() + for b in bins: + if b.source_name not in build_source_deps: + continue + if (b.source_name, source) in ignore_sort: + continue + dep_sources.add(b.source_name) + full_req[source] = dep_sources + + cycles = [] + while True: + try: + order = list(graphlib.TopologicalSorter(full_req).static_order()) + except graphlib.CycleError as ex: + LOG.warning("Cycle detected: %s", ex) + cycle = ex.args[1] + cycles.append(cycle) + # break the cycle and retry + full_req[cycle[1]].remove(cycle[0]) + else: + break + + reports.append('\n== SUMMARY ==\n') + for source in order: + reports.append('{}\t{:50} {} :: {}'.format( + '' if source in requested_source_names else 'N', + from_repo.sources[source].source_rpm.decode(), + b' '.join(build_source_deps[source]).decode(), + b' '.join(pkg_source_deps[source]).decode())) + + if cycles: + reports.append('\nCycles:') + reports.extend('\t%s' % ' '.join(source.decode() for source in c) + for c in cycles) + + special = set() + for source_name in order: + source = to_repo.sources.get(source_name) + if not source: + continue + if b'.mips' in source.release or b'.rv64' in source.release: + special.add(source.source_rpm) + + if special: + reports.append('\nThe followng packages are special:') + reports.extend('\t%s' % source.decode() + for source in sorted(special)) + + return '\n'.join(reports) + + +def requires_build_report(from_repo, to_repo, *requires, + ignore=(), ignore_sort=()): + missing_reqs = list(_missing_requires( + from_repo, to_repo, + [Dependency.wrap(r) for r in requires], + ignore, ignore_sort)) + reports = [format_triplet_report(missing_reqs, 'Requested')] + source_names = set(p.source_name for _k, _d, p in missing_reqs) + reports.append(recursive_build_report( + from_repo, to_repo, *source_names, + ignore=ignore, ignore_sort=ignore_sort)) + return '\n'.join(reports) + + +def _unmets(repo, old_repo=None): + old_unmets = set(old_repo.unmets()) if old_repo else set() + unmets = (u for u in repo.unmets() if u not in old_unmets) + + old_build_unmets = set(old_repo.build_unmets()) if old_repo else set() + build_unmets = (u for u in repo.build_unmets() + if u not in old_build_unmets) + + return itertools.chain( + (('unmet', d, b) for b, d in unmets), + (('BUILD', d, b) for b, d in build_unmets)) + + +def unmets_report(repo): + return format_triplet_report(_unmets(repo)) + + +def who_cares(repo, *source_names): + '''What new unmets deletion of the pacakge will produce?''' + next_repo = repo.copy('next_' + repo.name) + next_repo.delete_sources(*source_names) + return format_triplet_report(_unmets(next_repo, repo)) + + +_GOOD_EXTRAS_PREFIXES = (b'kernel-', b'u-boot', b'riscv', b'fu540', b'opensbi') + + +def extras_to_remove(base_repo, repo, ignore=()): + extras = set(repo.sources) - set(base_repo.sources) - set(ignore) + # filter out certain packages that must be left alone + filtered_extras = set( + name for name in extras + if (not any(name.startswith(p) for p in _GOOD_EXTRAS_PREFIXES) + and b'jpp' not in repo.sources[name].release)) + while True: + LOG.info('Checking %d filtered extras', len(filtered_extras)) + next_repo = repo.copy('removal_test_' + repo.name) + next_repo.delete_sources(*filtered_extras) + leave_alone = set() + for kind, req, pkg in _unmets(next_repo, repo): + for b in repo.binaries_from(*filtered_extras): + if b.source_name in leave_alone: + continue + if any(p.is_provide_for(req) for p in b.provides): + leave_alone.add(b.source_name) + if leave_alone: + filtered_extras -= leave_alone + else: + break + for name in sorted(filtered_extras): + print(name.decode()) + + +if __name__ == '__main__': + from repos_cmp.utils import interactive_setup + from pydoc import pager # noqa + CONFIG = interactive_setup() + riscv64 = Repository.load_from_config('sisyphus_riscv64', CONFIG) + x86_64 = Repository.load_from_config('sisyphus', CONFIG) + x86_64.bits = 64 + mipsel = Repository.load_from_config('sisyphus_mipsel', CONFIG) diff --git a/repos_cmp/rpm_ffi.py b/repos_cmp/rpm_ffi.py new file mode 100644 index 0000000..17e5b43 --- /dev/null +++ b/repos_cmp/rpm_ffi.py @@ -0,0 +1,101 @@ + +import cffi + + +_CDEF = """ +void +parseEVR(const char* evr, +const char **e, const char **v, const char **r); + +int +rpmEVRcmp(const char * const aE, const char * const aV, const char * const aR, + const char * const aDepend, + const char * const bE, const char * const bV, const char * const bR, + const char * const bDepend); + +int rpmRangesOverlap(const char * AName, const char * AEVR, int AFlags, + const char * BName, const char * BEVR, int BFlags); +""" + + +_FFI = cffi.FFI() +_FFI.cdef(_CDEF) +_LIBRPM = _FFI.dlopen('librpm-4.0.4.so') + +_EMPTY = _FFI.new('char[]', b'') + + +def _pp_str(p_str): + """Convert FFI's char** to Python string. + + Assumes that pointer points to a zero-terminated C-string. + Returns None if the pointer is NULL. + """ + if p_str and p_str[0]: + return _FFI.string(p_str[0]) + return None + + +def parse_evr(evr): + """Returns 3-tuple (epoch, version, release)""" + p_evr = _FFI.new('char[]', evr) + e = _FFI.new('char**') + v = _FFI.new('char**') + r = _FFI.new('char**') + _LIBRPM.parseEVR(p_evr, e, v, r) + epoch = _pp_str(e) + epoch = int(epoch) if epoch else None + return epoch, _pp_str(v), _pp_str(r) + + +def ranges_overlap(aname, aevr, aflags, bname, bevr, bflags): + return _LIBRPM.rpmRangesOverlap( + _FFI.new('char[]', aname), _FFI.new('char[]', aevr), aflags, + _FFI.new('char[]', bname), _FFI.new('char[]', bevr), bflags) + + +def _to_pchar(part): + if not part: + return _FFI.NULL + if not isinstance(part, bytes): + part = part.encode() + return _FFI.new('char[]', part) if part else _FFI.NULL + + +def evr_cmp(evr1, evr2): + # Each EVR is 3-tuple 'epoch, version, release'. + # If epoch is None we replace it with '0'. + return _LIBRPM.rpmEVRcmp( + _to_pchar(str(evr1[0]).encode() if evr1[0] else b'0'), + _to_pchar(evr1[1]), _to_pchar(evr1[2]), _EMPTY, + _to_pchar(str(evr2[0]).encode() if evr2[0] else b'0'), + _to_pchar(evr2[1]), _to_pchar(evr2[2]), _EMPTY) + + +def ver_cmp(ver1, ver2): + return evr_cmp((None, ver1, b'1'), (None, ver2, b'1')) + + +def _tests(): + assert parse_evr(b'3:42.8.4-alt1.mips0') == (3, b'42.8.4', b'alt1.mips0') + assert parse_evr(b'0:42.8.4-alt1.mips0') == (0, b'42.8.4', b'alt1.mips0') + assert parse_evr(b'42.8.4-alt1.mips0') == (None, b'42.8.4', b'alt1.mips0') + + assert parse_evr(b'1.20.0-alt1_1') == (None, b'1.20.0', b'alt1_1') + assert parse_evr(b'1:1.20.1-alt1') == (1, b'1.20.1', b'alt1') + + assert evr_cmp((3, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt2')) < 0 + assert evr_cmp((3, b'42.9.4', b'alt1'), (3, b'42.8.4', b'alt2')) > 0 + assert evr_cmp((3, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt1')) == 0 + assert evr_cmp((5, b'42.8.4', b'alt1'), (3, b'42.8.4', b'alt1')) > 0 + assert evr_cmp((3, b'42.9.4', b'alt1'), (5, b'42.8.4', b'alt1')) < 0 + + # unicode test + assert evr_cmp((3, u'42.8.4', u'alt1'), (3, b'42.8.4', b'alt1')) == 0 + + assert evr_cmp((1, b'1.2.0', b'alt1'), (None, b'1.2.0', b'alt1_1')) > 0 + assert evr_cmp((None, b'1.2.0', b'alt1_1'), (1, b'1.2.0', b'alt1')) < 0 + + +if __name__ == '__main__': + _tests() diff --git a/repos_cmp/utils.py b/repos_cmp/utils.py new file mode 100644 index 0000000..69d8e77 --- /dev/null +++ b/repos_cmp/utils.py @@ -0,0 +1,39 @@ + +import json +import logging +import os +import sys + +LOG = logging.getLogger(__name__) + + +def readline_setup(): + import atexit + import readline + + readline.parse_and_bind('tab:complete') + histfile = os.path.join(os.environ['HOME'], '.pythonhistory') + try: + readline.read_history_file(histfile) + except IOError: + pass + atexit.register(readline.write_history_file, histfile) + + +def logging_setup(loglevel=logging.INFO): + logging.basicConfig( + format='%(asctime)s %(levelname)-5s %(name)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr, level=loglevel) + + +def interactive_setup(readline=True, loglevel=logging.INFO): + logging_setup() + + if readline: + readline_setup() + + config = sys.argv[1] + LOG.info("Loading configuraition file: %s", config) + with open(config, 'r') as f: + return json.load(f) diff --git a/scripts/run.sh b/scripts/run.sh new file mode 100755 index 0000000..a261494 --- /dev/null +++ b/scripts/run.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +set -eux + +BASE_BASES="$(pwd)/bases" + +function get_base() { + local name="$1" remote="$2" arch="$3" + local dst="$BASE_BASES/$name/$arch" + + mkdir -p "$dst" + rsync -yavP "$remote/$arch/base" "$dst/" +} + +get_base x86_64 xrs:/space/ALT/Sisyphus x86_64 +get_base x86_64 xrs:/space/ALT/Sisyphus noarch + +get_base riscv64 xrs:/riscv64/repo_depot/repo/sisyphus_riscv64/release/latest riscv64 +get_base riscv64 xrs:/riscv64/repo_depot/repo/sisyphus_riscv64/release/latest noarch + +get_base mipsel xrs:/repo_depot/repo/sisyphus_mipsel/release/latest mipsel +get_base mipsel xrs:/repo_depot/repo/sisyphus_mipsel/release/latest noarch + +cat > "$BASE_BASES/config.json" << EOF +{ + "description": "Configuration for reports-secondary.office.basealt.ru", + "repos": { + "sisyphus": { + "path": "$BASE_BASES/x86_64", + "arch": ["x86_64", "noarch"], + "bits": 64 + }, + "sisyphus_riscv64": { + "path": "$BASE_BASES/riscv64", + "arch": ["riscv64", "noarch"], + "bits": 64 + }, + "sisyphus_mipsel": { + "path": "$BASE_BASES/mipsel", + "arch": ["mipsel", "noarch"], + "bits": 32 + } + } +} +EOF + +python3 -im repos_cmp.repos "$BASE_BASES/config.json"