From 2077d293f51260c9b7842d4f8c1a60e4ea47cdb9 Mon Sep 17 00:00:00 2001 From: "Ivan A. Melnikov" Date: Fri, 22 Mar 2019 17:13:07 +0400 Subject: [PATCH] side report! --- Makefile | 9 +- port_stats/rpm_ffi.py | 4 + port_stats/side_report.py | 178 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 port_stats/side_report.py diff --git a/Makefile b/Makefile index 551baef..ea57403 100644 --- a/Makefile +++ b/Makefile @@ -1,3 +1,10 @@ -all: + +all: flake8; @: + +flake8: python2-flake8 --ignore=F403,F405,W503 port_stats/ + +run: + python2 -im port_stats.interactive conf/local.json + diff --git a/port_stats/rpm_ffi.py b/port_stats/rpm_ffi.py index 1b3b37f..e4b18af 100644 --- a/port_stats/rpm_ffi.py +++ b/port_stats/rpm_ffi.py @@ -72,6 +72,10 @@ def evr_cmp(evr1, evr2, mode): p_e2, p_v2, p_r2, dep) +def ver_cmp(ver1, ver2): + return evr_cmp((None, ver1, 1), (None, ver2, 1), 'pkg') + + def _tests(): assert parse_evr('3:42.8.4-alt1.mipsel0') == (3, '42.8.4', 'alt1.mipsel0') assert parse_evr('0:42.8.4-alt1.mipsel0') == (0, '42.8.4', 'alt1.mipsel0') diff --git a/port_stats/side_report.py b/port_stats/side_report.py new file mode 100644 index 0000000..636af00 --- /dev/null +++ b/port_stats/side_report.py @@ -0,0 +1,178 @@ + +from __future__ import print_function + +import collections +import json +import logging +import sys +import time + + +from port_stats import lists +from port_stats import rpm_ffi +from port_stats import utils + +LOG = logging.getLogger('side_report') + +_SRPM_PREFIXES = [ + 'boost', + 'samba', + 'opensm', + 'scst', + 'nut', +] + +_SUFFIXES = [ + '-devel-static', + '-devel', + '-static', + '-libs', +] + + +_DISTROMAP = { + 'corosynclib': 'corosync', + 'dapl-utils': 'dapl', + 'device-mapper-multipath': 'multipath-tools', + 'drbd90-utils': 'drbd-utils', + 'e1000e': 'kernel-source-e1000e', + 'grub2': 'grub', + 'grub2-tools': 'grub', + 'krb5-workstation': 'krb5', + 'libibumad': 'rdma-core', + 'librdmacm': 'rdma-core', + 'librdmacm-devel': 'rdma-core', + 'librdmacm-utils': 'rdma-core', + 'libsmbclient': 'samba', + 'libwbclient': 'samba', + 'mpitests_openmpi': 'mpitests', + 'net-snmp': 'net-snmp30', + 'net-snmp-agent': 'net-snmp30', + 'net-snmp-perl': 'net-snmp30', + 'net-snmp-utils': 'net-snmp30', + 'nfs-utils': 'nfs', + 'nodejs': 'node', + 'nss-pam-ldapd': 'nss-ldapd', + 'openldap-clients': 'openldap', + 'openssl': 'openssl1.1', + 'perl-Socket': 'perl', + 'perl-Storable': 'perl', + 'perl-Test-Simple': 'perl', + 'perl-constant': 'perl', + 'python-websockify': 'python-module-websockify', + 'python34': 'python3', + 'python34-PyYAML': 'python-module-yaml', + 'python34-daemon': 'python-module-daemon', + 'python34-lockfile': 'python-module-lockfile', + 'python34-setproctitle': 'python-module-setproctitle', + 'python34-setuptools': 'python-module-setuptools', + 'python34-simplejson': 'python-module-simplejson', +} + + +def alt_srpm_name(name): + for s in _SUFFIXES: + if name.endswith(s): + name = name[:-len(s)] + for p in _SRPM_PREFIXES: + if name.startswith(p + '-'): + return p + return _DISTROMAP.get(name, name) + + +def repo_nv(repo): + result = {} + for nevr in repo: + result[nevr.name] = nevr.version + return result + + +_CATEGORIES = [ + ('UPDATED', 'Пакет в sisyphus_mipsel большей версии'), + ('SAME', 'Пакет в sisyphus_mipsel той же версии'), + ('OLDER', 'Пакет в sisyphus_mipsel меньшей версии'), + ('MISSING', 'Пакет есть в Sisyphus для x86_64, но не для mipsel'), + ('KERNEL', 'Ядро и модули'), + ('UNKNOWN', 'Сопоставить не удалось'), +] + +_HEADER = ''' +Сравнение списка с пакетной базой Sisyphus x86_64 и Sisyphus mipsel. +Время: %s + +Поля: +- пакет в оригинальном списке +- имя пакета из списка +- имя SRPM из Сизифа +- версия в списке +- версия в Sisyphus x86_64 +- версия в Sisyphus mipsel +''' + + +def category(srpm, side_v, sisyphus_v, mipsel_v): + if srpm.startswith('kmod-') or srpm == 'kernel': + return 'KERNEL' + if sisyphus_v is None: + return 'UNKNOWN' + if mipsel_v is None: + return 'MISSING' + c = rpm_ffi.ver_cmp(side_v, mipsel_v) + if c < 0: + return 'UPDATED' + if c == 0: + return 'SAME' + if c > 0: + return 'OLDER' + return 'OMG' + + +def main(config_path, side_list_path, *args): + logging.basicConfig( + format='%(asctime)s %(levelname)-5s %(name)s - %(message)s', + datefmt='%Y-%m-%d %H:%M:%S', + stream=sys.stderr, level=logging.INFO) + + with open(config_path) as f: + config = json.load(f) + repos = lists.read_all_srclists(config['repos']) + sisyphus_nv = repo_nv(repos['sisyphus']) + mipsel_nv = repo_nv(repos['sisyphus_mipsel']) + + with open(side_list_path) as f: + side_pkgs = f.read().split() + + LOG.info("Found %s pkgs in %s", len(side_pkgs), side_list_path) + # original package, name, version + side_nvs = sorted((x,) + tuple(x.rsplit('-', 2)[:-1]) for x in side_pkgs) + # original package, name, ALT SRPM, version + side_nvs = [(o, n, alt_srpm_name(n), v) + for o, n, v in side_nvs] + + # sanity check: each SRPM version should have 1 version in side list + s2v = collections.defaultdict(set) + for o, n, s, v in side_nvs: + s2v[s].add(v) + for s, vs in s2v.items(): + if len(vs) > 1: + LOG.warn('%s versions of %s: %s', len(vs), s, ' '.join(sorted(vs))) + + # add versions from sisyphus and sisyphus_mipsel + side_nvs = [(o, n, s, v, sisyphus_nv.get(s), mipsel_nv.get(s)) + for o, n, s, v in side_nvs] + + # categorize: + side_nvs = [(o, n, s, v, sv, mv, category(s, v, sv, mv)) + for o, n, s, v, sv, mv in side_nvs] + + print(_HEADER % utils.format_timestamp(time.time())) + for cat, descr in _CATEGORIES: + in_cat = sorted(t for t in side_nvs if t[-1] == cat) + print("\n\n%s: %s\n" % (len(in_cat), descr)) + for t in in_cat: + tl = (t[0], t[2]) if cat == 'UNKNOWN' else t[:-1] + print('\t'.join(x or 'MISSING' for x in tl)) + + +if __name__ == '__main__': + sys.exit(main(*sys.argv[1:]))