349cc65f61
It's hard to do 100% correctly, but the heuristics we employ here works quite well for now: when a package was rebuild by base repo (Sisyphus) after we've updated it last time, we probably need to rebuild it.
133 lines
4.3 KiB
Python
133 lines
4.3 KiB
Python
|
|
"""Form a daily report"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import collections
|
|
import datetime
|
|
import json
|
|
import logging
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
|
|
from port_stats import colorize
|
|
from port_stats import lists
|
|
from port_stats import reports
|
|
from port_stats import tasks
|
|
from port_stats import utils
|
|
|
|
LOG = logging.getLogger('daily')
|
|
|
|
|
|
def main(out_dir, config_path):
|
|
os.makedirs(out_dir, mode=0o755)
|
|
logging.basicConfig(
|
|
format='%(asctime)s %(levelname)-5s %(name)s - %(message)s',
|
|
datefmt='%Y-%m-%d %H:%M:%S',
|
|
stream=sys.stderr, level=logging.INFO)
|
|
LOG.info('Generating daily report to %s; configuration file is %s',
|
|
out_dir, config_path)
|
|
|
|
with open(config_path) as f:
|
|
config = json.load(f)
|
|
|
|
LOG.info("Loading everything")
|
|
repos = lists.read_all_srclists(config['repos'])
|
|
all_tasks = tasks.load_tasks(config['tasks'])
|
|
LOG.info("Loading complete")
|
|
|
|
for r in config['colorize']:
|
|
r_dir = os.path.join(out_dir, r['name'])
|
|
os.makedirs(r_dir, mode=0o755)
|
|
daily_report(r_dir, r, repos, all_tasks)
|
|
_unmets(config['repos'][r['new']], os.path.join(r_dir, 'unmets.txt'))
|
|
|
|
LOG.info("DONE")
|
|
|
|
|
|
def _unmets(repo, out_file):
|
|
fno, fname = tempfile.mkstemp('.list')
|
|
try:
|
|
LOG.debug('File is %s, fno is %s', fname, fno)
|
|
with os.fdopen(fno, 'w') as f:
|
|
for arch in repo['arch']:
|
|
f.write('rpm file:%s %s classic\n' % (repo['path'], arch))
|
|
with open(out_file, 'w') as out:
|
|
subprocess.check_call(
|
|
['unmets', '-a', repo['arch'][0], '-s', fname], stdout=out)
|
|
finally:
|
|
os.unlink(fname)
|
|
|
|
|
|
def daily_report(out_dir, what, repos, all_tasks):
|
|
LOG.info("Writing daily report for %s", what['name'])
|
|
now = datetime.datetime.utcnow()
|
|
base_name = what['base']
|
|
new_name = what['new']
|
|
|
|
task_list = [t for t in all_tasks if t['repo'] == new_name]
|
|
package_tasks = tasks.tasks_by_package(task_list)
|
|
update_times = tasks.last_update_times(task_list)
|
|
|
|
state_stats = collections.Counter(t['state'] for t in task_list)
|
|
state_stats['[TOTAL]'] = len(task_list)
|
|
state_stats['[PENDING]'] = sum(1 for t in task_list
|
|
if t['state'] not in ['DONE'])
|
|
|
|
base = repos[base_name]
|
|
new = repos[new_name]
|
|
|
|
base_size = len(base)
|
|
new_size = len(new)
|
|
|
|
LOG.info("%s: %s tasks; base repo size is %s; repo size is %s",
|
|
what['name'], len(task_list), base_size, new_size)
|
|
|
|
by_name, by_color = colorize.colorize(base, new)
|
|
color_stats = reports.color_stats(by_color)
|
|
|
|
stats_path = os.path.join(out_dir, 'stats.txt')
|
|
LOG.info("Writing stats to %s", stats_path)
|
|
p = reports.percent
|
|
summary = [
|
|
("Summary report for", what['name'], "at", now.isoformat(' '), 'UTC'),
|
|
(base_name, "SRPMs:", base_size),
|
|
(new_name, "SRPMs:", p(new_size, base_size)),
|
|
(" recent enough:", p(color_stats['[RECENT]'], new_size)),
|
|
(" update pending:", p(color_stats['[PENDING]'], new_size)),
|
|
|
|
("\nPackage statistics:"),
|
|
reports.text_totals(colorize.COLORS, color_stats, colorize.LEGEND),
|
|
("\nTask statistics:"),
|
|
reports.text_totals(sorted(state_stats.keys()), state_stats)
|
|
]
|
|
with open(stats_path, 'w') as stats:
|
|
for part in summary:
|
|
if isinstance(part, basestring):
|
|
print(part, file=stats)
|
|
else:
|
|
print(*part, file=stats)
|
|
|
|
# Totals in JSON
|
|
with open(os.path.join(out_dir, 'color_stats.json'), 'w') as out:
|
|
print(utils.format_dict(color_stats, indent=True), file=out)
|
|
with open(os.path.join(out_dir, 'task_stats.json'), 'w') as out:
|
|
print(utils.format_dict(state_stats, indent=True), file=out)
|
|
|
|
update_report = reports.update_days(by_name, package_tasks, update_times,
|
|
now=utils.to_timestamp(now))
|
|
with open(os.path.join(out_dir, 'update_days.txt'), 'w') as out:
|
|
print(update_report, file=out)
|
|
|
|
# TSV for packages
|
|
with open(os.path.join(out_dir, 'colorized.tsv'), 'w') as out:
|
|
for name in sorted(by_name):
|
|
print(reports.package_one_line(name, by_name, package_tasks),
|
|
file=out)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main(*sys.argv[1:]))
|