312 lines
8.8 KiB
Python
312 lines
8.8 KiB
Python
|
|
"""Interactive python shell with port_stats
|
|
|
|
What's in the box:
|
|
|
|
* loading:
|
|
- load() -- (re)loads the task and packages information
|
|
- use(repo) -- switch to using <repo>
|
|
- load(repo) -- load stuff and start using the given repo instead of default
|
|
* display packages:
|
|
- pt(name) -- prints information about the package tasks
|
|
- spt(name) -- prints information about the package tasks, short version
|
|
- spi(name) -- prints the short package information
|
|
- gspi(pattern), g -- prints the short package information for
|
|
all the packages with name matching pattern ('g' for 'grep')
|
|
- list_spi(pkgs_list) -- same as gspi, but for python lists
|
|
of strings instead of patterns
|
|
* display tasks:
|
|
- ti(num) -- prints information about the task #num
|
|
- fti(num) -- prints information about the task #num, full version (a dict)
|
|
- fresh(num) -- display subtasks that would update packages
|
|
* display logs:
|
|
- logs(num, [idx]), l -- print logs (logs/events.X.Y.log) for task #num
|
|
- build_log(num, [subtask_id], [arch], [kind]) -- print the build
|
|
logs for a subtask of given task
|
|
* reports:
|
|
- stats() -- packages statistics
|
|
- update_days() -- all the packages that need to be rebuild,
|
|
sorted by days pending.
|
|
- display_tasks([infos], [num], [min_id], [include], [exclude]) --
|
|
display tasks (all from current repo by default)
|
|
- next_tasks([min_id], [include], [exclude]) -- display
|
|
recycler's tasks that would update packages
|
|
* to= targets:
|
|
- dump
|
|
- append_to
|
|
- pager
|
|
* other:
|
|
- doc(): print this message.
|
|
|
|
You can also enjoy autocompletion with <TAB>.
|
|
"""
|
|
|
|
from __future__ import print_function
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
import rlcompleter # noqa
|
|
import sys
|
|
import time
|
|
|
|
from pydoc import pager
|
|
|
|
from port_stats import colorize
|
|
from port_stats import lists
|
|
from port_stats import logs as m_logs
|
|
from port_stats import reports
|
|
from port_stats import tasks
|
|
from port_stats import utils
|
|
|
|
|
|
LOG = logging.getLogger('port_stats.interactive')
|
|
|
|
|
|
def get_task(taskid, task_dict):
|
|
try:
|
|
return task_dict[int(taskid)]
|
|
except Exception:
|
|
raise ValueError('Task not found: %s' % taskid)
|
|
|
|
|
|
# {{{ to= targets
|
|
|
|
def dump(string):
|
|
print(string)
|
|
|
|
|
|
def append_to(filename):
|
|
def _write(string):
|
|
if os.path.exists(filename):
|
|
LOG.info('File %s already exists, appending', filename)
|
|
with open(filename, 'a') as f:
|
|
f.write(string)
|
|
f.write('\n')
|
|
return _write
|
|
|
|
|
|
# }}}
|
|
|
|
# The Interactive Part
|
|
|
|
CONFIG = None
|
|
CURRENT = None
|
|
REPOS = {}
|
|
TASKS = {}
|
|
PACKAGE_TASKS = {}
|
|
UPDATE_TIMES = {}
|
|
BY_NAME = {}
|
|
BY_COLOR = {}
|
|
|
|
|
|
def repo_tasks(task_dict=None):
|
|
for _, t in sorted(TASKS.iteritems()):
|
|
if t['repo'] == CURRENT['new']:
|
|
yield t
|
|
|
|
|
|
def use(repo=None):
|
|
global PACKAGE_TASKS, UPDATE_TIMES, BY_NAME, BY_COLOR, CURRENT
|
|
if repo is not None:
|
|
try:
|
|
CURRENT = (c for c in CONFIG['colorize']
|
|
if c['name'] == repo).next()
|
|
except StopIteration:
|
|
raise ValueError('Unknown repo: ' + repo)
|
|
elif CURRENT is None:
|
|
CURRENT = CONFIG['colorize'][0]
|
|
LOG.info("Updating data structures for %s...", CURRENT['name'])
|
|
sys.ps1 = CURRENT['name'] + '>>> '
|
|
PACKAGE_TASKS = tasks.tasks_by_package(repo_tasks())
|
|
UPDATE_TIMES = tasks.last_update_times(repo_tasks())
|
|
packages = (frozenset(PACKAGE_TASKS)
|
|
.union(p.name for p in REPOS[CURRENT['base']])
|
|
.union(p.name for p in REPOS[CURRENT['new']]))
|
|
BY_NAME, BY_COLOR = colorize.colorize(REPOS[CURRENT['base']],
|
|
REPOS[CURRENT['new']],
|
|
packages)
|
|
LOG.info("DONE")
|
|
|
|
|
|
def load(repo=None):
|
|
global TASKS, REPOS
|
|
TASKS = dict((t['taskid'], t)
|
|
for t in tasks.load_tasks(CONFIG['tasks'], cache=TASKS))
|
|
REPOS = lists.read_all_srclists(CONFIG['repos'])
|
|
LOG.info("Got %s tasks, %s repositories", len(TASKS), len(REPOS))
|
|
for name in sorted(REPOS):
|
|
LOG.info(" %s: %s srpms", name, len(REPOS[name]))
|
|
use(repo)
|
|
|
|
|
|
def pt(pkg, to=print):
|
|
# ts = sorted(PACKAGE_TASKS[pkg], key=lambda t: int(t['taskid']))
|
|
display_tasks(PACKAGE_TASKS[pkg], to=to)
|
|
|
|
|
|
def spt(pkg, to=print):
|
|
to(tasks.format_tasks_short(PACKAGE_TASKS.get(pkg)))
|
|
|
|
|
|
def spi(pkg, to=print):
|
|
to(reports.package_one_line(pkg, BY_NAME, PACKAGE_TASKS))
|
|
|
|
|
|
def update_days(to=pager):
|
|
to(reports.update_days(BY_NAME, PACKAGE_TASKS, UPDATE_TIMES, time.time()))
|
|
|
|
|
|
def _spi_by_predicate(pred, colors):
|
|
colors = colors or colorize.COLORS
|
|
return '\n'.join(
|
|
reports.package_one_line(name, BY_NAME, PACKAGE_TASKS)
|
|
for name in sorted(BY_NAME)
|
|
if pred(name) and BY_NAME[name][0] in colors)
|
|
|
|
|
|
def gspi(pattern, colors=None, to=print):
|
|
p = re.compile(pattern)
|
|
to(_spi_by_predicate(p.search, colors))
|
|
|
|
|
|
g = gspi # noqa
|
|
|
|
|
|
def list_spi(pkgs, colors=None, to=print):
|
|
if isinstance(pkgs, (str, bytes)):
|
|
pkgs = pkgs.split()
|
|
pset = frozenset(pkgs)
|
|
lines = _spi_by_predicate(pset.__contains__, colors)
|
|
unknown = pset - frozenset(BY_NAME)
|
|
if unknown:
|
|
lines += "\nNot in repos:\n\t" + "\n\t".join(sorted(unknown))
|
|
to(lines)
|
|
|
|
|
|
def _colorizer():
|
|
return colorize.package_colorizer(BY_NAME, 'NONE')
|
|
|
|
|
|
def ti(num, to=print):
|
|
to(tasks.format_task(get_task(num, TASKS), _colorizer()))
|
|
|
|
|
|
def display_tasks(infos=None, num=None, min_id=None,
|
|
include=None, exclude=None, to=pager):
|
|
infos = infos or repo_tasks()
|
|
if min_id is not None:
|
|
infos = (t for t in infos if t['taskid'] >= min_id)
|
|
czr = _colorizer()
|
|
infos = (tasks.format_task(t, czr) for t in infos)
|
|
if include:
|
|
pi = re.compile(include)
|
|
infos = (t for t in infos if pi.search(t))
|
|
if exclude:
|
|
pe = re.compile(exclude)
|
|
infos = (t for t in infos if not pe.search(t))
|
|
infos = list(infos)
|
|
total = len(infos)
|
|
if num is not None:
|
|
infos = infos[-num:]
|
|
to("%s of %s infos\n\n%s" % (len(infos), total, '\n\n'.join(infos)))
|
|
|
|
|
|
def _fresh_subtasks(info):
|
|
result = {}
|
|
for subtaskid, subtask in info['subtasks'].items():
|
|
p = tasks.subtask_package(subtask)
|
|
color, bp, np = BY_NAME.get(p, ('MISSING', None, None))
|
|
if not np:
|
|
continue
|
|
if color == 'EXTRA':
|
|
result[subtaskid] = subtask
|
|
continue
|
|
# for outdated packages, check for newer versions
|
|
# for SLATE packages, check for base repo version rebuilds
|
|
rebuild_nevr = np if color in ('ORANGE', 'YELLOW') else bp
|
|
strict = info['task_ctime'] < UPDATE_TIMES.get(p, 0)
|
|
if tasks.subtask_is_updating(subtask, rebuild_nevr, strict):
|
|
# subtask is interesting, let's save it
|
|
result[subtaskid] = subtask
|
|
return result
|
|
|
|
|
|
def fresh(num, color=True, to=print):
|
|
subtasks = _fresh_subtasks(get_task(num, TASKS))
|
|
if subtasks:
|
|
clr = _colorizer() if color else None
|
|
items = sorted((int(k), tasks.format_subtask(s, clr))
|
|
for k, s in subtasks.iteritems())
|
|
to('\n'.join('%6d %s' % item for item in items))
|
|
else:
|
|
to('Nothing interesting in task %s' % num)
|
|
|
|
|
|
def next_tasks(min_id=None, include=None, exclude=None, to=pager):
|
|
infos = (t for t in repo_tasks()
|
|
if (t['state'] != 'DONE'
|
|
and t['owner'] == 'recycler'
|
|
and _fresh_subtasks(t)))
|
|
display_tasks(infos, None, min_id, include, exclude, to)
|
|
|
|
|
|
def fti(num, to=print):
|
|
to(utils.format_dict(get_task(num, TASKS), indent=True))
|
|
|
|
|
|
def _page_file(log_file, to):
|
|
with open(log_file, 'r') as f:
|
|
log = f.read().decode('utf-8', errors='replace')
|
|
to(log_file + ':\n\n' + log)
|
|
|
|
|
|
def logs(num, idx=-1, to=pager):
|
|
log_file = m_logs.task_event_logs(get_task(num, TASKS))[idx]
|
|
_page_file(log_file, to)
|
|
|
|
|
|
l = logs # noqa
|
|
|
|
|
|
def _default_arch():
|
|
return CONFIG['repos'][CURRENT['new']]['arch'][0]
|
|
|
|
|
|
def build_log(task_id, subtask_id=None, arch=None, kind='log', to=pager):
|
|
task_id = int(task_id)
|
|
arch = (arch or _default_arch()).lower()
|
|
info = get_task(task_id, TASKS)
|
|
if subtask_id is None:
|
|
subtask_id = min(utils.maybe_int(x) for x in info['subtasks'].keys())
|
|
log_file = os.path.join(info['task_path'], 'build',
|
|
str(subtask_id), arch, kind)
|
|
_page_file(log_file, to)
|
|
|
|
|
|
bl = build_log # noqa
|
|
|
|
|
|
def stats(names=None, to=dump):
|
|
total = len(names) if names else len(REPOS[CURRENT['new']])
|
|
to(reports.color_totals(BY_COLOR, names, total, summary=True))
|
|
|
|
|
|
def inapt_reporter():
|
|
import inapt
|
|
d = inapt.DependencyInfo.load(CONFIG['repos'][CURRENT['base']])
|
|
|
|
def _reporter(name, to=pager):
|
|
to(d.recursive_srpm_report(name, _colorizer()))
|
|
|
|
return _reporter
|
|
|
|
|
|
def doc(to=dump):
|
|
to(__doc__)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
CONFIG = utils.interactive_setup()
|
|
doc()
|