port_stats: Tools for getting tasks information

You can try doing this within an interactive python shell:

    python -i -m port_stats.interactive

The default pathes work on basealt.

port_stats/interactive.py

@@ -0,0 +1,112 @@
+
+"""Interactive python shell with port_stats
+
+What's in the box:
+
+- load_tasks() -- (re)loads the task information from prefix
+- pt(name) -- prints information about the package tasks
+- ti(num) -- prints information about the task #num
+- logs(num, [idx]) -- print logs (logs/events.X.Y.log) for task #num
+
+You can also enjoy autocompletion with <TAB>.
+"""
+from __future__ import print_function
+
+import atexit
+import logging
+import os
+import readline
+import rlcompleter
+import sys
+
+from pydoc import pager
+
+from port_stats import lists
+from port_stats import tasks
+from port_stats import utils
+
+
+
+LOG = logging.getLogger('port_stats.interactive')
+
+
+def get_task(taskid, task_list):
+    for task in task_list:
+        if int(task['taskid']) == int(taskid):
+            return task
+
+def package_tasks(pkg, data):
+    ts = sorted(data[pkg], key=lambda t: int(t['taskid']))
+    return '\n'.join(tasks.format_task(t) for t in ts)
+
+
+def task_event_logs(num, task_list):
+    info = get_task(num, task_list)
+    if info is None:
+        return []
+    path = os.path.join(info['task_path'], 'logs')
+    res = sorted((f for f in os.listdir(path) if f.startswith('events')),
+                 key=lambda f: [utils.maybe_int(x) for x in f.split('.')])
+    return [os.path.join(path, x) for x in res]
+
+
+## The Interactive Part ##
+
+TASKS = []
+PACKAGE_TASKS = {}
+
+
+def load_tasks(prefix='/mnt/tasks_secondary'):
+    global TASKS, PACKAGE_TASKS
+    LOG.info("Loading tasks...")
+    TASKS = tasks.load_tasks(prefix)
+    LOG.info("Preparing the data structures...")
+    PACKAGE_TASKS = tasks.tasks_by_package(TASKS)
+    LOG.info("DONE")
+
+
+def pt(pkg, to=print):
+    to(package_tasks(pkg, PACKAGE_TASKS))
+
+
+def ti(num, to=print):
+    to(tasks.format_task(get_task(num, TASKS)))
+
+
+def fti(num, to=print):
+    to(utils.format_dict(get_task(num, TASKS), indent=True))
+
+
+def logs(num, idx=-1, to=pager):
+    log_file = task_event_logs(num, TASKS)[idx]
+    with open(log_file, 'r') as f:
+        log = f.read()
+    to(log_file + ':\n\n' + log)
+
+
+def interactive_setup():
+    # Bind ‘TAB’ to complete
+    readline.parse_and_bind('tab:complete')
+
+    # Set history file – ~\.pythonhistory
+    histfile = os.path.join(os.environ['HOME'], '.pythonhistory')
+
+    # Attempt read of histfile
+    try:
+        readline.read_history_file(histfile)
+    except IOError:
+        pass
+
+    # Write history file at shell exit
+    atexit.register(readline.write_history_file, histfile)
+
+    # Configure logging
+    logging.basicConfig(
+        format='%(asctime)s %(levelname)-5s %(name)s - %(message)s',
+        datefmt='%Y-%m-%d %H:%M:%S',
+        stream=sys.stderr, level=logging.INFO)
+    LOG.info("Test message")
+
+if __name__ == '__main__':
+    interactive_setup()
+    print(__doc__)

port_stats/tasks.py

@@ -0,0 +1,93 @@
+
+import json
+import logging
+import os
+import sys
+
+from collections import defaultdict
+
+from port_stats import utils
+
+
+LOG = logging.getLogger(__name__)
+
+
+def _task_dirs(prefix):
+    yield prefix
+    done_archive = os.path.join(prefix, 'archive', 'done')
+    for filename in os.listdir(done_archive):
+        if filename.startswith('_'):
+            yield os.path.join(done_archive, filename)
+
+
+def task_pathes(prefix):
+    for task_dir in _task_dirs(prefix):
+        for name in os.listdir(task_dir):
+            if name.isdigit():
+                yield os.path.join(task_dir, name)
+
+
+def load_task(path):
+    with open(os.path.join(path, 'info.json')) as f:
+        data = json.load(f)
+    data['task_path'] = path
+    data['task_time'] = utils.file_time_str(path)
+    data.setdefault('subtasks', {})
+    return data
+ 
+
+def load_tasks(prefix):
+    return [load_task(path) for path in task_pathes(prefix)]
+
+
+_FORMAT_SUBTASK = {
+    'srpm':     lambda st: 'srpm ' + st.get('srpm'),
+    'delete':   lambda st: 'delete ' + st.get('package'),
+    'repo':     lambda st: '%s %s' % (st.get('dir'),
+                                      st.get('tag') or st.get('tag_name')) 
+}
+
+def format_subtask(subtask):
+    stype = subtask.get('type')
+    formatter = _FORMAT_SUBTASK.get(stype, utils.format_dict)
+    return formatter(subtask)
+
+
+def format_task(info):
+    head =  '%(taskid)s %(state)s try=%(try)s %(owner)s ' % info
+    subtasks = sorted((int(k), format_subtask(s))
+                      for k, s in info['subtasks'].iteritems())
+    tail = ''.join('\n%12d %s' % item for item in subtasks)
+    return head + tail
+
+
+def task_packages(info):
+    for subtask in info['subtasks'].values():
+        if 'pkgname' in subtask:
+            yield subtask['pkgname']
+        elif 'package' in subtask:
+            yield subtask['package']
+        elif 'srpm' in subtask:
+            yield subtask['srpm'].rsplit('-', 2)[0]
+        elif 'dir' in subtask:
+            name = os.path.basename(subtask['dir'])
+            if name.endswith('.git'):
+                name = name[:-4]
+            yield name
+        elif list(subtask.keys()) == ['userid']:
+            continue
+        else:
+            LOG.error('Failed to parse subtask %s',
+                      utils.format_dict(subtask))
+
+
+def tasks_by_package(task_list):
+    result = defaultdict(list)
+    for task in task_list:
+        try:
+            for p in task_packages(task):
+                result[p].append(task)
+        except Exception as ex:
+            LOG.error('Failed to parse task: %s',
+                      utils.format_dict(task), exc_info=True)
+    return dict((p, sorted(l)) for p, l in result.iteritems())

port_stats/utils.py

@@ -0,0 +1,35 @@
+
+import collections
+import datetime
+import json
+import os
+
+
+def file_time_str(path):
+    return datetime.datetime.fromtimestamp(
+        os.path.getmtime(path)).strftime('%Y-%m-%d %H:%M:%S')
+
+
+def maybe_int(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+def _transform_to_format(value):
+    if isinstance(value, dict):
+        return dict((maybe_int(k), _transform_to_format(v))
+                    for k, v in value.iteritems())
+    if isinstance(value, (list, tuple)):
+        return [_transform_to_format(v) for v in value]
+    return value
+
+
+def format_dict(data, indent=False):
+    out = _transform_to_format(data)
+    if indent:
+        return json.dumps(out, sort_keys=True,
+                          indent=2, separators=(',', ': '))
+    else:
+        return json.dumps(out, sort_keys=True)