progress: Drop tqdm and revert back to local urlgrabber copy

I thought tqdm was available everywhere, but it does not seem like
it will be in RHEL9. Revert back to the old urlgrabber copy, now
stored in virtinst/_progresspriv.py. If we ever want to try tqdm
again, we can just revert this commit

Signed-off-by: Cole Robinson <crobinso@redhat.com>

.coveragerc

@@ -3,6 +3,7 @@ source=virtinst/
 
 [report]
 skip_covered = yes
+omit=virtinst/_progresspriv.py
 
 exclude_lines =
     # Have to re-enable the standard pragma

virt-manager.spec

@@ -70,7 +70,6 @@ Requires: python3-argcomplete
 Requires: python3-libvirt
 Requires: python3-libxml2
 Requires: python3-requests
-Requires: python3-tqdm
 Requires: libosinfo >= 0.2.10
 # Required for gobject-introspection infrastructure
 Requires: python3-gobject-base

virtManager/asyncjob.py

@@ -5,7 +5,6 @@
 # See the COPYING file in the top-level directory.
 
 import threading
-import time
 import traceback
 
 from gi.repository import Gdk
@@ -20,16 +19,11 @@ from .baseclass import vmmGObjectUI
 
 class _vmmMeter(virtinst.progress.Meter):
     def __init__(self, pbar_pulse, pbar_fraction, pbar_done):
-        bar_format = ("{percentage:3.0f}% "
+        virtinst.progress.Meter.__init__(self, quiet=True)
-                "{n_fmt}/{total_fmt} "
-                "[{elapsed}<{remaining} {rate_fmt}{postfix}]")
-        virtinst.progress.Meter.__init__(self,
-                quiet=True, bar_format=bar_format)
 
         self._pbar_pulse = pbar_pulse
         self._pbar_fraction = pbar_fraction
         self._pbar_done = pbar_done
-        self._last_print_t = 0
 
 
     #################
@@ -37,19 +31,15 @@ class _vmmMeter(virtinst.progress.Meter):
     #################
 
     def _write(self):
-        # Update the gtk progressbar in the async window with the
-        # content generated by tqdm.
-        cur_t = time.time()
-        if (cur_t - self._last_print_t) < self._tqdm.mininterval:
-            return
-
-        msg = str(self._tqdm)
         if self._size is None:
-            self._pbar_pulse(msg, self._text)
+            self._pbar_pulse("", self._text)
         else:
-            frac = float(self._total_read) / self._size
+            fread = virtinst.progress.Meter.format_number(self._total_read)
-            self._pbar_fraction(frac, msg, self._text)
+            rtime = virtinst.progress.Meter.format_time(
-        self._last_print_t = time.time()
+                    self._meter.re.remaining_time(), True)
+            frac = self._meter.re.fraction_read()
+            out = "%3i%% %5sB %s ETA" % (frac * 100, fread, rtime)
+            self._pbar_fraction(frac, out, self._text)
 
 
     #############################################
@@ -57,7 +47,7 @@ class _vmmMeter(virtinst.progress.Meter):
     #############################################
 
     def is_started(self):
-        return bool(self._tqdm)
+        return bool(self._meter.start_time)
 
 
     ###################

virtinst/_progresspriv.py

@@ -0,0 +1,508 @@
+# This work is licensed under the GNU GPLv2 or later.
+# See the COPYING file in the top-level directory.
+
+# This file is part of urlgrabber, a high-level cross-protocol url-grabber
+# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
+
+# This code is all straight from python-urlgrabber, which we historically
+# used the system installed version of. But since the project is in
+# maintenance mode upstream, and eventually we want to switch to python3,
+# we are just copying this for now.
+
+
+import sys
+import time
+import math
+import fcntl
+import struct
+import termios
+
+
+# Code from https://mail.python.org/pipermail/python-list/2000-May/033365.html
+def terminal_width(fd=1):
+    """ Get the real terminal width """
+    try:
+        buf = 'abcdefgh'
+        buf = fcntl.ioctl(fd, termios.TIOCGWINSZ, buf)
+        ret = struct.unpack('hhhh', buf)[1]
+        if ret == 0:
+            return 80
+        # Add minimum too?
+        return ret
+    except IOError:
+        return 80
+
+
+_term_width_val = None
+_term_width_last = None
+
+
+def terminal_width_cached(fd=1, cache_timeout=1.000):
+    """ Get the real terminal width, but cache it for a bit. """
+    global _term_width_val
+    global _term_width_last
+
+    now = time.time()
+    if _term_width_val is None or (now - _term_width_last) > cache_timeout:
+        _term_width_val = terminal_width(fd)
+        _term_width_last = now
+    return _term_width_val
+
+
+class TerminalLine:
+    """ Help create dynamic progress bars, uses terminal_width_cached(). """
+
+    def __init__(self, min_rest=0, beg_len=None, fd=1, cache_timeout=1.000):
+        if beg_len is None:
+            beg_len = min_rest
+        self._min_len = min_rest
+        self.llen = max(terminal_width_cached(fd, cache_timeout), beg_len)
+        self._fin = False
+
+    def __len__(self):
+        """ Usable length for elements. """
+        return self.llen - self._min_len
+
+    def rest_split(self, fixed, elements=2):
+        """ After a fixed length, split the rest of the line length among
+            a number of different elements (default=2). """
+        if self.llen < fixed:
+            return 0
+        return (self.llen - fixed) // elements
+
+    def add(self, element, full_len=None):
+        """ If there is room left in the line, above min_len, add element.
+            Note that as soon as one add fails all the rest will fail too. """
+
+        if full_len is None:
+            full_len = len(element)
+        if len(self) < full_len:
+            self._fin = True
+        if self._fin:
+            return ''
+
+        self.llen -= len(element)
+        return element
+
+    def rest(self):
+        """ Current rest of line, same as .rest_split(fixed=0, elements=1). """
+        return self.llen
+
+
+class BaseMeter:
+    def __init__(self):
+        self.update_period = 0.3  # seconds
+
+        self.url = None
+        self.basename = None
+        self.text = None
+        self.size = None
+        self.start_time = None
+        self.fsize = None
+        self.last_amount_read = 0
+        self.last_update_time = None
+        self.re = RateEstimator()
+
+    def set_text(self, text):
+        self.text = text
+
+    def start(self, text, size):
+        self.text = text
+
+        self.size = size
+        if size is not None:
+            self.fsize = format_number(size) + 'B'
+
+        now = time.time()
+        self.start_time = now
+        self.re.start(size, now)
+        self.last_amount_read = 0
+        self.last_update_time = now
+        self._do_start(now)
+
+    def _do_start(self, now=None):
+        pass
+
+    def update(self, amount_read, now=None):
+        # for a real gui, you probably want to override and put a call
+        # to your mainloop iteration function here
+        if now is None:
+            now = time.time()
+        if (not self.last_update_time or
+                (now >= self.last_update_time + self.update_period)):
+            self.re.update(amount_read, now)
+            self.last_amount_read = amount_read
+            self.last_update_time = now
+            self._do_update(amount_read, now)
+
+    def _do_update(self, amount_read, now=None):
+        pass
+
+    def end(self):
+        self._do_end(self.last_amount_read, self.last_update_time)
+
+    def _do_end(self, amount_read, now=None):
+        pass
+
+
+#  This is kind of a hack, but progress is gotten from grabber which doesn't
+# know about the total size to download. So we do this so we can get the data
+# out of band here. This will be "fixed" one way or anther soon.
+_text_meter_total_size = 0
+_text_meter_sofar_size = 0
+
+
+def text_meter_total_size(size, downloaded=0):
+    global _text_meter_total_size
+    global _text_meter_sofar_size
+    _text_meter_total_size = size
+    _text_meter_sofar_size = downloaded
+
+#
+#       update: No size (minimal: 17 chars)
+#       -----------------------------------
+# <text>                          <rate> | <current size> <elapsed time>
+#  8-48                          1    8  3             6 1            9 5
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<elapsed time>       (10, total: 27)
+#        3. +                     ( 5, total: 32)
+#        4. +<rate>               ( 9, total: 41)
+#
+#       update: Size, Single file
+#       -------------------------
+# <text>            <pc>  <bar> <rate> | <current size> <eta time> ETA
+#  8-25            1 3-4 1 6-16 1   8  3             6 1        9 1  3 1
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<eta time>           (10, total: 27)
+#        3. +ETA                  ( 5, total: 32)
+#        4. +<pc>                 ( 4, total: 36)
+#        5. +<rate>               ( 9, total: 45)
+#        6. +<bar>                ( 7, total: 52)
+#
+#       update: Size, All files
+#       -----------------------
+# <text> <total pc> <pc>  <bar> <rate> | <current size> <eta time> ETA
+#  8-22 1      5-7 1 3-4 1 6-12 1   8  3             6 1        9 1  3 1
+#
+# Order: 1. <text>+<current size> (17)
+#        2. +<eta time>           (10, total: 27)
+#        3. +ETA                  ( 5, total: 32)
+#        4. +<total pc>           ( 5, total: 37)
+#        4. +<pc>                 ( 4, total: 41)
+#        5. +<rate>               ( 9, total: 50)
+#        6. +<bar>                ( 7, total: 57)
+#
+#       end
+#       ---
+# <text>                                 | <current size> <elapsed time>
+#  8-56                                  3             6 1            9 5
+#
+# Order: 1. <text>                ( 8)
+#        2. +<current size>       ( 9, total: 17)
+#        3. +<elapsed time>       (10, total: 27)
+#        4. +                     ( 5, total: 32)
+#
+
+
+def _term_add_bar(tl, bar_max_length, pc):
+    bar_len = bar_max_length * pc
+    ibar_len = int(bar_len)
+    progressbar = '=' * ibar_len
+    if (bar_len - ibar_len) >= 0.5:
+        progressbar += '-'
+    return tl.add(' [%-*.*s]' % (bar_max_length, bar_max_length,
+                                 progressbar))
+
+
+def _term_add_end(tl, osize, size):
+    if osize:  # osize should be None or >0, but that's been broken.
+        if size > osize:  # Is ??? better? Really need something to say < vs >.
+            return tl.add(' !!! '), True
+        elif size != osize:
+            return tl.add(' ... '), True
+    return tl.add(' ' * 5), False
+
+
+class TextMeter(BaseMeter):
+    def __init__(self, output=sys.stderr):
+        BaseMeter.__init__(self)
+        self.output = output
+
+    def _do_update(self, amount_read, now=None):
+        etime = self.re.elapsed_time()
+        fread = format_number(amount_read)
+        # self.size = None
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+
+        ave_dl = format_number(self.re.average_rate())
+        sofar_size = None
+        if _text_meter_total_size:
+            sofar_size = _text_meter_sofar_size + amount_read
+            sofar_pc = (sofar_size * 100) // _text_meter_total_size
+
+        # Include text + ui_rate in minimal
+        tl = TerminalLine(8, 8 + 1 + 8)
+        # For big screens, make it more readable.
+        use_hours = bool(tl.llen > 80)
+        ui_size = tl.add(' | %5sB' % fread)
+        if self.size is None:
+            ui_time = tl.add('  %s' % format_time(etime, use_hours))
+            ui_end = tl.add(' ' * 5)
+            ui_rate = tl.add(' %5sB/s' % ave_dl)
+            out = '%-*.*s%s%s%s%s\r' % (tl.rest(), tl.rest(), text,
+                                        ui_rate, ui_size, ui_time, ui_end)
+        else:
+            rtime = self.re.remaining_time()
+            frtime = format_time(rtime, use_hours)
+            frac = self.re.fraction_read()
+
+            ui_time = tl.add('  %s' % frtime)
+            ui_end = tl.add(' ETA ')
+
+            if sofar_size is None:
+                ui_sofar_pc = ''
+            else:
+                ui_sofar_pc = tl.add(' (%i%%)' % sofar_pc,
+                                     full_len=len(" (100%)"))
+
+            ui_pc = tl.add(' %2i%%' % (frac * 100))
+            ui_rate = tl.add(' %5sB/s' % ave_dl)
+            # Make text grow a bit before we start growing the bar too
+            blen = 4 + tl.rest_split(8 + 8 + 4)
+            ui_bar = _term_add_bar(tl, blen, frac)
+            out = '\r%-*.*s%s%s%s%s%s%s%s\r' % (
+                tl.rest(), tl.rest(), text,
+                ui_sofar_pc, ui_pc, ui_bar,
+                ui_rate, ui_size, ui_time, ui_end
+            )
+
+        self.output.write(out)
+        self.output.flush()
+
+    def _do_end(self, amount_read, now=None):
+        global _text_meter_total_size
+        global _text_meter_sofar_size
+
+        total_size = format_number(amount_read)
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+
+        tl = TerminalLine(8)
+        # For big screens, make it more readable.
+        use_hours = bool(tl.llen > 80)
+        ui_size = tl.add(' | %5sB' % total_size)
+        ui_time = tl.add('  %s' % format_time(self.re.elapsed_time(),
+                                              use_hours))
+        ui_end, not_done = _term_add_end(tl, self.size, amount_read)
+        out = '\r%-*.*s%s%s%s\n' % (tl.rest(), tl.rest(), text,
+                                    ui_size, ui_time, ui_end)
+        self.output.write(out)
+        self.output.flush()
+
+        # Don't add size to the sofar size until we have all of it.
+        # If we don't have a size, then just pretend/hope we got all of it.
+        if not_done:
+            return
+
+        if _text_meter_total_size:
+            _text_meter_sofar_size += amount_read
+        if _text_meter_total_size <= _text_meter_sofar_size:
+            _text_meter_total_size = 0
+            _text_meter_sofar_size = 0
+
+
+text_progress_meter = TextMeter
+
+######################################################################
+# support classes and functions
+
+
+class RateEstimator:
+    def __init__(self, timescale=5.0):
+        self.timescale = timescale
+        self.total = None
+        self.start_time = None
+        self.last_update_time = None
+        self.last_amount_read = 0
+        self.ave_rate = None
+
+    def start(self, total=None, now=None):
+        if now is None:
+            now = time.time()
+        self.total = total
+        self.start_time = now
+        self.last_update_time = now
+        self.last_amount_read = 0
+        self.ave_rate = None
+
+    def update(self, amount_read, now=None):
+        if now is None:
+            now = time.time()
+        # libcurl calls the progress callback when fetching headers
+        # too, thus amount_read = 0 .. hdr_size .. 0 .. content_size.
+        # Occasionally we miss the 2nd zero and report avg speed < 0.
+        # Handle read_diff < 0 here. BZ 1001767.
+        if amount_read == 0 or amount_read < self.last_amount_read:
+            # if we just started this file, all bets are off
+            self.last_update_time = now
+            self.last_amount_read = amount_read
+            self.ave_rate = None
+            return
+
+        time_diff = now - self.last_update_time
+        read_diff = amount_read - self.last_amount_read
+        # First update, on reget is the file size
+        if self.last_amount_read:
+            self.last_update_time = now
+            self.ave_rate = self._temporal_rolling_ave(
+                time_diff, read_diff, self.ave_rate, self.timescale)
+        self.last_amount_read = amount_read
+
+    #####################################################################
+    # result methods
+    def average_rate(self):
+        "get the average transfer rate (in bytes/second)"
+        return self.ave_rate
+
+    def elapsed_time(self):
+        "the time between the start of the transfer and the most recent update"
+        return self.last_update_time - self.start_time
+
+    def remaining_time(self):
+        "estimated time remaining"
+        if not self.ave_rate or not self.total:
+            return None
+        return (self.total - self.last_amount_read) / self.ave_rate
+
+    def fraction_read(self):
+        """the fraction of the data that has been read
+        (can be None for unknown transfer size)"""
+        if self.total is None:
+            return None
+        elif self.total == 0:
+            return 1.0
+        else:
+            return float(self.last_amount_read) / self.total
+
+    #########################################################################
+    # support methods
+    def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
+        """a temporal rolling average performs smooth averaging even when
+        updates come at irregular intervals.  This is performed by scaling
+        the "epsilon" according to the time since the last update.
+        Specifically, epsilon = time_diff / timescale
+
+        As a general rule, the average will take on a completely new value
+        after 'timescale' seconds."""
+        epsilon = min(time_diff / timescale, 1.0)
+        return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
+
+    def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
+        """perform a "rolling average" iteration
+        a rolling average "folds" new data into an existing average with
+        some weight, epsilon.  epsilon must be between 0.0 and 1.0 (inclusive)
+        a value of 0.0 means only the old value (initial value) counts,
+        and a value of 1.0 means only the newest value is considered."""
+
+        try:
+            recent_rate = read_diff / time_diff
+        except ZeroDivisionError:
+            recent_rate = None
+        if last_ave is None:
+            return recent_rate
+        elif recent_rate is None:
+            return last_ave
+
+        # at this point, both last_ave and recent_rate are numbers
+        return epsilon * recent_rate + (1 - epsilon) * last_ave
+
+    def _round_remaining_time(self, rt, start_time=15.0):
+        """round the remaining time, depending on its size
+        If rt is between n*start_time and (n+1)*start_time round downward
+        to the nearest multiple of n (for any counting number n).
+        If rt < start_time, round down to the nearest 1.
+        For example (for start_time = 15.0):
+         2.7  -> 2.0
+         25.2 -> 25.0
+         26.4 -> 26.0
+         35.3 -> 34.0
+         63.6 -> 60.0
+        """
+
+        if rt < 0:
+            return 0.0
+        shift = int(math.log(rt / start_time) / math.log(2))
+        rt = int(rt)
+        if shift <= 0:
+            return rt
+        return float(int(rt) >> shift << shift)
+
+
+def format_time(seconds, use_hours=0):
+    if seconds is None or seconds < 0:
+        if use_hours:
+            return '--:--:--'
+        else:
+            return '--:--'
+    elif seconds == float('inf'):
+        return 'Infinite'
+    else:
+        seconds = int(seconds)
+        minutes = seconds // 60
+        seconds = seconds % 60
+        if use_hours:
+            hours = minutes // 60
+            minutes = minutes % 60
+            return '%02i:%02i:%02i' % (hours, minutes, seconds)
+        else:
+            return '%02i:%02i' % (minutes, seconds)
+
+
+def format_number(number, SI=0, space=' '):
+    """Turn numbers into human-readable metric-like numbers"""
+    symbols = ['',  # (none)
+               'k',  # kilo
+               'M',  # mega
+               'G',  # giga
+               'T',  # tera
+               'P',  # peta
+               'E',  # exa
+               'Z',  # zetta
+               'Y']  # yotta
+
+    if SI:
+        step = 1000.0
+    else:
+        step = 1024.0
+
+    thresh = 999
+    depth = 0
+    max_depth = len(symbols) - 1
+    number = number or 0
+
+    # we want numbers between 0 and thresh, but don't exceed the length
+    # of our list.  In that event, the formatting will be screwed up,
+    # but it'll still show the right number.
+    while number > thresh and depth < max_depth:
+        depth = depth + 1
+        number = number / step
+
+    if isinstance(number, int):
+        # it's an int or a long, which means it didn't get divided,
+        # which means it's already short enough
+        fmt = '%i%s%s'
+    elif number < 9.95:
+        # must use 9.95 for proper sizing.  For example, 9.99 will be
+        # rounded to 10.0 with the .1f format string (which is too long)
+        fmt = '%.1f%s%s'
+    else:
+        fmt = '%.0f%s%s'
+
+    return(fmt % (float(number or 0), space, symbols[depth]))

virtinst/progress.py

@@ -5,59 +5,41 @@
 # See the COPYING file in the top-level directory.
 #
 
-import tqdm
+import sys
 
-
+from . import _progresspriv
-class _fakewriter:
-    def write(self, msg):
-        pass
-    def flush(self):
-        pass
 
 
 class Meter:
     """
-    Compat API that matches what urlgrabber used to provide, so our
+    Meter class that hides the internals of the backend implementation
-    internal callers don't need to adapter
+    from virtinst and friends
     """
-    def __init__(self, quiet=False, bar_format=None):
+    # Used by virt-manager subclass
+    format_number = _progresspriv.format_number
+    format_time = _progresspriv.format_time
+
+    def __init__(self, quiet=False):
         self._text = None
         self._size = None
         self._total_read = 0
-        self._tqdm = None
+        if quiet:
-        self._quiet = quiet
+            self._meter = _progresspriv.BaseMeter()
-        self._bar_format = bar_format
+        else:
+            self._meter = _progresspriv.TextMeter(output=sys.stdout)
 
     def start(self, text, size):
         self._text = text
         self._size = size
         self._total_read = 0
-
+        self._meter.start(text, size)
-        if self._quiet:
-            fileobj = _fakewriter()
-        else:
-            fileobj = None
-
-        self._tqdm = tqdm.tqdm(
-                desc=self._text, total=self._size,
-                unit='B', unit_scale=True, unit_divisor=1024,
-                miniters=0, mininterval=0.25, leave=True, file=fileobj,
-                # Set bar_format to only print the text to start. We
-                # don't want it to print bytes info until first data comes in
-                bar_format="{desc}",
-        )
 
     def update(self, new_total):
-        update_amount = new_total - self._total_read
         self._total_read = new_total
-        # Data came in, use our requested bar_format
+        self._meter.update(new_total)
-        self._tqdm.bar_format = self._bar_format
-        self._tqdm.update(update_amount)
 
     def end(self):
-        if self._tqdm is not None:
+        self._meter.end()
-            self._tqdm.close()
-        self._tqdm = None
 
 
 def make_meter(quiet):