2018-04-04 16:35:41 +03:00
# This work is licensed under the GNU GPLv2 or later.
2018-03-20 22:00:02 +03:00
# See the COPYING file in the top-level directory.
2015-09-19 03:28:55 +03:00
# This file is part of urlgrabber, a high-level cross-protocol url-grabber
# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
# This code is all straight from python-urlgrabber, which we historically
# used the system installed version of. But since the project is in
# maintenance mode upstream, and eventually we want to switch to python3,
# we are just copying this for now.
import sys
import time
import math
import fcntl
import struct
import termios
2018-10-25 15:37:44 +03:00
2018-04-30 15:56:53 +03:00
# Code from https://mail.python.org/pipermail/python-list/2000-May/033365.html
2015-09-19 03:28:55 +03:00
def terminal_width ( fd = 1 ) :
""" Get the real terminal width """
try :
buf = ' abcdefgh '
buf = fcntl . ioctl ( fd , termios . TIOCGWINSZ , buf )
ret = struct . unpack ( ' hhhh ' , buf ) [ 1 ]
if ret == 0 :
return 80
# Add minimum too?
return ret
2018-10-25 15:37:48 +03:00
except IOError :
2015-09-19 03:28:55 +03:00
return 80
2018-10-25 15:37:44 +03:00
2018-10-25 15:37:45 +03:00
_term_width_val = None
2015-09-19 03:28:55 +03:00
_term_width_last = None
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
def terminal_width_cached ( fd = 1 , cache_timeout = 1.000 ) :
""" Get the real terminal width, but cache it for a bit. """
global _term_width_val
global _term_width_last
now = time . time ( )
if _term_width_val is None or ( now - _term_width_last ) > cache_timeout :
2018-10-25 15:37:45 +03:00
_term_width_val = terminal_width ( fd )
2015-09-19 03:28:55 +03:00
_term_width_last = now
return _term_width_val
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
class TerminalLine :
""" Help create dynamic progress bars, uses terminal_width_cached(). """
def __init__ ( self , min_rest = 0 , beg_len = None , fd = 1 , cache_timeout = 1.000 ) :
if beg_len is None :
beg_len = min_rest
self . _min_len = min_rest
2018-10-25 15:37:57 +03:00
self . llen = terminal_width_cached ( fd , cache_timeout )
if self . llen < beg_len :
self . llen = beg_len
2015-09-19 03:28:55 +03:00
self . _fin = False
def __len__ ( self ) :
""" Usable length for elements. """
2018-10-25 15:37:57 +03:00
return self . llen - self . _min_len
2015-09-19 03:28:55 +03:00
def rest_split ( self , fixed , elements = 2 ) :
""" After a fixed length, split the rest of the line length among
a number of different elements ( default = 2 ) . """
2018-10-25 15:37:57 +03:00
if self . llen < fixed :
2015-09-19 03:28:55 +03:00
return 0
2018-10-25 15:37:57 +03:00
return ( self . llen - fixed ) / / elements
2015-09-19 03:28:55 +03:00
def add ( self , element , full_len = None ) :
""" If there is room left in the line, above min_len, add element.
Note that as soon as one add fails all the rest will fail too . """
if full_len is None :
full_len = len ( element )
if len ( self ) < full_len :
self . _fin = True
if self . _fin :
return ' '
2018-10-25 15:37:57 +03:00
self . llen - = len ( element )
2015-09-19 03:28:55 +03:00
return element
def rest ( self ) :
""" Current rest of line, same as .rest_split(fixed=0, elements=1). """
2018-10-25 15:37:57 +03:00
return self . llen
2015-09-19 03:28:55 +03:00
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
class BaseMeter :
def __init__ ( self ) :
2018-10-25 15:37:47 +03:00
self . update_period = 0.3 # seconds
2015-09-19 03:28:55 +03:00
2018-10-25 15:37:45 +03:00
self . filename = None
self . url = None
self . basename = None
self . text = None
self . size = None
2015-09-19 03:28:55 +03:00
self . start_time = None
2018-10-25 15:37:45 +03:00
self . fsize = None
2015-09-19 03:28:55 +03:00
self . last_amount_read = 0
self . last_update_time = None
self . re = RateEstimator ( )
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
def start ( self , filename = None , url = None , basename = None ,
size = None , now = None , text = None ) :
self . filename = filename
2018-10-25 15:37:45 +03:00
self . url = url
2015-09-19 03:28:55 +03:00
self . basename = basename
2018-10-25 15:37:45 +03:00
self . text = text
2015-09-19 03:28:55 +03:00
self . size = size
2018-10-25 15:37:50 +03:00
if size is not None :
2018-10-25 15:37:46 +03:00
self . fsize = format_number ( size ) + ' B '
2015-09-19 03:28:55 +03:00
2018-10-25 15:37:46 +03:00
if now is None :
now = time . time ( )
2015-09-19 03:28:55 +03:00
self . start_time = now
self . re . start ( size , now )
self . last_amount_read = 0
self . last_update_time = now
self . _do_start ( now )
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
def _do_start ( self , now = None ) :
pass
def update ( self , amount_read , now = None ) :
# for a real gui, you probably want to override and put a call
# to your mainloop iteration function here
2018-10-25 15:37:46 +03:00
if now is None :
now = time . time ( )
2015-09-19 03:28:55 +03:00
if ( not self . last_update_time or
2018-10-25 15:37:51 +03:00
( now > = self . last_update_time + self . update_period ) ) :
2015-09-19 03:28:55 +03:00
self . re . update ( amount_read , now )
self . last_amount_read = amount_read
self . last_update_time = now
self . _do_update ( amount_read , now )
def _do_update ( self , amount_read , now = None ) :
pass
def end ( self , amount_read , now = None ) :
2018-10-25 15:37:46 +03:00
if now is None :
now = time . time ( )
2015-09-19 03:28:55 +03:00
self . re . update ( amount_read , now )
self . last_amount_read = amount_read
self . last_update_time = now
self . _do_end ( amount_read , now )
def _do_end ( self , amount_read , now = None ) :
pass
2017-10-11 14:35:40 +03:00
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
# This is kind of a hack, but progress is gotten from grabber which doesn't
# know about the total size to download. So we do this so we can get the data
# out of band here. This will be "fixed" one way or anther soon.
_text_meter_total_size = 0
_text_meter_sofar_size = 0
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
def text_meter_total_size ( size , downloaded = 0 ) :
global _text_meter_total_size
global _text_meter_sofar_size
_text_meter_total_size = size
_text_meter_sofar_size = downloaded
#
# update: No size (minimal: 17 chars)
# -----------------------------------
2017-10-11 14:35:40 +03:00
# <text> <rate> | <current size> <elapsed time>
2015-09-19 03:28:55 +03:00
# 8-48 1 8 3 6 1 9 5
#
# Order: 1. <text>+<current size> (17)
# 2. +<elapsed time> (10, total: 27)
# 3. + ( 5, total: 32)
# 4. +<rate> ( 9, total: 41)
#
# update: Size, Single file
# -------------------------
# <text> <pc> <bar> <rate> | <current size> <eta time> ETA
# 8-25 1 3-4 1 6-16 1 8 3 6 1 9 1 3 1
#
# Order: 1. <text>+<current size> (17)
# 2. +<eta time> (10, total: 27)
# 3. +ETA ( 5, total: 32)
# 4. +<pc> ( 4, total: 36)
# 5. +<rate> ( 9, total: 45)
# 6. +<bar> ( 7, total: 52)
#
# update: Size, All files
# -----------------------
# <text> <total pc> <pc> <bar> <rate> | <current size> <eta time> ETA
# 8-22 1 5-7 1 3-4 1 6-12 1 8 3 6 1 9 1 3 1
#
# Order: 1. <text>+<current size> (17)
# 2. +<eta time> (10, total: 27)
# 3. +ETA ( 5, total: 32)
# 4. +<total pc> ( 5, total: 37)
# 4. +<pc> ( 4, total: 41)
# 5. +<rate> ( 9, total: 50)
# 6. +<bar> ( 7, total: 57)
#
# end
# ---
2017-10-11 14:35:40 +03:00
# <text> | <current size> <elapsed time>
2015-09-19 03:28:55 +03:00
# 8-56 3 6 1 9 5
#
# Order: 1. <text> ( 8)
# 2. +<current size> ( 9, total: 17)
# 3. +<elapsed time> (10, total: 27)
# 4. + ( 5, total: 32)
#
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
def _term_add_bar ( tl , bar_max_length , pc ) :
2018-10-25 15:37:55 +03:00
bar_len = bar_max_length * pc
ibar_len = int ( bar_len )
progressbar = ' = ' * ibar_len
if ( bar_len - ibar_len ) > = 0.5 :
progressbar + = ' - '
return tl . add ( ' [ %-*.*s ] ' % ( bar_max_length , bar_max_length ,
progressbar ) )
2015-09-19 03:28:55 +03:00
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
def _term_add_end ( tl , osize , size ) :
2018-10-25 15:37:47 +03:00
if osize : # osize should be None or >0, but that's been broken.
if size > osize : # Is ??? better? Really need something to say < vs >.
2015-09-19 03:28:55 +03:00
return tl . add ( ' !!! ' ) , True
elif size != osize :
return tl . add ( ' ... ' ) , True
return tl . add ( ' ' * 5 ) , False
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
class TextMeter ( BaseMeter ) :
def __init__ ( self , fo = sys . stderr ) :
BaseMeter . __init__ ( self )
self . fo = fo
def _do_update ( self , amount_read , now = None ) :
etime = self . re . elapsed_time ( )
fread = format_number ( amount_read )
2018-10-25 15:37:49 +03:00
# self.size = None
2015-09-19 03:28:55 +03:00
if self . text is not None :
text = self . text
else :
text = self . basename
ave_dl = format_number ( self . re . average_rate ( ) )
sofar_size = None
if _text_meter_total_size :
sofar_size = _text_meter_sofar_size + amount_read
2018-10-25 15:37:45 +03:00
sofar_pc = ( sofar_size * 100 ) / / _text_meter_total_size
2015-09-19 03:28:55 +03:00
# Include text + ui_rate in minimal
2018-10-25 15:37:56 +03:00
tl = TerminalLine ( 8 , 8 + 1 + 8 )
2018-03-02 11:12:53 +03:00
# For big screens, make it more readable.
2018-10-25 15:37:57 +03:00
use_hours = bool ( tl . llen > 80 )
2015-09-19 03:28:55 +03:00
ui_size = tl . add ( ' | %5s B ' % fread )
if self . size is None :
ui_time = tl . add ( ' %s ' % format_time ( etime , use_hours ) )
2018-10-25 15:37:45 +03:00
ui_end = tl . add ( ' ' * 5 )
2015-09-19 03:28:55 +03:00
ui_rate = tl . add ( ' %5s B/s ' % ave_dl )
out = ' %-*.*s %s %s %s %s \r ' % ( tl . rest ( ) , tl . rest ( ) , text ,
ui_rate , ui_size , ui_time , ui_end )
else :
rtime = self . re . remaining_time ( )
frtime = format_time ( rtime , use_hours )
frac = self . re . fraction_read ( )
ui_time = tl . add ( ' %s ' % frtime )
2018-10-25 15:37:45 +03:00
ui_end = tl . add ( ' ETA ' )
2015-09-19 03:28:55 +03:00
if sofar_size is None :
ui_sofar_pc = ' '
else :
ui_sofar_pc = tl . add ( ' ( %i %% ) ' % sofar_pc ,
full_len = len ( " (100 % ) " ) )
2018-10-25 15:37:57 +03:00
ui_pc = tl . add ( ' %2i %% ' % ( frac * 100 ) )
2015-09-19 03:28:55 +03:00
ui_rate = tl . add ( ' %5s B/s ' % ave_dl )
# Make text grow a bit before we start growing the bar too
blen = 4 + tl . rest_split ( 8 + 8 + 4 )
ui_bar = _term_add_bar ( tl , blen , frac )
2018-10-25 15:37:53 +03:00
out = ' \r %-*.*s %s %s %s %s %s %s %s \r ' % (
tl . rest ( ) , tl . rest ( ) , text ,
ui_sofar_pc , ui_pc , ui_bar ,
ui_rate , ui_size , ui_time , ui_end
)
2015-09-19 03:28:55 +03:00
self . fo . write ( out )
self . fo . flush ( )
def _do_end ( self , amount_read , now = None ) :
global _text_meter_total_size
global _text_meter_sofar_size
total_size = format_number ( amount_read )
if self . text is not None :
text = self . text
else :
text = self . basename
tl = TerminalLine ( 8 )
2018-03-02 11:12:53 +03:00
# For big screens, make it more readable.
2018-10-25 15:37:57 +03:00
use_hours = bool ( tl . llen > 80 )
2015-09-19 03:28:55 +03:00
ui_size = tl . add ( ' | %5s B ' % total_size )
2018-10-25 15:37:53 +03:00
ui_time = tl . add ( ' %s ' % format_time ( self . re . elapsed_time ( ) ,
use_hours ) )
2015-09-19 03:28:55 +03:00
ui_end , not_done = _term_add_end ( tl , self . size , amount_read )
out = ' \r %-*.*s %s %s %s \n ' % ( tl . rest ( ) , tl . rest ( ) , text ,
ui_size , ui_time , ui_end )
self . fo . write ( out )
self . fo . flush ( )
# Don't add size to the sofar size until we have all of it.
# If we don't have a size, then just pretend/hope we got all of it.
if not_done :
return
if _text_meter_total_size :
_text_meter_sofar_size + = amount_read
if _text_meter_total_size < = _text_meter_sofar_size :
_text_meter_total_size = 0
_text_meter_sofar_size = 0
2018-10-25 15:37:53 +03:00
2015-09-19 03:28:55 +03:00
text_progress_meter = TextMeter
######################################################################
# support classes and functions
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
class RateEstimator :
def __init__ ( self , timescale = 5.0 ) :
self . timescale = timescale
2018-10-25 15:37:43 +03:00
self . total = None
self . start_time = None
self . last_update_time = None
self . last_amount_read = 0
self . ave_rate = None
2015-09-19 03:28:55 +03:00
def start ( self , total = None , now = None ) :
2018-10-25 15:37:46 +03:00
if now is None :
now = time . time ( )
2015-09-19 03:28:55 +03:00
self . total = total
self . start_time = now
self . last_update_time = now
self . last_amount_read = 0
self . ave_rate = None
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
def update ( self , amount_read , now = None ) :
2018-10-25 15:37:46 +03:00
if now is None :
now = time . time ( )
2015-09-19 03:28:55 +03:00
# libcurl calls the progress callback when fetching headers
# too, thus amount_read = 0 .. hdr_size .. 0 .. content_size.
2018-05-21 22:42:50 +03:00
# Occasionally we miss the 2nd zero and report avg speed < 0.
2015-09-19 03:28:55 +03:00
# Handle read_diff < 0 here. BZ 1001767.
if amount_read == 0 or amount_read < self . last_amount_read :
# if we just started this file, all bets are off
self . last_update_time = now
self . last_amount_read = amount_read
self . ave_rate = None
return
2018-10-25 15:37:45 +03:00
time_diff = now - self . last_update_time
2015-09-19 03:28:55 +03:00
read_diff = amount_read - self . last_amount_read
# First update, on reget is the file size
if self . last_amount_read :
self . last_update_time = now
2018-10-25 15:37:54 +03:00
self . ave_rate = self . _temporal_rolling_ave (
2015-09-19 03:28:55 +03:00
time_diff , read_diff , self . ave_rate , self . timescale )
self . last_amount_read = amount_read
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
#####################################################################
# result methods
def average_rate ( self ) :
" get the average transfer rate (in bytes/second) "
return self . ave_rate
def elapsed_time ( self ) :
" the time between the start of the transfer and the most recent update "
return self . last_update_time - self . start_time
def remaining_time ( self ) :
" estimated time remaining "
2018-10-25 15:37:46 +03:00
if not self . ave_rate or not self . total :
return None
2015-09-19 03:28:55 +03:00
return ( self . total - self . last_amount_read ) / self . ave_rate
def fraction_read ( self ) :
""" the fraction of the data that has been read
( can be None for unknown transfer size ) """
2018-10-25 15:37:46 +03:00
if self . total is None :
return None
elif self . total == 0 :
return 1.0
else :
return float ( self . last_amount_read ) / self . total
2015-09-19 03:28:55 +03:00
#########################################################################
# support methods
def _temporal_rolling_ave ( self , time_diff , read_diff , last_ave , timescale ) :
""" a temporal rolling average performs smooth averaging even when
updates come at irregular intervals . This is performed by scaling
the " epsilon " according to the time since the last update .
Specifically , epsilon = time_diff / timescale
As a general rule , the average will take on a completely new value
after ' timescale ' seconds . """
epsilon = time_diff / timescale
2018-10-25 15:37:46 +03:00
if epsilon > 1 :
epsilon = 1.0
2015-09-19 03:28:55 +03:00
return self . _rolling_ave ( time_diff , read_diff , last_ave , epsilon )
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
def _rolling_ave ( self , time_diff , read_diff , last_ave , epsilon ) :
""" perform a " rolling average " iteration
a rolling average " folds " new data into an existing average with
some weight , epsilon . epsilon must be between 0.0 and 1.0 ( inclusive )
a value of 0.0 means only the old value ( initial value ) counts ,
and a value of 1.0 means only the newest value is considered . """
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
try :
recent_rate = read_diff / time_diff
except ZeroDivisionError :
recent_rate = None
2018-10-25 15:37:46 +03:00
if last_ave is None :
return recent_rate
elif recent_rate is None :
return last_ave
2015-09-19 03:28:55 +03:00
# at this point, both last_ave and recent_rate are numbers
2018-10-25 15:37:45 +03:00
return epsilon * recent_rate + ( 1 - epsilon ) * last_ave
2015-09-19 03:28:55 +03:00
def _round_remaining_time ( self , rt , start_time = 15.0 ) :
""" round the remaining time, depending on its size
If rt is between n * start_time and ( n + 1 ) * start_time round downward
to the nearest multiple of n ( for any counting number n ) .
If rt < start_time , round down to the nearest 1.
For example ( for start_time = 15.0 ) :
2.7 - > 2.0
25.2 - > 25.0
26.4 - > 26.0
35.3 - > 34.0
63.6 - > 60.0
"""
2018-10-25 15:37:46 +03:00
if rt < 0 :
return 0.0
2017-10-11 14:35:55 +03:00
shift = int ( math . log ( rt / start_time ) / math . log ( 2 ) )
2015-09-19 03:28:55 +03:00
rt = int ( rt )
2018-10-25 15:37:46 +03:00
if shift < = 0 :
return rt
2015-09-19 03:28:55 +03:00
return float ( int ( rt ) >> shift << shift )
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
def format_time ( seconds , use_hours = 0 ) :
if seconds is None or seconds < 0 :
2018-10-25 15:37:46 +03:00
if use_hours :
return ' --:--:-- '
else :
return ' --:-- '
2015-09-19 03:28:55 +03:00
elif seconds == float ( ' inf ' ) :
return ' Infinite '
else :
seconds = int ( seconds )
2017-10-11 14:35:55 +03:00
minutes = seconds / / 60
2015-09-19 03:28:55 +03:00
seconds = seconds % 60
if use_hours :
2017-10-11 14:35:55 +03:00
hours = minutes / / 60
2015-09-19 03:28:55 +03:00
minutes = minutes % 60
return ' %02i : %02i : %02i ' % ( hours , minutes , seconds )
else :
return ' %02i : %02i ' % ( minutes , seconds )
2017-10-11 14:35:40 +03:00
2018-10-25 15:37:44 +03:00
2015-09-19 03:28:55 +03:00
def format_number ( number , SI = 0 , space = ' ' ) :
""" Turn numbers into human-readable metric-like numbers """
symbols = [ ' ' , # (none)
2018-10-25 15:37:47 +03:00
' k ' , # kilo
' M ' , # mega
' G ' , # giga
' T ' , # tera
' P ' , # peta
' E ' , # exa
' Z ' , # zetta
' Y ' ] # yotta
2017-10-11 14:35:40 +03:00
2018-10-25 15:37:46 +03:00
if SI :
step = 1000.0
else :
step = 1024.0
2015-09-19 03:28:55 +03:00
thresh = 999
depth = 0
max_depth = len ( symbols ) - 1
2018-01-27 23:23:22 +03:00
number = number or 0
2017-10-11 14:35:40 +03:00
2015-09-19 03:28:55 +03:00
# we want numbers between 0 and thresh, but don't exceed the length
# of our list. In that event, the formatting will be screwed up,
# but it'll still show the right number.
while number > thresh and depth < max_depth :
2018-10-25 15:37:45 +03:00
depth = depth + 1
2015-09-19 03:28:55 +03:00
number = number / step
2017-10-11 14:35:51 +03:00
if isinstance ( number , int ) :
2015-09-19 03:28:55 +03:00
# it's an int or a long, which means it didn't get divided,
# which means it's already short enough
2017-10-11 14:35:44 +03:00
fmt = ' %i %s %s '
2015-09-19 03:28:55 +03:00
elif number < 9.95 :
# must use 9.95 for proper sizing. For example, 9.99 will be
# rounded to 10.0 with the .1f format string (which is too long)
2017-10-11 14:35:44 +03:00
fmt = ' %.1f %s %s '
2015-09-19 03:28:55 +03:00
else :
2017-10-11 14:35:44 +03:00
fmt = ' %.0f %s %s '
2017-10-11 14:35:40 +03:00
2017-10-11 14:35:44 +03:00
return ( fmt % ( float ( number or 0 ) , space , symbols [ depth ] ) )
2019-06-08 00:32:51 +03:00
# virtinst additions
def make_meter ( quiet ) :
if quiet :
return BaseMeter ( )
return TextMeter ( fo = sys . stdout )
def ensure_meter ( meter ) :
if meter :
return meter
return make_meter ( quiet = True )