1 files changed, 530 insertions, 0 deletions

diff --git a/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py
new file mode 100644
index 0000000000..02db524e76
--- /dev/null
+++ b/scripts/lib/mic/3rdparty/pykickstart/urlgrabber/progress.py
@@ -0,0 +1,530 @@
+#   This library is free software; you can redistribute it and/or
+#   modify it under the terms of the GNU Lesser General Public
+#   License as published by the Free Software Foundation; either
+#   version 2.1 of the License, or (at your option) any later version.
+#
+#   This library is distributed in the hope that it will be useful,
+#   but WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#   Lesser General Public License for more details.
+#
+#   You should have received a copy of the GNU Lesser General Public
+#   License along with this library; if not, write to the 
+#      Free Software Foundation, Inc., 
+#      59 Temple Place, Suite 330, 
+#      Boston, MA  02111-1307  USA
+
+# This file is part of urlgrabber, a high-level cross-protocol url-grabber
+# Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
+
+# $Id: progress.py,v 1.7 2005/08/19 21:59:07 mstenner Exp $
+
+import sys
+import time
+import math
+import thread
+    
+class BaseMeter:
+    def __init__(self):
+        self.update_period = 0.3 # seconds
+
+        self.filename   = None
+        self.url        = None
+        self.basename   = None
+        self.text       = None
+        self.size       = None
+        self.start_time = None
+        self.last_amount_read = 0
+        self.last_update_time = None
+        self.re = RateEstimator()
+        
+    def start(self, filename=None, url=None, basename=None,
+              size=None, now=None, text=None):
+        self.filename = filename
+        self.url      = url
+        self.basename = basename
+        self.text     = text
+
+        #size = None #########  TESTING
+        self.size = size
+        if not size is None: self.fsize = format_number(size) + 'B'
+
+        if now is None: now = time.time()
+        self.start_time = now
+        self.re.start(size, now)
+        self.last_amount_read = 0
+        self.last_update_time = now
+        self._do_start(now)
+        
+    def _do_start(self, now=None):
+        pass
+
+    def update(self, amount_read, now=None):
+        # for a real gui, you probably want to override and put a call
+        # to your mainloop iteration function here
+        if now is None: now = time.time()
+        if (now >= self.last_update_time + self.update_period) or \
+               not self.last_update_time:
+            self.re.update(amount_read, now)
+            self.last_amount_read = amount_read
+            self.last_update_time = now
+            self._do_update(amount_read, now)
+
+    def _do_update(self, amount_read, now=None):
+        pass
+
+    def end(self, amount_read, now=None):
+        if now is None: now = time.time()
+        self.re.update(amount_read, now)
+        self.last_amount_read = amount_read
+        self.last_update_time = now
+        self._do_end(amount_read, now)
+
+    def _do_end(self, amount_read, now=None):
+        pass
+        
+class TextMeter(BaseMeter):
+    def __init__(self, fo=sys.stderr):
+        BaseMeter.__init__(self)
+        self.fo = fo
+
+    def _do_update(self, amount_read, now=None):
+        etime = self.re.elapsed_time()
+        fetime = format_time(etime)
+        fread = format_number(amount_read)
+        #self.size = None
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+        if self.size is None:
+            out = '\r%-60.60s    %5sB %s ' % \
+                  (text, fread, fetime)
+        else:
+            rtime = self.re.remaining_time()
+            frtime = format_time(rtime)
+            frac = self.re.fraction_read()
+            bar = '='*int(25 * frac)
+
+            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s ETA ' % \
+                  (text, frac*100, bar, fread, frtime)
+
+        self.fo.write(out)
+        self.fo.flush()
+
+    def _do_end(self, amount_read, now=None):
+        total_time = format_time(self.re.elapsed_time())
+        total_size = format_number(amount_read)
+        if self.text is not None:
+            text = self.text
+        else:
+            text = self.basename
+        if self.size is None:
+            out = '\r%-60.60s    %5sB %s ' % \
+                  (text, total_size, total_time)
+        else:
+            bar = '='*25
+            out = '\r%-25.25s %3i%% |%-25.25s| %5sB %8s     ' % \
+                  (text, 100, bar, total_size, total_time)
+        self.fo.write(out + '\n')
+        self.fo.flush()
+
+text_progress_meter = TextMeter
+
+class MultiFileHelper(BaseMeter):
+    def __init__(self, master):
+        BaseMeter.__init__(self)
+        self.master = master
+
+    def _do_start(self, now):
+        self.master.start_meter(self, now)
+
+    def _do_update(self, amount_read, now):
+        # elapsed time since last update
+        self.master.update_meter(self, now)
+
+    def _do_end(self, amount_read, now):
+        self.ftotal_time = format_time(now - self.start_time)
+        self.ftotal_size = format_number(self.last_amount_read)
+        self.master.end_meter(self, now)
+
+    def failure(self, message, now=None):
+        self.master.failure_meter(self, message, now)
+
+    def message(self, message):
+        self.master.message_meter(self, message)
+
+class MultiFileMeter:
+    helperclass = MultiFileHelper
+    def __init__(self):
+        self.meters = []
+        self.in_progress_meters = []
+        self._lock = thread.allocate_lock()
+        self.update_period = 0.3 # seconds
+        
+        self.numfiles         = None
+        self.finished_files   = 0
+        self.failed_files     = 0
+        self.open_files       = 0
+        self.total_size       = None
+        self.failed_size      = 0
+        self.start_time       = None
+        self.finished_file_size = 0
+        self.last_update_time = None
+        self.re = RateEstimator()
+
+    def start(self, numfiles=None, total_size=None, now=None):
+        if now is None: now = time.time()
+        self.numfiles         = numfiles
+        self.finished_files   = 0
+        self.failed_files     = 0
+        self.open_files       = 0
+        self.total_size       = total_size
+        self.failed_size      = 0
+        self.start_time       = now
+        self.finished_file_size = 0
+        self.last_update_time = now
+        self.re.start(total_size, now)
+        self._do_start(now)
+
+    def _do_start(self, now):
+        pass
+
+    def end(self, now=None):
+        if now is None: now = time.time()
+        self._do_end(now)
+        
+    def _do_end(self, now):
+        pass
+
+    def lock(self): self._lock.acquire()
+    def unlock(self): self._lock.release()
+
+    ###########################################################
+    # child meter creation and destruction
+    def newMeter(self):
+        newmeter = self.helperclass(self)
+        self.meters.append(newmeter)
+        return newmeter
+    
+    def removeMeter(self, meter):
+        self.meters.remove(meter)
+        
+    ###########################################################
+    # child functions - these should only be called by helpers
+    def start_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            if not meter in self.in_progress_meters:
+                self.in_progress_meters.append(meter)
+                self.open_files += 1
+        finally:
+            self._lock.release()
+        self._do_start_meter(meter, now)
+        
+    def _do_start_meter(self, meter, now):
+        pass
+        
+    def update_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        if (now >= self.last_update_time + self.update_period) or \
+               not self.last_update_time:
+            self.re.update(self._amount_read(), now)
+            self.last_update_time = now
+            self._do_update_meter(meter, now)
+
+    def _do_update_meter(self, meter, now):
+        pass
+
+    def end_meter(self, meter, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            try: self.in_progress_meters.remove(meter)
+            except ValueError: pass
+            self.open_files     -= 1
+            self.finished_files += 1
+            self.finished_file_size += meter.last_amount_read
+        finally:
+            self._lock.release()
+        self._do_end_meter(meter, now)
+
+    def _do_end_meter(self, meter, now):
+        pass
+
+    def failure_meter(self, meter, message, now):
+        if not meter in self.meters:
+            raise ValueError('attempt to use orphaned meter')
+        self._lock.acquire()
+        try:
+            try: self.in_progress_meters.remove(meter)
+            except ValueError: pass
+            self.open_files     -= 1
+            self.failed_files   += 1
+            if meter.size and self.failed_size is not None:
+                self.failed_size += meter.size
+            else:
+                self.failed_size = None
+        finally:
+            self._lock.release()
+        self._do_failure_meter(meter, message, now)
+
+    def _do_failure_meter(self, meter, message, now):
+        pass
+
+    def message_meter(self, meter, message):
+        pass
+
+    ########################################################
+    # internal functions
+    def _amount_read(self):
+        tot = self.finished_file_size
+        for m in self.in_progress_meters:
+            tot += m.last_amount_read
+        return tot
+
+
+class TextMultiFileMeter(MultiFileMeter):
+    def __init__(self, fo=sys.stderr):
+        self.fo = fo
+        MultiFileMeter.__init__(self)
+
+    # files: ###/### ###%  data: ######/###### ###%  time: ##:##:##/##:##:##
+    def _do_update_meter(self, meter, now):
+        self._lock.acquire()
+        try:
+            format = "files: %3i/%-3i %3i%%   data: %6.6s/%-6.6s %3i%%   " \
+                     "time: %8.8s/%8.8s"
+            df = self.finished_files
+            tf = self.numfiles or 1
+            pf = 100 * float(df)/tf + 0.49
+            dd = self.re.last_amount_read
+            td = self.total_size
+            pd = 100 * (self.re.fraction_read() or 0) + 0.49
+            dt = self.re.elapsed_time()
+            rt = self.re.remaining_time()
+            if rt is None: tt = None
+            else: tt = dt + rt
+
+            fdd = format_number(dd) + 'B'
+            ftd = format_number(td) + 'B'
+            fdt = format_time(dt, 1)
+            ftt = format_time(tt, 1)
+            
+            out = '%-79.79s' % (format % (df, tf, pf, fdd, ftd, pd, fdt, ftt))
+            self.fo.write('\r' + out)
+            self.fo.flush()
+        finally:
+            self._lock.release()
+
+    def _do_end_meter(self, meter, now):
+        self._lock.acquire()
+        try:
+            format = "%-30.30s %6.6s    %8.8s    %9.9s"
+            fn = meter.basename
+            size = meter.last_amount_read
+            fsize = format_number(size) + 'B'
+            et = meter.re.elapsed_time()
+            fet = format_time(et, 1)
+            frate = format_number(size / et) + 'B/s'
+            
+            out = '%-79.79s' % (format % (fn, fsize, fet, frate))
+            self.fo.write('\r' + out + '\n')
+        finally:
+            self._lock.release()
+        self._do_update_meter(meter, now)
+
+    def _do_failure_meter(self, meter, message, now):
+        self._lock.acquire()
+        try:
+            format = "%-30.30s %6.6s %s"
+            fn = meter.basename
+            if type(message) in (type(''), type(u'')):
+                message = message.splitlines()
+            if not message: message = ['']
+            out = '%-79s' % (format % (fn, 'FAILED', message[0] or ''))
+            self.fo.write('\r' + out + '\n')
+            for m in message[1:]: self.fo.write('  ' + m + '\n')
+            self._lock.release()
+        finally:
+            self._do_update_meter(meter, now)
+
+    def message_meter(self, meter, message):
+        self._lock.acquire()
+        try:
+            pass
+        finally:
+            self._lock.release()
+
+    def _do_end(self, now):
+        self._do_update_meter(None, now)
+        self._lock.acquire()
+        try:
+            self.fo.write('\n')
+            self.fo.flush()
+        finally:
+            self._lock.release()
+        
+######################################################################
+# support classes and functions
+
+class RateEstimator:
+    def __init__(self, timescale=5.0):
+        self.timescale = timescale
+
+    def start(self, total=None, now=None):
+        if now is None: now = time.time()
+        self.total = total
+        self.start_time = now
+        self.last_update_time = now
+        self.last_amount_read = 0
+        self.ave_rate = None
+        
+    def update(self, amount_read, now=None):
+        if now is None: now = time.time()
+        if amount_read == 0:
+            # if we just started this file, all bets are off
+            self.last_update_time = now
+            self.last_amount_read = 0
+            self.ave_rate = None
+            return
+
+        #print 'times', now, self.last_update_time
+        time_diff = now         - self.last_update_time
+        read_diff = amount_read - self.last_amount_read
+        self.last_update_time = now
+        self.last_amount_read = amount_read
+        self.ave_rate = self._temporal_rolling_ave(\
+            time_diff, read_diff, self.ave_rate, self.timescale)
+        #print 'results', time_diff, read_diff, self.ave_rate
+        
+    #####################################################################
+    # result methods
+    def average_rate(self):
+        "get the average transfer rate (in bytes/second)"
+        return self.ave_rate
+
+    def elapsed_time(self):
+        "the time between the start of the transfer and the most recent update"
+        return self.last_update_time - self.start_time
+
+    def remaining_time(self):
+        "estimated time remaining"
+        if not self.ave_rate or not self.total: return None
+        return (self.total - self.last_amount_read) / self.ave_rate
+
+    def fraction_read(self):
+        """the fraction of the data that has been read
+        (can be None for unknown transfer size)"""
+        if self.total is None: return None
+        elif self.total == 0: return 1.0
+        else: return float(self.last_amount_read)/self.total
+
+    #########################################################################
+    # support methods
+    def _temporal_rolling_ave(self, time_diff, read_diff, last_ave, timescale):
+        """a temporal rolling average performs smooth averaging even when
+        updates come at irregular intervals.  This is performed by scaling
+        the "epsilon" according to the time since the last update.
+        Specifically, epsilon = time_diff / timescale
+
+        As a general rule, the average will take on a completely new value
+        after 'timescale' seconds."""
+        epsilon = time_diff / timescale
+        if epsilon > 1: epsilon = 1.0
+        return self._rolling_ave(time_diff, read_diff, last_ave, epsilon)
+    
+    def _rolling_ave(self, time_diff, read_diff, last_ave, epsilon):
+        """perform a "rolling average" iteration
+        a rolling average "folds" new data into an existing average with
+        some weight, epsilon.  epsilon must be between 0.0 and 1.0 (inclusive)
+        a value of 0.0 means only the old value (initial value) counts,
+        and a value of 1.0 means only the newest value is considered."""
+        
+        try:
+            recent_rate = read_diff / time_diff
+        except ZeroDivisionError:
+            recent_rate = None
+        if last_ave is None: return recent_rate
+        elif recent_rate is None: return last_ave
+
+        # at this point, both last_ave and recent_rate are numbers
+        return epsilon * recent_rate  +  (1 - epsilon) * last_ave
+
+    def _round_remaining_time(self, rt, start_time=15.0):
+        """round the remaining time, depending on its size
+        If rt is between n*start_time and (n+1)*start_time round downward
+        to the nearest multiple of n (for any counting number n).
+        If rt < start_time, round down to the nearest 1.
+        For example (for start_time = 15.0):
+         2.7  -> 2.0
+         25.2 -> 25.0
+         26.4 -> 26.0
+         35.3 -> 34.0
+         63.6 -> 60.0
+        """
+
+        if rt < 0: return 0.0
+        shift = int(math.log(rt/start_time)/math.log(2))
+        rt = int(rt)
+        if shift <= 0: return rt
+        return float(int(rt) >> shift << shift)
+        
+
+def format_time(seconds, use_hours=0):
+    if seconds is None or seconds < 0:
+        if use_hours: return '--:--:--'
+        else:         return '--:--'
+    else:
+        seconds = int(seconds)
+        minutes = seconds / 60
+        seconds = seconds % 60
+        if use_hours:
+            hours = minutes / 60
+            minutes = minutes % 60
+            return '%02i:%02i:%02i' % (hours, minutes, seconds)
+        else:
+            return '%02i:%02i' % (minutes, seconds)
+            
+def format_number(number, SI=0, space=' '):
+    """Turn numbers into human-readable metric-like numbers"""
+    symbols = ['',  # (none)
+               'k', # kilo
+               'M', # mega
+               'G', # giga
+               'T', # tera
+               'P', # peta
+               'E', # exa
+               'Z', # zetta
+               'Y'] # yotta
+    
+    if SI: step = 1000.0
+    else: step = 1024.0
+
+    thresh = 999
+    depth = 0
+    max_depth = len(symbols) - 1
+    
+    # we want numbers between 0 and thresh, but don't exceed the length
+    # of our list.  In that event, the formatting will be screwed up,
+    # but it'll still show the right number.
+    while number > thresh and depth < max_depth:
+        depth  = depth + 1
+        number = number / step
+
+    if type(number) == type(1) or type(number) == type(1L):
+        # it's an int or a long, which means it didn't get divided,
+        # which means it's already short enough
+        format = '%i%s%s'
+    elif number < 9.95:
+        # must use 9.95 for proper sizing.  For example, 9.99 will be
+        # rounded to 10.0 with the .1f format string (which is too long)
+        format = '%.1f%s%s'
+    else:
+        format = '%.0f%s%s'
+        
+    return(format % (float(number or 0), space, symbols[depth]))