jpayne@68: """ jpayne@68: Customisable progressbar decorator for iterators. jpayne@68: Includes a default `range` iterator printing to `stderr`. jpayne@68: jpayne@68: Usage: jpayne@68: >>> from tqdm import trange, tqdm jpayne@68: >>> for i in trange(10): jpayne@68: ... ... jpayne@68: """ jpayne@68: import sys jpayne@68: from collections import OrderedDict, defaultdict jpayne@68: from contextlib import contextmanager jpayne@68: from datetime import datetime, timedelta, timezone jpayne@68: from numbers import Number jpayne@68: from time import time jpayne@68: from warnings import warn jpayne@68: from weakref import WeakSet jpayne@68: jpayne@68: from ._monitor import TMonitor jpayne@68: from .utils import ( jpayne@68: CallbackIOWrapper, Comparable, DisableOnWriteError, FormatReplace, SimpleTextIOWrapper, jpayne@68: _is_ascii, _screen_shape_wrapper, _supports_unicode, _term_move_up, disp_len, disp_trim, jpayne@68: envwrap) jpayne@68: jpayne@68: __author__ = "https://github.com/tqdm/tqdm#contributions" jpayne@68: __all__ = ['tqdm', 'trange', jpayne@68: 'TqdmTypeError', 'TqdmKeyError', 'TqdmWarning', jpayne@68: 'TqdmExperimentalWarning', 'TqdmDeprecationWarning', jpayne@68: 'TqdmMonitorWarning'] jpayne@68: jpayne@68: jpayne@68: class TqdmTypeError(TypeError): jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: class TqdmKeyError(KeyError): jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: class TqdmWarning(Warning): jpayne@68: """base class for all tqdm warnings. jpayne@68: jpayne@68: Used for non-external-code-breaking errors, such as garbled printing. jpayne@68: """ jpayne@68: def __init__(self, msg, fp_write=None, *a, **k): jpayne@68: if fp_write is not None: jpayne@68: fp_write("\n" + self.__class__.__name__ + ": " + str(msg).rstrip() + '\n') jpayne@68: else: jpayne@68: super().__init__(msg, *a, **k) jpayne@68: jpayne@68: jpayne@68: class TqdmExperimentalWarning(TqdmWarning, FutureWarning): jpayne@68: """beta feature, unstable API and behaviour""" jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: class TqdmDeprecationWarning(TqdmWarning, DeprecationWarning): jpayne@68: # not suppressed if raised jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: class TqdmMonitorWarning(TqdmWarning, RuntimeWarning): jpayne@68: """tqdm monitor errors which do not affect external functionality""" jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: def TRLock(*args, **kwargs): jpayne@68: """threading RLock""" jpayne@68: try: jpayne@68: from threading import RLock jpayne@68: return RLock(*args, **kwargs) jpayne@68: except (ImportError, OSError): # pragma: no cover jpayne@68: pass jpayne@68: jpayne@68: jpayne@68: class TqdmDefaultWriteLock(object): jpayne@68: """ jpayne@68: Provide a default write lock for thread and multiprocessing safety. jpayne@68: Works only on platforms supporting `fork` (so Windows is excluded). jpayne@68: You must initialise a `tqdm` or `TqdmDefaultWriteLock` instance jpayne@68: before forking in order for the write lock to work. jpayne@68: On Windows, you need to supply the lock from the parent to the children as jpayne@68: an argument to joblib or the parallelism lib you use. jpayne@68: """ jpayne@68: # global thread lock so no setup required for multithreading. jpayne@68: # NB: Do not create multiprocessing lock as it sets the multiprocessing jpayne@68: # context, disallowing `spawn()`/`forkserver()` jpayne@68: th_lock = TRLock() jpayne@68: jpayne@68: def __init__(self): jpayne@68: # Create global parallelism locks to avoid racing issues with parallel jpayne@68: # bars works only if fork available (Linux/MacOSX, but not Windows) jpayne@68: cls = type(self) jpayne@68: root_lock = cls.th_lock jpayne@68: if root_lock is not None: jpayne@68: root_lock.acquire() jpayne@68: cls.create_mp_lock() jpayne@68: self.locks = [lk for lk in [cls.mp_lock, cls.th_lock] if lk is not None] jpayne@68: if root_lock is not None: jpayne@68: root_lock.release() jpayne@68: jpayne@68: def acquire(self, *a, **k): jpayne@68: for lock in self.locks: jpayne@68: lock.acquire(*a, **k) jpayne@68: jpayne@68: def release(self): jpayne@68: for lock in self.locks[::-1]: # Release in inverse order of acquisition jpayne@68: lock.release() jpayne@68: jpayne@68: def __enter__(self): jpayne@68: self.acquire() jpayne@68: jpayne@68: def __exit__(self, *exc): jpayne@68: self.release() jpayne@68: jpayne@68: @classmethod jpayne@68: def create_mp_lock(cls): jpayne@68: if not hasattr(cls, 'mp_lock'): jpayne@68: try: jpayne@68: from multiprocessing import RLock jpayne@68: cls.mp_lock = RLock() jpayne@68: except (ImportError, OSError): # pragma: no cover jpayne@68: cls.mp_lock = None jpayne@68: jpayne@68: @classmethod jpayne@68: def create_th_lock(cls): jpayne@68: assert hasattr(cls, 'th_lock') jpayne@68: warn("create_th_lock not needed anymore", TqdmDeprecationWarning, stacklevel=2) jpayne@68: jpayne@68: jpayne@68: class Bar(object): jpayne@68: """ jpayne@68: `str.format`-able bar with format specifiers: `[width][type]` jpayne@68: jpayne@68: - `width` jpayne@68: + unspecified (default): use `self.default_len` jpayne@68: + `int >= 0`: overrides `self.default_len` jpayne@68: + `int < 0`: subtract from `self.default_len` jpayne@68: - `type` jpayne@68: + `a`: ascii (`charset=self.ASCII` override) jpayne@68: + `u`: unicode (`charset=self.UTF` override) jpayne@68: + `b`: blank (`charset=" "` override) jpayne@68: """ jpayne@68: ASCII = " 123456789#" jpayne@68: UTF = u" " + u''.join(map(chr, range(0x258F, 0x2587, -1))) jpayne@68: BLANK = " " jpayne@68: COLOUR_RESET = '\x1b[0m' jpayne@68: COLOUR_RGB = '\x1b[38;2;%d;%d;%dm' jpayne@68: COLOURS = {'BLACK': '\x1b[30m', 'RED': '\x1b[31m', 'GREEN': '\x1b[32m', jpayne@68: 'YELLOW': '\x1b[33m', 'BLUE': '\x1b[34m', 'MAGENTA': '\x1b[35m', jpayne@68: 'CYAN': '\x1b[36m', 'WHITE': '\x1b[37m'} jpayne@68: jpayne@68: def __init__(self, frac, default_len=10, charset=UTF, colour=None): jpayne@68: if not 0 <= frac <= 1: jpayne@68: warn("clamping frac to range [0, 1]", TqdmWarning, stacklevel=2) jpayne@68: frac = max(0, min(1, frac)) jpayne@68: assert default_len > 0 jpayne@68: self.frac = frac jpayne@68: self.default_len = default_len jpayne@68: self.charset = charset jpayne@68: self.colour = colour jpayne@68: jpayne@68: @property jpayne@68: def colour(self): jpayne@68: return self._colour jpayne@68: jpayne@68: @colour.setter jpayne@68: def colour(self, value): jpayne@68: if not value: jpayne@68: self._colour = None jpayne@68: return jpayne@68: try: jpayne@68: if value.upper() in self.COLOURS: jpayne@68: self._colour = self.COLOURS[value.upper()] jpayne@68: elif value[0] == '#' and len(value) == 7: jpayne@68: self._colour = self.COLOUR_RGB % tuple( jpayne@68: int(i, 16) for i in (value[1:3], value[3:5], value[5:7])) jpayne@68: else: jpayne@68: raise KeyError jpayne@68: except (KeyError, AttributeError): jpayne@68: warn("Unknown colour (%s); valid choices: [hex (#00ff00), %s]" % ( jpayne@68: value, ", ".join(self.COLOURS)), jpayne@68: TqdmWarning, stacklevel=2) jpayne@68: self._colour = None jpayne@68: jpayne@68: def __format__(self, format_spec): jpayne@68: if format_spec: jpayne@68: _type = format_spec[-1].lower() jpayne@68: try: jpayne@68: charset = {'a': self.ASCII, 'u': self.UTF, 'b': self.BLANK}[_type] jpayne@68: except KeyError: jpayne@68: charset = self.charset jpayne@68: else: jpayne@68: format_spec = format_spec[:-1] jpayne@68: if format_spec: jpayne@68: N_BARS = int(format_spec) jpayne@68: if N_BARS < 0: jpayne@68: N_BARS += self.default_len jpayne@68: else: jpayne@68: N_BARS = self.default_len jpayne@68: else: jpayne@68: charset = self.charset jpayne@68: N_BARS = self.default_len jpayne@68: jpayne@68: nsyms = len(charset) - 1 jpayne@68: bar_length, frac_bar_length = divmod(int(self.frac * N_BARS * nsyms), nsyms) jpayne@68: jpayne@68: res = charset[-1] * bar_length jpayne@68: if bar_length < N_BARS: # whitespace padding jpayne@68: res = res + charset[frac_bar_length] + charset[0] * (N_BARS - bar_length - 1) jpayne@68: return self.colour + res + self.COLOUR_RESET if self.colour else res jpayne@68: jpayne@68: jpayne@68: class EMA(object): jpayne@68: """ jpayne@68: Exponential moving average: smoothing to give progressively lower jpayne@68: weights to older values. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: smoothing : float, optional jpayne@68: Smoothing factor in range [0, 1], [default: 0.3]. jpayne@68: Increase to give more weight to recent values. jpayne@68: Ranges from 0 (yields old value) to 1 (yields new value). jpayne@68: """ jpayne@68: def __init__(self, smoothing=0.3): jpayne@68: self.alpha = smoothing jpayne@68: self.last = 0 jpayne@68: self.calls = 0 jpayne@68: jpayne@68: def __call__(self, x=None): jpayne@68: """ jpayne@68: Parameters jpayne@68: ---------- jpayne@68: x : float jpayne@68: New value to include in EMA. jpayne@68: """ jpayne@68: beta = 1 - self.alpha jpayne@68: if x is not None: jpayne@68: self.last = self.alpha * x + beta * self.last jpayne@68: self.calls += 1 jpayne@68: return self.last / (1 - beta ** self.calls) if self.calls else self.last jpayne@68: jpayne@68: jpayne@68: class tqdm(Comparable): jpayne@68: """ jpayne@68: Decorate an iterable object, returning an iterator which acts exactly jpayne@68: like the original iterable, but prints a dynamically updating jpayne@68: progressbar every time a value is requested. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: iterable : iterable, optional jpayne@68: Iterable to decorate with a progressbar. jpayne@68: Leave blank to manually manage the updates. jpayne@68: desc : str, optional jpayne@68: Prefix for the progressbar. jpayne@68: total : int or float, optional jpayne@68: The number of expected iterations. If unspecified, jpayne@68: len(iterable) is used if possible. If float("inf") or as a last jpayne@68: resort, only basic progress statistics are displayed jpayne@68: (no ETA, no progressbar). jpayne@68: If `gui` is True and this parameter needs subsequent updating, jpayne@68: specify an initial arbitrary large positive number, jpayne@68: e.g. 9e9. jpayne@68: leave : bool, optional jpayne@68: If [default: True], keeps all traces of the progressbar jpayne@68: upon termination of iteration. jpayne@68: If `None`, will leave only if `position` is `0`. jpayne@68: file : `io.TextIOWrapper` or `io.StringIO`, optional jpayne@68: Specifies where to output the progress messages jpayne@68: (default: sys.stderr). Uses `file.write(str)` and `file.flush()` jpayne@68: methods. For encoding, see `write_bytes`. jpayne@68: ncols : int, optional jpayne@68: The width of the entire output message. If specified, jpayne@68: dynamically resizes the progressbar to stay within this bound. jpayne@68: If unspecified, attempts to use environment width. The jpayne@68: fallback is a meter width of 10 and no limit for the counter and jpayne@68: statistics. If 0, will not print any meter (only stats). jpayne@68: mininterval : float, optional jpayne@68: Minimum progress display update interval [default: 0.1] seconds. jpayne@68: maxinterval : float, optional jpayne@68: Maximum progress display update interval [default: 10] seconds. jpayne@68: Automatically adjusts `miniters` to correspond to `mininterval` jpayne@68: after long display update lag. Only works if `dynamic_miniters` jpayne@68: or monitor thread is enabled. jpayne@68: miniters : int or float, optional jpayne@68: Minimum progress display update interval, in iterations. jpayne@68: If 0 and `dynamic_miniters`, will automatically adjust to equal jpayne@68: `mininterval` (more CPU efficient, good for tight loops). jpayne@68: If > 0, will skip display of specified number of iterations. jpayne@68: Tweak this and `mininterval` to get very efficient loops. jpayne@68: If your progress is erratic with both fast and slow iterations jpayne@68: (network, skipping items, etc) you should set miniters=1. jpayne@68: ascii : bool or str, optional jpayne@68: If unspecified or False, use unicode (smooth blocks) to fill jpayne@68: the meter. The fallback is to use ASCII characters " 123456789#". jpayne@68: disable : bool, optional jpayne@68: Whether to disable the entire progressbar wrapper jpayne@68: [default: False]. If set to None, disable on non-TTY. jpayne@68: unit : str, optional jpayne@68: String that will be used to define the unit of each iteration jpayne@68: [default: it]. jpayne@68: unit_scale : bool or int or float, optional jpayne@68: If 1 or True, the number of iterations will be reduced/scaled jpayne@68: automatically and a metric prefix following the jpayne@68: International System of Units standard will be added jpayne@68: (kilo, mega, etc.) [default: False]. If any other non-zero jpayne@68: number, will scale `total` and `n`. jpayne@68: dynamic_ncols : bool, optional jpayne@68: If set, constantly alters `ncols` and `nrows` to the jpayne@68: environment (allowing for window resizes) [default: False]. jpayne@68: smoothing : float, optional jpayne@68: Exponential moving average smoothing factor for speed estimates jpayne@68: (ignored in GUI mode). Ranges from 0 (average speed) to 1 jpayne@68: (current/instantaneous speed) [default: 0.3]. jpayne@68: bar_format : str, optional jpayne@68: Specify a custom bar string formatting. May impact performance. jpayne@68: [default: '{l_bar}{bar}{r_bar}'], where jpayne@68: l_bar='{desc}: {percentage:3.0f}%|' and jpayne@68: r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' jpayne@68: '{rate_fmt}{postfix}]' jpayne@68: Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, jpayne@68: percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, jpayne@68: rate, rate_fmt, rate_noinv, rate_noinv_fmt, jpayne@68: rate_inv, rate_inv_fmt, postfix, unit_divisor, jpayne@68: remaining, remaining_s, eta. jpayne@68: Note that a trailing ": " is automatically removed after {desc} jpayne@68: if the latter is empty. jpayne@68: initial : int or float, optional jpayne@68: The initial counter value. Useful when restarting a progress jpayne@68: bar [default: 0]. If using float, consider specifying `{n:.3f}` jpayne@68: or similar in `bar_format`, or specifying `unit_scale`. jpayne@68: position : int, optional jpayne@68: Specify the line offset to print this bar (starting from 0) jpayne@68: Automatic if unspecified. jpayne@68: Useful to manage multiple bars at once (eg, from threads). jpayne@68: postfix : dict or *, optional jpayne@68: Specify additional stats to display at the end of the bar. jpayne@68: Calls `set_postfix(**postfix)` if possible (dict). jpayne@68: unit_divisor : float, optional jpayne@68: [default: 1000], ignored unless `unit_scale` is True. jpayne@68: write_bytes : bool, optional jpayne@68: Whether to write bytes. If (default: False) will write unicode. jpayne@68: lock_args : tuple, optional jpayne@68: Passed to `refresh` for intermediate output jpayne@68: (initialisation, iterating, and updating). jpayne@68: nrows : int, optional jpayne@68: The screen height. If specified, hides nested bars outside this jpayne@68: bound. If unspecified, attempts to use environment height. jpayne@68: The fallback is 20. jpayne@68: colour : str, optional jpayne@68: Bar colour (e.g. 'green', '#00ff00'). jpayne@68: delay : float, optional jpayne@68: Don't display until [default: 0] seconds have elapsed. jpayne@68: gui : bool, optional jpayne@68: WARNING: internal parameter - do not use. jpayne@68: Use tqdm.gui.tqdm(...) instead. If set, will attempt to use jpayne@68: matplotlib animations for a graphical output [default: False]. jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : decorated iterator. jpayne@68: """ jpayne@68: jpayne@68: monitor_interval = 10 # set to 0 to disable the thread jpayne@68: monitor = None jpayne@68: _instances = WeakSet() jpayne@68: jpayne@68: @staticmethod jpayne@68: def format_sizeof(num, suffix='', divisor=1000): jpayne@68: """ jpayne@68: Formats a number (greater than unity) with SI Order of Magnitude jpayne@68: prefixes. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: num : float jpayne@68: Number ( >= 1) to format. jpayne@68: suffix : str, optional jpayne@68: Post-postfix [default: '']. jpayne@68: divisor : float, optional jpayne@68: Divisor between prefixes [default: 1000]. jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : str jpayne@68: Number with Order of Magnitude SI unit postfix. jpayne@68: """ jpayne@68: for unit in ['', 'k', 'M', 'G', 'T', 'P', 'E', 'Z']: jpayne@68: if abs(num) < 999.5: jpayne@68: if abs(num) < 99.95: jpayne@68: if abs(num) < 9.995: jpayne@68: return f'{num:1.2f}{unit}{suffix}' jpayne@68: return f'{num:2.1f}{unit}{suffix}' jpayne@68: return f'{num:3.0f}{unit}{suffix}' jpayne@68: num /= divisor jpayne@68: return f'{num:3.1f}Y{suffix}' jpayne@68: jpayne@68: @staticmethod jpayne@68: def format_interval(t): jpayne@68: """ jpayne@68: Formats a number of seconds as a clock time, [H:]MM:SS jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: t : int jpayne@68: Number of seconds. jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : str jpayne@68: [H:]MM:SS jpayne@68: """ jpayne@68: mins, s = divmod(int(t), 60) jpayne@68: h, m = divmod(mins, 60) jpayne@68: return f'{h:d}:{m:02d}:{s:02d}' if h else f'{m:02d}:{s:02d}' jpayne@68: jpayne@68: @staticmethod jpayne@68: def format_num(n): jpayne@68: """ jpayne@68: Intelligent scientific notation (.3g). jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: n : int or float or Numeric jpayne@68: A Number. jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : str jpayne@68: Formatted number. jpayne@68: """ jpayne@68: f = f'{n:.3g}'.replace('e+0', 'e+').replace('e-0', 'e-') jpayne@68: n = str(n) jpayne@68: return f if len(f) < len(n) else n jpayne@68: jpayne@68: @staticmethod jpayne@68: def status_printer(file): jpayne@68: """ jpayne@68: Manage the printing and in-place updating of a line of characters. jpayne@68: Note that if the string is longer than a line, then in-place jpayne@68: updating may not work (it will print a new line at each refresh). jpayne@68: """ jpayne@68: fp = file jpayne@68: fp_flush = getattr(fp, 'flush', lambda: None) # pragma: no cover jpayne@68: if fp in (sys.stderr, sys.stdout): jpayne@68: getattr(sys.stderr, 'flush', lambda: None)() jpayne@68: getattr(sys.stdout, 'flush', lambda: None)() jpayne@68: jpayne@68: def fp_write(s): jpayne@68: fp.write(str(s)) jpayne@68: fp_flush() jpayne@68: jpayne@68: last_len = [0] jpayne@68: jpayne@68: def print_status(s): jpayne@68: len_s = disp_len(s) jpayne@68: fp_write('\r' + s + (' ' * max(last_len[0] - len_s, 0))) jpayne@68: last_len[0] = len_s jpayne@68: jpayne@68: return print_status jpayne@68: jpayne@68: @staticmethod jpayne@68: def format_meter(n, total, elapsed, ncols=None, prefix='', ascii=False, unit='it', jpayne@68: unit_scale=False, rate=None, bar_format=None, postfix=None, jpayne@68: unit_divisor=1000, initial=0, colour=None, **extra_kwargs): jpayne@68: """ jpayne@68: Return a string-based progress bar given some parameters jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: n : int or float jpayne@68: Number of finished iterations. jpayne@68: total : int or float jpayne@68: The expected total number of iterations. If meaningless (None), jpayne@68: only basic progress statistics are displayed (no ETA). jpayne@68: elapsed : float jpayne@68: Number of seconds passed since start. jpayne@68: ncols : int, optional jpayne@68: The width of the entire output message. If specified, jpayne@68: dynamically resizes `{bar}` to stay within this bound jpayne@68: [default: None]. If `0`, will not print any bar (only stats). jpayne@68: The fallback is `{bar:10}`. jpayne@68: prefix : str, optional jpayne@68: Prefix message (included in total width) [default: '']. jpayne@68: Use as {desc} in bar_format string. jpayne@68: ascii : bool, optional or str, optional jpayne@68: If not set, use unicode (smooth blocks) to fill the meter jpayne@68: [default: False]. The fallback is to use ASCII characters jpayne@68: " 123456789#". jpayne@68: unit : str, optional jpayne@68: The iteration unit [default: 'it']. jpayne@68: unit_scale : bool or int or float, optional jpayne@68: If 1 or True, the number of iterations will be printed with an jpayne@68: appropriate SI metric prefix (k = 10^3, M = 10^6, etc.) jpayne@68: [default: False]. If any other non-zero number, will scale jpayne@68: `total` and `n`. jpayne@68: rate : float, optional jpayne@68: Manual override for iteration rate. jpayne@68: If [default: None], uses n/elapsed. jpayne@68: bar_format : str, optional jpayne@68: Specify a custom bar string formatting. May impact performance. jpayne@68: [default: '{l_bar}{bar}{r_bar}'], where jpayne@68: l_bar='{desc}: {percentage:3.0f}%|' and jpayne@68: r_bar='| {n_fmt}/{total_fmt} [{elapsed}<{remaining}, ' jpayne@68: '{rate_fmt}{postfix}]' jpayne@68: Possible vars: l_bar, bar, r_bar, n, n_fmt, total, total_fmt, jpayne@68: percentage, elapsed, elapsed_s, ncols, nrows, desc, unit, jpayne@68: rate, rate_fmt, rate_noinv, rate_noinv_fmt, jpayne@68: rate_inv, rate_inv_fmt, postfix, unit_divisor, jpayne@68: remaining, remaining_s, eta. jpayne@68: Note that a trailing ": " is automatically removed after {desc} jpayne@68: if the latter is empty. jpayne@68: postfix : *, optional jpayne@68: Similar to `prefix`, but placed at the end jpayne@68: (e.g. for additional stats). jpayne@68: Note: postfix is usually a string (not a dict) for this method, jpayne@68: and will if possible be set to postfix = ', ' + postfix. jpayne@68: However other types are supported (#382). jpayne@68: unit_divisor : float, optional jpayne@68: [default: 1000], ignored unless `unit_scale` is True. jpayne@68: initial : int or float, optional jpayne@68: The initial counter value [default: 0]. jpayne@68: colour : str, optional jpayne@68: Bar colour (e.g. 'green', '#00ff00'). jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : Formatted meter and stats, ready to display. jpayne@68: """ jpayne@68: jpayne@68: # sanity check: total jpayne@68: if total and n >= (total + 0.5): # allow float imprecision (#849) jpayne@68: total = None jpayne@68: jpayne@68: # apply custom scale if necessary jpayne@68: if unit_scale and unit_scale not in (True, 1): jpayne@68: if total: jpayne@68: total *= unit_scale jpayne@68: n *= unit_scale jpayne@68: if rate: jpayne@68: rate *= unit_scale # by default rate = self.avg_dn / self.avg_dt jpayne@68: unit_scale = False jpayne@68: jpayne@68: elapsed_str = tqdm.format_interval(elapsed) jpayne@68: jpayne@68: # if unspecified, attempt to use rate = average speed jpayne@68: # (we allow manual override since predicting time is an arcane art) jpayne@68: if rate is None and elapsed: jpayne@68: rate = (n - initial) / elapsed jpayne@68: inv_rate = 1 / rate if rate else None jpayne@68: format_sizeof = tqdm.format_sizeof jpayne@68: rate_noinv_fmt = ((format_sizeof(rate) if unit_scale else f'{rate:5.2f}') jpayne@68: if rate else '?') + unit + '/s' jpayne@68: rate_inv_fmt = ( jpayne@68: (format_sizeof(inv_rate) if unit_scale else f'{inv_rate:5.2f}') jpayne@68: if inv_rate else '?') + 's/' + unit jpayne@68: rate_fmt = rate_inv_fmt if inv_rate and inv_rate > 1 else rate_noinv_fmt jpayne@68: jpayne@68: if unit_scale: jpayne@68: n_fmt = format_sizeof(n, divisor=unit_divisor) jpayne@68: total_fmt = format_sizeof(total, divisor=unit_divisor) if total is not None else '?' jpayne@68: else: jpayne@68: n_fmt = str(n) jpayne@68: total_fmt = str(total) if total is not None else '?' jpayne@68: jpayne@68: try: jpayne@68: postfix = ', ' + postfix if postfix else '' jpayne@68: except TypeError: jpayne@68: pass jpayne@68: jpayne@68: remaining = (total - n) / rate if rate and total else 0 jpayne@68: remaining_str = tqdm.format_interval(remaining) if rate else '?' jpayne@68: try: jpayne@68: eta_dt = (datetime.now() + timedelta(seconds=remaining) jpayne@68: if rate and total else datetime.fromtimestamp(0, timezone.utc)) jpayne@68: except OverflowError: jpayne@68: eta_dt = datetime.max jpayne@68: jpayne@68: # format the stats displayed to the left and right sides of the bar jpayne@68: if prefix: jpayne@68: # old prefix setup work around jpayne@68: bool_prefix_colon_already = (prefix[-2:] == ": ") jpayne@68: l_bar = prefix if bool_prefix_colon_already else prefix + ": " jpayne@68: else: jpayne@68: l_bar = '' jpayne@68: jpayne@68: r_bar = f'| {n_fmt}/{total_fmt} [{elapsed_str}<{remaining_str}, {rate_fmt}{postfix}]' jpayne@68: jpayne@68: # Custom bar formatting jpayne@68: # Populate a dict with all available progress indicators jpayne@68: format_dict = { jpayne@68: # slight extension of self.format_dict jpayne@68: 'n': n, 'n_fmt': n_fmt, 'total': total, 'total_fmt': total_fmt, jpayne@68: 'elapsed': elapsed_str, 'elapsed_s': elapsed, jpayne@68: 'ncols': ncols, 'desc': prefix or '', 'unit': unit, jpayne@68: 'rate': inv_rate if inv_rate and inv_rate > 1 else rate, jpayne@68: 'rate_fmt': rate_fmt, 'rate_noinv': rate, jpayne@68: 'rate_noinv_fmt': rate_noinv_fmt, 'rate_inv': inv_rate, jpayne@68: 'rate_inv_fmt': rate_inv_fmt, jpayne@68: 'postfix': postfix, 'unit_divisor': unit_divisor, jpayne@68: 'colour': colour, jpayne@68: # plus more useful definitions jpayne@68: 'remaining': remaining_str, 'remaining_s': remaining, jpayne@68: 'l_bar': l_bar, 'r_bar': r_bar, 'eta': eta_dt, jpayne@68: **extra_kwargs} jpayne@68: jpayne@68: # total is known: we can predict some stats jpayne@68: if total: jpayne@68: # fractional and percentage progress jpayne@68: frac = n / total jpayne@68: percentage = frac * 100 jpayne@68: jpayne@68: l_bar += f'{percentage:3.0f}%|' jpayne@68: jpayne@68: if ncols == 0: jpayne@68: return l_bar[:-1] + r_bar[1:] jpayne@68: jpayne@68: format_dict.update(l_bar=l_bar) jpayne@68: if bar_format: jpayne@68: format_dict.update(percentage=percentage) jpayne@68: jpayne@68: # auto-remove colon for empty `{desc}` jpayne@68: if not prefix: jpayne@68: bar_format = bar_format.replace("{desc}: ", '') jpayne@68: else: jpayne@68: bar_format = "{l_bar}{bar}{r_bar}" jpayne@68: jpayne@68: full_bar = FormatReplace() jpayne@68: nobar = bar_format.format(bar=full_bar, **format_dict) jpayne@68: if not full_bar.format_called: jpayne@68: return nobar # no `{bar}`; nothing else to do jpayne@68: jpayne@68: # Formatting progress bar space available for bar's display jpayne@68: full_bar = Bar(frac, jpayne@68: max(1, ncols - disp_len(nobar)) if ncols else 10, jpayne@68: charset=Bar.ASCII if ascii is True else ascii or Bar.UTF, jpayne@68: colour=colour) jpayne@68: if not _is_ascii(full_bar.charset) and _is_ascii(bar_format): jpayne@68: bar_format = str(bar_format) jpayne@68: res = bar_format.format(bar=full_bar, **format_dict) jpayne@68: return disp_trim(res, ncols) if ncols else res jpayne@68: jpayne@68: elif bar_format: jpayne@68: # user-specified bar_format but no total jpayne@68: l_bar += '|' jpayne@68: format_dict.update(l_bar=l_bar, percentage=0) jpayne@68: full_bar = FormatReplace() jpayne@68: nobar = bar_format.format(bar=full_bar, **format_dict) jpayne@68: if not full_bar.format_called: jpayne@68: return nobar jpayne@68: full_bar = Bar(0, jpayne@68: max(1, ncols - disp_len(nobar)) if ncols else 10, jpayne@68: charset=Bar.BLANK, colour=colour) jpayne@68: res = bar_format.format(bar=full_bar, **format_dict) jpayne@68: return disp_trim(res, ncols) if ncols else res jpayne@68: else: jpayne@68: # no total: no progressbar, ETA, just progress stats jpayne@68: return (f'{(prefix + ": ") if prefix else ""}' jpayne@68: f'{n_fmt}{unit} [{elapsed_str}, {rate_fmt}{postfix}]') jpayne@68: jpayne@68: def __new__(cls, *_, **__): jpayne@68: instance = object.__new__(cls) jpayne@68: with cls.get_lock(): # also constructs lock if non-existent jpayne@68: cls._instances.add(instance) jpayne@68: # create monitoring thread jpayne@68: if cls.monitor_interval and (cls.monitor is None jpayne@68: or not cls.monitor.report()): jpayne@68: try: jpayne@68: cls.monitor = TMonitor(cls, cls.monitor_interval) jpayne@68: except Exception as e: # pragma: nocover jpayne@68: warn("tqdm:disabling monitor support" jpayne@68: " (monitor_interval = 0) due to:\n" + str(e), jpayne@68: TqdmMonitorWarning, stacklevel=2) jpayne@68: cls.monitor_interval = 0 jpayne@68: return instance jpayne@68: jpayne@68: @classmethod jpayne@68: def _get_free_pos(cls, instance=None): jpayne@68: """Skips specified instance.""" jpayne@68: positions = {abs(inst.pos) for inst in cls._instances jpayne@68: if inst is not instance and hasattr(inst, "pos")} jpayne@68: return min(set(range(len(positions) + 1)).difference(positions)) jpayne@68: jpayne@68: @classmethod jpayne@68: def _decr_instances(cls, instance): jpayne@68: """ jpayne@68: Remove from list and reposition another unfixed bar jpayne@68: to fill the new gap. jpayne@68: jpayne@68: This means that by default (where all nested bars are unfixed), jpayne@68: order is not maintained but screen flicker/blank space is minimised. jpayne@68: (tqdm<=4.44.1 moved ALL subsequent unfixed bars up.) jpayne@68: """ jpayne@68: with cls._lock: jpayne@68: try: jpayne@68: cls._instances.remove(instance) jpayne@68: except KeyError: jpayne@68: # if not instance.gui: # pragma: no cover jpayne@68: # raise jpayne@68: pass # py2: maybe magically removed already jpayne@68: # else: jpayne@68: if not instance.gui: jpayne@68: last = (instance.nrows or 20) - 1 jpayne@68: # find unfixed (`pos >= 0`) overflow (`pos >= nrows - 1`) jpayne@68: instances = list(filter( jpayne@68: lambda i: hasattr(i, "pos") and last <= i.pos, jpayne@68: cls._instances)) jpayne@68: # set first found to current `pos` jpayne@68: if instances: jpayne@68: inst = min(instances, key=lambda i: i.pos) jpayne@68: inst.clear(nolock=True) jpayne@68: inst.pos = abs(instance.pos) jpayne@68: jpayne@68: @classmethod jpayne@68: def write(cls, s, file=None, end="\n", nolock=False): jpayne@68: """Print a message via tqdm (without overlap with bars).""" jpayne@68: fp = file if file is not None else sys.stdout jpayne@68: with cls.external_write_mode(file=file, nolock=nolock): jpayne@68: # Write the message jpayne@68: fp.write(s) jpayne@68: fp.write(end) jpayne@68: jpayne@68: @classmethod jpayne@68: @contextmanager jpayne@68: def external_write_mode(cls, file=None, nolock=False): jpayne@68: """ jpayne@68: Disable tqdm within context and refresh tqdm when exits. jpayne@68: Useful when writing to standard output stream jpayne@68: """ jpayne@68: fp = file if file is not None else sys.stdout jpayne@68: jpayne@68: try: jpayne@68: if not nolock: jpayne@68: cls.get_lock().acquire() jpayne@68: # Clear all bars jpayne@68: inst_cleared = [] jpayne@68: for inst in getattr(cls, '_instances', []): jpayne@68: # Clear instance if in the target output file jpayne@68: # or if write output + tqdm output are both either jpayne@68: # sys.stdout or sys.stderr (because both are mixed in terminal) jpayne@68: if hasattr(inst, "start_t") and (inst.fp == fp or all( jpayne@68: f in (sys.stdout, sys.stderr) for f in (fp, inst.fp))): jpayne@68: inst.clear(nolock=True) jpayne@68: inst_cleared.append(inst) jpayne@68: yield jpayne@68: # Force refresh display of bars we cleared jpayne@68: for inst in inst_cleared: jpayne@68: inst.refresh(nolock=True) jpayne@68: finally: jpayne@68: if not nolock: jpayne@68: cls._lock.release() jpayne@68: jpayne@68: @classmethod jpayne@68: def set_lock(cls, lock): jpayne@68: """Set the global lock.""" jpayne@68: cls._lock = lock jpayne@68: jpayne@68: @classmethod jpayne@68: def get_lock(cls): jpayne@68: """Get the global lock. Construct it if it does not exist.""" jpayne@68: if not hasattr(cls, '_lock'): jpayne@68: cls._lock = TqdmDefaultWriteLock() jpayne@68: return cls._lock jpayne@68: jpayne@68: @classmethod jpayne@68: def pandas(cls, **tqdm_kwargs): jpayne@68: """ jpayne@68: Registers the current `tqdm` class with jpayne@68: pandas.core. jpayne@68: ( frame.DataFrame jpayne@68: | series.Series jpayne@68: | groupby.(generic.)DataFrameGroupBy jpayne@68: | groupby.(generic.)SeriesGroupBy jpayne@68: ).progress_apply jpayne@68: jpayne@68: A new instance will be created every time `progress_apply` is called, jpayne@68: and each instance will automatically `close()` upon completion. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: tqdm_kwargs : arguments for the tqdm instance jpayne@68: jpayne@68: Examples jpayne@68: -------- jpayne@68: >>> import pandas as pd jpayne@68: >>> import numpy as np jpayne@68: >>> from tqdm import tqdm jpayne@68: >>> from tqdm.gui import tqdm as tqdm_gui jpayne@68: >>> jpayne@68: >>> df = pd.DataFrame(np.random.randint(0, 100, (100000, 6))) jpayne@68: >>> tqdm.pandas(ncols=50) # can use tqdm_gui, optional kwargs, etc jpayne@68: >>> # Now you can use `progress_apply` instead of `apply` jpayne@68: >>> df.groupby(0).progress_apply(lambda x: x**2) jpayne@68: jpayne@68: References jpayne@68: ---------- jpayne@68: jpayne@68: """ jpayne@68: from warnings import catch_warnings, simplefilter jpayne@68: jpayne@68: from pandas.core.frame import DataFrame jpayne@68: from pandas.core.series import Series jpayne@68: try: jpayne@68: with catch_warnings(): jpayne@68: simplefilter("ignore", category=FutureWarning) jpayne@68: from pandas import Panel jpayne@68: except ImportError: # pandas>=1.2.0 jpayne@68: Panel = None jpayne@68: Rolling, Expanding = None, None jpayne@68: try: # pandas>=1.0.0 jpayne@68: from pandas.core.window.rolling import _Rolling_and_Expanding jpayne@68: except ImportError: jpayne@68: try: # pandas>=0.18.0 jpayne@68: from pandas.core.window import _Rolling_and_Expanding jpayne@68: except ImportError: # pandas>=1.2.0 jpayne@68: try: # pandas>=1.2.0 jpayne@68: from pandas.core.window.expanding import Expanding jpayne@68: from pandas.core.window.rolling import Rolling jpayne@68: _Rolling_and_Expanding = Rolling, Expanding jpayne@68: except ImportError: # pragma: no cover jpayne@68: _Rolling_and_Expanding = None jpayne@68: try: # pandas>=0.25.0 jpayne@68: from pandas.core.groupby.generic import SeriesGroupBy # , NDFrameGroupBy jpayne@68: from pandas.core.groupby.generic import DataFrameGroupBy jpayne@68: except ImportError: # pragma: no cover jpayne@68: try: # pandas>=0.23.0 jpayne@68: from pandas.core.groupby.groupby import DataFrameGroupBy, SeriesGroupBy jpayne@68: except ImportError: jpayne@68: from pandas.core.groupby import DataFrameGroupBy, SeriesGroupBy jpayne@68: try: # pandas>=0.23.0 jpayne@68: from pandas.core.groupby.groupby import GroupBy jpayne@68: except ImportError: # pragma: no cover jpayne@68: from pandas.core.groupby import GroupBy jpayne@68: jpayne@68: try: # pandas>=0.23.0 jpayne@68: from pandas.core.groupby.groupby import PanelGroupBy jpayne@68: except ImportError: jpayne@68: try: jpayne@68: from pandas.core.groupby import PanelGroupBy jpayne@68: except ImportError: # pandas>=0.25.0 jpayne@68: PanelGroupBy = None jpayne@68: jpayne@68: tqdm_kwargs = tqdm_kwargs.copy() jpayne@68: deprecated_t = [tqdm_kwargs.pop('deprecated_t', None)] jpayne@68: jpayne@68: def inner_generator(df_function='apply'): jpayne@68: def inner(df, func, *args, **kwargs): jpayne@68: """ jpayne@68: Parameters jpayne@68: ---------- jpayne@68: df : (DataFrame|Series)[GroupBy] jpayne@68: Data (may be grouped). jpayne@68: func : function jpayne@68: To be applied on the (grouped) data. jpayne@68: **kwargs : optional jpayne@68: Transmitted to `df.apply()`. jpayne@68: """ jpayne@68: jpayne@68: # Precompute total iterations jpayne@68: total = tqdm_kwargs.pop("total", getattr(df, 'ngroups', None)) jpayne@68: if total is None: # not grouped jpayne@68: if df_function == 'applymap': jpayne@68: total = df.size jpayne@68: elif isinstance(df, Series): jpayne@68: total = len(df) jpayne@68: elif (_Rolling_and_Expanding is None or jpayne@68: not isinstance(df, _Rolling_and_Expanding)): jpayne@68: # DataFrame or Panel jpayne@68: axis = kwargs.get('axis', 0) jpayne@68: if axis == 'index': jpayne@68: axis = 0 jpayne@68: elif axis == 'columns': jpayne@68: axis = 1 jpayne@68: # when axis=0, total is shape[axis1] jpayne@68: total = df.size // df.shape[axis] jpayne@68: jpayne@68: # Init bar jpayne@68: if deprecated_t[0] is not None: jpayne@68: t = deprecated_t[0] jpayne@68: deprecated_t[0] = None jpayne@68: else: jpayne@68: t = cls(total=total, **tqdm_kwargs) jpayne@68: jpayne@68: if len(args) > 0: jpayne@68: # *args intentionally not supported (see #244, #299) jpayne@68: TqdmDeprecationWarning( jpayne@68: "Except func, normal arguments are intentionally" + jpayne@68: " not supported by" + jpayne@68: " `(DataFrame|Series|GroupBy).progress_apply`." + jpayne@68: " Use keyword arguments instead.", jpayne@68: fp_write=getattr(t.fp, 'write', sys.stderr.write)) jpayne@68: jpayne@68: try: # pandas>=1.3.0 jpayne@68: from pandas.core.common import is_builtin_func jpayne@68: except ImportError: jpayne@68: is_builtin_func = df._is_builtin_func jpayne@68: try: jpayne@68: func = is_builtin_func(func) jpayne@68: except TypeError: jpayne@68: pass jpayne@68: jpayne@68: # Define bar updating wrapper jpayne@68: def wrapper(*args, **kwargs): jpayne@68: # update tbar correctly jpayne@68: # it seems `pandas apply` calls `func` twice jpayne@68: # on the first column/row to decide whether it can jpayne@68: # take a fast or slow code path; so stop when t.total==t.n jpayne@68: t.update(n=1 if not t.total or t.n < t.total else 0) jpayne@68: return func(*args, **kwargs) jpayne@68: jpayne@68: # Apply the provided function (in **kwargs) jpayne@68: # on the df using our wrapper (which provides bar updating) jpayne@68: try: jpayne@68: return getattr(df, df_function)(wrapper, **kwargs) jpayne@68: finally: jpayne@68: t.close() jpayne@68: jpayne@68: return inner jpayne@68: jpayne@68: # Monkeypatch pandas to provide easy methods jpayne@68: # Enable custom tqdm progress in pandas! jpayne@68: Series.progress_apply = inner_generator() jpayne@68: SeriesGroupBy.progress_apply = inner_generator() jpayne@68: Series.progress_map = inner_generator('map') jpayne@68: SeriesGroupBy.progress_map = inner_generator('map') jpayne@68: jpayne@68: DataFrame.progress_apply = inner_generator() jpayne@68: DataFrameGroupBy.progress_apply = inner_generator() jpayne@68: DataFrame.progress_applymap = inner_generator('applymap') jpayne@68: DataFrame.progress_map = inner_generator('map') jpayne@68: DataFrameGroupBy.progress_map = inner_generator('map') jpayne@68: jpayne@68: if Panel is not None: jpayne@68: Panel.progress_apply = inner_generator() jpayne@68: if PanelGroupBy is not None: jpayne@68: PanelGroupBy.progress_apply = inner_generator() jpayne@68: jpayne@68: GroupBy.progress_apply = inner_generator() jpayne@68: GroupBy.progress_aggregate = inner_generator('aggregate') jpayne@68: GroupBy.progress_transform = inner_generator('transform') jpayne@68: jpayne@68: if Rolling is not None and Expanding is not None: jpayne@68: Rolling.progress_apply = inner_generator() jpayne@68: Expanding.progress_apply = inner_generator() jpayne@68: elif _Rolling_and_Expanding is not None: jpayne@68: _Rolling_and_Expanding.progress_apply = inner_generator() jpayne@68: jpayne@68: # override defaults via env vars jpayne@68: @envwrap("TQDM_", is_method=True, types={'total': float, 'ncols': int, 'miniters': float, jpayne@68: 'position': int, 'nrows': int}) jpayne@68: def __init__(self, iterable=None, desc=None, total=None, leave=True, file=None, jpayne@68: ncols=None, mininterval=0.1, maxinterval=10.0, miniters=None, jpayne@68: ascii=None, disable=False, unit='it', unit_scale=False, jpayne@68: dynamic_ncols=False, smoothing=0.3, bar_format=None, initial=0, jpayne@68: position=None, postfix=None, unit_divisor=1000, write_bytes=False, jpayne@68: lock_args=None, nrows=None, colour=None, delay=0.0, gui=False, jpayne@68: **kwargs): jpayne@68: """see tqdm.tqdm for arguments""" jpayne@68: if file is None: jpayne@68: file = sys.stderr jpayne@68: jpayne@68: if write_bytes: jpayne@68: # Despite coercing unicode into bytes, py2 sys.std* streams jpayne@68: # should have bytes written to them. jpayne@68: file = SimpleTextIOWrapper( jpayne@68: file, encoding=getattr(file, 'encoding', None) or 'utf-8') jpayne@68: jpayne@68: file = DisableOnWriteError(file, tqdm_instance=self) jpayne@68: jpayne@68: if disable is None and hasattr(file, "isatty") and not file.isatty(): jpayne@68: disable = True jpayne@68: jpayne@68: if total is None and iterable is not None: jpayne@68: try: jpayne@68: total = len(iterable) jpayne@68: except (TypeError, AttributeError): jpayne@68: total = None jpayne@68: if total == float("inf"): jpayne@68: # Infinite iterations, behave same as unknown jpayne@68: total = None jpayne@68: jpayne@68: if disable: jpayne@68: self.iterable = iterable jpayne@68: self.disable = disable jpayne@68: with self._lock: jpayne@68: self.pos = self._get_free_pos(self) jpayne@68: self._instances.remove(self) jpayne@68: self.n = initial jpayne@68: self.total = total jpayne@68: self.leave = leave jpayne@68: return jpayne@68: jpayne@68: if kwargs: jpayne@68: self.disable = True jpayne@68: with self._lock: jpayne@68: self.pos = self._get_free_pos(self) jpayne@68: self._instances.remove(self) jpayne@68: raise ( jpayne@68: TqdmDeprecationWarning( jpayne@68: "`nested` is deprecated and automated.\n" jpayne@68: "Use `position` instead for manual control.\n", jpayne@68: fp_write=getattr(file, 'write', sys.stderr.write)) jpayne@68: if "nested" in kwargs else jpayne@68: TqdmKeyError("Unknown argument(s): " + str(kwargs))) jpayne@68: jpayne@68: # Preprocess the arguments jpayne@68: if ( jpayne@68: (ncols is None or nrows is None) and (file in (sys.stderr, sys.stdout)) jpayne@68: ) or dynamic_ncols: # pragma: no cover jpayne@68: if dynamic_ncols: jpayne@68: dynamic_ncols = _screen_shape_wrapper() jpayne@68: if dynamic_ncols: jpayne@68: ncols, nrows = dynamic_ncols(file) jpayne@68: else: jpayne@68: _dynamic_ncols = _screen_shape_wrapper() jpayne@68: if _dynamic_ncols: jpayne@68: _ncols, _nrows = _dynamic_ncols(file) jpayne@68: if ncols is None: jpayne@68: ncols = _ncols jpayne@68: if nrows is None: jpayne@68: nrows = _nrows jpayne@68: jpayne@68: if miniters is None: jpayne@68: miniters = 0 jpayne@68: dynamic_miniters = True jpayne@68: else: jpayne@68: dynamic_miniters = False jpayne@68: jpayne@68: if mininterval is None: jpayne@68: mininterval = 0 jpayne@68: jpayne@68: if maxinterval is None: jpayne@68: maxinterval = 0 jpayne@68: jpayne@68: if ascii is None: jpayne@68: ascii = not _supports_unicode(file) jpayne@68: jpayne@68: if bar_format and ascii is not True and not _is_ascii(ascii): jpayne@68: # Convert bar format into unicode since terminal uses unicode jpayne@68: bar_format = str(bar_format) jpayne@68: jpayne@68: if smoothing is None: jpayne@68: smoothing = 0 jpayne@68: jpayne@68: # Store the arguments jpayne@68: self.iterable = iterable jpayne@68: self.desc = desc or '' jpayne@68: self.total = total jpayne@68: self.leave = leave jpayne@68: self.fp = file jpayne@68: self.ncols = ncols jpayne@68: self.nrows = nrows jpayne@68: self.mininterval = mininterval jpayne@68: self.maxinterval = maxinterval jpayne@68: self.miniters = miniters jpayne@68: self.dynamic_miniters = dynamic_miniters jpayne@68: self.ascii = ascii jpayne@68: self.disable = disable jpayne@68: self.unit = unit jpayne@68: self.unit_scale = unit_scale jpayne@68: self.unit_divisor = unit_divisor jpayne@68: self.initial = initial jpayne@68: self.lock_args = lock_args jpayne@68: self.delay = delay jpayne@68: self.gui = gui jpayne@68: self.dynamic_ncols = dynamic_ncols jpayne@68: self.smoothing = smoothing jpayne@68: self._ema_dn = EMA(smoothing) jpayne@68: self._ema_dt = EMA(smoothing) jpayne@68: self._ema_miniters = EMA(smoothing) jpayne@68: self.bar_format = bar_format jpayne@68: self.postfix = None jpayne@68: self.colour = colour jpayne@68: self._time = time jpayne@68: if postfix: jpayne@68: try: jpayne@68: self.set_postfix(refresh=False, **postfix) jpayne@68: except TypeError: jpayne@68: self.postfix = postfix jpayne@68: jpayne@68: # Init the iterations counters jpayne@68: self.last_print_n = initial jpayne@68: self.n = initial jpayne@68: jpayne@68: # if nested, at initial sp() call we replace '\r' by '\n' to jpayne@68: # not overwrite the outer progress bar jpayne@68: with self._lock: jpayne@68: # mark fixed positions as negative jpayne@68: self.pos = self._get_free_pos(self) if position is None else -position jpayne@68: jpayne@68: if not gui: jpayne@68: # Initialize the screen printer jpayne@68: self.sp = self.status_printer(self.fp) jpayne@68: if delay <= 0: jpayne@68: self.refresh(lock_args=self.lock_args) jpayne@68: jpayne@68: # Init the time counter jpayne@68: self.last_print_t = self._time() jpayne@68: # NB: Avoid race conditions by setting start_t at the very end of init jpayne@68: self.start_t = self.last_print_t jpayne@68: jpayne@68: def __bool__(self): jpayne@68: if self.total is not None: jpayne@68: return self.total > 0 jpayne@68: if self.iterable is None: jpayne@68: raise TypeError('bool() undefined when iterable == total == None') jpayne@68: return bool(self.iterable) jpayne@68: jpayne@68: def __len__(self): jpayne@68: return ( jpayne@68: self.total if self.iterable is None jpayne@68: else self.iterable.shape[0] if hasattr(self.iterable, "shape") jpayne@68: else len(self.iterable) if hasattr(self.iterable, "__len__") jpayne@68: else self.iterable.__length_hint__() if hasattr(self.iterable, "__length_hint__") jpayne@68: else getattr(self, "total", None)) jpayne@68: jpayne@68: def __reversed__(self): jpayne@68: try: jpayne@68: orig = self.iterable jpayne@68: except AttributeError: jpayne@68: raise TypeError("'tqdm' object is not reversible") jpayne@68: else: jpayne@68: self.iterable = reversed(self.iterable) jpayne@68: return self.__iter__() jpayne@68: finally: jpayne@68: self.iterable = orig jpayne@68: jpayne@68: def __contains__(self, item): jpayne@68: contains = getattr(self.iterable, '__contains__', None) jpayne@68: return contains(item) if contains is not None else item in self.__iter__() jpayne@68: jpayne@68: def __enter__(self): jpayne@68: return self jpayne@68: jpayne@68: def __exit__(self, exc_type, exc_value, traceback): jpayne@68: try: jpayne@68: self.close() jpayne@68: except AttributeError: jpayne@68: # maybe eager thread cleanup upon external error jpayne@68: if (exc_type, exc_value, traceback) == (None, None, None): jpayne@68: raise jpayne@68: warn("AttributeError ignored", TqdmWarning, stacklevel=2) jpayne@68: jpayne@68: def __del__(self): jpayne@68: self.close() jpayne@68: jpayne@68: def __str__(self): jpayne@68: return self.format_meter(**self.format_dict) jpayne@68: jpayne@68: @property jpayne@68: def _comparable(self): jpayne@68: return abs(getattr(self, "pos", 1 << 31)) jpayne@68: jpayne@68: def __hash__(self): jpayne@68: return id(self) jpayne@68: jpayne@68: def __iter__(self): jpayne@68: """Backward-compatibility to use: for x in tqdm(iterable)""" jpayne@68: jpayne@68: # Inlining instance variables as locals (speed optimisation) jpayne@68: iterable = self.iterable jpayne@68: jpayne@68: # If the bar is disabled, then just walk the iterable jpayne@68: # (note: keep this check outside the loop for performance) jpayne@68: if self.disable: jpayne@68: for obj in iterable: jpayne@68: yield obj jpayne@68: return jpayne@68: jpayne@68: mininterval = self.mininterval jpayne@68: last_print_t = self.last_print_t jpayne@68: last_print_n = self.last_print_n jpayne@68: min_start_t = self.start_t + self.delay jpayne@68: n = self.n jpayne@68: time = self._time jpayne@68: jpayne@68: try: jpayne@68: for obj in iterable: jpayne@68: yield obj jpayne@68: # Update and possibly print the progressbar. jpayne@68: # Note: does not call self.update(1) for speed optimisation. jpayne@68: n += 1 jpayne@68: jpayne@68: if n - last_print_n >= self.miniters: jpayne@68: cur_t = time() jpayne@68: dt = cur_t - last_print_t jpayne@68: if dt >= mininterval and cur_t >= min_start_t: jpayne@68: self.update(n - last_print_n) jpayne@68: last_print_n = self.last_print_n jpayne@68: last_print_t = self.last_print_t jpayne@68: finally: jpayne@68: self.n = n jpayne@68: self.close() jpayne@68: jpayne@68: def update(self, n=1): jpayne@68: """ jpayne@68: Manually update the progress bar, useful for streams jpayne@68: such as reading files. jpayne@68: E.g.: jpayne@68: >>> t = tqdm(total=filesize) # Initialise jpayne@68: >>> for current_buffer in stream: jpayne@68: ... ... jpayne@68: ... t.update(len(current_buffer)) jpayne@68: >>> t.close() jpayne@68: The last line is highly recommended, but possibly not necessary if jpayne@68: `t.update()` will be called in such a way that `filesize` will be jpayne@68: exactly reached and printed. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: n : int or float, optional jpayne@68: Increment to add to the internal counter of iterations jpayne@68: [default: 1]. If using float, consider specifying `{n:.3f}` jpayne@68: or similar in `bar_format`, or specifying `unit_scale`. jpayne@68: jpayne@68: Returns jpayne@68: ------- jpayne@68: out : bool or None jpayne@68: True if a `display()` was triggered. jpayne@68: """ jpayne@68: if self.disable: jpayne@68: return jpayne@68: jpayne@68: if n < 0: jpayne@68: self.last_print_n += n # for auto-refresh logic to work jpayne@68: self.n += n jpayne@68: jpayne@68: # check counter first to reduce calls to time() jpayne@68: if self.n - self.last_print_n >= self.miniters: jpayne@68: cur_t = self._time() jpayne@68: dt = cur_t - self.last_print_t jpayne@68: if dt >= self.mininterval and cur_t >= self.start_t + self.delay: jpayne@68: cur_t = self._time() jpayne@68: dn = self.n - self.last_print_n # >= n jpayne@68: if self.smoothing and dt and dn: jpayne@68: # EMA (not just overall average) jpayne@68: self._ema_dn(dn) jpayne@68: self._ema_dt(dt) jpayne@68: self.refresh(lock_args=self.lock_args) jpayne@68: if self.dynamic_miniters: jpayne@68: # If no `miniters` was specified, adjust automatically to the jpayne@68: # maximum iteration rate seen so far between two prints. jpayne@68: # e.g.: After running `tqdm.update(5)`, subsequent jpayne@68: # calls to `tqdm.update()` will only cause an update after jpayne@68: # at least 5 more iterations. jpayne@68: if self.maxinterval and dt >= self.maxinterval: jpayne@68: self.miniters = dn * (self.mininterval or self.maxinterval) / dt jpayne@68: elif self.smoothing: jpayne@68: # EMA miniters update jpayne@68: self.miniters = self._ema_miniters( jpayne@68: dn * (self.mininterval / dt if self.mininterval and dt jpayne@68: else 1)) jpayne@68: else: jpayne@68: # max iters between two prints jpayne@68: self.miniters = max(self.miniters, dn) jpayne@68: jpayne@68: # Store old values for next call jpayne@68: self.last_print_n = self.n jpayne@68: self.last_print_t = cur_t jpayne@68: return True jpayne@68: jpayne@68: def close(self): jpayne@68: """Cleanup and (if leave=False) close the progressbar.""" jpayne@68: if self.disable: jpayne@68: return jpayne@68: jpayne@68: # Prevent multiple closures jpayne@68: self.disable = True jpayne@68: jpayne@68: # decrement instance pos and remove from internal set jpayne@68: pos = abs(self.pos) jpayne@68: self._decr_instances(self) jpayne@68: jpayne@68: if self.last_print_t < self.start_t + self.delay: jpayne@68: # haven't ever displayed; nothing to clear jpayne@68: return jpayne@68: jpayne@68: # GUI mode jpayne@68: if getattr(self, 'sp', None) is None: jpayne@68: return jpayne@68: jpayne@68: # annoyingly, _supports_unicode isn't good enough jpayne@68: def fp_write(s): jpayne@68: self.fp.write(str(s)) jpayne@68: jpayne@68: try: jpayne@68: fp_write('') jpayne@68: except ValueError as e: jpayne@68: if 'closed' in str(e): jpayne@68: return jpayne@68: raise # pragma: no cover jpayne@68: jpayne@68: leave = pos == 0 if self.leave is None else self.leave jpayne@68: jpayne@68: with self._lock: jpayne@68: if leave: jpayne@68: # stats for overall rate (no weighted average) jpayne@68: self._ema_dt = lambda: None jpayne@68: self.display(pos=0) jpayne@68: fp_write('\n') jpayne@68: else: jpayne@68: # clear previous display jpayne@68: if self.display(msg='', pos=pos) and not pos: jpayne@68: fp_write('\r') jpayne@68: jpayne@68: def clear(self, nolock=False): jpayne@68: """Clear current bar display.""" jpayne@68: if self.disable: jpayne@68: return jpayne@68: jpayne@68: if not nolock: jpayne@68: self._lock.acquire() jpayne@68: pos = abs(self.pos) jpayne@68: if pos < (self.nrows or 20): jpayne@68: self.moveto(pos) jpayne@68: self.sp('') jpayne@68: self.fp.write('\r') # place cursor back at the beginning of line jpayne@68: self.moveto(-pos) jpayne@68: if not nolock: jpayne@68: self._lock.release() jpayne@68: jpayne@68: def refresh(self, nolock=False, lock_args=None): jpayne@68: """ jpayne@68: Force refresh the display of this bar. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: nolock : bool, optional jpayne@68: If `True`, does not lock. jpayne@68: If [default: `False`]: calls `acquire()` on internal lock. jpayne@68: lock_args : tuple, optional jpayne@68: Passed to internal lock's `acquire()`. jpayne@68: If specified, will only `display()` if `acquire()` returns `True`. jpayne@68: """ jpayne@68: if self.disable: jpayne@68: return jpayne@68: jpayne@68: if not nolock: jpayne@68: if lock_args: jpayne@68: if not self._lock.acquire(*lock_args): jpayne@68: return False jpayne@68: else: jpayne@68: self._lock.acquire() jpayne@68: self.display() jpayne@68: if not nolock: jpayne@68: self._lock.release() jpayne@68: return True jpayne@68: jpayne@68: def unpause(self): jpayne@68: """Restart tqdm timer from last print time.""" jpayne@68: if self.disable: jpayne@68: return jpayne@68: cur_t = self._time() jpayne@68: self.start_t += cur_t - self.last_print_t jpayne@68: self.last_print_t = cur_t jpayne@68: jpayne@68: def reset(self, total=None): jpayne@68: """ jpayne@68: Resets to 0 iterations for repeated use. jpayne@68: jpayne@68: Consider combining with `leave=True`. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: total : int or float, optional. Total to use for the new bar. jpayne@68: """ jpayne@68: self.n = 0 jpayne@68: if total is not None: jpayne@68: self.total = total jpayne@68: if self.disable: jpayne@68: return jpayne@68: self.last_print_n = 0 jpayne@68: self.last_print_t = self.start_t = self._time() jpayne@68: self._ema_dn = EMA(self.smoothing) jpayne@68: self._ema_dt = EMA(self.smoothing) jpayne@68: self._ema_miniters = EMA(self.smoothing) jpayne@68: self.refresh() jpayne@68: jpayne@68: def set_description(self, desc=None, refresh=True): jpayne@68: """ jpayne@68: Set/modify description of the progress bar. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: desc : str, optional jpayne@68: refresh : bool, optional jpayne@68: Forces refresh [default: True]. jpayne@68: """ jpayne@68: self.desc = desc + ': ' if desc else '' jpayne@68: if refresh: jpayne@68: self.refresh() jpayne@68: jpayne@68: def set_description_str(self, desc=None, refresh=True): jpayne@68: """Set/modify description without ': ' appended.""" jpayne@68: self.desc = desc or '' jpayne@68: if refresh: jpayne@68: self.refresh() jpayne@68: jpayne@68: def set_postfix(self, ordered_dict=None, refresh=True, **kwargs): jpayne@68: """ jpayne@68: Set/modify postfix (additional stats) jpayne@68: with automatic formatting based on datatype. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: ordered_dict : dict or OrderedDict, optional jpayne@68: refresh : bool, optional jpayne@68: Forces refresh [default: True]. jpayne@68: kwargs : dict, optional jpayne@68: """ jpayne@68: # Sort in alphabetical order to be more deterministic jpayne@68: postfix = OrderedDict([] if ordered_dict is None else ordered_dict) jpayne@68: for key in sorted(kwargs.keys()): jpayne@68: postfix[key] = kwargs[key] jpayne@68: # Preprocess stats according to datatype jpayne@68: for key in postfix.keys(): jpayne@68: # Number: limit the length of the string jpayne@68: if isinstance(postfix[key], Number): jpayne@68: postfix[key] = self.format_num(postfix[key]) jpayne@68: # Else for any other type, try to get the string conversion jpayne@68: elif not isinstance(postfix[key], str): jpayne@68: postfix[key] = str(postfix[key]) jpayne@68: # Else if it's a string, don't need to preprocess anything jpayne@68: # Stitch together to get the final postfix jpayne@68: self.postfix = ', '.join(key + '=' + postfix[key].strip() jpayne@68: for key in postfix.keys()) jpayne@68: if refresh: jpayne@68: self.refresh() jpayne@68: jpayne@68: def set_postfix_str(self, s='', refresh=True): jpayne@68: """ jpayne@68: Postfix without dictionary expansion, similar to prefix handling. jpayne@68: """ jpayne@68: self.postfix = str(s) jpayne@68: if refresh: jpayne@68: self.refresh() jpayne@68: jpayne@68: def moveto(self, n): jpayne@68: # TODO: private method jpayne@68: self.fp.write('\n' * n + _term_move_up() * -n) jpayne@68: getattr(self.fp, 'flush', lambda: None)() jpayne@68: jpayne@68: @property jpayne@68: def format_dict(self): jpayne@68: """Public API for read-only member access.""" jpayne@68: if self.disable and not hasattr(self, 'unit'): jpayne@68: return defaultdict(lambda: None, { jpayne@68: 'n': self.n, 'total': self.total, 'elapsed': 0, 'unit': 'it'}) jpayne@68: if self.dynamic_ncols: jpayne@68: self.ncols, self.nrows = self.dynamic_ncols(self.fp) jpayne@68: return { jpayne@68: 'n': self.n, 'total': self.total, jpayne@68: 'elapsed': self._time() - self.start_t if hasattr(self, 'start_t') else 0, jpayne@68: 'ncols': self.ncols, 'nrows': self.nrows, 'prefix': self.desc, jpayne@68: 'ascii': self.ascii, 'unit': self.unit, 'unit_scale': self.unit_scale, jpayne@68: 'rate': self._ema_dn() / self._ema_dt() if self._ema_dt() else None, jpayne@68: 'bar_format': self.bar_format, 'postfix': self.postfix, jpayne@68: 'unit_divisor': self.unit_divisor, 'initial': self.initial, jpayne@68: 'colour': self.colour} jpayne@68: jpayne@68: def display(self, msg=None, pos=None): jpayne@68: """ jpayne@68: Use `self.sp` to display `msg` in the specified `pos`. jpayne@68: jpayne@68: Consider overloading this function when inheriting to use e.g.: jpayne@68: `self.some_frontend(**self.format_dict)` instead of `self.sp`. jpayne@68: jpayne@68: Parameters jpayne@68: ---------- jpayne@68: msg : str, optional. What to display (default: `repr(self)`). jpayne@68: pos : int, optional. Position to `moveto` jpayne@68: (default: `abs(self.pos)`). jpayne@68: """ jpayne@68: if pos is None: jpayne@68: pos = abs(self.pos) jpayne@68: jpayne@68: nrows = self.nrows or 20 jpayne@68: if pos >= nrows - 1: jpayne@68: if pos >= nrows: jpayne@68: return False jpayne@68: if msg or msg is None: # override at `nrows - 1` jpayne@68: msg = " ... (more hidden) ..." jpayne@68: jpayne@68: if not hasattr(self, "sp"): jpayne@68: raise TqdmDeprecationWarning( jpayne@68: "Please use `tqdm.gui.tqdm(...)`" jpayne@68: " instead of `tqdm(..., gui=True)`\n", jpayne@68: fp_write=getattr(self.fp, 'write', sys.stderr.write)) jpayne@68: jpayne@68: if pos: jpayne@68: self.moveto(pos) jpayne@68: self.sp(self.__str__() if msg is None else msg) jpayne@68: if pos: jpayne@68: self.moveto(-pos) jpayne@68: return True jpayne@68: jpayne@68: @classmethod jpayne@68: @contextmanager jpayne@68: def wrapattr(cls, stream, method, total=None, bytes=True, **tqdm_kwargs): jpayne@68: """ jpayne@68: stream : file-like object. jpayne@68: method : str, "read" or "write". The result of `read()` and jpayne@68: the first argument of `write()` should have a `len()`. jpayne@68: jpayne@68: >>> with tqdm.wrapattr(file_obj, "read", total=file_obj.size) as fobj: jpayne@68: ... while True: jpayne@68: ... chunk = fobj.read(chunk_size) jpayne@68: ... if not chunk: jpayne@68: ... break jpayne@68: """ jpayne@68: with cls(total=total, **tqdm_kwargs) as t: jpayne@68: if bytes: jpayne@68: t.unit = "B" jpayne@68: t.unit_scale = True jpayne@68: t.unit_divisor = 1024 jpayne@68: yield CallbackIOWrapper(t.update, stream, method) jpayne@68: jpayne@68: jpayne@68: def trange(*args, **kwargs): jpayne@68: """Shortcut for tqdm(range(*args), **kwargs).""" jpayne@68: return tqdm(range(*args), **kwargs)