Python progressbar 模块，RotatingMarker() 实例源码

def collect_mailids(server):
    folders = server.list_folders()

    #construct progressbar
    progressbar_widgets = [
        '[Searching for mails on server] ',
        progressbar.Percentage(),
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ']
    progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=len(folders)).start()

    #collect all mailids for all folders
    folder_contents = {}
    folder_progress = 0
    for flags, delimiter, folder in folders:
        #read all mailids for the folder
        server.select_folder(folder, readonly=True)
        folder_contents[folder] = server.search()

        #update progrssbar
        folder_progress += 1
        progressbar_instance.update(folder_progress)

    progressbar_instance.finish()
    return folder_contents

def download(download_list, total_download_size):
    progressbar_widgets = [
        '[Downloading mails            ] ',
        progressbar.Percentage(),
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ',
        progressbar.ETA(), ' ',
        bitmath.integrations.BitmathFileTransferSpeed()]
    progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=int(total_download_size)).start()

    downloaded_size = bitmath.Byte(0)
    for folder, mails in download_list.items():
        server.select_folder(folder, readonly=True)
        for mailid, mailfilename, mailsize in mails:
            #make parent directory
            if not os.path.isdir(os.path.dirname(mailfilename)):
                os.makedirs(os.path.dirname(mailfilename))

            #download mail
            with open(mailfilename, 'wb') as mailfile:
                mailfile.write(server.fetch([mailid], ['RFC822'])[mailid][b'RFC822'])

            #update progressbar
            downloaded_size += mailsize
            progressbar_instance.update(int(downloaded_size))
    progressbar_instance.finish()

def deleteHostsByHostgroup(groupname):
    hostgroup = zapi.hostgroup.get(output=['groupid'],filter={'name': groupname})
    if hostgroup.__len__() != 1:
        logger.error('Hostgroup not found: %s\n\tFound this: %s' % (groupname,hostgroup))
    groupid = int(hostgroup[0]['groupid'])
    hosts = zapi.host.get(output=['name','hostid'],groupids=groupid)
    total = len(hosts)
    logger.info('Hosts found: %d' % (total))
    if ( args.run ):
        x = 0
        bar = ProgressBar(maxval=total,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
        logger.echo = False
        for host in hosts:
            x = x + 1
            bar.update(x)
            logger.debug('(%d/%d) >> Removing >> %s' % (x, total, host))
            out = zapi.globo.deleteMonitors(host['name'])
        bar.finish()
        logger.echo = True
    else:
        logger.info('No host removed due to --no-run arg. Full list of hosts:')
        for host in hosts:
            logger.info('%s' % host['name'])
    return

def hosts_disable_all():
  """
  status de host 0 = enabled
  status de host 1 = disabled
  """
  logger.info('Disabling all hosts, in blocks of 1000')
  hosts = zapi.host.get(output=[ 'hostid' ], search={ 'status': 0 })
  maxval = int(ceil(hosts.__len__())/1000+1)
  bar = ProgressBar(maxval=maxval,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
  i = 0
  for i in xrange(maxval):
    block = hosts[:1000]
    del hosts[:1000]
    result = zapi.host.massupdate(hosts=[ x for x in block ], status=1)
    i += 1
    bar.update(i)
  bar.finish()
  logger.info('Done')
  return

def proxy_passive_to_active():
  """
  status de prxy 5 = active
  status de prxy 6 = passive
  """
  logger.info('Change all proxys to active')
  proxys = zapi.proxy.get(output=[ 'shorten', 'host' ],
    filter={ 'status': 6 })
  if ( proxys.__len__() == 0 ):
    logger.info('Done')
    return
  bar = ProgressBar(maxval=proxys.__len__(),widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
  i = 0
  for x in proxys:
    i += 1
    proxyid = x['proxyid']
    result = zapi.proxy.update(proxyid=proxyid, status=5)
    logger.echo = False
    logger.debug('Changed from passive to active proxy: %s' % (x['host']))
    bar.update(i)
  bar.finish()
  logger.echo = True
  logger.info('Done')
  return

def getProgress(self, url, fileSize):
        status = json.loads(urllib.urlopen(url).read())
        if len(status["data"]) ==0 :
            logger.info(url + " upload done ")
            return  True
        widgets = ['Progress: ', Percentage(), ' ', Bar(
            marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
        pbar = ProgressBar(widgets=widgets, maxval=fileSize).start()
        upload_size = 0
        while upload_size < fileSize:
            _response = self.doGet(url)
            _data = json.loads(_response)
            upload_size = long(_data["data"]["upload_size"])
            total_size = long(_data["data"]["total_size"])
            if upload_size == 0 and total_size == 0:
                break
            pbar.update(upload_size)
            time.sleep(1)
        pbar.finish()
        logger.info(url + " upload done")
        return True

        """
        ??????
        """

def download(number, save_dir='./'):
    """Download pre-trained word vector
    :param number: integer, default ``None``
    :param save_dir: str, default './'
    :return: file path for downloaded file
    """
    df = load_datasets()

    row = df.iloc[[number]]
    url = ''.join(row.URL)
    if not url:
        print('The word vector you specified was not found. Please specify correct name.')

    widgets = ['Test: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ', ETA(), ' ', FileTransferSpeed()]
    pbar = ProgressBar(widgets=widgets)

    def dlProgress(count, blockSize, totalSize):
        if pbar.max_value is None:
            pbar.max_value = totalSize
            pbar.start()

        pbar.update(min(count * blockSize, totalSize))

    file_name = url.split('/')[-1]
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)
    save_path = os.path.join(save_dir, file_name)
    path, _ = urlretrieve(url, save_path, reporthook=dlProgress)
    pbar.finish()
    return path

def collect_mailinfos(server, folder_contents, outpath_format):
    #construct progressbar
    progressbar_widgets = [
        '[Choosing mails for download  ] ',
        progressbar.Percentage(),
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ', progressbar.ETA()]
    total_count = 0
    for folder, mailids in folder_contents.items():
        total_count += len(mailids)
    progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=total_count).start()

    #collect all mailinfos
    mailinfos = {}
    mailinfo_count = 0
    for folder, mailids in folder_contents.items():
        mailinfos[folder] = []

        #get mailinfo bit by bit
        server.select_folder(folder, readonly=True)
        for mailid in mailids:
            #fetch mail information
            mailinfo = server.fetch([mailid], ['ENVELOPE', 'INTERNALDATE', 'RFC822.SIZE'])[mailid]
            mailsize = bitmath.Byte(mailinfo[b'RFC822.SIZE'])
            mailfilename = construct_mailfilename(outpath_format, mailinfo, args.outdir, folder, mailid)                

            #only add if mailfilename can be constructed
            if mailfilename:
                mailinfos[folder].append((mailid, mailfilename, mailsize))

            mailinfo_count += 1
            progressbar_instance.update(mailinfo_count)

    progressbar_instance.finish()
    return mailinfos

def cleanup(stored_files, stored_dirs, download_list, outdir):
    #create list of files to keep
    keep_list = []
    for folder, mails in download_list.items():
        for mailid, mailfilename, mailsize in mails:
            keep_list.append(mailfilename)

    progressbar_widgets = [
        '[Cleaning up outdir           ] ',
        progressbar.Percentage(),
        progressbar.Bar(marker=progressbar.RotatingMarker()), ' ']
    progressbar_instance = progressbar.ProgressBar(widgets=progressbar_widgets, maxval=len(stored_files)).start()
    file_count = 0

    #delete all files we don't need to keep
    for file in stored_files:
        #delete if not on server
        if not file in keep_list:
            os.remove(file)

        #progressbar
        file_count += 1
        progressbar_instance.update(file_count)
    progressbar_instance.finish()

    #remove empty folders
    possible_empty_folders = True
    while possible_empty_folders:
        #find all subfolders
        stored_dirs = []
        for root, dirs, files in os.walk(outdir):
            for name in dirs:
                stored_dirs.append(os.path.join(root, name))

        #delete empty folders indicate next run if one folder was deleted
        possible_empty_folders = False
        for folder in stored_dirs:
            if not os.listdir(folder):
                shutil.rmtree(folder)
                possible_empty_folders = True

def print_status_stream(title, stream):
    widgets = [title, FormatLabel(''), ' ', Percentage(), ' ', Bar(), ' ', RotatingMarker()]
    bar = None
    if sys.stderr.isatty():
        bar = progressbar.ProgressBar(widgets=widgets, max_value=255)

    def print_error(status):
        print(status['error'])

    def print_status(status):
        progress = status.get('progressDetail')
        if progress:
            widgets[1] = FormatLabel("%12s" % (status['status']))
            prog = int(round(255 * ((progress['current'] / progress['total']))))
            if bar is not None:
                bar.update(prog)

    def print_unknown(status):
        print(status)

    for line in stream:
        status = json.loads(line.decode('utf8'))
        if 'error' in status:
            print_error(status)
        elif 'status' in status:
            print_status(status)
        else:
            print_unknown(status)

def createSQL(table,values,name='insert'):
    '''
    Generate the SQL insert line, breaking each insert to up to ~1k values
    and up to ~1k insert's (~1M values total for each SQL file)
    '''
    logger.info('Generating SQL file')
    queryInsert='INSERT INTO %s (itemid,clock,num,value_min,value_avg,value_max) VALUES' % table
    i=0 # Controls the progress bar
    x=0 # Controls number of inserts in one line
    y=0 # Controls number of lines in one file
    z=0 # Controls number of file name
    valuesLen=values.__len__()
    sqlFile='%s.sql.%d' % (name,z)
    logger.debug('Total itens for %s: %d' % (name,valuesLen))

    if valuesLen > 0:
        bar=ProgressBar(maxval=valuesLen,widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
        for value in values:
            i+=1
            x+=1
            if x != 1: # First line only
                sqlInsert='%s,%s' % (sqlInsert,value)
            else:
                sqlInsert=value
            if y >= 1000: # If there is more than 1k lines, write to new file
                z+=1
                y=0
            if x >= 1000 or i == valuesLen: # If there is more than 1k values or we finished our list, write to file
                sqlFile='%s.sql.%d' % (name,z)
                fileAppend(f=sqlFile,content='%s %s;\n' % (queryInsert,sqlInsert))
                x=0
                y+=1
                sqlInsert=''
            if args.loglevel.upper() != 'DEBUG': # Dont print progressbar if in debug mode
                bar.update(i)
        bar.finish()
    else:
        logger.warning('No values received')

def discovery_disable_all(status=0):
  """
  Alterar status de todos os discoveries *auto*
  Status 0 = enable
  Status 1 = disable
  """ 
  logger.info('Disabling all network discoveries')
  druleids = zapi.drule.get(output=[ 'druleid', 'iprange', 'name', 'proxy_hostid', 'status' ],
      selectDChecks='extend', filter={ 'status': 0 })
  if ( druleids.__len__() == 0 ):
    logger.info('Done')
    return
  bar = ProgressBar(maxval=druleids.__len__(),widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
  i = 0
  for x in druleids:
    params_disable = {
      'druleid': x['druleid'],
      'iprange': x['iprange'],
      'name': x['name'],
      'dchecks': x['dchecks'],
      'status': 1
    }
    out = zapi.drule.update(**params_disable)
    logger.echo = False
    if out:
      logger.debug('\tNew status: %s (%s) --> %d' % (x['name'],out['druleids'],status))
    else:
      logger.warning('\tFAILED to change status: %s (%s) --> %d' % (x['name'],out['druleids'],status))
    i += 1
    bar.update(i)
  logger.echo = True
  bar.finish()
  logger.info('Done')
  return

def desabilitaItensNaoSuportados():
    query = {
            "output": "extend",
            "filter": {
                "state": 1
            },
            "monitored": True
        }

    filtro = raw_input('Qual a busca para key_? [NULL = ENTER] ')
    if filtro.__len__() > 0:
        query['search']={'key_': filtro}

    limite = raw_input('Qual o limite de itens? [NULL = ENTER] ')
    if limite.__len__() > 0:
        try:
            query['limit']=int(limite)
        except:
            print 'Limite invalido'
            raw_input("Pressione ENTER para voltar")
            main()

    opcao = raw_input("Confirma operação? [s/n]")
    if opcao == 's' or opcao == 'S':
        itens = zapi.item.get(query)
        print 'Encontramos {} itens'.format(itens.__len__())
        bar = ProgressBar(maxval=itens.__len__(),widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
        i = 0
        for x in itens:
            result = zapi.item.update({"itemid": x['itemid'], "status": 1})
            i += 1
            bar.update(i)
        bar.finish()
        print "Itens desabilitados!!!"
        raw_input("Pressione ENTER para continuar")
    main()

def __init__(self):
        self.stop_running = threading.Event()
        self.progress_thread = threading.Thread(target=self.init_progress)
        self.progress_thread.daemon = True
        spinner = RotatingMarker()
        spinner.INTERVAL = datetime.timedelta(milliseconds=100)
        self.widgets = [spinner, ' ', Percentage(), ' ', FormatLabel('Calculating patch requirements'), ' ', Bar(), ' ', FormatLabel('')]
        self.progress = ProgressBar(redirect_stdout=True, widgets=self.widgets, max_value=100)
        self.progress.update(0)

def doUploadFileProgress(self,filePath, url):
        startTime = getNow()
        result = False
        try:
            widgets = ['Progress: ', Percentage(), ' ', Bar(
                marker=RotatingMarker('>-=')), ' ', ETA(), ' ', FileTransferSpeed()]
            pbar = ProgressBar(widgets=widgets, maxval=os.path.getsize(filePath)).start()
            progress = Progress()
            fileSizeStr = formatSize(os.path.getsize(filePath))
            logger.info("??????{0} ?? {1}".format(filePath,fileSizeStr))
            stream = file_with_callback(filePath, 'rb', progress.update,os.path.basename(filePath),pbar)
            params = {"filedata": stream}
            datagen, headers = multipart_encode(params)
            upload_request =urllib2.Request(url, datagen, headers)
            response = urllib2.urlopen(upload_request).read()
            endTime = getNow()
            totlaTime = caltime(startTime, endTime)
            logger.info("??????{0} ????{1} ????{2} ????{3} ??{4} ????{5}"
                        .format(filePath,startTime, endTime, fileSizeStr, totlaTime,response))
            #???????????'4b ? 0'???json ??errmsg("Extra data", s, end, len(s) ??????,??????????
            if "code\":0" in response.replace(' ', ''):
                result = True
            else:
                result = json.loads(response)["code"] == 0
        except Exception as e:
            logger.error("??????{0} exception: {1}".format(filePath,e))
        return result

def _addresses_to_check_with_caching(self, show_progress=True):
        num_addrs = len(list(self._addresses_to_check()))
        widgets = ['ROP: ', progressbar.Percentage(), ' ',
                   progressbar.Bar(marker=progressbar.RotatingMarker()),
                   ' ', progressbar.ETA(), ' ', progressbar.FileTransferSpeed()]
        progress = progressbar.ProgressBar(widgets=widgets, maxval=num_addrs)
        if show_progress:
            progress.start()
        self._cache = dict()
        seen = dict()
        for i, a in enumerate(self._addresses_to_check()):
            if show_progress:
                progress.update(i)
            try:
                bl = self.project.factory.block(a)
                if bl.size > self._max_block_size:
                    continue
                block_data = bl.bytes
            except (SimEngineError, SimMemoryError):
                continue
            if block_data in seen:
                self._cache[seen[block_data]].add(a)
                continue
            else:
                if len(bl.vex.constant_jump_targets) == 0 and not self._block_has_ip_relative(a, bl):
                    seen[block_data] = a
                    self._cache[a] = set()
                yield a
        if show_progress:
            progress.finish()

def main():
    '''
    Controls general flow of operations
    '''
    # If it exists, use the cached data of hosts and items
    if (os.path.isfile(move_items_file)):
        with open(move_items_file) as infile:
            hosts=json.load(infile)
            logger.info('Cache loaded from file (%s)' % move_items_file)
    else:
        hosts=getItems()
        with open(move_items_file, 'w') as outfile:
            json.dump(hosts, outfile)
            logger.info('Cache written to file (%s)' % move_items_file)

    for host in hosts:
        logger.info('Geting trends data of host: %s' % host['name'])
        host['trends']=list()
        host['trends_uint']=list()
        if host['itens'].__len__() > 0:
            bar=ProgressBar(maxval=host['itens'].__len__(),widgets=[Percentage(), ReverseBar(), ETA(), RotatingMarker(), Timer()]).start()
            i=0
            for item in host['itens']:
                temp=getTrends(hostname=host['name'],item=item)
                i+=1
                if args.loglevel.upper() != 'DEBUG':
                    bar.update(i)
                if temp['table'] == 'trends':
                    for value in temp['values']:
                        host['trends'].append('(%d, %d, %d, %d, %d, %d)' % (int(item['itemid']), int(value[1]), int(value[2]), int(value[3]), int(value[4]), int(value[5])))
                elif temp['table'] == 'trends_uint':
                    for value in temp['values']:
                        host['trends_uint'].append('(%d, %d, %d, %d, %d, %d)' % (int(item['itemid']), int(value[1]), int(value[2]), int(value[3]), int(value[4]), int(value[5])))
                else:
                    logger.warning('Unknown value type: %s' % temp['table'])
            bar.finish()
        '''
        Now, we send in blocks of up to ~1M values to generate the SQL files
        '''
        if host['trends'].__len__() > 0:
            createSQL(table='trends',values=host['trends'],name=host['name'])
        elif host['trends_uint'].__len__() > 0:
            createSQL(table='trends_uint',values=host['trends_uint'],name=host['name'])
        else:
            logger.warning('No data from %s found to be sent.' % host['name'])



# Start DB connection