Python selenium.webdriver 模块，FirefoxProfile() 实例源码

def chrome(self):
        # https://github.com/SeleniumHQ/selenium/blob/master/py/selenium/webdriver/remote/webdriver.py
        # http://www.guguncube.com/2983/python-testing-selenium-with-google-chrome
        # https://gist.github.com/addyosmani/5336747
        # http://blog.likewise.org/2015/01/setting-up-chromedriver-and-the-selenium-webdriver-python-bindings-on-ubuntu-14-dot-04/
        # https://sites.google.com/a/chromium.org/chromedriver/getting-started
        # http://stackoverflow.com/questions/8255929/running-webdriver-chrome-with-selenium
        chrome = webdriver.Chrome()
        return chrome

#     @property
#     def firefox(self):
#         profile = webdriver.FirefoxProfile()
#         #firefox = webdriver.Firefox(firefox_profile=profile)
#         firefox = WebDriver(firefox_profile=profile)
#         return firefox

def main(username, account_password,destination):
    profile = webdriver.FirefoxProfile()
    profile.set_preference('browser.download.folderList', 2) # custom location
    profile.set_preference('browser.download.manager.showWhenStarting', False)
    profile.set_preference('browser.download.dir', destination)
    profile.set_preference('browser.helperApps.neverAsk.saveToDisk', "image/png,image/jpeg")

    if not username == "NONE" and not account_password == "NONE" and not destination == "NONE":
        display = Display(visible=0, size=(800, 600))
            display.start()
        driver = webdriver.Firefox(firefox_profile=profile)
        driver.get("https://www.facebook.com")

        email_id = driver.find_element_by_id("email")
        password = driver.find_element_by_id("pass")
        email_id.send_keys(username)
        password.send_keys(account_password)
        driver.find_element_by_id("loginbutton").click()
        # driver.find_element_by_css_selector("._5afe.sortableItem").click()
        driver.find_element_by_id("navItem_2305272732").click()
        time.sleep(3)
        list_of_images = driver.find_elements_by_css_selector(".uiMediaThumbImg")
        list_of_images[0].click()
        # print list_of_images
        for image in list_of_images:
            time.sleep(3)
            driver.find_element_by_xpath("//div[@class = 'overlayBarButtons rfloat _ohf']/div/div/a").click()
            time.sleep(3)
            option = driver.find_element_by_xpath("//div[@class = 'uiContextualLayerPositioner uiLayer']/div/div/div[@class = '_54ng']/ul[@class = '_54nf']/li[4]/a")
            option_name = option.find_element_by_xpath(".//*")
            option_name = option_name.find_element_by_xpath(".//*")
            if option_name.get_attribute('innerHTML').lower() == "download":
                option_name.click()
            # print option.get_attribute('innerHTML')
            driver.find_element_by_css_selector(".snowliftPager.next.hilightPager").click()

            display.stop()

    else:
        print "\nIncomplete Parameters, Program is Shutting Down."

def set_firefoxprofile(proxy_ip, proxy_port):
        """method to update the given preferences in Firefox profile"""
        ff_profile = webdriver.FirefoxProfile()
        if proxy_ip is not None and proxy_port is not None:
            proxy_port = int(proxy_port)
            ff_profile.set_preference("network.proxy.type", 1)
            ff_profile.set_preference("network.proxy.http", proxy_ip)
            ff_profile.set_preference("network.proxy.http_port", proxy_port)
            ff_profile.set_preference("network.proxy.ssl", proxy_ip)
            ff_profile.set_preference("network.proxy.ssl_port", proxy_port)
            ff_profile.set_preference("network.proxy.ftp", proxy_ip)
            ff_profile.set_preference("network.proxy.ftp_port", proxy_port)
            ff_profile.update_preferences()
        else:
            ff_profile = None
        return ff_profile

    # private methods

def getFirefox(tempDir='/tmp', showImage=1):
    """get Firefox Webdriver object
    :param showImage: 2 = don't show, 1=show
    """
    proxy = Proxy(dict(proxyType=ProxyType.AUTODETECT))
    profile = webdriver.FirefoxProfile()
    profile.set_preference("plugin.state.flash", 0)
    profile.set_preference("plugin.state.java", 0)
    profile.set_preference("media.autoplay.enabled", False)
    # 2=dont_show, 1=normal
    profile.set_preference("permissions.default.image", showImage)
    profile.set_preference("webdriver.load.strategy", "unstable")
    # automatic download
    # 2 indicates a custom (see: browser.download.dir) folder.
    profile.set_preference("browser.download.folderList", 2)
    # whether or not to show the Downloads window when a download begins.
    profile.set_preference("browser.download.manager.showWhenStarting", False)
    profile.set_preference("browser.download.dir", tempDir)
    profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
                           "application/octet-stream"+
                           ",application/zip"+
                           ",application/x-rar-compressed"+
                           ",application/x-gzip"+
                           ",application/msword")
    return webdriver.Firefox(firefox_profile=profile, proxy=proxy)

def __init__(self, proxy):
        """init the webdriver by setting the proxy and user-agent

        Args:
            proxy (str): proxy in the form of ip:port
        """
        # set proxy
        ip, port = proxy.split(':')
        profile = webdriver.FirefoxProfile()
        profile.set_preference("network.proxy.type", 1)
        profile.set_preference("network.proxy.http", ip)
        profile.set_preference("network.proxy.http_port", port)
        # set user_agent
        profile.set_preference("general.useragent.override", generate_user_agent())

        profile.update_preferences()
        self.driver = webdriver.Firefox(firefox_profile=profile)

        print 'current proxy: %s'%proxy

def driver():

    if exists(screenshot_dir):
        shutil.rmtree(screenshot_dir)
    os.mkdir(screenshot_dir)

    firefox_path = '{0}/firefox/firefox'.format(DIR)
    caps = DesiredCapabilities.FIREFOX
    caps["marionette"] = True
    caps['acceptSslCerts'] = True

    binary = FirefoxBinary(firefox_path)

    profile = webdriver.FirefoxProfile()
    profile.add_extension('{0}/JSErrorCollector.xpi'.format(DIR))
    profile.set_preference('app.update.auto', False)
    profile.set_preference('app.update.enabled', False)
    driver = webdriver.Firefox(profile,
                               capabilities=caps, log_path="{0}/firefox.log".format(LOG_DIR),
                               firefox_binary=binary, executable_path=join(DIR, 'geckodriver/geckodriver'))
    # driver.set_page_load_timeout(30)
    # print driver.capabilities['version']
    return driver

def firefox(headless=True):
    """
    Context manager returning Selenium webdriver.
    Instance is reused and must be cleaned up on exit.
    """
    from selenium import webdriver
    from selenium.webdriver.firefox.options import Options
    if headless:
        driver_key = 'headless'
        firefox_options = Options()
        firefox_options.add_argument('-headless')
    else:
        driver_key = 'headed'
        firefox_options = None
    # Load profile, if it exists:
    if os.path.isdir(PROFILE_DIR):
        firefox_profile = webdriver.FirefoxProfile(PROFILE_DIR)
    else:
        firefox_profile = None
    if FIREFOX_INSTANCE[driver_key] is None:
        FIREFOX_INSTANCE[driver_key] = webdriver.Firefox(
            firefox_profile=firefox_profile,
            firefox_options=firefox_options,
        )
    yield FIREFOX_INSTANCE[driver_key]

def start_selenium_driver(module):

    profile = webdriver.FirefoxProfile()
    profile.set_preference("browser.download.folderList", 2)
    profile.set_preference("browser.download.manager.showWhenStarting", False)
    profile.set_preference("browser.helperApps.neverAsk.saveToDisk", "application/x-gzip")
    profile.set_preference("browser.helperApps.alwaysAsk.force", False);
    profile.set_preference("browser.download.dir", module.params['download_directory'])

    driver = webdriver.Firefox(profile)
    # Lets make sure that firefox is closed at the exit of the module
    atexit.register(driver.close)

    driver.implicitly_wait(30)
    driver.get(module.params['url'])
    return driver

def __init__(self):
        # disable the js of firefox to speed up. it is not necessary to run
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        # get the webdriver of the opened firefox and open the url
        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)
        self.driver.get("http://en.swisswebcams.ch/verzeichnis/traffic/schweiz/beliebt")

        # open the file to store the list and write the format of the list at the first line
        self.f = open('list_swissWebcam_traffic.txt', 'w')
        self.f.write("country#city#snapshot_url#latitude#longitude" + "\n")

        # wait object to use
        self.wait = ui.WebDriverWait(self.driver, 10)

def __init__(self):
        # store the url of homepage and the country code
        self.home_url = "http://en.swisswebcams.ch"
        self.geo_url = "http://map.topin.travel/?p=swc&id="
        self.country = "CH"

        # open the file to store the list and write the format of the list at the first line
        self.f = open('list_swissWebcam.txt', 'w')
        self.f.write("country#city#snapshot_url#latitude#longitude" + "\n")

        # open the Firefox
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)

        # wait object to use
        self.wait = ui.WebDriverWait(self.driver, 10)

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url = "http://infotrafego.pbh.gov.br"
        self.traffic_url = "http://infotrafego.pbh.gov.br/info_trafego_cameras.html"
        self.country = "BR"
        self.state = ""

        # open the file to store the list and write the format of the list at the first line
        self.list_file = open("list_infotrafego_traffic.txt", "w")
        self.list_file.write("city#country#snapshot_url#latitude#longitude" + "\n")

        # get the webdriver of the opened firefox and open the url
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)

        # gps module
        self.gps = Geocoding('Google', None)

        reload(sys)  
        sys.setdefaultencoding('utf8')

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url = "http://www.phillytraffic.com"
        self.traffic_url = "http://www.phillytraffic.com/#!traffic-updates/cjn9"
        self.country = "USA"
        self.state = "PA"

        # open the file to store the list and write the format of the list at the first line
        self.file = open('list_Philadelphia_PA.txt', 'w')
        self.file.write("city#country#state#snapshot_url#latitude#longitude" + "\n")

        # open the brwoser
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)
        self.wait = ui.WebDriverWait(self.driver, 10)

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url = "http://tmc.baycountyfl.gov"
        self.traffic_url = "http://tmc.baycountyfl.gov/"
        self.country = "USA"
        self.state = "FL"

        # open the file to store the list and write the format of the list at the first line
        self.f = open('list_FL_baycounty.txt', 'w')
        self.f.write("city#country#state#snapshot_url#latitude#longitude" + "\n")

        # open the web-browser
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox()
        self.wait = ui.WebDriverWait(self.driver, 10)

        # gps module
        self.gps = Geocoding('Google', None)

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url   = "https://www.theweathernetwork.com"
        self.json_url   = "https://www.theweathernetwork.com/api/maps/trafficcameras/9/43.84598317236631/-80.71453475531251/43.048384299427234/-78.72600936468751"
        self.country    = "CA"
        self.state      = ""

        # open the file to store the list and write the format of the list at the first line
        self.list_file = open('list_Canada_weatherNetwork.txt', 'w')
        self.list_file.write("city#country#snapshot_url#latitude#longitude" + "\n")

        # open the browser
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url = "http://www.wsoctv.com"
        self.traffic_url = "http://www.wsoctv.com/traffic/nc-cams"
        self.country = "USA"
        self.state = "NC"

        # open the file to store the list and write the format of the list at the first line
        self.f = open('list_NorthCarolina_wsoctv.txt', 'w')
        self.f.write("city#country#state#snapshot_url#latitude#longitude" + "\n")

        # open the web-browser
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox()
        self.wait = ui.WebDriverWait(self.driver, 10)

def getProxyDriver(self):

        PROXY_ADDRESS = random.choice(self.proxies.keys())

        address = PROXY_ADDRESS.replace("http://", "").replace("https://", "")

        host = address.split(':')[0]
        port = int(address.split(':')[1])
        profile = webdriver.FirefoxProfile()
        profile.set_preference("network.proxy.type", 1)
        profile.set_preference("network.proxy.http", host)
        profile.set_preference("network.proxy.http_port", port)
        profile.update_preferences()

        self.driver = webdriver.Firefox(firefox_profile=profile)
        self.logger.info("creating driver: [%s] using proxy [%s]" % (self.driver.name, PROXY_ADDRESS))
        self.driver.maximize_window()

def firefox():

        import os

        profile = webdriver.FirefoxProfile()

        profile.set_preference('browser.download.folderList', 2)
        profile.set_preference('browser.download.manager.showWhenStarting', False)
        profile.set_preference('browser.download.dir', os.getcwd())
        profile.set_preference('app.update.auto', False)
        profile.set_preference('app.update.enabled', False)
        profile.set_preference('app.update.silent', False)
        profile.set_preference('browser.helperApps.neverAsk.saveToDisk', 'text/csv/xls/zip/exe/msi')
        profile.set_preference('xpinstall.signatures.required', False)

        return webdriver.Firefox(profile)

def get_browser(self, proxy):
        """ ???????????firefox """
        # ?????
        firefox_profile = webdriver.FirefoxProfile()
        # ????image
        #firefox_profile.set_preference('permissions.default.stylesheet', 2)
        #firefox_profile.set_preference('permissions.default.image', 2)
        #firefox_profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so', 'false')
        # ??
        if proxy.is_valid():
            myProxy = '%s:%s' % (proxy.host, proxy.port)
            ff_proxy = Proxy({
                'proxyType': ProxyType.MANUAL,
                'httpProxy': myProxy,
                'ftpProxy': myProxy,
                'sslProxy': myProxy,
                'noProxy': ''})

            browser = webdriver.Firefox(firefox_profile=firefox_profile, proxy=ff_proxy)
        else:
            browser = webdriver.Firefox(firefox_profile=firefox_profile)

        return browser

def proxy(PROXY_HOST,PROXY_PORT):
    fp = webdriver.FirefoxProfile()
    print "[" + t.green("+") + "]Proxy host set to: " + PROXY_HOST
    print "[" + t.green("+") + "]Proxy port set to: " + PROXY_PORT
    print "\n[" + t.green("+") + "]Establishing connection..."
    fp.set_preference("network.proxy.type", 1)
    fp.set_preference("network.proxy.http",PROXY_HOST)
    fp.set_preference("network.proxy.http_port",int(PROXY_PORT))
    fp.set_preference("general.useragent.override","'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36'")
    fp.update_preferences()
    return webdriver.Firefox(firefox_profile=fp)


# Function to generate and process results based on input

def firefox_driver():
    # Doesn't work with geckodriver! :(
    capabilities = webdriver.DesiredCapabilities().FIREFOX
    capabilities['acceptSslCerts'] = True

    profile = webdriver.FirefoxProfile()
    profile.accept_untrusted_certs = True

    return webdriver.Firefox(firefox_profile=profile, capabilities=capabilities)

def start_foxdr(thread):
    uaList = [line[:-1] for line in open('Base_Data\\ualist.txt')]
    open('Base_Data\\ualist.txt').close()
    i = random.choice(uaList)
    profile = webdriver.FirefoxProfile('c:\\Users\\'+thread)
    profile.set_preference('permissions.default.image', 2)
    profile.set_preference("general.useragent.override", i)
    path1 = 'C:\\Program Files (x86)\\Mozilla Firefox\\geckodriver.exe'
    path2 = 'C:\\Program Files\\Mozilla Firefox\\geckodriver.exe'
    try:
        dr= webdriver.Firefox(executable_path = path1,firefox_profile = profile )
    except:
        dr= webdriver.Firefox(executable_path = path2,firefox_profile = profile )
    return dr,uaList

def getGift(roomid):
    fp = webdriver.FirefoxProfile(
        r'/Users/eclipse/Library/Application Support/Firefox/Profiles/tmsbsjpg.default')
    browser = webdriver.Firefox(fp)
    browser.implicitly_wait(15)  # seconds
    browser.get("http://www.douyu.com/" + roomid)
    try:
        indexvideo = browser.find_element_by_class_name('cs-textarea')
        print type(indexvideo)
        indexvideo.send_keys('2333333333333')
        print indexvideo
        time.sleep(5)
        sendbut = browser.find_element_by_class_name('b-btn')
        ActionChains(browser).move_to_element(
            indexvideo).click(sendbut).perform()
        gift = browser.find_element_by_class_name('peck-cdn')
    except Exception, e:
        print str(e)
        browser.quit()

    times = 0
    while True:
        try:
            ActionChains(browser).move_to_element(gift).click(gift).perform()
            time.sleep(1)
            print times
            times += 1
        except Exception, e:
            print 'completed by an error'
            browser.quit()

def __init__(self, download_dir, *mime_types):
        """
        Create a Firefox webdriver that downloads into the path download_dir,
        and initiates downloads automatically for any of the given MIME types.
        """
        self.download_dir = download_dir
        self.profile = webdriver.FirefoxProfile()
        self.profile.set_preference('browser.helperApps.neverAsk.saveToDisk',
                                    ', '.join(mime_types))

        # Enable setting the default download directory to a custom path.
        self.profile.set_preference('browser.download.folderList', 2)

        self.profile.set_preference('browser.download.dir', self.download_dir)
        super().__init__(self.profile)

def getFirefoxBrowser():
    proxy = {'host': "proxy.abuyun.com", 'port': 9020, 'usr': "HWJB1R49VGL78Q3D", 'pwd': "0C29FFF1CB8308C4"}

    # userProfileFilePath = r"C:\Users\LZ\AppData\Roaming\Mozilla\Firefox\Profiles\vbqy66hj.default"
    # kadUserProfileFilePath = r"C:\Users\zml\AppData\Roaming\Mozilla\Firefox\Profiles\lotur5zd.default"
    fp = webdriver.FirefoxProfile(localData.FirefoxUserProfileFilePath)

    fp.add_extension('resource/closeproxy.xpi')

    fp.set_preference('network.proxy.type', 1)
    fp.set_preference('network.proxy.http', proxy['host'])
    fp.set_preference('network.proxy.http_port', int(proxy['port']))
    fp.set_preference('network.proxy.ssl', proxy['host'])
    fp.set_preference('network.proxy.ssl_port', int(proxy['port']))
    fp.set_preference('network.proxy.ftp', proxy['host'])
    fp.set_preference('network.proxy.ftp_port', int(proxy['port']))       
    fp.set_preference('network.proxy.no_proxies_on', 'localhost, 127.0.0.1')



# ???????????
    fp.set_preference('permissions.default.image', 2)

    credentials = '{usr}:{pwd}'.format(**proxy)
    credentials = b64encode(credentials.encode('ascii')).decode('utf-8')
    fp.set_preference('extensions.closeproxyauth.authtoken', credentials)
    browser = webdriver.Firefox(executable_path="resource/geckodriver", firefox_profile=fp)
    # browser = webdriver.Firefox(executable_path="geckodriver")

    browser.set_page_load_timeout(1)
    return browser

def load_firefox(config):
    """Start Firefox webdriver with the given configuration.

    Args:
        config (dict): The configuration loaded previously in Cabu.

    Returns:
        webdriver (selenium.webdriver): An instance of Firefox webdriver.

    """
    binary = None
    profile = webdriver.FirefoxProfile()

    if os.environ.get('HTTPS_PROXY') or os.environ.get('HTTP_PROXY'):
        proxy_address = os.environ.get('HTTPS_PROXY', os.environ.get('HTTP_PROXY'))
        proxy_port = re.search('\:([0-9]+)$', proxy_address).group(1)

        profile.set_preference('network.proxy.type', 1)
        profile.set_preference(
            'network.proxy.http',
            proxy_address
        )
        profile.set_preference('network.proxy.http_port', proxy_port)
        profile.update_preferences()

    if 'HEADERS' in config and config['HEADERS']:
        profile = Headers(config).set_headers(profile)

    if config['DRIVER_BINARY_PATH']:
        from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
        binary = FirefoxBinary(config['DRIVER_BINARY_PATH'])

    return webdriver.Firefox(firefox_binary=binary, firefox_profile=profile)

def test_firefox_headers_loading(self):
        self.app.config['DRIVER_NAME'] = 'Firefox'
        profile = webdriver.FirefoxProfile()
        headers = Headers(self.app.config)
        profile = headers.set_headers(profile)
        self.assertEquals(
            profile.__dict__['default_preferences']['general.useragent.override'],
            'Mozilla/6.0 (Macintosh; Intel Mac OS X 10_11_3) AppleWebKit/537.36'
            ' (KHTML, like Gecko) Chrome/48.0.2564.103 Safari/537.36'
        )

def __init__(self, filename):
        config = ConfigParser.ConfigParser()
        config.read(filename)
        self.LOGIN_USER_VALUE = config.get('credentials', 'login_user_value')
        self.LOGIN_PASS_VALUE = config.get('credentials', 'login_pass_value')
        # self.client = fbchat.Client(self.LOGIN_USER_VALUE, self.LOGIN_PASS_VALUE)
        profile = webdriver.FirefoxProfile()
        profile.set_preference("javascript.enabled", False)
        self.driver = webdriver.Firefox(profile)
        # self.driver = webdriver.PhantomJS()
        # self.driver = webdriver.Chrome('./chromedriver')
        self.driver.set_page_load_timeout(self.TIMEOUT)

def get_proxy_browser():
    """ ?????????????? """
    global proxies
    global meta_info

    if not proxies:
        proxies = get_proxy(if_force=True)

    while 1:
        _, meta_info = generate_proxy(proxies)
        host, port, http_method = meta_info
        try:
            profile = webdriver.FirefoxProfile()
            profile.set_preference('network.proxy.type', 1)   # 0 => direct connect, 1 => use config defautl to 0
            if http_method == 'HTTP':
                profile.set_preference('network.proxy.socks', host)
                profile.set_preference('network.proxy.socks_port', port)
            elif http_method == 'HTTPS':
                profile.set_preference('network.proxy.ssl', host)
                profile.set_preference('network.proxy.ssl_port', port)
            profile.update_preferences()
            browser = webdriver.Firefox(profile)
            browser.get('http://weixin.sogou.com')
            return browser
        except:
            print meta_info, 'was failed, now is going to choose another one'
            proxies.remove(meta_info)
            print 'Still have ', len(proxies), 'proxies'
            if not proxies:
                proxies = get_proxy(if_force=True)
            _, meta_info = generate_proxy(proxies)

def setProxy(self, proxy):
        profile = FirefoxProfile()
        profile.accept_untrusted_certs = True
        profile.assume_untrusted_cert_issuer = True
        prefix = "network.proxy."
        profile.set_preference("%stype" % prefix, 1)
        for type in ["http", "ssl", "ftp", "socks"]:
            profile.set_preference("%s%s" % (prefix, type), proxy.getHost())
            profile.set_preference("%s%s_port" % (prefix, type), int(proxy.getPort()))
        return profile

def new_firefox_driver(self, javascript_enabled=True):
        assert javascript_enabled, 'Cannot disable javascript anymore, see: https://github.com/seleniumhq/selenium/issues/635'
        from selenium.webdriver import FirefoxProfile, DesiredCapabilities

        # FF does not fire events when its window is not in focus.
        # Native events used to fix this.
        # After FF34 FF does not support native events anymore
        # We're on 48.0 now on local machines, but on 31 on travis

        fp = FirefoxProfile()
#        fp.set_preference("focusmanager.testmode", False)
#        fp.set_preference('plugins.testmode', False)
        fp.set_preference('webdriver_enable_native_events', True)
        fp.set_preference('webdriver.enable.native.events', True)
        fp.set_preference('enable.native.events', True)
        fp.native_events_enabled = True

        fp.set_preference('network.http.max-connections-per-server', 1)
        fp.set_preference('network.http.max-persistent-connections-per-server', 0)
        fp.set_preference('network.http.spdy.enabled', False)
        fp.set_preference('network.http.pipelining', True)
        fp.set_preference('network.http.pipelining.maxrequests', 8)
        fp.set_preference('network.http.pipelining.ssl', True)
        fp.set_preference('html5.offmainthread', False)

        dc = DesiredCapabilities.FIREFOX.copy()

        if not javascript_enabled:
            fp.set_preference('javascript.enabled', False)
            dc['javascriptEnabled'] = False

        wd = webdriver.Firefox(firefox_profile=fp, capabilities=dc)
        self.reahl_server.install_handler(wd)
        return wd

def browse(url, cookie, ua):
    domain = ".".join(url.split("/")[2].split(".")[-2:])
    cookies = cookie_dict(cookie, domain)
    profile = webdriver.FirefoxProfile()
    profile.set_preference("general.useragent.override", ua)
    browser = webdriver.Firefox(profile)
    browser.get(url)
    browser.delete_all_cookies()
    for c in cookies:
        try:
            browser.add_cookie(c)
        except:
            pass
    browser.get(url)

def _make_ff(self , remote , desired_capabilites , profile_dir):

        if not profile_dir: profile_dir = FIREFOX_PROFILE_DIR
        profile = webdriver.FirefoxProfile(profile_dir)
        if remote:
            browser = self._create_remote_web_driver(webdriver.DesiredCapabilities.FIREFOX  ,
                        remote , desired_capabilites , profile)
        else:
            browser = webdriver.Firefox(firefox_profile=profile)
        return browser

def getWebDriverProfile(self):
        profile = webdriver.FirefoxProfile()
        #profile.set_preference("browser.cache.disk.enable", False)
        #profile.set_preference("browser.cache.memory.enable", False)
        #profile.set_preference("browser.cache.offline.enable", False)
        #profile.set_preference("network.http.use-cache", False)
        return profile

def add_proxy_to_profile(self, profile):
        if profile is None:
            self.profile = webdriver.FirefoxProfile()
            self.profile.set_proxy(self.proxy.selenium_proxy())
        else:
            self.profile = profile
            self.profile.set_proxy(self.proxy.selenium_proxy())

def __init__(self):
        # store the url of homepage, traffic page, the country code, and the state code
        self.home_url = "http://www.insecam.org"
        self.traffic_url = "http://www.insecam.org/"

        # open the file to store the list and write the format of the list at the first line
        self.us = open('list_insecam_US.txt', 'w')
        self.us.write("city#country#state#snapshot_url#latitude#longitude" + "\n")

        self.ot = open('list_insecam_Other.txt', 'w')
        self.ot.write("city#country#snapshot_url#latitude#longitude" + "\n")

        # open the web-driver
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference("browser.download.folderList", 2)
        firefox_profile.set_preference("javascript.enabled", False)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile)

        # list of categories that it will parse and list of image URLs that should not be parsed
        self.categories = ['City', 'Village', 'River' 'Square', 'Construction', 'Bridge', 'Nature', 'Mountain', 'Traffic', 'Street', 'Road', 'Architecture', 'Port', 'Beach']
        self.invalid = ['http://admin:@50.30.102.221:85/videostream.cgi',
                        'http://198.1.4.43:80/mjpg/video.mjpg?COUNTER',
                        'http://97.76.101.212:80/mjpg/video.mjpg?COUNTER',
                        'http://24.222.206.98:1024/img/video.mjpeg?COUNTER',
                        'http://71.43.210.90:80/SnapshotJPEG?Resolution=640x480&Quality=Clarity&1467044876',
                        'http://61.149.161.158:82/mjpg/video.mjpg?COUNTER',
                        "http://213.126.67.202:1024/oneshotimage1",
                        "http://71.90.110.144:8080/img/video.mjpeg?COUNTER",
                        "http://95.63.206.142:80/mjpg/video.mjpg?COUNTER",
                        "http://201.229.94.197:80/mjpg/video.mjpg?COUNTER"
                       ]

def test_make_profile(self):
        profile = BingAutomater.make_profile()

        profile_type = type(profile)
        is_it_equal = profile_type == type(webdriver.FirefoxProfile())

        javascript_enabled = profile.default_preferences.get('javascript.enabled', None)
        max_script_run_time = profile.default_preferences.get('dom.max_script_run_time', None) 


        self.assertTrue(is_it_equal)
        self.assertFalse(javascript_enabled)
        self.assertEqual(max_script_run_time, 0)

def initialize_driver(url, userInfo, passInfo, prof = None ):
    """ signs into outlook and returns driver
        Optional argument of prof can change UA of driver
    """

    default_prof = webdriver.FirefoxProfile()
    default_prof.set_preference("dom.max_chrome_script_run_time", 0)
    default_prof.set_preference("dom.max_script_run_time", 0)
    default_prof.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
                                      'false')
    default_prof.set_preference("javascript.enabled", False);

    profile_to_use = prof if prof is not None else default_prof
    profile_to_use.add_extension(extension=adblock_xpi)


    driver = webdriver.Firefox(profile_to_use)

    time.sleep(10)
    driver.get("http://www.outlook.com")

    try:
        user = driver.find_element_by_name("loginfmt")
        pass_ = driver.find_element_by_name("passwd")
        user.send_keys(userInfo)
        pass_.send_keys(passInfo)
        time.sleep(5)
        user.submit()
    except (NoSuchElementException, TimeoutException) as err:
        print("Couldn't initialize browser: %s", err)

    time.sleep(10)
    return driver

def setup_mobile_profile():
    """ Sets up a profile to use with driver, returns profile"""
    prof = webdriver.FirefoxProfile()
    ua_string = MOBILE_UA
    prof.set_preference("general.useragent.override", ua_string)
    prof.set_preference("dom.max_chrome_script_run_time", 0)
    prof.set_preference("dom.max_script_run_time", 0)
    prof.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
                                      'false')
    return prof

def make_profile():
    """ Set up a profile and return it """

    profile = webdriver.FirefoxProfile()
    profile.set_preference("dom.max_chrome_script_run_time", 0)
    profile.set_preference("dom.max_script_run_time", 0)
    profile.set_preference('dom.ipc.plugins.enabled.libflashplayer.so',
                                      'false')
    profile.set_preference("javascript.enabled", False)


    #profile_to_use.add_extension(extension=adblock_xpi)
    return profile

def init_tbb_profile(self, version):
        profile_directory = cm.get_tbb_profile_path(version)
        self.prof_dir_path = clone_dir_with_timestap(profile_directory)
        if self.capture_screen and self.page_url:
            self.add_canvas_permission()
        try:
            tbb_profile = webdriver.FirefoxProfile(self.prof_dir_path)
        except Exception:
            wl_log.error("Error creating the TB profile", exc_info=True)
        else:
            return tbb_profile

def init_tbb_profile(self, version):
        profile_directory = cm.get_tbb_profile_path(version)
        self.prof_dir_path = clone_dir_with_timestap(profile_directory)
        if self.capture_screen and self.page_url:
            self.add_canvas_permission()
        try:
            tbb_profile = webdriver.FirefoxProfile(self.prof_dir_path)
        except Exception:
            wl_log.error("Error creating the TB profile", exc_info=True)
        else:
            return tbb_profile

def get_firefox_driver(path = None, selenium_grid_hub = None, no_proxy=False):

    if selenium_grid_hub:
        desired_capabilities={
            "browserName": "firefox",
            "javascriptEnabled": True,
            "proxy": {
                "proxyType": "direct" if no_proxy else "system"
            }
        }

        profile = webdriver.FirefoxProfile()
        profile.set_preference("network.http.phishy-userpass-length", 255);

        return webdriver.Remote(command_executor=selenium_grid_hub, desired_capabilities=desired_capabilities, browser_profile=profile)
    else:
        binary = None
        if path:
            binary = FirefoxBinary(path) #, log_file=open("/tmp/bat_firefox", 'a'))
        profile = webdriver.FirefoxProfile()
        profile.set_preference("network.http.phishy-userpass-length", 255);
        profile.set_preference("network.proxy.type", 0)
        capabilities = None
        if USE_MARIONETTE:
            # tell webdriver to use Marionette instead of plain Firefox
            capabilities = DesiredCapabilities.FIREFOX
            capabilities["marionette"] = True
        return webdriver.Firefox(firefox_profile=profile, firefox_binary=binary, capabilities=capabilities)

def login(email, password):
    """
    Performs a Login to the Facebook platform.
    :param email: The used email account.
    :param password: Its password
    :return: Returns the logged Selenium web driver and the user name string.
    """
    logger.info('Init Firefox Browser')
    profile = webdriver.FirefoxProfile()
    profile.set_preference('dom.disable_beforeunload', True)
    driver = webdriver.Firefox(profile)

    driver.get('https://www.facebook.com')

    logger.info('Log in - Searching for the email input')
    get_by_xpath(driver, '//input[@id="email"]').send_keys(email)

    logger.info('Log in - Searching for the password input')
    get_by_xpath(driver, '//input[@id="pass"]').send_keys(password)

    logger.info('Log in - Searching for the submit button')
    get_by_xpath(driver, '//input[@type="submit"]').click()

    logger.info('Log in - get the user name')
    user_name = get_by_xpath(driver, '//a[@class="fbxWelcomeBoxName"]').text

    logger.info('Log in - Saving the username, which is: %s' % user_name)
    return driver, user_name

def get_driver(self):
        if self.driver_name == 'PhantomJS':
            # These capabilities were part of a failed attempt to make the
            # PhantomJS driver work.
            cap = webdriver.DesiredCapabilities.PHANTOMJS
            cap["phantomjs.page.settings.resourceTimeout"] = 20000
            cap["phantomjs.page.settings.userAgent"] = \
                ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5)'
                 ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116'
                 ' Safari/537.36')
            return webdriver.PhantomJS(desired_capabilities=cap)
        elif self.driver_name == 'Chrome':
            driver = webdriver.Chrome()
            driver.set_window_size(1700, 900)
        elif self.driver_name == 'Chrome-Hub':
            capabilities = DesiredCapabilities.CHROME.copy()
            capabilities["chrome.switches"] = [
                "--start-maximized",
                '--ignore-certificate-errors',
                '--test-type']
            driver = webdriver.Remote(
                command_executor=os.environ.get('HUB_ADDRESS'),
                desired_capabilities=capabilities)
            driver.set_window_size(1200, 900)
        elif self.driver_name == 'Firefox':
            fp = webdriver.FirefoxProfile()
            fp.set_preference("dom.max_chrome_script_run_time", 0)
            fp.set_preference("dom.max_script_run_time", 0)
            driver = webdriver.Firefox(firefox_profile=fp)
        elif self.driver_name == 'Firefox-Hub':
            driver = webdriver.Remote(
                command_executor=os.environ.get('HUB_ADDRESS'),
                desired_capabilities=DesiredCapabilities.FIREFOX)
        else:
            driver = getattr(webdriver, self.driver_name)()
        driver.set_script_timeout(10)
        self.all_drivers.append(driver)
        return driver

def FirefoxProfileSettings():
    profile = webdriver.FirefoxProfile()
    profile.set_preference('network.proxy.type', 1)
    profile.set_preference('network.proxy.socks', '127.0.0.1')
    profile.set_preference('network.proxy.socks_port', 9050)

    return profile

def FirefoxProfileSettings():
    profile=webdriver.FirefoxProfile()
    profile.set_preference('network.proxy.type', 1)
    profile.set_preference('network.proxy.socks', '127.0.0.1')
    profile.set_preference('network.proxy.socks_port', 9050)

    return profile

def init_driver():
    profile = webdriver.FirefoxProfile('/Users/MinWooKim/Library/Application Support/Firefox/Profiles/axkhqz5b.default')
    profile.set_preference('browser.download.folderList', 2)
    profile.set_preference('browser.download.dir', '~/Desktop/polls')
    driver = webdriver.Firefox(profile)
    driver.wait = WebDriverWait(driver, 2)
    return driver

def _get_Firefox(self):
        try:
            if self.proxy:
                profile = webdriver.FirefoxProfile()
                profile.set_preference(
                    "network.proxy.type",
                    1
                )  # this means that the proxy is user set
                if self.proxy.proto.lower().startswith('socks'):
                    profile.set_preference(
                        "network.proxy.socks",
                        self.proxy.host
                    )
                    profile.set_preference(
                        "network.proxy.socks_port",
                        self.proxy.port
                    )
                    profile.set_preference(
                        "network.proxy.socks_version",
                        5 if self.proxy.proto[-1] == '5' else 4
                    )
                    profile.update_preferences()
                elif self.proxy.proto == 'http':
                    profile.set_preference(
                        "network.proxy.http",
                        self.proxy.host
                    )
                    profile.set_preference(
                        "network.proxy.http_port",
                        self.proxy.port
                    )
                else:
                    raise ValueError('Invalid protocol given in proxyfile.')
                profile.update_preferences()
                self.webdriver = webdriver.Firefox(firefox_profile=profile)
            else:
                self.webdriver = webdriver.Firefox()
            return True
        except WebDriverException as e:
            # no available webdriver instance.
            logger.error(e)
        return False

def __init__(self, running, browser, send_mails, format_list):

        super(CollectorProcess, self).__init__()

        self.format_list = format_list
        self.running = running
        self.send_mails = send_mails

        # Setting "Downloads" path up
        self.download_path = os.path.abspath(os.path.join(os.curdir, "RAW-collection"))

        if not os.path.exists(self.download_path):
            os.mkdir(self.download_path)

        # Selecting browser
        self.driver = None
        if browser == "chrome":
            chrome_options = webdriver.ChromeOptions()
            preferences = {"download.default_directory": self.download_path}
            chrome_options.add_experimental_option("prefs", preferences)

            # Loading the page
            self.driver = webdriver.Chrome(chrome_options=chrome_options)
        elif browser == "firefox":
            profile = webdriver.FirefoxProfile()

            profile.set_preference("browser.download.folderList", 2)
            profile.set_preference("browser.download.dir", self.download_path)
            profile.set_preference("browser.download.manager.alertOnEXEOpen", False)
            profile.set_preference("browser.helperApps.neverAsk.saveToDisk",
                                   "application/msword, application/csv, application/ris, text/csv, image/png, " +
                                   "application/pdf, text/html, text/plain, application/zip, application/x-zip, " +
                                   "application/x-zip-compressed, application/download, application/octet-stream")
            profile.set_preference("browser.download.manager.showWhenStarting", False)
            profile.set_preference("browser.download.manager.focusWhenStarting", False)
            profile.set_preference("browser.download.useDownloadDir", True)
            profile.set_preference("browser.helperApps.alwaysAsk.force", False)
            profile.set_preference("browser.download.manager.alertOnEXEOpen", False)
            profile.set_preference("browser.download.manager.closeWhenDone", True)
            profile.set_preference("browser.download.manager.showAlertOnComplete", False)
            profile.set_preference("browser.download.manager.useWindow", False)
            profile.set_preference("services.sync.prefs.sync.browser.download.manager.showWhenStarting", False)
            profile.set_preference("pdfjs.disabled", True)

            self.driver = webdriver.Firefox(firefox_profile=profile)
        else:
            raise ValueError('Browser can only be either "chrome" or "firefox"')

        self.currently_open_battles = set()
        self.leave_que = []

        self.format_index = -1
        self.format_length = len(self.format_list)

def start(self,testhost='',browsertype='',implicity_wait_timeout=IMPLICITY_WAIT_TIMEOUT):
        '''
        To open a browser
        '''
        browser = None
        # lists={
        #      'http://192.168.195.2:8888/wd/hub':'firefox',
        #      #'http://172.16.142.241:7777/wd/hub':'chrome',
        #      }
        # for host,bsr in lists.items():
        #     print host,bsr
        if testhost=='':
            if browsertype.startswith('Chrome'):
                chromedriver = os.path.join(os.environ["AUTODIR"], "webdriver", "chromedriver.exe")
                #Cloud_DebugLog.debug_print("To print the chromedirver path: " + chromedriver)
                chromedriver = os.path.abspath(chromedriver)
                os.environ["webdriver.chrome.driver"] = chromedriver
                chrome_options = Options()
                chrome_options.add_argument("--ignore-certificate-errors")
                chrome_options.add_argument("--disable-popup-blocking")
                options = webdriver.ChromeOptions()
                # set some options
                #driver = webdriver.Remote(desired_capabilities=options.to_capabilities())
                options.add_argument("--always-authorize-plugins")            
                #for opt in options.arguments():
                #    Cloud_DebugLog.info_print("option : " + opt)

                browser_chrome = webdriver.Chrome(chromedriver, chrome_options=options)            
                browser = browser_chrome
            else:
                fp = webdriver.FirefoxProfile()
                fp.set_preference("browser.download.folderList",2)
                fp.set_preference("browser.download.manager.showWhenStarting",False)
                fp.set_preference("browser.download.dir","d:\\test") 
                fp.set_preference("browser.helperApps.neverAsk.saveToDisk","application/binary")
                #fp.set_preference("browser.helperApps.alwaysAsk.force", False);
                browser = webdriver.Firefox(firefox_profile=fp)

            if not browser:
                raise(TestError("No browser opened"))
        else:
            browser = Remote(
                          command_executor=testhost+"/wd/hub",
                          desired_capabilities={'platform':'ANY',
                                                'browserName':browsertype,
                                                'version': '', 
                                                'javascriptEnabled':True
                                            }
                                )
            #driver = Remote(command_executor='http://127.0.0.1:4444/wd/hub',desired_capabilities={'platform': 'ANY','browserName':'chrome',  'version': '', 'javascriptEnabled':True})


        browser.implicitly_wait(implicity_wait_timeout)
        browser.maximize_window()
        browser.get(self.url)

        return browser