Python selenium.webdriver 模块，Proxy() 实例源码

def get_html_by_webdirver(url, proxies = ''):
    html = None
    try:

        driver = webdriver.PhantomJS()

        if proxies:
            proxy=webdriver.Proxy()
            proxy.proxy_type=ProxyType.MANUAL
            proxy.http_proxy= proxies  #'220.248.229.45:3128'
            #????????webdriver.DesiredCapabilities.PHANTOMJS?
            proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
            driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)

        driver.get(url)
        html = driver.page_source
        # driver.save_screenshot('1.png')   #????
        driver.close()
    except Exception as e:
        log.error(e)
    return html and len(html) < 1024 * 1024 and html or None

def process_request(self, request, spider):
        if spider.name == "gsxt":
            # print("PhantomJS is starting...")
            # driver = webdriver.PhantomJS(r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")   # OK
            driver = webdriver.Chrome(r"/home/lxw/Software/chromedirver_selenium/chromedriver") # OK

            """
            # Using IP Proxies:
            # ????chrome?????chrome???IP?????????????????
            # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
            proxy = webdriver.Proxy()
            proxy.proxy_type = ProxyType.MANUAL
            req = requests.get("http://datazhiyuan.com:60001/plain", timeout=10)
            print("Get an IP proxy:", req.text)

            if req.text:
                proxy.http_proxy = req.text  # "1.9.171.51:800"
            # ????????webdriver.DesiredCapabilities.PHANTOMJS?
            proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
            driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
            """

            driver.get(request.url) # ????????????, ??http://roll.news.qq.com/??
            time.sleep(2)
            js = "var q=document.documentElement.scrollTop=10000"
            driver.execute_script(js)   # ???js????????????????????
            time.sleep(3)
            body = driver.page_source
            print("??" + request.url)
            return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
        else:
            return

def reflashProxy(caps, driver, pIPs):
    if len(pIPs) < minPIPCount:
        # ??ip???????
        pIPs = getAvailableIPs()
    # pipObj = random.choice(pIPs)
    randomPIpIndex = random.randint(0, len(pIPs))
    pipObj = pIPs[randomPIpIndex]
    pIp = pipObj[0]
    pPort = pipObj[1]
    ua = random.choice(USER_AGENTS)
    caps["phantomjs.page.settings.userAgent"] = ua
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    proxy.http_proxy = pIp + ':' + str(pPort)
    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    proxy.add_to_capabilities(caps)
    driver.start_session(caps)
    return pIPs, pIp, randomPIpIndex

def start_browser():
    webdriver.DesiredCapabilities.PHANTOMJS['phantomjs.page.customHeaders.{}'.format('User-Agent')] = random.choice(user_agent_list)
    browser = webdriver.PhantomJS()
    # proxy = webdriver.Proxy()
    # proxy.proxy_type = ProxyType.MANUAL
    # proxy.http_proxy = '127.0.0.1:56923'
    # proxy.add_to_capabilities( webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.implicitly_wait(120)
    browser.set_page_load_timeout(120)
    return browser

def main():
    # browser = webdriver.PhantomJS()   # Be OK in command line, but not in PyCharm.
    # browser = webdriver.PhantomJS(r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")
    browser = webdriver.Chrome(r"/home/lxw/Software/chromedirver_selenium/chromedriver") # OK
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n")

    # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    # req = requests.get("http://datazhiyuan.com:60001/plain", timeout=10)
    req = requests.get("http://localhost:60001/plain", timeout=10)
    print("Get an IP proxy:", req.text)
    if req.text:
        proxy.http_proxy = req.text    # '1.9.171.51:800'
    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n")

    # ???????
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.DIRECT
    proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
    browser.get("http://ipecho.net/plain")
    print('session_id: ', browser.session_id)
    print('page_source: ', browser.page_source)
    print('cookie: ', browser.get_cookies())
    print("----"*10, "\n")

def get_driver_phantomjs():
    """
    References:
    PhantomJS:
    1. [??PHANTOMJS?USER-AGENT](http://smilejay.com/2013/12/set-user-agent-for-phantomjs/)
    2. [Selenium 2 - Setting user agent for IE and Chrome](http://stackoverflow.com/questions/6940477/selenium-2-setting-user-agent-for-ie-and-chrome)
    """
    dcap = dict(DesiredCapabilities.PHANTOMJS)

    # Setting User-Agent
    ua = random.choice(RotateUserAgentMiddleware.user_agent_list)
    if ua:
        print("Current User-Agent is:", ua)
        dcap["phantomjs.page.settings.userAgent"] = ua

    driver = webdriver.PhantomJS(executable_path=r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs", desired_capabilities=dcap)

    """
    # Setting IP Proxies
    # ??DesiredCapabilities(????)??????????sessionId????????????????????????????url
    proxy = webdriver.Proxy()
    proxy.proxy_type = ProxyType.MANUAL
    ip_proxy = get_proxy()
    if ip_proxy:
        proxy.http_proxy = ip_proxy

    # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    # proxy.add_to_capabilities(DesiredCapabilities.PHANTOMJS)
    # driver.start_session(DesiredCapabilities.PHANTOMJS)
    proxy.add_to_capabilities(dcap)
    driver.start_session(dcap)
    """

    # ??????
    driver.set_page_load_timeout(TIMEOUT)
    driver.set_script_timeout(TIMEOUT)  # ???????????

    return driver

def get_driver_chrome(self):
        # chromedriver
        options = webdriver.ChromeOptions()
        proxy = get_proxy()
        # NOTE: ??"http"?"https"??????????http?????https
        self.proxies["http"] = proxy
        self.proxies["https"] = proxy
        if proxy:
            options.add_argument('--proxy-server=' + proxy)

        display = Display(visible=0, size=(800, 800))
        display.start()
        driver = webdriver.Chrome(executable_path=r"/home/lxw/Software/chromedriver_selenium/chromedriver", chrome_options=options)

        """
        # PhantomJS: Not working. why?
        driver = webdriver.PhantomJS(executable_path=r"/home/lxw/Downloads/phantomjs/phantomjs-2.1.1-linux-x86_64/bin/phantomjs")
        proxy = webdriver.Proxy()
        proxy.proxy_type = ProxyType.MANUAL
        proxy_str = get_proxy()
        if proxy_str:
            proxy.http_proxy = proxy_str
        # ????????webdriver.DesiredCapabilities.PHANTOMJS?
        proxy.add_to_capabilities(webdriver.DesiredCapabilities.PHANTOMJS)
        driver.start_session(webdriver.DesiredCapabilities.PHANTOMJS)
        """

        # ??????
        driver.set_page_load_timeout(self.TIMEOUT)
        driver.set_script_timeout(self.TIMEOUT)  # ???????????
        return driver

def getNewBrowserArgs():
    pIPs = getAvailableIPs()
    pipObj = random.choice(pIPs)
    # randomPIpIndex = random.randint(0, len(pIPs))
    # pipObj = pIPs[randomPIpIndex]
    pIp = pipObj[0]
    pPort = pipObj[1]
    # ua = random.choice(USER_AGENTS)
    # caps["phantomjs.page.settings.userAgent"] = ua
    # proxy = webdriver.Proxy()
    # proxy.proxy_type = ProxyType.MANUAL
    # proxy.http_proxy = pIp + ':' + str(pPort)
    # # ????????webdriver.DesiredCapabilities.PHANTOMJS?
    # proxy.add_to_capabilities(caps)
    # driver.start_session(caps)

    # if driver:
    #     try:
    #         # driver.close()
    #         driver.quit()
    #     except Exception as er:
    #         print er
    caps = webdriver.DesiredCapabilities.PHANTOMJS

    ua = random.choice(USER_AGENTS)
    caps["phantomjs.page.settings.userAgent"] = ua

    service_args = [
        '--proxy=' + pIp + ':' + str(pPort),
        '--proxy-type=http',
    ]

    return service_args,caps

def test_browser(browser, stats, binary, urls,
                 per_tab_pause, settle_wait_time,
                 proxy, process_count):

    test_options = {
        'per_tab_pause': per_tab_pause,
        'settle_wait_time': settle_wait_time
    }

    if browser == 'Chrome':
        options = webdriver.chrome.options.Options()
        options.binary_location = binary
        caps = options.to_capabilities()

        if proxy:
            webdriver_proxy = webdriver.Proxy()
            webdriver_proxy.http_proxy = proxy
            webdriver_proxy.add_to_capabilities(caps)

        driver = webdriver.Chrome(desired_capabilities=caps)

        test = MultiTabTest(driver, stats, **test_options)
        test.open_urls(urls)

        driver.quit()
    elif browser == 'Firefox':
        for count in process_count:
            print "FIREFOX WITH %d CONTENT PROCESSES" % count
            test = FirefoxMultiTabTest(binary, stats, proxy=proxy, process_count=count, **test_options)
            test.open_urls(urls)
    elif browser in ('Safari', 'IE'):
        # Currently this is a manual test, sorry.
        manual_test = os.path.abspath(os.path.join(
            os.path.dirname(__file__), 'comp_analysis_manual_test.htm'))
        test = ManualMultiTabTest(binary, stats, **test_options)
        prefix = 'file://' if browser == 'IE' else ''
        test.open_urls([prefix + manual_test])
    elif browser == 'Edge':
        # Currently this is even more manual than IE and Safari. Edge won't
        # let us provide a path to launch.
        print "Open up explorer, find 'atsy/example/comp_analysis_manual_test.htm'"
        print "Right-click, 'Open with' -> 'Microsoft Edge'"
        print "Run the test, press enter when it's done."
        import sys
        sys.stdin.read(1)
        stats.print_stats()
    else:
        raise Exception("Unhandled browser: %s" % browser)