Python selenium.webdriver 模块，Chrome() 实例源码

def process_request(self, request, spider):
        if spider.name == "jobbole":
            self.browser.get(request.url)
            import time
            time.sleep(3)
            print ("??:{0}".format(request.url))

            return HtmlResponse(url=self.browser.current_url, body=self.browser.page_source, encoding="utf-8", request=request)

#linux?

# from pyvirtualdisplay import Display
# display = Display(visible=0, size=(800, 600))
# display.start()
#
# browser = webdriver.Chrome()
# browser.get()

def populate_text_field(driver, element_locator, text):
    '''Populates text field with provided text

    Args:
        element_locator ((selenium.webdriver.common.by.By., str)): element locator described using `By`. Take a look at `Locate elements By <http://selenium-python.readthedocs.io/api.html#locate-elements-by>`_ for more info.
        text (str): text to populate text field with.

    Example:
        ::

            from selenium import webdriver
            from selenium.webdriver.common.by import By
            from selenium_extensions.core import populate_text_field


            driver = webdriver.Chrome()
            ...
            populate_text_field(driver, (By.CLASS_NAME, 'textbox'), 'some text')
    '''
    input_element = driver.find_element(*element_locator)
    input_element.send_keys(text)

def __init__(self, username, passwd, playground, groupname):
        self.playground = playground

        chrome_options = webdriver.ChromeOptions()
        #chrome_options.add_argument('--start-maximized')
        chrome_options.add_argument('--proxy-server=http://xx.mioffice.cn:8888')
        self.driver = webdriver.Chrome(chrome_options = chrome_options)
        driver = self.driver

        driver.implicitly_wait(300) # seconds
        driver.get('https://rankade.com/')
        assert 'rankade' in driver.title

        driver.find_element_by_css_selector("a.sign-button.sign-in-button").click()
        assert 'Sign in' in driver.title

        input = driver.find_element_by_name("email")
        input.send_keys(username)
        input = driver.find_element_by_name("password")
        input.send_keys(passwd)
        driver.find_element_by_name("submit").click()
        assert 'rankade' in driver.title
        driver.find_element_by_id("dashboardLink").click()
        # assert 'rankade - My dashboard' in driver.title
        driver.find_element_by_link_text(groupname).click()

def chrome(self):
        # https://github.com/SeleniumHQ/selenium/blob/master/py/selenium/webdriver/remote/webdriver.py
        # http://www.guguncube.com/2983/python-testing-selenium-with-google-chrome
        # https://gist.github.com/addyosmani/5336747
        # http://blog.likewise.org/2015/01/setting-up-chromedriver-and-the-selenium-webdriver-python-bindings-on-ubuntu-14-dot-04/
        # https://sites.google.com/a/chromium.org/chromedriver/getting-started
        # http://stackoverflow.com/questions/8255929/running-webdriver-chrome-with-selenium
        chrome = webdriver.Chrome()
        return chrome

#     @property
#     def firefox(self):
#         profile = webdriver.FirefoxProfile()
#         #firefox = webdriver.Firefox(firefox_profile=profile)
#         firefox = WebDriver(firefox_profile=profile)
#         return firefox

def loadCartAndCheckout(self):
        #Import Cookies
        driver = webdriver.Chrome(executable_path="./chromedriver")
        driver.delete_all_cookies()
        driver.get(self.URL_cart)

        cookies = requests.utils.dict_from_cookiejar(self.user_session.cookies)

        for cookie in cookies.items():
            cookie_dict = {'name': '',
                           'value': '',
                           'path': '/'}
            cookie_dict['name'] = cookie[0]
            cookie_dict['value'] = cookie[1]
            driver.add_cookie(cookie_dict)

        driver.get(self.URL_cart)
        #time.sleep(5)
        #driver.quit()

def get_js_errors(self):
        '''
        Uses the JSErrorCollector plugin for Chrome / Firefox to get any JS errors.

        [
            {
                'sourceName': u'tests/html/js_error.html',
                'pageUrl': u'tests/html/js_error.html',
                'errorMessage': 'ReferenceError: b is not defined',
                'lineNumber': 7
            }
        ]
        '''
        if self.driver in ('Chrome', 'Firefox'):
            return self.js('return window.JSErrorCollector_errors ? window.JSErrorCollector_errors.pump() : []')
        else:
            print("Checking for JS errors with this method only works in Firefox or Chrome")
            return []

def screenshot(self, path=None):
        '''
        Saves a screenshot. Takes a path as a parameter.

        Parameters
        ----------
        path: str
            Defaults to: /tmp/selenium-screenshot.png

        '''
        if not path:
            path = '/tmp/selenium-screenshot.png'

        # if isinstance(self.browser, webdriver.remote.webdriver.WebDriver):
        #     # Get base64 screenshot from the remote.
        #     base64_data = self.browser.get_screenshot_as_base64()
        #     ss_data = base64.decodestring(base64_data)
        #     with open(path, 'w') as f:
        #         f.write(ss_data)
        #         f.close()
        # else:
        if self.browser == 'chrome-headless':
            print("You are running Chrome in headless mode. Screenshots will be blank.")
        else:
            self.browser.save_screenshot(path)

def gethtml(zurl,str_fname):
    mobileEmulation = {'deviceName': 'Apple iPhone 6'}
    options = webdriver.ChromeOptions()
    options.add_experimental_option('mobileEmulation', mobileEmulation)
    driver = webdriver.Chrome(executable_path='chromedriver.exe', chrome_options=options)


    driver.get(zurl)
    time.sleep(5)
    result = []


    # for i in range(0,300):  #???0?20?????i?
    for i in range(0, 1):  # ???0?3?????i?
        print('????' + str(i))
        myscroll(driver)
        time.sleep(2)

    st=time.strftime("%Y%m%d",time.localtime())
    # print(driver.page_source, file=open('itg201703.html', 'w', encoding='utf-8'))
    print(driver.page_source, file=open(str_fname+"-"+st+".html", 'w', encoding='utf-8'))
    print("?????????")
    print(driver.title)
    driver.quit()

def _get_webdriver(self):
        """Return a webdriver instance and set it up
        with the according profile/ proxies.
        Chrome is quite fast, but not as stealthy as PhantomJS.
        Returns:
            The appropriate webdriver mode according to self.browser_type.
            If no webdriver mode could be found, return False.
        """
        if self.browser_type == 'chrome':
            return self._get_Chrome()
        elif self.browser_type == 'firefox':
            return self._get_Firefox()
        elif self.browser_type == 'phantomjs':
            return self._get_PhantomJS()

        return False

def setup():

    global browser
    driverPath = os.getcwd()+'/chromedriver'
    url = r'https://www.instagram.com/accounts/login/'
    chromeOptions = webdriver.ChromeOptions()
    #chromeOptions.binary_location='/opt/google/chrome/google-chrome'

    '''                                             #These arguments make chrome run headless.Unfortunately the chrome headless is in beta and hence considerably slow.
    chromeOptions.add_argument("--headless")            
    chromeOptions.add_argument("--disable-gpu")
    chromeOptions.add_argument("--start-fullscreen")
    '''

    prefs = {"profile.managed_default_content_settings.images":2}
    chromeOptions.add_experimental_option("prefs",prefs)
    print('reached 1')
    browser = webdriver.Chrome(driverPath,chrome_options=chromeOptions)
    print('reached')
    #browser.set_window_position(-10000000, 0)       #move chrome away from view
    print('Fetching login page..')
    browser.get(url)
    print('reached login page')

def getHtml(url, loadmore = False, waittime = 2):
    browser = webdriver.Chrome('chromedriver')
    browser.get(url)
    time.sleep(waittime)
    if loadmore:
        while True:
            try:
                next_button = browser.find_element_by_class_name("more")
                next_button.click()
                time.sleep(waittime)
            except:
                break
    html = browser.page_source
    browser.quit()
    return html

# for test
#url = "https://movie.douban.com/tag/#/?sort=S&range=9,10&tags=??,??,??"
#html = getHtml(url)
#print(html)

def glassdoor_login():
    '''
    Function to create a selenium Chrome driver and login using my credentials

    INPUT:
        None

    OUTPUT:
        webdriver.Chrome object
    '''
    url = 'https://www.glassdoor.com/profile/login_input.htm'
    driver = webdriver.Chrome()
    soup = get_soup(driver, url)

    user = driver.find_element_by_name('username')
    user.click()
    user.send_keys(USER_ID)

    pwrd = driver.find_element_by_xpath('//*[@id="signInPassword"]')
    pwrd.click()
    pwrd.send_keys(PASSWORD)

    sign_in = driver.find_element_by_id('signInBtn')
    sign_in.click()
    return driver

def give_me_the_page(n, user_name, password, broswer, pt = None):
    if not pt:
        if broswer=='Chrome':
            pt = webdriver.Chrome()
        elif broswer=='Safari':
            pt = webdriver.Safari()
        else:
            pt = webdriver.PhantomJS()
    pt.get('http://electsys.sjtu.edu.cn/edu/login.aspx')
    time.sleep(1)
    pt.execute_script("""var img=document.getElementById('form-input').getElementsByTagName('div')[2].getElementsByTagName('img')[0];
        var d=document.createElement('CANVAS');
        var cxt=d.getContext('2d');
        d.width=img.width;
        d.height=img.height;
        cxt.drawImage(img,0,0);
        img.src=d.toDataURL('png');""")

def get_driver_path():

        chrome_driver_folder_name = ""
        if platform == "linux" or platform == "linux2":
            # linux
            chrome_driver_folder_name = "linux_x64"
        elif platform == "darwin":
            # OS X
            chrome_driver_folder_name = "mac_x64"
        else:
            raise ValueError("Platform not identified")

        chrome_driver_path = os.path.normpath(
            os.path.join(os.path.dirname(os.path.abspath(__file__)),
                         os.pardir, os.pardir, os.pardir,
                         "resources", "chrome", chrome_driver_folder_name,
                         "chromedriver"))
        assert os.path.isfile(chrome_driver_path), \
            "Chrome driver must exists: %s" % chrome_driver_path

        return chrome_driver_path

def strat_isml(thread):
    uaList = []
    for line in open('Base_Data\\ualist.txt'):
        uaList.append(line[:-1])
    open('Base_Data\\ualist.txt').close()
    i = random.choice(uaList)
    option = webdriver.ChromeOptions()
    option.add_argument('--user-agent={}'.format(i))
    option.add_argument('--profile-directory=Default')
    option.add_argument('--user-data-dir=c:\\Users\\{}'.format(thread))
    with open("Base_Data\\ChromeOptions.txt") as a:
        for line in a:
            option.add_argument(line)
    path1 = 'C:\\Program Files (x86)\\Google\\Chrome\\Application\\chromedriver.exe'
    path2 = 'C:\\Program Files\\Google\\Chrome\\Application\\chromedriver.exe'
    try:
        dr = webdriver.Chrome(path1,chrome_options=option)
    except:
        dr = webdriver.Chrome(path2,chrome_options=option)
    return dr,uaList

def fb_login(self):
    usr, pwd = get_details()
    try:
        driver = webdriver.Chrome('/usr/bin/chromedriver')
    except:
        driver = webdriver.Chrome('/usr/lib/chromium-browser/chromedriver')
    driver.get('https://www.facebook.com/')

    user_id = driver.find_element_by_id('email')
    user_id.send_keys(usr)
    sleep(2)

    password = driver.find_element_by_id('pass')
    password.send_keys(pwd)
    sleep(2)

    submit = driver.find_element_by_id('loginbutton')
    submit.click()
    if six.PY2:
        raw_input('Enter anything to end the session: ')
    else:
        input('Enter anything to end the session: ')
    driver.quit()

def netease():
    options = webdriver.ChromeOptions()
    options.add_argument(
        '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36')
    driver = webdriver.Chrome(executable_path=r'C:\software\chrome\chromedriver.exe',
                              chrome_options=options)
    driver.implicitly_wait(40)
    driver.get("http://30daydo.com/")
    elem_user = driver.find_element_by_tag_name("??")
    elem_user.click()
    ''''
    elem_pwd = driver.find_element_by_name("password")
    elem_pwd.send_keys("123456")
    elem_pwd.send_keys(Keys.RETURN)
    '''
    time.sleep(5)
    assert "baidu" in driver.title
    driver.close()
    driver.quit()

def key_operation():
    # ????
    options = webdriver.ChromeOptions()
    options.add_argument(
        '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36')
    browser = webdriver.Chrome(executable_path=r'C:\software\chrome\chromedriver.exe',
                               chrome_options=options)  #
    browser.implicitly_wait(60)

    browser.get('https://m.fang.com/fangjia/sz_list_pinggu/')
    #browser.send_keys(Keys.DOWN)
    count=0
    while count<190:

        browser.find_element_by_xpath("//body[@class='whitebg']").send_keys(Keys.PAGE_DOWN)
        time.sleep(5)
        count=count+1
    raw_input('enter')

def __init__(self, username, passwd):
        self.driver = webdriver.Chrome()
        driver = self.driver

        driver.implicitly_wait(300) # seconds
        driver.get('https://rankade.com/')
        assert 'rankade' in driver.title

        driver.find_element_by_css_selector("a.sign-button.sign-in-button").click()
        assert 'Sign in' in driver.title

        input = driver.find_element_by_name("email")
        input.send_keys(username)
        input = driver.find_element_by_name("password")
        input.send_keys(passwd)
        driver.find_element_by_name("submit").click()
        assert 'rankade' in driver.title
        driver.find_element_by_id("dashboardLink").click()
        # assert 'rankade - My dashboard' in driver.title
        driver.find_element_by_link_text("mifoosball").click()
        # driver.save_screenshot('mifoosball.png')
        # assert 'mifoosball' in driver.title

def takeSnapshot(report_name, root_path, script_name=None, *args, **kwargs):
  """ """

  options = webdriver.ChromeOptions()
  options.add_argument("headless")
  driver = webdriver.Chrome(os.path.join(root_path, 'system', 'webDrivers', 'chromedriver'), chrome_options=options)
  if not script_name:
    script_name = report_name

  url_str = url_for('ares.run_report', report_name=report_name, script_name=script_name, **kwargs)
  if report_name.startswith('_'):
    report_dir = os.path.join(root_path, config.ARES_FOLDER, 'reports', report_name)
    if report_name == '_AresTemplates':
      url_str = url_for('ares.run_template', template=script_name)
  else:
    report_dir = os.path.join(root_path, config.ARES_USERS_LOCATION, report_name)
  driver.get(url_str)
  driver.save_screenshot(os.path.join(report_dir, '%s.png' % script_name ))
  driver.quit()

def scrape(screen_name, since_date, until_date, include_retweets=True, wait_secs=5):
    log.info("Scraping %s since %s until %s", screen_name, since_date, until_date)
    driver = webdriver.Chrome()
    try:
        driver.implicitly_wait(wait_secs)
        url = "https://twitter.com/search?f=tweets&vertical=default&q=from:{}+since:{}+until:{}&src=typd".format(screen_name, since_date.isoformat(),
                                                                              until_date.isoformat())
        if include_retweets:
            url += "+include:retweets"
        log.debug("Getting %s", url)
        driver.get(url)

        scroll_count = 0
        last_tweet_count = 0
        while last_tweet_count != len(driver.find_elements_by_class_name("original-tweet")):
            scroll_count += 1
            last_tweet_count = len(driver.find_elements_by_class_name("original-tweet"))
            log.debug("Scrolling down %s. Found %s tweets.", scroll_count, last_tweet_count)
            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
            time.sleep(wait_secs)

        return set([e.get_attribute("data-tweet-id") for e in driver.find_elements_by_class_name("original-tweet")])
    finally:
        driver.close()
        driver.quit()

def create_selenium_driver(browser='chrome'):
    # set default browser string based on env (if available)
    env_browser = os.environ.get('TOASTER_TESTS_BROWSER')
    if env_browser:
        browser = env_browser

    if browser == 'chrome':
        return webdriver.Chrome(
            service_args=["--verbose", "--log-path=selenium.log"]
        )
    elif browser == 'firefox':
        return webdriver.Firefox()
    elif browser == 'marionette':
        capabilities = DesiredCapabilities.FIREFOX
        capabilities['marionette'] = True
        return webdriver.Firefox(capabilities=capabilities)
    elif browser == 'ie':
        return webdriver.Ie()
    elif browser == 'phantomjs':
        return webdriver.PhantomJS()
    else:
        msg = 'Selenium driver for browser %s is not available' % browser
        raise RuntimeError(msg)

def __scrollbars_hide(self):
        """Hides Chrome's scrollbars.

        Creates a new <style> element to contain the CSS rule for hiding
        the browser scrollbars. `::-webkit-scrollbar {width: 0px;}`

        Args:
            None

        Returns:
            None
        """
        self.execute_script(
            "var sheet = document.createElement('style'); "
            "sheet.id = 'chrome_screenshot_fix'; "
            "sheet.innerHTML = '::-webkit-scrollbar {width: 0px;}'; "
            "document.body.appendChild(sheet); ")

def __screenshot_png(self, func):
        """Helper function that produces the screenshot.

        Produces a stitched together screenshot of the current webpage.
        Automatically hides and restores Chrome's scrollbars.

        Args:
            func: A helper function which will be passed the finalized
                screenshot. Whatever is returned by `func` is returned
                by this function.

        Returns:
            Whatever is returned by func(screenshot).
        """
        self.__scrollbars_hide()

        doc_width = self.__document_width
        doc_height = self.__document_height
        with Image(width=doc_width*2, height=doc_height*2) as screenshot:
            for data, rect in self.__iter_screenshots((doc_width, doc_height)):

                with Image(blob=base64.b64decode(data),
                           format='png') as shot:
                    screenshot.composite(image=shot,
                                         left=rect[LEFT]*2,
                                         top=rect[TOP]*2)
                del data

            _ret = func(screenshot)

        self.__scrollbars_restore()
        return _ret

def login(url):
    login_name = input('???QQ?\n')
    login_password = input('???QQ??\n')
    driver = webdriver.Chrome()
    driver.get(url)
    time.sleep(3)

    login_type = driver.find_element_by_id('switcher_plogin')

    login_type.click()

    username = driver.find_element_by_id('u')
    username.clear()
    password = driver.find_element_by_id('p')
    password.clear()
    username.send_keys(login_name)
    password.send_keys(login_password)

    submit = driver.find_element_by_id('login_button')
    submit.click()
    time.sleep(5)

    cookies = driver.get_cookies()
    driver.close()
    return cookies

def login():
    acount_num = input('?????:\n')
    passwd_str = input('?????:\n')
    driver = webdriver.Chrome(executable_path='/Users/resolvewang/Documents/program/driver/chromedriver')
    url = 'http://mail.163.com/'
    driver.get(url)
    time.sleep(5)
    # 163??????iframe???????????????iframe
    driver.switch_to.frame('x-URS-iframe')

    acount = driver.find_element_by_name('email')
    acount.clear()
    acount.send_keys(acount_num)

    passwd = driver.find_element_by_name('password')
    passwd.clear()
    passwd.send_keys(passwd_str)

    time.sleep(3)
    click_button = driver.find_element_by_id('dologin')
    click_button.click()
    time.sleep(5)
    cur_cookies = driver.get_cookies()[0]
    return cur_cookies

def login(login_url, login_name, login_passwd):
    driver = webdriver.Chrome()
    driver.get(login_url)
    time.sleep(5)

    login_tab_right = driver.find_element_by_class_name('login-tab-r')
    login_tab_right.click()

    account = driver.find_element_by_id('loginname')
    password = driver.find_element_by_id('nloginpwd')
    submit = driver.find_element_by_id('loginsubmit')

    account.clear()
    password.clear()
    account.send_keys(login_name)
    password.send_keys(login_passwd)

    submit.click()
    time.sleep(5)

    jd_cookies = driver.get_cookies()
    driver.close()
    return jd_cookies

def login(name, passwd):
    url = 'https://pan.baidu.com/'
    # ?????Chrome?Phantomjs??????????????????????
    driver = webdriver.Chrome(executable_path='/Users/resolvewang/Documents/program/driver/chromedriver')
    driver.maximize_window()
    driver.get(url)
    print('????')
    chg_field = driver.find_element_by_class_name('pass-login-tab').find_element_by_class_name('account-title')
    chg_field.click()

    name_field = driver.find_element_by_id('TANGRAM__PSP_4__userName')
    name_field.send_keys(name)
    passwd_field = driver.find_element_by_id('TANGRAM__PSP_4__password')
    passwd_field.send_keys(passwd)
    login_button = driver.find_element_by_id('TANGRAM__PSP_4__submit')
    login_button.click()
    time.sleep(20)
    return driver.get_cookies()

def load_driver(config, vdisplay=None):
    """Initialize a weddriver selected in config with given config.

    Args:
        config (dict): The configuration loaded previously in Cabu.

    Returns:
        webdriver (selenium.webdriver): An instance of selenium webdriver or None.
    """

    if config['DRIVER_NAME'] == 'Firefox':
        driver = load_firefox(config)
    elif config['DRIVER_NAME'] == 'Chrome':
        driver = load_chrome(config)
    elif config['DRIVER_NAME'] == 'PhantomJS':
        driver = load_phantomjs(config)
    elif not config.get('DRIVER_NAME'):
        return None
    else:
        raise DriverException(vdisplay, 'Driver unrecognized.')

    driver.set_page_load_timeout(config['DRIVER_PAGE_TIMEOUT'])
    driver.set_window_size(config['DRIVER_WINDOWS_WIDTH'], config['DRIVER_WINDOWS_HEIGHT'])

    return driver

def init_driver(self):
        global driver

        if self.is_initialized:
            return

        if self.driver_name == 'chrome':
            driver = webdriver.Chrome(executable_path=self.driver_path)
        elif self.driver_name == 'phantomjs':
            driver = webdriver.PhantomJS(executable_path=self.driver_path)
        elif self.driver_name == 'firefox':
            driver = webdriver.Firefox(executable_path=self.driver_path)
        else:
            raise Exception(
                'Driver "{}" is not supported'.format(self.driver_name))

        self.is_initialized = True
        driver.set_window_size(self.width, self.height)
        driver.implicitly_wait(5)

def brInit(adb_crx=None):
    """
    Initialises Selenium's webdriver (Chrome)
    This version uses an adblock extension to load pages faster.
    Download the crx file and save it in the same folder.
    You can use any extension you want.

    return: webdriver object
    """
    if adb_crx == None:
        driver = driver = webdriver.Chrome()
    else:
        chop = webdriver.ChromeOptions()
        chop.add_extension(adb_crx)
        driver = webdriver.Chrome(chrome_options = chop)
    return driver

def scrape():
    print((os.path.dirname(os.path.abspath(__file__)) + '/' + cdname))
    browser = webdriver.Chrome(os.path.dirname(os.path.abspath(__file__)) + '/' + cdname)
    browser.get('https://my.unt.edu/psp/papd01/EMPLOYEE/EMPL/h/?tab=NTPA_GUEST')

    euid = input('What is your EUID? ')
    password = getpass.getpass('What is your password? ')

    euid_field = browser.find_element_by_name('userid')
    password_field = browser.find_element_by_name('pwd')
    euid_field.send_keys(euid)
    password_field.send_keys(password)

    login_field = browser.find_element_by_css_selector('input[value="Login"]')
    login_field.click()

    browser.get('https://my.unt.edu/psp/papd01/EMPLOYEE/EMPL/h/?cmd=getCachedPglt&pageletname=GBPA_STUDENT_CLASSES&tab=GBPA_STUDENT&PORTALPARAM_COMPWIDTH=Narrow')

    classes = browser.find_elements_by_css_selector('p')
    return build_datetimes(parse_times(format(classes)))

def test():
    profile_dir = r"D:\MyChrome\Default"
    # ?????
    # "Referer": "http://weixin.sogou.com"
    chrome_options = webdriver.ChromeOptions()
    chrome_options.add_argument("--user-data-dir=" + os.path.abspath(profile_dir))
    PROXY = "123.56.238.200:8123"
    # j = random.randint(0, len(proxys)-1)
    # proxy = proxys[j]
    chrome_options.add_argument('--proxy-server=%s' % PROXY)
    # chrome_options.add_extension('')??crx??
    # service_args = ['--proxy=localhost:9050', '--proxy-type=socks5', ]
    driver = webdriver.Chrome(r'C:\Python27\chromedriver', chrome_options=chrome_options)
    driver.get('http://icanhazip.com')
    driver.refresh()
    print(driver.page_source)
    driver.quit()

def __init__(self):
        self._ocr = RClient(config.dama_name, config.dama_pswd, config.dama_soft_id, config.dama_soft_key)
        self._cache = filecache.WechatCache(config.cache_dir, 60 * 60)
        self._session = self._cache.get(config.cache_session_name) if self._cache.get(
            config.cache_session_name) else requests.session()
        self.cookie = self.maintain_cookies_ph()
        self.agents = [
            "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0; Acoo Browser; SLCC1; .NET CLR 2.0.50727; Media Center PC 5.0; .NET CLR 3.0.04506)",
            "Mozilla/4.0 (compatible; MSIE 7.0; AOL 9.5; AOLBuild 4337.35; Windows NT 5.1; .NET CLR 1.1.4322; .NET CLR 2.0.50727)",
            "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)",
            "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
            "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
            "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
            "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
            "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
            "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5",
            "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6",
            "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11",
            "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20",
            "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52",
            ]

def update_page_source(self):
        url = self.build_search_url()

        driver = Chrome()
        driver.get(url)


        num_scrolls = 0
        try:

            while num_scrolls < self.scroll_max:
                driver.execute_script(random_js_scroll())
                self.page_source = driver.page_source
                random_sleep()
                num_scrolls+=1

        except Exception as e:
            l.WARN(e)

        driver.close()

def __init__(self, user, pwd, repeat, debug):
        self.debug = debug
        self.set_debug()        # if self.debug = yes view procedure step by step in the browser

        # don't loading the images to optimize the speed of requests
        chromeOptions = webdriver.ChromeOptions()
        prefs = {"profile.managed_default_content_settings.images":2}
        chromeOptions.add_experimental_option("prefs",prefs)

        self.browser = webdriver.Chrome(chrome_options=chromeOptions)
        self.login(user, pwd)

        self.go_to_internet()

        i = 1
        while i <= repeat:
            i += 1
            self.go_to_service()
            self.confirmed()

        self.logout()

def login(username, password):
    driver_directory = os.path.dirname(__file__)
    if (platform.system() == "Windows"):
        relative_path = "chromedriver.exe"
    else:
        relative_path = "chromedriver"
    absolute_file_path = os.path.join(driver_directory, relative_path)
    driver = webdriver.Chrome(executable_path=absolute_file_path)
    driver.get("http://www.ilearn.sexy")  # Shortcut to itslearning

    username_field = driver.find_element_by_name("feidename")
    username_field.send_keys(username)
    password_field = driver.find_element_by_name("password")
    password_field.send_keys(password)
    password_field.submit()
    login_success_field = driver.find_element_by_name("mainmenu")

    driver.close()
    driver.quit()

def main():
    print 'hello'

    print sys.argv
    print len(sys.argv)
    dper= sys.argv[1]
    print "your dper is:"+dper

    opts = Options()
    opts.add_argument("user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.86 Safari/537.36")
    driver = webdriver.Chrome(chrome_options=opts)
    driver.maximize_window()

    driver.get("http://s.dianping.com/event/119124")
    driver.add_cookie({'name':'dper', 'value':dper,'path':'/'})
    category_urls=[]
    category_urls.append("http://s.dianping.com/event/shanghai/c1")
    category_urls.append("http://s.dianping.com/event/shanghai/c6")
    for url in category_urls:
        process_category(url, driver)
    driver.quit()

def start_driver(self):
        try:
            driver = self.find_chromedriver()
        except WebDriverException:
            logger.error('Unable to locate chromedriver')
            time.sleep(0.1)
            download = raw_input('\nNo chrome driver found.  Download? (y/n): ')
            if download.lower().startswith('y'):
                self.download_chromedriver()
                try:
                    driver = webdriver.Chrome(executable_path=self.chrome_path)
                except WebDriverException as e:
                    if 'cannot find' in e.msg:
                        logger.critical('Could not start Chrome browser')
                        raise SystemExit('Forms log cannot be retrieved without Chrome and chromedriver.')
                    else:
                        logger.exception('Cannot start the Chrome browser')
                        raise SystemExit('Forms log cannot be retrieved without Chrome and chromedriver.')

            else:
                raise SystemExit('Forms log cannot be retrieved without Chrome and chromedriver.')

        return driver

def test_dynamic_request_browser_actions(self):
        cm = CookiesMiddleware(self.spider, self.spider.settings)
        self.driver = webdriver.Chrome()
        dh = DownloadHandler(self.spider, self.driver, self.driver_sem)

        def _actions(driver):
            driver.find_element_by_name('account').send_keys("username")
            driver.find_element_by_name('password').send_keys("pwd")
            driver.find_element_by_xpath('/html/body/div[1]/div/div[2]/div[2]/form/div[2]/button').click()
            gevent.sleep(5)

        request = Request('https://www.zhihu.com/#signin',
                          dynamic=True, meta={'cookiejar': 'test'},
                          browser_actions=[_actions],
                          )
        cm.process_request(request)
        response = dh.fetch(request)
        cm.process_response(request, response)

        request = Request('https://www.zhihu.com', dynamic=True, meta={'cookiejar': 'test'})
        cm.process_request(request)
        response = dh.fetch(request)
        cm.process_response(request, response)
        print response.body
        self.driver.close()

def shut_down(driver):
    '''Shuts down the driver and its virtual display

    Args:
        driver (selenium.webdriver.): Selenium webdriver to stop.

    Example:
        ::

            from selenium import webdriver
            from selenium_extensions.core import shut_down


            driver = webdriver.Chrome()
            ...
            shut_down(driver)
    '''
    driver.quit()
    try:
        kill_virtual_display(driver.display)
    except (AttributeError, TypeError):
        # Display is either None or there is no display at all
        pass

def click_on_element(driver, element_locator):
    '''Clicks on a Selenium element represented by ``element_locator``

    Args:
        element_locator ((selenium.webdriver.common.by.By., str)): element locator described using `By`. Take a look at `Locate elements By <http://selenium-python.readthedocs.io/api.html#locate-elements-by>`_ for more info.

    Example:
        ::

            from selenium import webdriver
            from selenium.webdriver.common.by import By
            from selenium_extensions.core import click_on_element


            driver = webdriver.Chrome()
            ...
            click_on_element(driver, (By.ID, 'form-submit-button'))
    '''
    element = driver.find_element(*element_locator)
    element.click()

def __init__(self, savedir='./', screenshot=False):
        """
        Initialize ScreenScraper.

        :param savedir: directory to save OFX in
        :type savedir: str
        :param screenshot: whether or not to take screenshots throughout the
          process
        :type screenshot: bool
        """
        self._savedir = os.path.abspath(os.path.expanduser(savedir))
        if not os.path.exists(self._savedir):
            os.makedirs(self._savedir)
        self._cookie_file = os.path.join(self._savedir, 'cookies.txt')
        logger.debug('Using savedir: %s', self._savedir)
        self._screenshot_num = 1
        self._screenshot = screenshot
        if self._screenshot:
            logger.warning("screenshotting all actions")
        self.user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36' \
                          ' (KHTML, like Gecko) Chrome/62.0.3202.62 ' \
                          'Safari/537.36'

def get_selenium_driver(browser_name: str) -> Union[Chrome, Firefox]:
    """
    Return the same instance to the Selenium driver.

    :param browser_name: the name of the browser: chrome or mozilla
    :type browser_name: str
    :return: an instance of the required driver.
    :rtype: Union[Chrome, Mozilla]
    """
    if browser_name.upper() == 'CHROME':
        return SingletonFactory.build(Chrome)

    elif browser_name.upper() == 'Mozilla':
        return SingletonFactory.build(Firefox)

    else:
        raise NotImplementedError

def __init__(self):
        self.browser = webdriver.Chrome(executable_path="C:/chromedriver.exe")
        super (JSPageMiddleware,self).__init__()
    #??chrome??????

def before_all(context):
    context.browser = webdriver.Chrome()

def checkCart(self):
        print self.cj

        #driver = webdriver.Firefox()
        #driver.add_cookie(cj_dict)
        #driver.get('http://shop.bdgastore.com/cart')
        #print(driver.get_cookies())
        #for cookie in self.cj:
        #    print cookie.name, cookie.value, cookie.domain
        #try importing cookies manually vs using the requests util

        #driver = webdriver.Chrome('./chromedriver')

def setUp(self):
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')
        self.page = MyPage(browser=webdriver.Chrome(chrome_options=chrome_options))
        self.page.browser.get('http://localhost:3000')

def setUp(self):
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')
        self.page = MyPage(browser=webdriver.Chrome(chrome_options=chrome_options))
        self.page.browser.get('http://localhost:3000')

def setUp(self):
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--headless')
        self.page = MyPage(browser=webdriver.Chrome(chrome_options=chrome_options))
        self.page.browser.get('http://localhost:3000')