Code trial:
#coding=utf-8 from selenium import webdriver from selenium.webdriver.common.keys import Keys from selenium.webdriver.common.by import By from selenium.webdriver.support.ui import Select, WebDriverWait from selenium.webdriver.support import expected_conditions as EC import time import pandas as pd # Below is to crawl data from a webpage with two different dropdown try: driver = webdriver.Chrome('./chromedriver') driver.get('https://price.joinsland.joins.com/theme/index_theme.asp?sisaegbn=T05') select1 = Select(WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, "//select[@name='sido']")))) for item1 in select1.options: item1.click() select2 = Select(WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, "//select[@name='gugun']")))) for item2 in select2.options: item2.click() time.sleep(2) # below is to get the attained date into excel file table = driver.find_element_by_class_name('tbl_box') tbody = table.find_element_by_tag_name('tbody') rows=tbody.find_elements_by_tag_name('tr') total = [] result = [] for index, value in enumerate(rows): body = value.find_elements_by_tag_name('td') for i in range(len(body)): data = body[i].text result.append(data) total.append(result) result=[] df = pd.DataFrame.from_records(total) df.to_excel('text.xlsx') except Exception as e: print(e) finally: driver.quit()
I have edited this code thanks to the lovely comment below but I get the same error saying as below.:
Message: stale element reference: element is not attached to the page document
I roughly understand why this message shows up but still have no clear idea on how to fix this. I would deeply appreciate any comment! Many thanks in advance!
This is what I figured out. But I’m not sure if code is right or not. I don’t know python.
#get select select1 = Select(driver.find_element_by_xpath('//select[@name="sido"]')) #get all options from select options1 = select1.options for opt1 is options1: #select the option which has the value of opt1 select1.select_by_value(opt1.get_attribute("value")) time.sleep(5) select2 = Select(driver.find_element_by_xpath('//select[@name="gugu"]')) options2 = select2.options for opt2 in options2: select1.select_by_value(opt2.get_attribute("value")) time.sleep(4)