您需要申请显式等待 https://selenium-python.readthedocs.org/waits.html#explicit-waits概念。例如。等待元素变得可见:
wait = WebDriverWait(driver, 10)
wait.until(EC.visibility_of_element_located((By.CLASS_NAME, 'searchbox')))
在这里,它会等待to 10每 500 毫秒检查一次元素的可见性。
有一组内置的预期条件需要等待,并且也很容易编写您的自定义预期条件 https://stackoverflow.com/a/29377790/771848.
仅供参考,以下是我们在聊天中集思广益后的处理方法。我们引入了一个自定义的预期条件等待元素文本改变。它帮助我们确定新搜索结果何时出现:
import re
import pandas as pd
from selenium import webdriver
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support.expected_conditions import _find_element
class text_to_change(object):
def __init__(self, locator, text):
self.locator = locator
self.text = text
def __call__(self, driver):
actual_text = _find_element(driver, self.locator).text
return actual_text != self.text
#Load URL
driver = webdriver.Firefox()
driver.get(url)
#Load DataFrame of terms to search for
df = pd.read_csv("searchkey.csv")
#Crawling function
def crawl(searchkey):
try:
text_before = driver.find_element_by_class_name("ac_results").text
except NoSuchElementException:
text_before = ""
searchbox = driver.find_element_by_name("searchbox")
searchbox.clear()
searchbox.send_keys(searchkey)
print "\nSearching for %s ..." % searchkey
WebDriverWait(driver, 10).until(
text_to_change((By.CLASS_NAME, "ac_results"), text_before)
)
search_result = driver.find_element_by_class_name("ac_results")
if search_result.text != "none":
names = re.match(r"^.*(?=(\())", search_result.text).group().encode("utf-8")
insrefs = re.findall(r"((?<=\()[0-9]*)", search_result.text)
if search_result.text == "none":
names = re.match(r"^.*(?=(\())", search_result.text)
insrefs = re.findall(r"((?<=\()[0-9]*)", search_result.text)
return pd.Series([insrefs, names])
#Run crawl
df[["Insref", "Name"]] = df["ISIN"].apply(crawl)
#Print DataFrame
print df