selenium: Can not find element even with full xpath

January 4, 2022

I am trying to extract data using selenium from [website][1]. But I can not access the button I want to click. I have tried many methods by no use. I don’t know why this is happening.
Here is the code.

from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

option = webdriver.ChromeOptions()
option.add_argument("start-maximized")
option.add_experimental_option("excludeSwitches", ["enable-automation"])
option.add_experimental_option('useAutomationExtension', False)
option.add_argument("--disable-blink-features")
option.add_argument("--disable-gpu")
#option.add_argument("--headless")
option.add_argument('--disable-blink-features=AutomationControlled')
wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=option)


url = "https://greyhoundbet.racingpost.com/#result-meeting/track_id=4&r_date=2021-01-01&r_time=13:24"
wd.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
wd.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
wd.get(url)

button = wd.find_elements(By.XPATH, "/html/body/div[3]/div[2]/div[2]/div[2]/div[1]/div/span")
#button = wd.find_elements(By.CLASSNAME, 'btnLeft btnBlueText')
#button = wd.find_element_by_id('resultsDateBtn')
#used above methods along with relative xpath

print(button)
for i in button:
    if i.get_attribute('id') == "resultsDateBtn":
        i.click()

wd.quit()

I always get empty list or could not find element error
[1]: https://greyhoundbet.racingpost.com/#results-list/r_date=2021-01-01

>Solution :

You are missing a wait / delay.
You should let the page loaded before accessing the web element.
The best way to do that is to use expected conditions, as following:

from selenium import webdriver
from selenium.webdriver.chrome import service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager

option = webdriver.ChromeOptions()
option.add_argument("start-maximized")
option.add_experimental_option("excludeSwitches", ["enable-automation"])
option.add_experimental_option('useAutomationExtension', False)
option.add_argument("--disable-blink-features")
option.add_argument("--disable-gpu")
#option.add_argument("--headless")
option.add_argument('--disable-blink-features=AutomationControlled')
wd = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=option)
url = "https://greyhoundbet.racingpost.com/#results-list/r_date=2021-01-01"
wd.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36'})
wd.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
wd.get(url)
wait = WebDriverWait(wd, 20)
button = wait.until(EC.visibility_of_element_located((By.ID, "resultsDateBtn")))
#now you can click this button etc
button.click()