The results are printing to the screen using Selenium. Is it possible to put these results in a pandas dataframe and if so could someone show me how? Thanks
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.common.exceptions import TimeoutException
from time import sleep
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import pandas as pd
import openpyxl
from openpyxl import load_workbook
driver = webdriver.Firefox()
driver.maximize_window()
driver.get('https://tradingview.com/screener/')
wait = WebDriverWait(driver, 35)
wait.until(EC.visibility_of_element_located((By.XPATH, "//div[@data-set='income_statement']"))).click()
# Scroll down 100 times
i = 1
while i < 75:
print(i)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
i += 1
sleep(1)
delay = 5 #seconds
# table updated, get the data
for row in driver.find_elements(By.CSS_SELECTOR,".tv-data-table__tbody tr"):
print(row.text)
Example of Results
AAPLD
APPLE INC
5.67USD 5.66USD 5.61USD 121.463BUSD 35.174BUSD 152.836BUSD 365.817BUSD 5.61USD 365.817BUSD 94.68BUSD
MSFTD
MICROSOFT CORP.
8.12USD 9.01USD 8.95USD 86.531BUSD 31.671BUSD 115.856BUSD 168.088BUSD 8.05USD 168.088BUSD 61.271BUSD
>Solution :
- Install pandas.
pip install pandas
- Create an array of texts and then create a dataframe from that.
texts = []
for row in driver.find_elements(By.CSS_SELECTOR,".tv-data-table__tbody tr"):
texts.append(row.text)
import pandas as pd
df = pd.DataFrame(dict(text=texts))