how do I webscrape this site properly?

from bs4 import BeautifulSoup
import requests

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0'
}

product = input("What do you want? ")

def eurika(product):

 eurika_url = f'https://www.eureka.com.kw/?instant_records%5Bquery%5D={product}'

    response = requests.get(eurika_url, headers=headers)
    soup = BeautifulSoup(response.content, 'html.parser')

    product_span = soup.find_all('div', class_='caption')
   
    if product_span:
     for product_spa in product_span:
        title = product_spa.find('span', class_='display-block fwbold')
        price = product_spa.find('span', style='color:red')
        parent = soup.find('a', class_='sobrTxt')
         
        if title  :
            title_text = title.text.strip()
            price_text = price.text.strip()
            link = parent['href']
           
        
           
           
            print("Product Title:", title_text)
            print("Product Price:", price_text)
            print("Product Link:", link)
            print()
                 
        else:
                print("Parent <a> tag not found")
         
    else:
     print("No products found on the page")


    response.close()

website is https://www.eureka.com.kw/

I want to get the title price and href

>Solution :

The data about items you see on the page is loaded via JavaScript from different URL. To get this info you can use next example:

import requests

params = {
    "x-algolia-agent": "Algolia for vanilla JavaScript (lite) 3.32.0;instantsearch.js (4.3.0);JS Helper (3.1.1)",
    "x-algolia-application-id": "5GPHMAA239",
    "x-algolia-api-key": "3d7dbc330852592da244c87ae924a221",
}

# Change the `query=charger` to put your query:
payload = {
    "requests": [
        {
            "indexName": "instant_records",
            "params": "hitsPerPage=25&distinct=true&clickAnalytics=true&query=charger&highlightPreTag=__ais-highlight__&highlightPostTag=__%2Fais-highlight__&maxValuesPerFacet=300&page=0&facets=%5B%22bn%22%2C%22clprc%22%2C%22rmn%22%5D&tagFilters=",
        }
    ]
}

api_url = "https://5gphmaa239-dsn.algolia.net/1/indexes/*/queries"

data = requests.post(api_url, params=params, json=payload).json()

for r in data["results"][0]["hits"]:
    print(
        f'{r["itmn"][:50]:<50} {r["clprcv"]:<10} https://www.eureka.com.kw/products/details/{r["objectID"]}'
    )

Prints:

Belkin 3-in-1 Wireless Charger With MagSafe WIZ016 39.9       https://www.eureka.com.kw/products/details/199129
Belkin 3-in-1 Wireless Charger With MagSafe WIZ016 39.9       https://www.eureka.com.kw/products/details/199128
BELKIN 3-IN-1 WIRELESS CHARGER WITH MAGSAFE WIZ009 39.9       https://www.eureka.com.kw/products/details/118761
BELKIN 3-IN-1 WIRELESS CHARGER WITH MAGSAFE WIZ009 39.9       https://www.eureka.com.kw/products/details/118760
ANKER 633 MAGNETIC WIRELESS CHARGER 5,000MAH B25A7 39.0       https://www.eureka.com.kw/products/details/68137
Anker 737 Charger GaN Prime 120W A2148211 Black    36.0       https://www.eureka.com.kw/products/details/177181
ANKER MAGGO WIRELESS MAGNETIC CHARGER B2568211     26.0       https://www.eureka.com.kw/products/details/120495
Zens Aluminium 4 in 1 Magsafe Wireless Charger 45W 23.9       https://www.eureka.com.kw/products/details/164567
Anker 735 Charger GaN Prime 65W A2668211 Black     23.0       https://www.eureka.com.kw/products/details/177183
ANKER POWERPORT III 60W TRAVEL ADAPTER CHARGER A26 22.5       https://www.eureka.com.kw/products/details/67330
RAVPOWER DESKTOP CHARGER 65W 4-PORTS RP-PC136 BLAC 22.0       https://www.eureka.com.kw/products/details/162054
Belkin BOOSTCHARGE PRO 108W Charger BKN-WCH010MYW  21.45      https://www.eureka.com.kw/products/details/188443
ANKER CAR MOUNT & CAR CHARGER B2551H13 BLACK       19.9       https://www.eureka.com.kw/products/details/67304
SAMSUNG SUPER FAST WIRELESS CHARGER DUO 15W EF-P54 19.9       https://www.eureka.com.kw/products/details/161444
Momax OnePlug 100W 4-Port Desktop Charger UM33UKW  19.5       https://www.eureka.com.kw/products/details/234340
ANKER POWERPORT ATOM III 65W CHARGER 4 PORTS A2045 19.0       https://www.eureka.com.kw/products/details/67356
Energea Travelite Standard Wall Charger 66WCHR-TL- 18.75      https://www.eureka.com.kw/products/details/199171
HYPERDRIVE 66W GAN USB TYPE-C CHARGER HJ265 WHITE  18.0       https://www.eureka.com.kw/products/details/119389
HYPERDRIVE 66W GAN USB TYPE-C CHARGER HJ265 BLACK  18.0       https://www.eureka.com.kw/products/details/119387
Zens Aluminium 4-1 Wireless Charger 45W USD PD ZE- 16.9       https://www.eureka.com.kw/products/details/164569
ANKER CHARGER NANO II 65W A2663K11 BLACK           16.5       https://www.eureka.com.kw/products/details/68124
Feeltek 30W Home Charger 2 ports PKL180UKB309 Whit 16.0       https://www.eureka.com.kw/products/details/165402
RAVPOWER WALL CHARGER 65W 3-PORTS PC172 BLACK      15.9       https://www.eureka.com.kw/products/details/162052
Elago Wireless Fast Charger Car Phone Mount  EMOUN 15.0       https://www.eureka.com.kw/products/details/199463
Cellairis 3 In 1 Foldable Wireless Charger 15W LC0 14.95      https://www.eureka.com.kw/products/details/221188