I’ve developed a code to access a supermarket’s API and pull product information, the problem is the info I pull seems to differ from that seen on the webpage’s Network -> Response tab.
This is my code:
import requests
import numpy as np
def fetch_data(base_url):
params = {
'take': 30,
'skip': 0,
'page': 1
}
headers = {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/112.0.1722.64',
'Accept': 'application/json, text/plain, */*',
'Accept-Language': 'en-US,en;q=0.9',
'Origin': 'https://shop.supervalu.ie',
'Referer': 'https://shop.supervalu.ie',
'X-Site-Host': 'https://shop.supervalu.ie',
}
with requests.Session() as s:
# Initial request to get the entire product response
response = s.get(
base_url,
params=params,
headers=headers
).json()
product_name = response['items'][1]['name']
print(product_name) #Prints 'SuperValu Large Loose Oranges (1 Piece)'
print(response['items'][1]['promotions']) #List comes back empty?
#deal = response['items'][1]['promotions']['name']
#print(deal)
fetch_data('https://storefrontgateway.supervalu.ie/api/stores/5550/categories/O200001/search?')
The request used is pulling info on the products located at https://shop.supervalu.ie/sm/delivery/rsid/5550/categories/fruit-%26-vegetables/fruit-id-O200001?page=1&skip=0.
Everything works as expected, except that response['items'][1]['promotions'] comes back as an empty list. But the Network tab on the webpage suggests that it should return:
"promotions": [
{
"id": "11PR3969",
"name": "2 for €5",
"description": "2 for €5",
"additionalInformation": "SuperValu Fruit",
"startDate": "02/11/2023",
"endDate": "31/12/2049",
"limit": 0,
"threshold": null,
"minimumQuantity": 2,
"pointsBased": false,
"imageUrl": null,
"limitPerSku": null
}
]
Just searching ‘https://storefrontgateway.supervalu.ie/api/stores/5550/categories/O200001/search?&take=30&page=1&skip=0’, shows me the list I’m getting (e.g. empty promotions list).
I’m no expert in this area, but it is my understanding then that perhaps I’m requesting the data differently to the webpage, or the webpage is performing an extra operation (?).
I appreciate any help/direction you can give me with this! Also, bonus points if you can show me how you figured out the correct approach, so I can better approach these problems in the future.
>Solution :
It seems you need to put X-Shopping-Mode to the HTTP requests headers to get promotions back:
import numpy as np
import requests
def fetch_data(base_url):
params = {"take": 30, "skip": 0, "page": 1}
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36 Edg/112.0.1722.64",
"Accept": "application/json, text/plain, */*",
"Accept-Language": "en-US,en;q=0.9",
"Origin": "https://shop.supervalu.ie",
"Referer": "https://shop.supervalu.ie",
"X-Site-Host": "https://shop.supervalu.ie",
"X-Shopping-Mode": "22222222-2222-2222-2222-222222222222", # <-- put shopping mode here for promotions
}
with requests.Session() as s:
# Initial request to get the entire product response
response = s.get(base_url, params=params, headers=headers).json()
for i in response["items"]:
print(i["name"], i["promotions"])
fetch_data(
"https://storefrontgateway.supervalu.ie/api/stores/5550/categories/O200001/search?"
)
Prints:
SuperValu Bananas Snack Pack (7 Piece) []
SuperValu Large Loose Oranges (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Loose Pink Lady Apples (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
Signature Tastes Raspberries (125 g) [{'id': '11PR3969', 'name': '2 for €5', 'description': '2 for €5', 'additionalInformation': 'SuperValu Fruit', 'startDate': '02/11/2023', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 2, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
Signature Tastes Ripe Avocado (1 Piece) [{'id': '11PR3625', 'name': '2 for €2.50', 'description': '2 for €2.50', 'additionalInformation': 'Signature Taste Avocado ', 'startDate': '08/06/2023', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 2, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
Signature Tastes Blueberry (150 g) [{'id': '11PR3969', 'name': '2 for €5', 'description': '2 for €5', 'additionalInformation': 'SuperValu Fruit', 'startDate': '02/11/2023', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 2, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Organic Fairtrade Bananas (5 Piece) []
SuperValu Seedless Red Grapes (500 g) []
SuperValu Loose Bananas (1 kg) []
SuperValu Loose Lemons (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Easy Peelers (750 g) []
SuperValu Loose Pears (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
Signature Tastes ClemenGold Mandarins (500 g) []
SuperValu Kids Apple Bag (10 Piece) []
SuperValu Lemons (3 Piece) []
SuperValu Mixed Seedless Grapes (500 g) []
SuperValu Pears (6 Piece) []
SuperValu Oranges (7 Piece) []
Freddy's Fyffes Bananas Bag (7 Piece) []
SuperValu Seedless White Grapes (500 g) []
SuperValu Loose Granny Smith Apples (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
Keelings Blueberries Family Pack (200 g) []
Signature Tastes Blackberries (125 g) [{'id': '11PR3969', 'name': '2 for €5', 'description': '2 for €5', 'additionalInformation': 'SuperValu Fruit', 'startDate': '02/11/2023', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 2, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Limes (3 Piece) []
SuperValu Loose Red Grapefruit (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Loose Limes (1 Piece) [{'id': '11PR3027', 'name': '4 for €2', 'description': '4 for €2', 'additionalInformation': 'SuperValu Loose Fruit Multibuy', 'startDate': '08/12/2022', 'endDate': '31/12/2049', 'limit': 0, 'threshold': None, 'minimumQuantity': 4, 'pointsBased': False, 'imageUrl': None, 'limitPerSku': None}]
SuperValu Lemon & Lime Pack (3 Piece) []
SuperValu Yellow Melon (1 Piece) []
SuperValu Fruit Salad (200 g) []
SuperValu Kiwi Fruit (6 Piece) []