How do I write data from python line by line to csv file?

December 27, 2021

I’m just learning python. I want to improve myself with examples. sorry for my English. I’m in the process of learning a new language. 🙂

The program pulls data from an e-commerce site.

when I want to save it as a csv file, each new data overwrites the previous data. I tried several examples but it didn’t work.

Thanks for your help.

import requests 
import gettext
from bs4 import BeautifulSoup
import pandas as pd
import openpyxl as xls 
import xlsxwriter`

baseurl = "https://www.trendyol.com"

headers = {'User-Agent': 
           'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36 OPR/38.0.2220.41'
          }

for x in range(1,62):
    r = requests.get(f'https://www.trendyol.com/cep-telefonu-x-c103498?pi={x}', headers=headers)
    soup = BeautifulSoup(r.content, 'lxml')
    
    productlist = soup.find_all('div', class_='p-card-wrppr')

    for item in productlist:
        
        productname = item.find('span', class_='prdct-desc-cntnr-name').getText()
        productprice_old = item.find('div', class_='prc-box-sllng').getText()
        productprice_discount = item.find('div', class_='prc-box-dscntd') 
        
        for productlink in item.find_all('a'):
            productlink = baseurl+productlink.get('href') 

            if productprice_discount == None:
                productprice_discount = productprice_old
            else: 
                productprice_discount = productprice_discount.getText()          
                
            for merchant_name in productlink:
                r = requests.get(productlink, headers=headers)
                soup = BeautifulSoup(r.content, 'lxml')   

                merchant_name = soup.find('a', class_='merchant-text')

                if merchant_name == None: 
                    merchant_name = soup.find('a', class_='title')
                    
                    if merchant_name == None:
                        merchant_name = soup.find('span', class_='product-description-market-place')
                    
                    if merchant_name == None:
                            merchant_name = ('NULL')
                else: 
                    merchant_name = merchant_name.getText()
                
                break

            for product_image in productlink:
                r = requests.get(productlink, headers=headers)
                soup = BeautifulSoup(r.content, 'lxml')
                product_image = soup.find_all('img', attrs={'class':'detail-section-img'})   
                
            
                image_src = [x['src'] for x in product_image]
                image_src = [x for x in image_src if x.endswith('.jpg' or '.png')]
                
            break


        data = [ [productname,productlink,productprice_old,productprice_discount,merchant_name,image_src] ]
        df = pd.DataFrame(data, columns = ["Product Name", "URL", "Price", "D-Price", "Store", "Image Url"])
        df.to_csv('trendyol3.csv')