Why am I not getting the results while scraping the website

Question

While I try to scrape the items individually I get the results, but when I use the try except method I don't get the results.

from bs4 import BeautifulSoup as soup
import pandas as pd
import requests
import urllib
import requests, random

data =[]
url = 'https://www.flipkart.com/search?q=iphone'

def getdata (url):
    user_agents = [
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0',
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36'
    ]
    user_agent = random.choice(user_agents)
    header_ = {'User-Agent': user_agent}
    req = urllib.request.Request(url, headers=header_)
    amazon_html = urllib.request.urlopen(req).read()
    f_soup = soup(amazon_html,'html.parser')
    
    for e in f_soup.select('div[data-tkid="2f9a5ad9-8f64-4327-8a23-5069ba20a68b.MOBFWBYZBZ7Y56WD.SEARCH"]'):
        
        try:
            title = e.find('div',{'class':'_4rR01T'}).text
        except:
            title = None
            
        try:
            rating = e.find('span',{'class':'a-size-base s-underline-text'}).text
        except:
            rating = 0
            
        data.append({
            'Title':title,
            'Rating':rating
        })
        
    return data

getdata (url)

OUTPUT

[]

output = pd.DataFrame(data)
output

WEBSITE LINK : https://www.flipkart.com/search?q=iphone&page=1

That page does not contain the `div` with `data-tkid` you are looking for. That `select` returns nothing, so `data` never changes — Tim Roberts, Apr 19 '22 at 06:50
What fields do you want to receive? I don't see the field - rating — Sergey K, Apr 19 '22 at 07:14

F.Hoque · Answer 1 · 2022-04-19T08:34:22.570

The problem was in title and rating elements selection. Now it's working.

from bs4 import BeautifulSoup 
import pandas as pd
import requests


    data =[]
    url = 'https://www.flipkart.com/search?q=iphone'
    
    headers = {
        'User-Agent':'mozila/5.0'}
    req =requests.get(url, headers=headers)
    print(req)
    
    soup=BeautifulSoup(req.content,'lxml')
    for e in soup.select('.col.col-7-12'):
            
        try:
            title = e.select_one('._4rR01T').text
        except:
            title = None
                
        try:
            rating = e.select_one('._3LWZlK').find(text=True)
        except:
            rating = 0
                
        data.append({
            'Title':title,
            'Rating':rating
            
        })
            
      
    print(data)

Output:

[{'Title': 'APPLE iPhone SE (White, 64 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone SE 
(Red, 64 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 12 Mini (Blue, 64 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 13 mini (Green, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 11 (Red, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 12 Mini (Black, 64 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 12 Mini (White, 64 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 12 Mini (Black, 128 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 13 Mini (Midnight, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone SE (White, 128 
GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 13 Mini ((PRODUCT)RED, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone SE (Red, 128 GB)', 'Rating': '4.5'}, {'Title': 'APPLE iPhone 13 Mini (Blue, 256 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 Mini (Pink, 128 
GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 Mini (Blue, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 mini (Green, 256 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 (Blue, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 Mini (Midnight, 256 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 12 Mini (Black, 256 GB)', 'Rating': '4.5'}, 
{'Title': 'APPLE iPhone 13 (Pink, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 (Midnight, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone XR (White, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 12 (Green, 128 GB)', 'Rating': '4.6'}, {'Title': 'APPLE iPhone 13 ((PRODUCT)RED, 256 GB)', 'Rating': '4.6'}]

Why am I not getting the results while scraping the website

1 Answers1