How to Scrape Amazon Product Reviews using Scrapy Splash

import scrapy

from scrapy_splash import SplashRequest

from ..items import AmazonreviewsItem

# lua script to automate the login part by imitating email input & password input

lua_script = “””

function main(splash, args)

   splash:init_cookies(splash.args.cookies)

   splash:set_viewport_full()

   assert(splash:go(args.url))

   assert(splash:wait(5))

   local email_input = splash:select(‘input[name=email]’)  

   email_input:send_text(“nishant@growthromeo.com”)

   assert(splash:wait(5))

   local email_submit = splash:select(‘input[id=continue]’)

   email_submit:click()

   assert(splash:wait(7))

   local password_input = splash:select(‘input[name=password]’)  

   password_input:send_text(“Amz$ucks012”)

   assert(splash:wait(6))

   local password_submit = splash:select(‘input[id=signInSubmit]’)

   password_submit:click()

   assert(splash:wait(7))

   return {

       html=splash:html(),

       url = splash:url(),

       cookies = splash:get_cookies(),

       }

   end

“””

class AmazonReviewsSpider(scrapy.Spider):

   # inherit scrapy base class

   # give the name to use for ‘scrapy crawl projectName’ command

   name = “amazonReview”

   # entry point for scrapy crawler execution

   def start_requests(self):

       # enter the amazon sign in url

       login_url = ‘https://www.amazon.com/ap/signin?openid.pape.max_auth_age=0&openid.return_to=https%3A%2F%2Fwww.amazon.com%2F%3Fref_%3Dnav_custrec_signin&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.assoc_handle=usflex&openid.mode=checkid_setup&openid.claimed_id=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0&’

       # splash request that will execute lua script and pass the response to crawl_product

       yield SplashRequest(

           url=login_url,

           callback=self.crawl_product,

           endpoint=‘execute’,

           args={

               ‘width’: 1000,

               ‘lua_source’: lua_script,

               ‘timeout’: 3600,

               ‘ua’: “Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.36”

               },

           )

   def crawl_product(self, response):

       # create a cookie dictionary to be used in scrapy request object that fetches the products page

       cookies_dict = {cookie[‘name’]: cookie[‘value’] for cookie in response.data[‘cookies’]}

       # the product we use to demonstrate review scraping

       url_list = [‘https://www.amazon.com/ENHANCE-Headphone-Customizable-Lighting-Flexible/dp/B07DR59JLP/’]

       # create requests for the specified product urls

       for url in url_list:

           yield scrapy.Request(url=url, cookies=cookies_dict, callback=self.parse_product)

   def parse_product(self, response):

       # using xpath identify and get the data for different reviews

       try:

           review_list = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div’)

           review_count = len(review_list)

           for review_number in range (1, review_count+1):

               items = AmazonreviewsItem()

               items[‘authorname’] = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div[‘ + str(review_number) + ‘]//span[contains(@class,”name”)]/text()’).get()

               items[‘reviewRating’] = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div[‘ + str(review_number) + ‘]//i[contains(@class,”rating”)]/span/text()’).get()

               items[‘commentTitle’] = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div[‘ + str(review_number) + ‘]//a[contains(@class,”title”)]/span/text()’).get()

               items[‘commentDate’] = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div[‘ + str(review_number) + ‘]//span[contains(@class,”date”)]/text()’).get()

               items[‘commentText’] = response.xpath(‘//div[contains(@id, “dp-review-list”)]/div[‘ + str(review_number) + ‘]//div[contains(@class,”reviewText”)]/span/text()’).get()

               yield items

       except Exception as e:

           self.log(f‘An error occurred: {str(e)}‘)