Skip to content

Question about follow Requests #129

@KauSaal

Description

@KauSaal

Hello,
I have a short question about follow Requests in Scrapy-Selenium,
In my code, I loop through all options in nested select elements. On the last select element the page is reloaded and I want to build a Scrapy Request out of a Subpage/Div ('canvas_post') to parse it:

import bs4
from scrapy import Request
from scrapy import Spider
from scrapy_selenium import SeleniumRequest
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select
from time import sleep

from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader

class EdimaxDESpider(Spider):
    name = "edimax_de"
    vendor = "Edimax"

    start_urls = ["https://www.edimax.com/edimax/download/download/data/edimax/de/download/"]

    def start_requests(self):
        url = "https://www.edimax.com/edimax/download/download/data/edimax/de/download/"
        yield SeleniumRequest(url=url, callback=self.parse, wait_time = 10)
    def parse(self, response):
        driver = response.request.meta['driver']
        # Find the select element by its class name
        solution_box = driver.find_element(By.CLASS_NAME, 'step1_select_cb')
        solution_select = Select(driver.find_element(By.CLASS_NAME, 'step1_select_cb'))
        # Get all option elements within the select element
        option_elements = solution_box.find_elements(By.TAG_NAME, ('option'))
        # Extract the value attribute from each option element
        options = [option_element.get_attribute('value') for option_element in option_elements]
        for option in options:
            if option != '':
                solution_select.select_by_value(option)
                sleep(1)
                # find the category box and select an option
                category_box = Select(driver.find_element(By.CLASS_NAME, 'step2_select_cb'))
                category_element = driver.find_element(By.CLASS_NAME, 'step2_select_cb')
                # Get all option elements within the category element
                option_elements = category_element.find_elements(By.TAG_NAME, ('option'))
                # Extract the value attribute from each option element
                options = [option_element.get_attribute('value') for option_element in option_elements]
                # loop through option
                for option in options:
                    if option != "":
                        category_box.select_by_value(option)
                        sleep(1)
                        # find the modelNo box and select an option
                        modelNo_box = Select(driver.find_element(By.CLASS_NAME, 'step3_select_cb'))
                        modelNo_element = driver.find_element(By.CLASS_NAME, 'step3_select_cb')
                        # Get all option elements within the modelNo element
                        option_elements = modelNo_element.find_elements(By.TAG_NAME, ('option'))
                        # Extract the value attribute from each option element
                        options = [option_element.get_attribute('value') for option_element in option_elements]
                        # loop through options
                        for option in options:
                            if option != '':
                                modelNo_box.select_by_value(option)
                                sleep(5)
                                html_from_page = driver.page_source
                                soup = bs4.BeautifulSoup(html_from_page, 'html.parser')
                                yield Request(soup,
                                                      callback=self.parse_product)
    def parse_product(self, response):
        print("IM HERE")
        canvas = response.css("#side2 > div.canvas_post").getall()
        print("ELEMENT CANVAS POST")`

The print statements in parse_product are never printed and I also dont get scrapy request log messages like when I'm using scrapy without selenium
Hope someone can give me an hint and thanks in advance
KauSaal

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions