Playwright Chromium/Javascript: how to pull multiple snippets from a single DIV

[natecovington]

I'm trying to scrape content off a Javascript based site, it renders public notices into a DIV with this xpath:

/html/body/div/div[1]/div[5]/div/div/div[2]/div/p[2]

Somewhere else on the page, a series of 20 (twenty) DIV's with xpath's in series like this:

/html/body/div/div[1]/div[5]/div/div/div[1]/div[1]/div[1]/div/div
/html/body/div/div[1]/div[5]/div/div/div[1]/div[1]/div[2]/div/div
/html/body/div/div[1]/div[5]/div/div/div[1]/div[1]/div[3]/div/div
(...)
/html/body/div/div[1]/div[5]/div/div/div[1]/div[1]/div[20]/div/div

How do I set it up so that it clicks on all 20 DIV's, gets the content from this box after each:
/html/body/div/div[1]/div[5]/div/div/div[2]/div/p[2]

Thanks!

[natecovington]

Here's an example Python script that generally shows what I'm trying to do

import asyncio
import re
from playwright.async_api import async_playwright

async def main():
    notices = []  # List to store notices

    async with async_playwright() as p:
        browser = await p.chromium.launch()
        page = await browser.new_page()

        # Hardcoded URL to scrape
        url = 'https://www.example.com/public-notices'

        try:
            # Navigate to the page
            await page.goto(url)

            # Wait for the page to load
            await page.wait_for_load_state('networkidle')

            # Find the heading "Most Recent Notices"
            heading = await page.query_selector('h2:has-text("Most Recent Notices")')

            if heading:
                # Scroll to the bottom of the page to load all content
                await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')

                # Find all buttons inside div with class "css-asl9ok" and click on them
                buttons = await page.query_selector_all('div.css-asl9ok div[role="button"]')
                for i, button in enumerate(buttons, start=1):
                    print(f"Clicking on button {i}")
                    await button.click(timeout=10000)  # Increase timeout to 10 seconds

                    # Wait for the content to load
                    await page.wait_for_selector('div.css-imz8u')

                    # Get the full text from the DIVs with class "css-e9gc8s"
                    full_text_elements = await page.query_selector_all('div.css-e9gc8s')
                    for element in full_text_elements:
                        # Get the text content of each full text element
                        full_text = await element.evaluate('(element) => element.textContent')
                        
                        # Clean up the text
                        full_text = full_text.replace('Public Notices', '')
                        full_text = full_text.replace('SharePrint', '')
                        full_text = full_text.replace('Originally published at example.com on ', '')

                        # Extract date from anywhere in the text
                        date_match = re.search(r'\d{2}/\d{2}/\d{4}', full_text)
                        date = date_match.group(0) if date_match else None

                        # Remove the date from anywhere in the text
                        if date:
                            full_text = full_text.replace(date, '').strip()

                        # Append notice and date to the list
                        notices.append({'Date': date, 'Notice': full_text.strip()})  

        except Exception as e:
            print(f"An error occurred: {e}")

        finally:
            # Write notices to HTML file
            with open('public_notices.html', 'w', encoding='utf-8') as html_file:
                for notice in notices:
                    if notice['Date'] and notice['Notice']:
                        html_file.write(f"<h3>{notice['Date']}</h3>")
                        html_file.write(f"<p>{notice['Notice']}</p>")
                        html_file.write("<hr>")

            # Take a screenshot for verification
            await page.screenshot(path='public_notices.png')

            # Close the browser
            await browser.close()

asyncio.run(main())