add scrape workfow

Author: ks6088ts

.github/workflows/scrape-visasq.yaml

@@ -0,0 +1,34 @@
+name: scrape-visasq
+
+on:
+  push:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 3" # Every Wednesday at 00:00 UTC
+
+jobs:
+  scrape-visasq:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Docker Login
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - name: Run Docker container and generate output file
+        run: |
+          mkdir -p assets
+          echo "test" > assets/test.txt
+          docker run \
+            --rm \
+            -v $(pwd)/assets:/app/assets \
+            ks6088ts/workshop-playwright-python:latest \
+            python scripts/visasq.py scrape --max-page 20
+
+      - name: Upload output file as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: assets
+          path: ./assets/
+          retention-days: 14

Dockerfile

@@ -19,6 +19,8 @@ COPY --from=requirements-stage /tmp/requirements.txt /app/requirements.txt
 COPY . .
 
 # Install dependencies
-RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
+RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt && \
+    playwright install --with-deps
+
 
 CMD ["python", "workshop_playwright_python/core.py"]

Makefile

@@ -57,7 +57,7 @@ update: ## update packages
 # ---
 DOCKER_REPO_NAME ?= ks6088ts
 DOCKER_IMAGE_NAME ?= workshop-playwright-python
-DOCKER_COMMAND ?=
+DOCKER_COMMAND ?= python scripts/visasq.py scrape --help
 
 # Tools
 TOOLS_DIR ?= /usr/local/bin
@@ -88,6 +88,13 @@ docker-scan: ## scan Docker image
 .PHONY: ci-test-docker
 ci-test-docker: docker-lint docker-build docker-scan docker-run ## run CI test for Docker
 
+.PHONY: docker-visasq-scrape
+docker-visasq-scrape: ## scrape visasq entries using Docker
+	docker run --rm \
+		-v $(PWD)/assets:/app/assets \
+		$(DOCKER_REPO_NAME)/$(DOCKER_IMAGE_NAME):$(GIT_TAG) \
+		python scripts/visasq.py scrape --max-page 20
+
 # ---
 # Docs
 # ---

scripts/visasq_scraper.py renamed to scripts/visasq.py

@@ -3,8 +3,12 @@
 import datetime
 import os
 
+import typer
+
 from playwright.async_api import Page, async_playwright
 
+app = typer.Typer(help="VisaSQ スクレイパー CLI ツール")
+
 
 async def dump_csv(entries, filepath="assets/visasq_entries.csv"):
     """CSV ファイルにエントリを保存するヘルパー関数"""
@@ -26,6 +30,7 @@ async def dump_csv(entries, filepath="assets/visasq_entries.csv"):
 
 async def retrieve_visasq_entries(page: Page, url: str):
     entries = []
+    print(f"Retrieving entries from {url}...")
     await page.goto(url)
     await page.wait_for_load_state("networkidle")
 
@@ -55,26 +60,31 @@ async def retrieve_visasq_entries(page: Page, url: str):
     return entries
 
 
-async def main():
+async def run_scraper(
+    base_url: str,
+    max_page: int,
+    keyword: str = "",
+    is_started_only: bool = True,
+    output_dir: str = "assets",
+):
     async with async_playwright() as p:
         browser = await p.chromium.launch()
         page = await browser.new_page()
 
-        BASE_URL = "https://expert.visasq.com"
         all_entries = []
-        max_page = 15
 
         try:
             for page_number in range(1, max_page + 1):
                 print(f"Retrieving entries from page {page_number}...")
-                entries = await retrieve_visasq_entries(
-                    page=page,
-                    url=f"{BASE_URL}/issue/?keyword=&is_started_only=true&page={page_number}",
-                )
+
+                # キーワードとフィルター条件を URL に追加
+                url = f"{base_url}/issue/?keyword={keyword}&is_started_only={'true' if is_started_only else 'false'}&page={page_number}"  # noqa: E501
+
+                entries = await retrieve_visasq_entries(page=page, url=url)
 
                 # entries の url を絶対 URL に変換
                 for entry in entries:
-                    entry["url"] = f"{BASE_URL}{entry['url']}"
+                    entry["url"] = f"{base_url}{entry['url']}"
 
                 all_entries.extend(entries)
                 print(f"Found {len(entries)} entries on page {page_number}")
@@ -88,16 +98,40 @@ async def main():
 
         # 現在の日時をファイル名に含める
         now = datetime.datetime.now()
-        filepath = "assets/visasq_entries_" + now.strftime("%Y%m%d_%H%M%S") + ".csv"
+        filepath = f"{output_dir}/visasq_entries_" + now.strftime("%Y%m%d_%H%M%S") + ".csv"
 
-        await dump_csv(
-            entries=all_entries,
-            filepath=filepath,
-        )
+        await dump_csv(entries=all_entries, filepath=filepath)
 
         print(f"Scraping completed. Total entries: {len(all_entries)}")
         print(f"Results saved to: {filepath}")
 
+        return all_entries
+
+
+@app.command()
+def scrape(
+    max_page: int = typer.Option(15, "--max-page", "-m", help="スクレイピングする最大ページ数"),
+    keyword: str = typer.Option("", "--keyword", "-k", help="検索キーワード"),
+    is_started_only: bool = typer.Option(
+        True, "--started-only/--not-started-only", help="進行中の案件のみを表示するかどうか"
+    ),
+    base_url: str = typer.Option("https://expert.visasq.com", "--base-url", "-u", help="VisaSQ の基本 URL"),
+    output_dir: str = typer.Option("assets", "--output-dir", "-o", help="出力ディレクトリ"),
+):
+    """
+    VisaSQ からデータをスクレイピングし、CSV ファイルに保存します
+    """
+    typer.echo(f"スクレイピングを開始します。最大ページ数: {max_page}")
+    asyncio.run(run_scraper(base_url, max_page, keyword, is_started_only, output_dir))
+
+
+@app.callback()
+def callback():
+    """
+    VisaSQ ウェブサイトから案件情報をスクレイピングするツール
+    """
+    pass
+
 
 if __name__ == "__main__":
-    asyncio.run(main())
+    app()