Merge-build: 2.9.0-beta

Author: jasoneri

.github/scripts/generate_hitomi_manifest.py

@@ -0,0 +1,86 @@
+"""Generate hitomi-manifest.json for client-side download."""
+import sys
+import json
+import hashlib
+import sqlite3
+import argparse
+from pathlib import Path
+from datetime import datetime, timezone
+from contextlib import closing
+
+
+SOURCES = [
+    {
+        "id": "github",
+        "priority": 10,
+        "url": "https://github.com/jasoneri/ComicGUISpider/releases/download/preset/hitomi.db"
+    },
+    {
+        "id": "gitee",
+        "priority": 20,
+        "url": "https://gitee.com/json_eri/ComicGUISpider/releases/download/preset/hitomi.db"
+    },
+]
+
+
+def generate(db_path, output_path):
+    db_file = Path(db_path)
+    if not db_file.exists():
+        print(f"[ERROR] {db_path} not found")
+        return False
+
+    # sha256
+    h = hashlib.sha256()
+    with open(db_file, 'rb') as f:
+        for chunk in iter(lambda: f.read(8192), b''):
+            h.update(chunk)
+    sha256 = h.hexdigest()
+    size = db_file.stat().st_size
+
+    # stats
+    table_count = 0
+    row_count = 0
+    with closing(sqlite3.connect(db_path)) as conn:
+        cursor = conn.cursor()
+        tables = cursor.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' AND name != 'language' AND name NOT LIKE 'sqlite_%'"
+        ).fetchall()
+        table_count = len(tables)
+        for (t,) in tables:
+            count = cursor.execute(f"SELECT COUNT(*) FROM `{t}`").fetchone()[0]
+            row_count += count
+
+    now = datetime.now(timezone.utc)
+    manifest = {
+        "schema_version": 1,
+        "version": now.strftime("%Y.%m.%d.%H%M"),
+        "generated_at": now.isoformat(),
+        "file": {
+            "name": "hitomi.db",
+            "size": size,
+            "sha256": sha256,
+            "table_count": table_count,
+            "row_count": row_count,
+        },
+        "sources": SOURCES,
+    }
+
+    out = Path(output_path)
+    out.parent.mkdir(parents=True, exist_ok=True)
+    with open(out, 'w', encoding='utf-8') as f:
+        json.dump(manifest, f, indent=2, ensure_ascii=False)
+
+    print(f"[OK] manifest written to {out}")
+    print(f"  version: {manifest['version']}")
+    print(f"  sha256:  {sha256[:16]}...")
+    print(f"  size:    {size} bytes")
+    print(f"  tables:  {table_count}, rows: {row_count}")
+    return True
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate hitomi-manifest.json")
+    parser.add_argument("db_path", help="Path to hitomi.db")
+    parser.add_argument("--output", "-o", default="hitomi-manifest.json", help="Output manifest path")
+    args = parser.parse_args()
+    sys.exit(0 if generate(args.db_path, args.output) else 1)

.github/scripts/hitomi_db_gate.py

@@ -0,0 +1,75 @@
+"""Quality gate for hitomi.db - validates integrity before release."""
+import sys
+import sqlite3
+from contextlib import closing
+
+EXPECTED_TABLES = 108  # 4 categories x 27 letters
+MIN_TOTAL_ROWS = 5000
+
+CATEGORIES = ['tags', 'artists', 'series', 'characters']
+LETTERS = [*[chr(i) for i in range(97, 123)], '123']
+
+
+def check(db_path):
+    errors = []
+
+    with closing(sqlite3.connect(db_path)) as conn:
+        cursor = conn.cursor()
+
+        # integrity check
+        result = cursor.execute("PRAGMA integrity_check").fetchone()
+        if not result or result[0] != "ok":
+            errors.append(f"integrity_check failed: {result}")
+            print("\n".join(errors))
+            return False
+
+        # table existence + non-empty
+        total_rows = 0
+        missing = []
+        empty = []
+
+        for cat in CATEGORIES:
+            for letter in LETTERS:
+                table_name = f"all{cat}-{letter}"
+                try:
+                    count = cursor.execute(f"SELECT COUNT(*) FROM `{table_name}`").fetchone()[0]
+                    total_rows += count
+                    if count == 0:
+                        empty.append(table_name)
+                except sqlite3.OperationalError:
+                    missing.append(table_name)
+
+        if missing:
+            errors.append(f"missing tables ({len(missing)}): {missing[:5]}...")
+        if empty:
+            # empty tables are warnings, not errors (some letters may legitimately have no data)
+            print(f"[WARN] empty tables ({len(empty)}): {empty[:10]}...")
+
+        found = EXPECTED_TABLES - len(missing)
+        if found < EXPECTED_TABLES:
+            errors.append(f"table count: {found}/{EXPECTED_TABLES}")
+
+        if total_rows < MIN_TOTAL_ROWS:
+            errors.append(f"total rows {total_rows} < minimum {MIN_TOTAL_ROWS}")
+
+        # language table
+        try:
+            cursor.execute("SELECT COUNT(*) FROM `language`")
+        except sqlite3.OperationalError:
+            errors.append("language table missing")
+
+    if errors:
+        print("[FAIL] Quality gate errors:")
+        for e in errors:
+            print(f"  - {e}")
+        return False
+
+    print(f"[PASS] {found} tables, {total_rows} total rows")
+    return True
+
+
+if __name__ == "__main__":
+    if len(sys.argv) < 2:
+        print("Usage: hitomi_db_gate.py <db_path>")
+        sys.exit(1)
+    sys.exit(0 if check(sys.argv[1]) else 1)

.github/scripts/smoke_wheel.py

@@ -0,0 +1,36 @@
+import os
+import subprocess
+import sys
+
+
+def run_test(name, cmd, env):
+    print(f"::group::{name}")
+    result = subprocess.run(cmd, env=env, text=True, capture_output=True)
+    if result.stdout:
+        print(result.stdout)
+    if result.stderr:
+        print(result.stderr, file=sys.stderr)
+    print(f"exit_code={result.returncode}")
+    print("::endgroup::")
+    return result.returncode
+
+
+def main():
+    env = os.environ.copy()
+    tests = [
+        ("entrypoint import", [sys.executable, "-c", "import CGS"]),
+        ("check mode startup", [sys.executable, "-c", "import CGS; CGS.start()"]),
+        ("cli help", ["cgs-cli", "--help"]),
+    ]
+
+    failures = []
+    for name, cmd in tests:
+        if run_test(name, cmd, env) != 0:
+            failures.append(name)
+
+    if failures:
+        raise SystemExit(f"Smoke tests failed: {', '.join(failures)}")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/hitomi-db.yml

@@ -0,0 +1,85 @@
+name: Hitomi DB Build
+
+on:
+  schedule:
+    - cron: '0 3 * * 1'  # UTC Monday 03:00
+  workflow_dispatch:
+    inputs:
+      force_full:
+        description: 'Force full scrape (ignore previous errors)'
+        required: false
+        default: 'false'
+        type: boolean
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install httpx[http2] lxml loguru PyYAML polib PyQt5
+
+      - name: Scrape dataset
+        id: scrape
+        continue-on-error: true
+        run: |
+          python -m utils.website.hitomi.scape_dataset \
+            --db-path assets/hitomi.db \
+            --workers 2 \
+            --max-retries 5 \
+            --timeout 15
+
+      - name: Retry failed (if first attempt had errors)
+        id: retry
+        if: steps.scrape.outcome == 'failure'
+        run: |
+          python -m utils.website.hitomi.scape_dataset \
+            --db-path assets/hitomi.db \
+            --workers 1 \
+            --max-retries 8 \
+            --timeout 20 \
+            --only-failed
+
+      - name: Quality gate
+        run: python .github/scripts/hitomi_db_gate.py assets/hitomi.db
+
+      - name: Generate manifest
+        run: |
+          python .github/scripts/generate_hitomi_manifest.py \
+            assets/hitomi.db \
+            --output assets/hitomi-manifest.json
+
+      - name: Upload to release
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          TAG="preset"
+
+          # Upload assets to existing release (or create if not exists)
+          if gh release view "$TAG" &>/dev/null; then
+            # Delete old assets then re-upload
+            gh release delete-asset "$TAG" hitomi.db --yes 2>/dev/null || true
+            gh release delete-asset "$TAG" hitomi-manifest.json --yes 2>/dev/null || true
+            gh release upload "$TAG" \
+              assets/hitomi.db \
+              assets/hitomi-manifest.json
+          else
+            gh release create "$TAG" \
+              --title "Preset Assets" \
+              --notes "Auto-managed preset assets (hitomi.db, etc.)" \
+              --latest=false \
+              assets/hitomi.db \
+              assets/hitomi-manifest.json
+          fi

.github/workflows/release.yml

@@ -34,34 +34,7 @@ jobs:
         uv venv .venv-smoke
         source .venv-smoke/bin/activate
         uv pip install dist/*.whl
-        python - <<'PY'
-        import os
-        import subprocess
-        import sys
-
-        env = os.environ.copy()
-        tests = [
-            ("entrypoint import", [sys.executable, "-c", "import CGS"]),
-            ("check mode startup", [sys.executable, "-c", "import CGS; CGS.start()"]),
-            ("cli help", ["cgs-cli", "--help"]),
-        ]
-
-        failures = []
-        for name, cmd in tests:
-            print(f"::group::{name}")
-            result = subprocess.run(cmd, env=env, text=True, capture_output=True)
-            if result.stdout:
-                print(result.stdout)
-            if result.stderr:
-                print(result.stderr, file=sys.stderr)
-            print(f"exit_code={result.returncode}")
-            print("::endgroup::")
-            if result.returncode != 0:
-                failures.append(name)
-
-        if failures:
-            raise SystemExit(f"Smoke tests failed: {', '.join(failures)}")
-        PY
+        uv run .github/scripts/smoke_wheel.py
 
     - name: Publish to PyPI
       run: uv publish

.gitignore

@@ -186,6 +186,8 @@ custom/start/*
 *.ico
 *.qrc
 src/
+!GUI/src/
+GUI/src/*
 !GUI/src/preview_format/
 !GUI/src/material_ct.py
 !GUI/src/__init__.py
@@ -204,7 +206,7 @@ version.json
 # self/conf
 gitee_t.json
 record.db
-hitomi.db
+hitomi.db*
 # self/desc created html
 desc.html
 docs/*.html
@@ -214,6 +216,8 @@ _bug_log
 # AI
 .cursor
 .claude
+.codex
+.gemini
 .kilocode
 /plans
 .ace-tool/