Merge pull request #217 from digitalghost-dev/1.8.2

1.8.2

Author: digitalghost-dev

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ on:
       - main
 
 env:
-  VERSION_NUMBER: 'v1.8.1'
+  VERSION_NUMBER: 'v1.8.2'
   DOCKERHUB_REGISTRY_NAME: 'digitalghostdev/poke-cli'
   AWS_REGION: 'us-west-2'
 

.gitignore

@@ -63,3 +63,6 @@ card_data/pipelines/poke_cli_dbt/.user.yml
 /card_data/sample_scripts/
 
 card_data/~/
+card_data/storage/
+/.claude/
+CLAUDE.md

.goreleaser.yml

@@ -14,7 +14,7 @@ builds:
       - windows
       - darwin
     ldflags:
-      - -s -w -X main.version=v1.8.1
+      - -s -w -X main.version=v1.8.2
 
 archives:
   - formats: [ 'zip' ]

Dockerfile

@@ -1,5 +1,5 @@
 # build 1
-FROM golang:1.24.9-alpine3.22 AS build
+FROM golang:1.24.11-alpine3.23 AS build
 
 WORKDIR /app
 
@@ -8,13 +8,13 @@ RUN go mod download
 
 COPY . .
 
-RUN go build -ldflags "-X main.version=v1.8.1" -o poke-cli .
+RUN go build -ldflags "-X main.version=v1.8.2" -o poke-cli .
 
 # build 2
-FROM --platform=$BUILDPLATFORM alpine:3.22
+FROM --platform=$BUILDPLATFORM alpine:3.23
 
 # Installing only necessary packages and remove them after use
-RUN apk add --no-cache shadow=4.17.3-r0 && \
+RUN apk add --no-cache shadow=4.18.0-r0 && \
     addgroup -S poke_group && adduser -S poke_user -G poke_group && \
     sed -i 's/^root:.*/root:!*:0:0:root:\/root:\/sbin\/nologin/' /etc/passwd && \
     apk del shadow

README.md

@@ -2,7 +2,7 @@
     <img height="250" width="350" src="pokemon.svg" alt="pokemon-logo"/>
     <h1>Pokémon CLI</h1>
     <img src="https://img.shields.io/github/v/release/digitalghost-dev/poke-cli?style=flat-square&logo=git&logoColor=FFCC00&label=Release%20Version&labelColor=EEE&color=FFCC00" alt="version-label">
-    <img src="https://img.shields.io/docker/image-size/digitalghostdev/poke-cli/v1.8.1?arch=arm64&style=flat-square&logo=docker&logoColor=FFCC00&labelColor=EEE&color=FFCC00" alt="docker-image-size">
+    <img src="https://img.shields.io/docker/image-size/digitalghostdev/poke-cli/v1.8.2?arch=arm64&style=flat-square&logo=docker&logoColor=FFCC00&labelColor=EEE&color=FFCC00" alt="docker-image-size">
     <img src="https://img.shields.io/github/actions/workflow/status/digitalghost-dev/poke-cli/ci.yml?branch=main&style=flat-square&logo=github&logoColor=FFCC00&label=CI&labelColor=EEE&color=FFCC00" alt="ci-status-badge">
 </div>
 <div align="center">
@@ -95,11 +95,11 @@ Cloudsmith is a fully cloud-based service that lets you easily create, store, an
 3. Choose how to interact with the container:
    * Run a single command and exit:
     ```bash
-    docker run --rm -it digitalghostdev/poke-cli:v1.8.1 <command> [subcommand] [flag]
+    docker run --rm -it digitalghostdev/poke-cli:v1.8.2 <command> [subcommand] [flag]
     ```
    * Enter the container and use its shell:
     ```bash
-    docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.8.1 -c "cd /app && exec sh"
+    docker run --rm -it --name poke-cli --entrypoint /bin/sh digitalghostdev/poke-cli:v1.8.2 -c "cd /app && exec sh"
    # placed into the /app directory, run the program with './poke-cli'
    # example: ./poke-cli ability swift-swim
     ```
@@ -192,7 +192,7 @@ Below is a list of the planned/completed commands and flags:
 - [ ] `card`: get data about a TCG card.
     - [x] add mega evolution data
     - [x] add scarlet & violet data
-    - [ ] add sword & shield data
+    - [x] add sword & shield data
     - [ ] add sun & moon data
     - [ ] add x & y data
 - [x] `item`: get data about an item.

card_data/pipelines/defs/extract/extract_data.py

@@ -10,6 +10,7 @@
 
 import requests
 
+
 class Series(BaseModel):
     id: str
     name: str
@@ -42,16 +43,18 @@ def extract_series_data() -> pl.DataFrame:
         print(e)
         raise
 
-    filtered = [s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"]]
+    filtered = [
+        s.model_dump(mode="json") for s in validated if s.id in ["swsh", "sv", "me"]
+    ]
     return pl.DataFrame(filtered)
 
 
 @dg.asset(kinds={"API", "Polars", "Pydantic"}, name="extract_set_data")
 def extract_set_data() -> pl.DataFrame:
     url_list = [
-        "https://api.tcgdex.net/v2/en/series/swsh",
-        "https://api.tcgdex.net/v2/en/series/sv",
         "https://api.tcgdex.net/v2/en/series/me",
+        "https://api.tcgdex.net/v2/en/series/sv",
+        "https://api.tcgdex.net/v2/en/series/swsh",
     ]
 
     flat: list[dict] = []
@@ -68,17 +71,14 @@ def extract_set_data() -> pl.DataFrame:
                 "official_card_count": s.get("cardCount", {}).get("official"),
                 "total_card_count": s.get("cardCount", {}).get("total"),
                 "logo": s.get("logo"),
-                "symbol": s.get("symbol")
+                "symbol": s.get("symbol"),
             }
             flat.append(entry)
 
     # Pydantic validation
     try:
         validated: list[Set] = [Set(**item) for item in flat]
-        print(
-            colored(" ✓", "green"),
-            "Pydantic validation passed for all set entries."
-        )
+        print(colored(" ✓", "green"), "Pydantic validation passed for all set entries.")
     except ValidationError as e:
         print(colored(" ✖", "red"), "Pydantic validation failed.")
         print(e)
@@ -89,9 +89,7 @@ def extract_set_data() -> pl.DataFrame:
 
 @dg.asset(kinds={"API"}, name="extract_card_url_from_set_data")
 def extract_card_url_from_set() -> list:
-    urls = [
-        "https://api.tcgdex.net/v2/en/sets/me02"
-    ]
+    urls = ["https://api.tcgdex.net/v2/en/sets/me02"]
 
     all_card_urls = []
 
@@ -102,7 +100,11 @@ def extract_card_url_from_set() -> list:
 
             data = r.json()["cards"]
 
-            set_card_urls = [f"https://api.tcgdex.net/v2/en/cards/{card['id']}" for card in data]
+            set_card_urls = [
+                f"https://api.tcgdex.net/v2/en/cards/{card['id']}"
+                for card in data
+                if "-TG" not in card["id"]
+            ]
             all_card_urls.extend(set_card_urls)
 
             time.sleep(0.1)
@@ -113,9 +115,9 @@ def extract_card_url_from_set() -> list:
     return all_card_urls
 
 
-@dg.asset(deps=[extract_card_url_from_set], kinds={"API"}, name="extract_card_info")
-def extract_card_info() -> list:
-    card_url_list = extract_card_url_from_set()
+@dg.asset(kinds={"API"}, name="extract_card_info")
+def extract_card_info(extract_card_url_from_set_data: list) -> list:
+    card_url_list = extract_card_url_from_set_data
     cards_list = []
 
     for url in card_url_list:
@@ -124,26 +126,37 @@ def extract_card_info() -> list:
             r.raise_for_status()
             data = r.json()
             cards_list.append(data)
-            # print(f"Retrieved card: {data['id']} - {data.get('name', 'Unknown')}")
+            print(f"Retrieved card: {data['id']} - {data.get('name', 'Unknown')}")
             time.sleep(0.1)
         except requests.RequestException as e:
             print(f"Failed to fetch {url}: {e}")
 
     return cards_list
 
 
-@dg.asset(deps=[extract_card_info], kinds={"Polars"}, name="create_card_dataframe")
-def create_card_dataframe() -> pl.DataFrame:
-    cards_list = extract_card_info()
+@dg.asset(kinds={"Polars"}, name="create_card_dataframe")
+def create_card_dataframe(extract_card_info: list) -> pl.DataFrame:
+    cards_list = extract_card_info
 
     all_flat_cards = []
 
     for card in cards_list:
         flat = {}
 
         # Copy top-level scalar values
-        scalar_keys = ['category', 'hp', 'id', 'illustrator', 'image', 'localId',
-                       'name', 'rarity', 'regulationMark', 'retreat', 'stage']
+        scalar_keys = [
+            "category",
+            "hp",
+            "id",
+            "illustrator",
+            "image",
+            "localId",
+            "name",
+            "rarity",
+            "regulationMark",
+            "retreat",
+            "stage",
+        ]
         for key in scalar_keys:
             flat[key] = card.get(key)
 
@@ -165,7 +178,7 @@ def create_card_dataframe() -> pl.DataFrame:
 
         attacks = card.get("attacks", [])
         for i, atk in enumerate(attacks):
-            prefix = f"attack_{i+1}"
+            prefix = f"attack_{i + 1}"
             flat[f"{prefix}_name"] = atk.get("name")
             flat[f"{prefix}_damage"] = atk.get("damage")
             flat[f"{prefix}_effect"] = atk.get("effect")

card_data/pipelines/defs/extract/extract_pricing_data.py

@@ -1,4 +1,6 @@
 from typing import Optional
+import re
+import unicodedata
 
 import dagster as dg
 import polars as pl
@@ -8,24 +10,42 @@
 
 
 SET_PRODUCT_MATCHING = {
-    "sv01": "22873",
-    "sv02": "23120",
-    "sv03": "23228",
-    "sv03.5": "23237",
-    "sv04": "23286",
-    "sv04.5": "23353",
-    "sv05": "23381",
-    "sv06": "23473",
-    "sv06.5": "23529",
-    "sv07": "23537",
-    "sv08": "23651",
-    "sv08.5": "23821",
-    "sv09": "24073",
-    "sv10": "24269",
+    "me02": "24448",
+    "me01": "24380",
+    # Scarlet & Violet
     "sv10.5b": "24325",
     "sv10.5w": "24326",
-    "me01": "24380",
-    "me02": "24448"
+    "sv10": "24269",
+    "sv09": "24073",
+    "sv08.5": "23821",
+    "sv08": "23651",
+    "sv07": "23537",
+    "sv06.5": "23529",
+    "sv06": "23473",
+    "sv05": "23381",
+    "sv04.5": "23353",
+    "sv04": "23286",
+    "sv03.5": "23237",
+    "sv03": "23228",
+    "sv02": "23120",
+    "sv01": "22873",
+    # Sword & Shield
+    "swsh12.5": "17688",
+    "swsh12": "3170",
+    "swsh11": "3118",
+    "swsh10.5": "3064",
+    "swsh10": "3040",
+    "swsh9": "2948",
+    "swsh8": "2906",
+    "swsh7": "2848",
+    "swsh6": "2807",
+    "swsh5": "2765",
+    "swsh4.5": "2754",
+    "swsh4": "2701",
+    "swsh3.5": "2685",
+    "swsh3": "2675",
+    "swsh2": "2626",
+    "swsh1": "2585",
 }
 
 
@@ -53,8 +73,31 @@ def get_card_number(card: dict) -> Optional[str]:
 
 
 def extract_card_name(full_name: str) -> str:
-    """Extract clean card name, removing variant information after dash"""
-    return full_name.partition("-")[0].strip() if "-" in full_name else full_name
+    """Extract clean card name, removing variant information after dash and parenthetical suffixes"""
+
+    name = full_name.partition("-")[0].strip() if "-" in full_name else full_name
+
+    # Remove parenthetical card numbers like "(010)" or "(1)"
+    # Pattern: space followed by parentheses containing only digits
+    name = re.sub(r"\s+\(\d+\)$", "", name)
+
+    # Remove known variant types in parentheses
+    # e.g., "(Secret)", "(Full Art)", "(Reverse Holofoil)", etc.
+    variant_types = [
+        "Full Art",
+        "Secret",
+        "Reverse Holofoil",
+        "Rainbow Rare",
+        "Gold",
+    ]
+    for variant in variant_types:
+        name = name.replace(f" ({variant})", "")
+
+    # Normalize accented characters (é → e, ñ → n, etc.)
+    name = unicodedata.normalize("NFD", name)
+    name = "".join(char for char in name if unicodedata.category(char) != "Mn")
+
+    return name.strip()
 
 
 def pull_product_information(set_number: str) -> pl.DataFrame:
@@ -83,9 +126,14 @@ def pull_product_information(set_number: str) -> pl.DataFrame:
         if not is_card(card):
             continue
 
+        # Skip ball pattern variants (unique to Prismatic Evolutions)
+        card_name = card.get("name", "")
+        if "(Poke Ball Pattern)" in card_name or "(Master Ball Pattern)" in card_name:
+            continue
+
         card_info = {
             "product_id": card["productId"],
-            "name": extract_card_name(card["name"]),
+            "name": extract_card_name(card_name),
             "card_number": get_card_number(card),
             "market_price": price_dict.get(card["productId"]),
         }
@@ -115,8 +163,10 @@ def build_dataframe() -> pl.DataFrame:
 
         # Raise error if any DataFrame is empty
         if df is None or df.shape[1] == 0 or df.is_empty():
-            error_msg = f"Empty DataFrame returned for set '{set_number}'. " \
-                       f"Cannot proceed with drop+replace operation to avoid data loss."
+            error_msg = (
+                f"Empty DataFrame returned for set '{set_number}'. "
+                f"Cannot proceed with drop+replace operation to avoid data loss."
+            )
             print(colored(" ✖", "red"), error_msg)
             raise ValueError(error_msg)