chore: add missing data (#128)

This pull request adds more data to the GitHub remote cache index.

While there, improve logging and docs.

Author: bassosimone

data/README.md

@@ -64,18 +64,21 @@ Then manually:
 
 - Python 3.13 using `uv` as documented in the toplevel [README.md](../README.md)
 
-**Complete Pipeline** (recommended):
+**Complete Pipeline**:
 
 ```bash
-uv run python ./data/generate_data.py
+uv run python ./data/generate_data.py -B
 ```
 
 This orchestrates the complete pipeline:
 
 1. Loads `./data/pipeline.yaml` to determine dates and granularities (edit this
    file to change the matrix)
-2. Queries both download and upload metrics for each dataset
-3. Saves results to v1 Parquet cache with query metadata
+2. Attempts to fetch from the GitHub cache first
+3. Otherwise, if `-B` is present, queries both download and upload metrics for each dataset
+4. Saves results to v1 Parquet cache with query metadata
+
+Omit the `-B` flag to avoid querying BigQuery.
 
 ## Future Improvements (Phase 2+)
 

data/ghcache.py

@@ -217,7 +217,7 @@ def cmd_scan(args) -> int:
 
         # Prepare manifest entry (URL will need to be filled in manually or via script)
         # For now, use placeholder URL
-        url_placeholder = f"https://github.com/m-lab/iqb/releases/download/v0.2.0/{mangled_name}"
+        url_placeholder = f"https://github.com/m-lab/iqb/releases/download/v0.5.0/{mangled_name}"
 
         files_dict[rel_path] = {"sha256": sha256, "url": url_placeholder}
         files_to_upload.append(str(dest_path))
@@ -231,7 +231,7 @@ def cmd_scan(args) -> int:
         for f in files_to_upload:
             print(f"  {f}")
         print("\nNext steps:")
-        print("1. Upload mangled files to GitHub release v0.2.0")
+        print("1. Upload mangled files to GitHub release v0.5.0")
         print("2. Update URLs in state/ghremote/manifest.json if needed")
         print("3. Commit updated state/ghremote/manifest.json to repository")
 

data/pipeline.yaml

@@ -17,7 +17,20 @@ matrix:
       end: "2025-07-01"
     - start: "2025-07-01"
       end: "2025-08-01"
+    - start: "2025-08-01"
+      end: "2025-09-01"
+    - start: "2025-09-01"
+      end: "2025-10-01"
     - start: "2025-10-01"
       end: "2025-11-01"
+    - start: "2025-11-01"
+      end: "2025-12-01"
+    - start: "2025-12-01"
+      end: "2026-01-01"
   granularities:
     - country
+    - country_asn
+    - subdivision1
+    - subdivision1_asn
+    - city
+    - city_asn

data/state/ghremote/manifest.json

@@ -25,7 +25,7 @@
 
 from ..pipeline.cache import PipelineCacheEntry, data_dir_or_default
 
-log = logging.getLogger("ghremote/cache")
+log = logging.getLogger("iqb.ghremote.cache")
 
 
 @dataclass(frozen=True, kw_only=True)

library/src/iqb/ghremote/cache.py

@@ -23,7 +23,7 @@
     TimeRemainingColumn,
 )
 
-log = logging.getLogger("pipeline/bqpq")
+log = logging.getLogger("iqb.pipeline.bqpq")
 
 
 def _rows_progress_columns(total_rows: int | None):

library/src/iqb/pipeline/bqpq.py

@@ -77,7 +77,7 @@ def sync(self) -> None:
         """
         if self.syncers:
             if not any(sync(self) for sync in self.syncers):
-                raise PipelineEntrySyncError(f"Cannot sync {self}: see above logs")
+                raise PipelineEntrySyncError(f"Cannot sync {self}: see above warnings")
             return
 
         if not self.exists():

library/src/iqb/pipeline/cache.py

@@ -17,7 +17,7 @@
     PipelineRemoteCache,
 )
 
-log = logging.getLogger("pipeline/bq")
+log = logging.getLogger("iqb.pipeline.bq")
 
 
 class IQBPipeline:

library/src/iqb/pipeline/pipeline.py

@@ -44,7 +44,8 @@ def configure(verbose: bool) -> None:
         handlers=[handler],
         force=True,
     )
+    logging.captureWarnings(True)
 
 
-log = logging.getLogger("scripting")
+log = logging.getLogger("iqb.scripting")
 """Logger that the scripting package should use."""

library/src/iqb/scripting/iqb_logging.py

@@ -52,4 +52,4 @@ def fake_log_render(*_args, **kwargs):
 
 
 def test_log_is_named_scripting() -> None:
-    assert iqb_logging.log.name == "scripting"
+    assert iqb_logging.log.name == "iqb.scripting"