refactor(data): move SQL queries to library package (#29)

Move BigQuery query templates from `./data/` to
`./library/src/iqb/queries/` to make them officially part of the
library.

Benefits:

1. Queries are now package resources (can be imported)
2. Cleaner separation: queries belong with library code
3. Prepares for future pipeline refactoring

No functional changes - same queries, same results confirmed.

While there, maintain the documentation of the `./data` directory
syncing it up with reality.

Author: bassosimone

data/README.md

@@ -19,21 +19,14 @@ the IQB prototype for Phase 1 development.
 
 - `br_2024_10.json` - Brazil, ~5M download samples, ~3M upload samples
 
+- ... (more files)
+
 ### Data Structure
 
 Each JSON file contains:
 
 ```JavaScript
 {
-  "metadata": {
-    "country_code": "US",
-    "country_name": "United States",
-    "period": "2024-10",
-    "period_description": "October 2024",
-    "dataset": "M-Lab NDT",
-    "download_samples": 31443312,
-    "upload_samples": 24288961
-  },
   "metrics": {
     "download_throughput_mbps": {"p1": 0.38, /* ... */, "p99": 891.82},
     "upload_throughput_mbps": {"p1": 0.06, /* ... */, "p99": 813.73},
@@ -49,21 +42,8 @@ Each JSON file contains:
 
 ### BigQuery Queries
 
-The data was extracted from M-Lab's public BigQuery tables using two queries:
-
-1. **Downloads** (`query_downloads.sql`): Queries
-`measurement-lab.ndt.unified_downloads` for:
-
-   - Download throughput (`a.MeanThroughputMbps`)
-
-   - Latency (`a.MinRTT`)
-
-   - Packet loss (`a.LossRate`)
-
-2. **Uploads** (`query_uploads.sql`): Queries
-`measurement-lab.ndt.unified_uploads` for:
-
-   - Upload throughput (`a.MeanThroughputMbps`)
+The data was extracted from M-Lab's public BigQuery tables using queries
+inside the [../library/src/iqb/queries](../library/src/iqb/queries) package.
 
 ### Running the Data Generation Pipeline
 
@@ -76,13 +56,13 @@ The data was extracted from M-Lab's public BigQuery tables using two queries:
 - `gcloud`-authenticated with an account subscribed to
 [M-Lab Discuss mailing list](https://groups.google.com/a/measurementlab.net/g/discuss)
 
-- Python 3.11+
+- Python 3.13 using `uv` as documented in the toplevel [README.md](../README.md)
 
 **Complete Pipeline** (recommended):
 
 ```bash
 cd data/
-python3 generate_data.py
+uv run python generate_data.py
 ```
 
 This orchestrates the complete pipeline:
@@ -101,13 +81,13 @@ Generated files: `us_2024_10.json`, `de_2024_10.json`, `br_2024_10.json`.
 cd data/
 
 # Stage 1a: Query downloads
-python3 run_query.py query_downloads.sql -o downloads.json
+uv run python run_query.py query_downloads.sql -o downloads.json
 
 # Stage 1b: Query uploads
-python3 run_query.py query_uploads.sql -o uploads.json
+uv run python run_query.py query_uploads.sql -o uploads.json
 
 # Stage 2: Merge data
-python3 merge_data.py
+uv run python merge_data.py
 ```
 
 **Pipeline Scripts**:
@@ -119,19 +99,6 @@ python3 merge_data.py
 - [merge_data.py](merge_data.py) - Merges download and upload data into
 per-country files
 
-### Modifying Queries
-
-To change the time period or countries, edit the SQL files:
-
-```sql
-WHERE
-    date BETWEEN "2024-10-01" AND "2024-10-31"  -- Change dates here
-    AND client.Geo.CountryCode IN ("US", "DE", "BR")  -- Change countries here
-```
-
-Country codes follow the
-[ISO 3166-1 alpha-2](https://en.wikipedia.org/wiki/ISO_3166-1_alpha-2) standard.
-
 ## Notes
 
 - **Static data**: These files contain pre-aggregated percentiles

data/generate_data.py

@@ -49,7 +49,7 @@ def generate_for_period(
         [
             "python3",
             str(data_dir / "run_query.py"),
-            str(data_dir / "query_downloads_template.sql"),
+            "downloads_by_country",
             "--start-date",
             start_date,
             "--end-date",
@@ -65,7 +65,7 @@ def generate_for_period(
         [
             "python3",
             str(data_dir / "run_query.py"),
-            str(data_dir / "query_uploads_template.sql"),
+            "uploads_by_country",
             "--start-date",
             start_date,
             "--end-date",

data/run_query.py

@@ -5,8 +5,13 @@
 import subprocess
 import sys
 from datetime import datetime
+from importlib.resources import files
 from pathlib import Path
 
+# Add library to path so we can import iqb.queries
+sys.path.insert(0, str(Path(__file__).parent.parent / "library" / "src"))
+import iqb.queries
+
 
 def validate_date(date_str: str) -> str:
     """
@@ -31,7 +36,7 @@ def validate_date(date_str: str) -> str:
 
 
 def run_bq_query(
-    query_file: Path,
+    query_name: str,
     output_file: Path | None,
     project_id: str,
     start_date: str,
@@ -55,7 +60,7 @@ def run_bq_query(
     Template becomes: date >= '2024-10-01' AND date < '2024-11-01'
 
     Args:
-        query_file: Path to SQL query template file
+        query_name: Name of SQL query template (e.g., "downloads_by_country")
         output_file: Path where to save JSON output (None = stdout)
         project_id: GCP project ID for billing
         start_date: Start date in YYYY-MM-DD format (inclusive)
@@ -70,12 +75,12 @@ def run_bq_query(
             f"start_date must be <= end_date, got: {start_date} > {end_date}"
         )
 
-    print(f"Running query: {query_file}", file=sys.stderr)
+    print(f"Running query: {query_name}", file=sys.stderr)
     print(f"  Date range: {start_date} to {end_date}", file=sys.stderr)
 
-    # Read query template
-    with open(query_file) as f:
-        query = f.read()
+    # Load query template from iqb.queries package
+    query_file = files(iqb.queries).joinpath(f"{query_name}.sql")
+    query = query_file.read_text()
 
     # Substitute template variables
     query = query.replace("{START_DATE}", start_date)
@@ -125,7 +130,10 @@ def main():
     parser = argparse.ArgumentParser(
         description="Execute BigQuery query template and save results"
     )
-    parser.add_argument("query_file", type=Path, help="Path to SQL query template file")
+    parser.add_argument(
+        "query_name",
+        help="Name of SQL query template (e.g., 'downloads_by_country', 'uploads_by_country')",
+    )
     parser.add_argument(
         "-o", "--output", type=Path, help="Path to output JSON file (default: stdout)"
     )
@@ -147,12 +155,8 @@ def main():
 
     args = parser.parse_args()
 
-    if not args.query_file.exists():
-        print(f"Error: Query file not found: {args.query_file}", file=sys.stderr)
-        sys.exit(1)
-
     run_bq_query(
-        args.query_file,
+        args.query_name,
         args.output,
         args.project_id,
         args.start_date,

library/src/iqb/queries/__init__.py

@@ -0,0 +1,5 @@
+"""SQL query templates for IQB data collection.
+
+This module contains SQL query templates for fetching measurement data
+from BigQuery NDT tables.
+"""

data/query_downloads_template.sql …src/iqb/queries/downloads_by_country.sqldata/query_downloads_template.sql renamed to library/src/iqb/queries/downloads_by_country.sql data/query_downloads_template.sql renamed to library/src/iqb/queries/downloads_by_country.sql

@@ -0,0 +1,138 @@
+"""Tests for the iqb.queries SQL query templates module."""
+
+from importlib.resources import files
+
+import iqb.queries
+
+
+class TestQueriesPackage:
+    """Tests for iqb.queries package structure."""
+
+    def test_queries_package_can_be_imported(self):
+        """Test that iqb.queries package can be imported."""
+        assert iqb.queries is not None
+
+    def test_queries_package_has_files(self):
+        """Test that queries package provides access to files."""
+        query_files = files(iqb.queries)
+        assert query_files is not None
+
+
+class TestDownloadsByCountryQuery:
+    """Tests for downloads_by_country.sql query template."""
+
+    def test_downloads_by_country_exists(self):
+        """Test that downloads_by_country.sql query file exists."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        assert query_file.is_file()
+
+    def test_downloads_by_country_can_be_read(self):
+        """Test that downloads_by_country.sql can be read."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        content = query_file.read_text()
+        assert content is not None
+        assert len(content) > 0
+
+    def test_downloads_by_country_has_date_placeholders(self):
+        """Test that downloads_by_country.sql contains date placeholders."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        content = query_file.read_text()
+        assert "{START_DATE}" in content
+        assert "{END_DATE}" in content
+
+    def test_downloads_by_country_queries_unified_downloads_table(self):
+        """Test that downloads_by_country.sql queries the correct table."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        content = query_file.read_text()
+        assert "measurement-lab.ndt.unified_downloads" in content
+
+    def test_downloads_by_country_groups_by_country_code(self):
+        """Test that downloads_by_country.sql groups by country code."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        content = query_file.read_text()
+        assert "GROUP BY country_code" in content
+
+    def test_downloads_by_country_calculates_percentiles(self):
+        """Test that downloads_by_country.sql calculates percentiles."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        content = query_file.read_text()
+        # Should calculate p1, p5, p10, p25, p50, p75, p90, p95, p99
+        assert "APPROX_QUANTILES" in content
+        assert "download_p95" in content or "download_p99" in content
+
+
+class TestUploadsByCountryQuery:
+    """Tests for uploads_by_country.sql query template."""
+
+    def test_uploads_by_country_exists(self):
+        """Test that uploads_by_country.sql query file exists."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        assert query_file.is_file()
+
+    def test_uploads_by_country_can_be_read(self):
+        """Test that uploads_by_country.sql can be read."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        content = query_file.read_text()
+        assert content is not None
+        assert len(content) > 0
+
+    def test_uploads_by_country_has_date_placeholders(self):
+        """Test that uploads_by_country.sql contains date placeholders."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        content = query_file.read_text()
+        assert "{START_DATE}" in content
+        assert "{END_DATE}" in content
+
+    def test_uploads_by_country_queries_unified_uploads_table(self):
+        """Test that uploads_by_country.sql queries the correct table."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        content = query_file.read_text()
+        assert "measurement-lab.ndt.unified_uploads" in content
+
+    def test_uploads_by_country_groups_by_country_code(self):
+        """Test that uploads_by_country.sql groups by country code."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        content = query_file.read_text()
+        assert "GROUP BY country_code" in content
+
+    def test_uploads_by_country_calculates_percentiles(self):
+        """Test that uploads_by_country.sql calculates percentiles."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        content = query_file.read_text()
+        # Should calculate p1, p5, p10, p25, p50, p75, p90, p95, p99
+        assert "APPROX_QUANTILES" in content
+        assert "upload_p95" in content or "upload_p99" in content
+
+
+class TestQueryTemplateSubstitution:
+    """Tests for query template placeholder substitution."""
+
+    def test_downloads_query_date_substitution(self):
+        """Test that date placeholders can be substituted in downloads query."""
+        query_file = files(iqb.queries).joinpath("downloads_by_country.sql")
+        template = query_file.read_text()
+
+        # Substitute placeholders
+        query = template.replace("{START_DATE}", "2024-10-01")
+        query = query.replace("{END_DATE}", "2024-11-01")
+
+        # Verify substitution worked
+        assert "{START_DATE}" not in query
+        assert "{END_DATE}" not in query
+        assert "2024-10-01" in query
+        assert "2024-11-01" in query
+
+    def test_uploads_query_date_substitution(self):
+        """Test that date placeholders can be substituted in uploads query."""
+        query_file = files(iqb.queries).joinpath("uploads_by_country.sql")
+        template = query_file.read_text()
+
+        # Substitute placeholders
+        query = template.replace("{START_DATE}", "2024-10-01")
+        query = query.replace("{END_DATE}", "2024-11-01")
+
+        # Verify substitution worked
+        assert "{START_DATE}" not in query
+        assert "{END_DATE}" not in query
+        assert "2024-10-01" in query
+        assert "2024-11-01" in query