Merge pull request #14 from JarvusInnovations/themightychris/dbt-inventory-model

Add stg_available_feeds model for inventory.json

Author: themightychris

models/staging/_staging.yml

@@ -90,3 +90,32 @@ models:
       - name: effect
         description: Alert effect code
       - *_uuid
+
+  - name: stg_available_feeds
+    description: >
+      Feed inventory from parquet.gtfsrt.io. Lists all available GTFS-RT
+      feeds with metadata including agency info, feed type, date range,
+      and size statistics. Reads directly from remote JSON.
+    columns:
+      - name: feed_url
+        description: Original GTFS-RT feed URL
+      - name: feed_base64
+        description: Base64url-encoded feed URL (used in parquet file paths)
+      - name: agency_id
+        description: Transit agency identifier (e.g., "septa", "actransit")
+      - name: agency_name
+        description: Transit agency display name
+      - name: feed_type
+        description: Feed type - service_alerts, trip_updates, or vehicle_positions
+      - name: date_min
+        description: Earliest date with data available
+      - name: date_max
+        description: Latest date with data available
+      - name: total_records
+        description: Total number of records across all dates
+      - name: total_bytes
+        description: Total size of all parquet files in bytes
+      - name: days_available
+        description: Number of days with data (computed)
+      - name: avg_bytes_per_day
+        description: Average bytes per day (computed)

models/staging/stg_available_feeds.sql

@@ -0,0 +1,30 @@
+{{
+    config(
+        materialized='table'
+    )
+}}
+
+/*
+    Staging model for parquet.gtfsrt.io feed inventory.
+    Reads directly from the public inventory JSON file.
+
+    This provides metadata about all available GTFS-RT feeds
+    including agency info, feed types, date ranges, and sizes.
+*/
+
+SELECT
+    url AS feed_url,
+    base64url AS feed_base64,
+    agency_id,
+    agency_name,
+    feed_type,
+    date_min,
+    date_max,
+    total_records,
+    total_bytes,
+
+    -- Computed fields
+    date_max::date - date_min::date + 1 AS days_available,
+    total_bytes / NULLIF(date_max::date - date_min::date + 1, 0) AS avg_bytes_per_day
+
+FROM read_json_auto('gs://parquet.gtfsrt.io/inventory.json')

profiles.yml

@@ -6,5 +6,6 @@ gtfsrt_sandbox:
       path: sandbox.duckdb
       extensions:
         - parquet
+        - httpfs
       settings:
         threads: 4