Merge pull request #25 from VectorInstitute/add_automatic_workflow

Add automatic forecast workflow

Author: amrit110

.github/workflows/deploy-gcp.yml

@@ -588,8 +588,9 @@ jobs:
           **Endpoints:**
           - Health: ${{ steps.deploy-backend.outputs.url }}/health
           - Model Info: ${{ steps.deploy-backend.outputs.url }}/model/info
-          - Predict: ${{ steps.deploy-backend.outputs.url }}/predict
-          - WebSocket: ${{ steps.deploy-backend.outputs.url }}/ws/predict
+          - Latest Forecasts: ${{ steps.deploy-backend.outputs.url }}/forecasts/latest
+          - Forecast Status: ${{ steps.deploy-backend.outputs.url }}/forecasts/status
+          - Logs: ${{ steps.deploy-backend.outputs.url }}/logs/forecast-runs
 
           ### Frontend Service
           - **URL:** ${{ steps.deploy-frontend.outputs.url }}

.gitignore

@@ -71,3 +71,4 @@ TEST/
 
 # predictions
 historical_predictions/
+forecasts/

README.md

@@ -1,4 +1,6 @@
-# 🚨 GACA Early Warning System
+# Global AI Alliance for Climate Action
+
+# High-Resolution Temperature Forecasting
 
 ----------------------------------------------------------------------------------------
 
@@ -11,9 +13,14 @@
 
 ### **NOAA URMA → Graph Neural Network (GCN-GRU) Temperature Forecasting**
 
-This repository contains the **modular inference pipeline** for the **GACA Early Warning System**, a high-resolution Graph Neural Network framework that generates localized temperature forecasts from **NOAA URMA** meteorological data.
+Automated production forecasting system for Southwestern Ontario. Features hourly GCNGRU predictions stored in BigQuery with rolling 30-day evaluation metrics.
 
-The pipeline takes the most recent URMA hourly fields, preprocesses them into graph-structured tensors, and produces **multi-horizon (1–48h)** temperature predictions across thousands of spatial nodes in Southwestern Ontario.
+**Key Features:**
+- **Automated Forecasts**: Hourly execution at :15 past each hour
+- **Multi-Horizon**: 1, 6, 12, 18, 24, 36, 48-hour predictions
+- **Live Dashboard**: Real-time forecast visualization with auto-refresh
+- **CLI Tools**: Manual inference and batch prediction capabilities
+- **Evaluation**: Daily rolling 30-day RMSE/MAE metrics
 
 ---
 
@@ -44,6 +51,55 @@ source .venv/bin/activate
 
 ---
 
+## Automated Forecasting System
+
+The backend runs automated hourly forecasts and daily evaluations using APScheduler:
+
+### Production Deployment
+
+**Forecasting Schedule:**
+- Runs hourly at :15 (after NOAA data availability)
+- Automatically stores predictions to BigQuery
+- CSV outputs saved to configurable directory
+
+**Evaluation Schedule:**
+- Runs daily at 00:30 UTC
+- Computes rolling 30-day metrics (RMSE/MAE)
+- SQL-based aggregation for efficiency
+
+**Dashboard:**
+- Auto-refreshes every 30 minutes
+- Smart polling checks for new data before fetching
+- Displays scheduler status and last update time
+- Manual refresh button available
+
+**Environment Variables:**
+```bash
+FORECAST_OUTPUT_DIR=forecasts/         # Output directory for CSVs
+GCP_PROJECT_ID=your-project-id         # BigQuery project (optional)
+BIGQUERY_DATASET=gaca_evaluation       # BigQuery dataset name
+```
+
+### CLI Usage
+
+The CLI remains unchanged and works independently of the automated system:
+
+```bash
+# Single forecast
+gaca-ews predict --config config.yaml --output results/
+
+# Batch predictions for date range
+gaca-ews batch-predict --start-date "2024-02-06 12:00" --end-date "2024-02-10 12:00" --interval 24
+
+# Model information
+gaca-ews info
+
+# Version
+gaca-ews version
+```
+
+---
+
 ## Author
 
 **Joud El-Shawa** - Vector Institute for AI & Western University

app/app/components/empty-state.tsx

@@ -9,7 +9,10 @@ export function EmptyState() {
       <div className="text-center text-slate-400 bg-slate-900/80 backdrop-blur-md rounded-2xl p-8 border border-slate-700/50">
         <Cloud className="w-16 h-16 mx-auto mb-4 opacity-50" />
         <p className="mb-2 text-lg">No predictions loaded</p>
-        <p className="text-sm">Click "Run Forecast" to generate predictions</p>
+        <p className="text-sm">Forecasts run hourly at :15 past the hour</p>
+        <p className="text-xs text-slate-500 mt-2">
+          Dashboard will auto-refresh when data is available
+        </p>
       </div>
     </div>
   );

app/app/components/empty-stats.tsx

@@ -7,9 +7,16 @@ export function EmptyStats() {
   return (
     <div className="bg-slate-800/50 backdrop-blur-sm rounded-2xl border border-slate-700/50 p-12 text-center">
       <Gauge className="w-16 h-16 text-slate-600 mx-auto mb-4" />
-      <p className="text-slate-400">
-        Statistics will appear here after running forecast
-      </p>
+      <div className="space-y-2">
+        <p className="text-slate-300 font-medium">Waiting for forecast data</p>
+        <p className="text-sm text-slate-400">
+          Temperature statistics and visualizations will appear when the next hourly
+          forecast completes
+        </p>
+        <p className="text-xs text-slate-500 mt-3">
+          Forecasts run automatically every hour at :15
+        </p>
+      </div>
     </div>
   );
 }

app/app/components/forecast-status.tsx

@@ -0,0 +1,100 @@
+import { Activity, Clock, RefreshCw } from "lucide-react";
+import type { ForecastStatus } from "../hooks/use-auto-predictions";
+
+interface ForecastStatusProps {
+  status: ForecastStatus | null;
+  lastFetchTime: Date | null;
+  onRefresh: () => void;
+  loading: boolean;
+}
+
+/**
+ * Compact unified forecast status display with inline refresh
+ */
+export function ForecastStatusDisplay({
+  status,
+  lastFetchTime,
+  onRefresh,
+  loading,
+}: ForecastStatusProps) {
+  const formatTime = (timestamp: string | null) => {
+    if (!timestamp) return null;
+    const date = new Date(timestamp);
+    return date.toLocaleTimeString("en-US", {
+      hour: "2-digit",
+      minute: "2-digit",
+      hour12: false,
+    });
+  };
+
+  const formatRelativeTime = (date: Date | null) => {
+    if (!date) return null;
+    const now = new Date();
+    const diff = Math.floor((now.getTime() - date.getTime()) / 1000);
+
+    if (diff < 60) return `${diff}s`;
+    if (diff < 3600) return `${Math.floor(diff / 60)}m`;
+    return `${Math.floor(diff / 3600)}h`;
+  };
+
+  const isRunning = status?.scheduler?.is_running || false;
+  const nextRun = status?.scheduler?.next_scheduled_run;
+  // Use scheduler timestamp first, fallback to last forecast from BigQuery
+  const lastRun =
+    status?.scheduler?.last_run_timestamp || status?.last_forecast?.run_timestamp;
+  const relativeTime = formatRelativeTime(lastFetchTime);
+
+  return (
+    <div className="absolute top-4 left-4">
+      <div className="bg-slate-900/95 backdrop-blur-sm rounded-lg border border-slate-700/50 shadow-lg">
+        <div className="flex items-center gap-3 px-3 py-2">
+          {/* Status Icon & Text */}
+          <div className="flex items-center gap-2">
+            <Activity
+              className={`w-3.5 h-3.5 ${isRunning ? "text-green-400 animate-pulse" : "text-blue-400"}`}
+            />
+            <div className="text-xs text-white font-medium">
+              {isRunning ? "Running" : "Auto"}
+            </div>
+          </div>
+
+          {/* Divider */}
+          <div className="h-4 w-px bg-slate-700/50" />
+
+          {/* Time Info */}
+          <div className="flex items-center gap-1.5 text-[10px] text-slate-400">
+            {lastRun ? (
+              <>
+                <Clock className="w-3 h-3" />
+                {relativeTime && <span>{relativeTime}</span>}
+                {nextRun && !isRunning && (
+                  <>
+                    <span className="text-slate-600">•</span>
+                    <span>Next {formatTime(nextRun)}</span>
+                  </>
+                )}
+              </>
+            ) : (
+              <span>Waiting for first run</span>
+            )}
+          </div>
+
+          {/* Divider */}
+          <div className="h-4 w-px bg-slate-700/50" />
+
+          {/* Refresh Button */}
+          <button
+            onClick={onRefresh}
+            disabled={loading}
+            className="hover:bg-slate-800/80 disabled:opacity-50 disabled:cursor-not-allowed rounded p-1 transition-all group"
+            title="Refresh predictions"
+          >
+            <RefreshCw
+              className={`w-3.5 h-3.5 text-blue-400 ${loading ? "animate-spin" : "group-hover:rotate-180 transition-transform duration-500"}`}
+            />
+          </button>
+        </div>
+      </div>
+    </div>
+  );
+}

app/app/components/index.ts

@@ -1,11 +1,10 @@
 export { Header } from "./header";
 export { CollapsedSidebar } from "./sidebar";
 export { StatCard } from "./stat-card";
-export { MapControls } from "./map-controls";
+export { ForecastStatusDisplay } from "./forecast-status";
 export { MapLegend } from "./map-legend";
 export { ForecastInfo } from "./forecast-info";
 export { HoverTooltip } from "./hover-tooltip";
-export { LoadingOverlay } from "./loading-overlay";
 export { ErrorOverlay } from "./error-overlay";
 export { EmptyState } from "./empty-state";
 export { TemperatureChart } from "./temperature-chart";
@@ -14,3 +13,5 @@ export { ModelInfoPanel } from "./model-info-panel";
 export { LoadingSkeleton } from "./loading-skeleton";
 export { EmptyStats } from "./empty-stats";
 export { HorizonSlider } from "./horizon-slider";
+export { LogTable } from "./log-table";
+export { LoadingOverlay } from "./loading-overlay";