Merge remote-tracking branch 'origin/main'

Author: Exploding-Tacocat

data-analysis/01_extract_source.py

@@ -2,11 +2,24 @@
 from pathlib import Path
 from typing import Any, Dict, List
 
+from bson import ObjectId
 from pymongo import MongoClient
 
 from common import ROOT, coerce_bool, coerce_float, coerce_int, load_config, parse_args, parse_json_field, write_csv, write_json
 
 
+def convert_objectid_to_str(obj: Any) -> Any:
+    """Recursively convert MongoDB ObjectId objects to strings."""
+    if isinstance(obj, ObjectId):
+        return str(obj)
+    elif isinstance(obj, dict):
+        return {key: convert_objectid_to_str(value) for key, value in obj.items()}
+    elif isinstance(obj, list):
+        return [convert_objectid_to_str(item) for item in obj]
+    else:
+        return obj
+
+
 def flatten_match_row(entry: Dict[str, Any]) -> Dict[str, Any]:
     metadata = entry.get('metadata') or {}
     return {
@@ -16,26 +29,26 @@ def flatten_match_row(entry: Dict[str, Any]) -> Dict[str, Any]:
         'robotPosition': metadata.get('robotPosition', ''),
         'robotAbsent': coerce_bool(entry.get('robotAbsent', False)),
         'autoStartingPosition': entry.get('autoStartingPosition'),
-        'autoPathJson': json.dumps(entry.get('autoPath') or {}, separators=(',', ':')),
-        'shootTimeBySegmentJson': json.dumps(entry.get('shootTimeBySegment') or {}, separators=(',', ':')),
-        'passTimeBySegmentJson': json.dumps(entry.get('passTimeBySegment') or {}, separators=(',', ':')),
-        'actionTimelineJson': json.dumps(entry.get('actionTimeline') or {}, separators=(',', ':')),
+        'autoPathJson': json.dumps(convert_objectid_to_str(entry.get('autoPath') or {}), separators=(',', ':')),
+        'shootTimeBySegmentJson': json.dumps(convert_objectid_to_str(entry.get('shootTimeBySegment') or {}), separators=(',', ':')),
+        'passTimeBySegmentJson': json.dumps(convert_objectid_to_str(entry.get('passTimeBySegment') or {}), separators=(',', ':')),
+        'actionTimelineJson': json.dumps(convert_objectid_to_str(entry.get('actionTimeline') or {}), separators=(',', ':')),
         'ballsPerSecondUsed': coerce_float(entry.get('ballsPerSecondUsed', 0)),
         'autoFuelScored': coerce_float(entry.get('autoFuelScored', 0)),
-        'teleFuelBySegmentJson': json.dumps(entry.get('teleFuelBySegment') or {}, separators=(',', ':')),
+        'teleFuelBySegmentJson': json.dumps(convert_objectid_to_str(entry.get('teleFuelBySegment') or {}), separators=(',', ':')),
         'teleTower': entry.get('teleTower', 'None'),
         'breakdown': entry.get('breakdown', 'None'),
         'driverQuality': entry.get('driverQuality', 'ok'),
         'defenseProvided': entry.get('defenseProvided', 'None'),
         'defenseReceived': coerce_bool(entry.get('defenseReceived', False)),
-        'foulsJson': json.dumps(entry.get('fouls') or {}, separators=(',', ':')),
-        'breaksJson': json.dumps(entry.get('breaks') or {}, separators=(',', ':')),
+        'foulsJson': json.dumps(convert_objectid_to_str(entry.get('fouls') or {}), separators=(',', ':')),
+        'breaksJson': json.dumps(convert_objectid_to_str(entry.get('breaks') or {}), separators=(',', ':')),
         'freeText': entry.get('freeText', ''),
     }
 
 
 def flatten_pit_row(entry: Dict[str, Any]) -> Dict[str, Any]:
-    intake = parse_json_field(entry.get('intakeSources'), {})
+    intake = parse_json_field(convert_objectid_to_str(entry.get('intakeSources')), {})
     if not isinstance(intake, dict):
         intake = {}
 

data-analysis/output/00_pipeline_report.json

@@ -0,0 +1,15 @@
+{
+  "startedAt": "2026-03-25T03:42:24.475532Z",
+  "finishedAt": "2026-03-25T03:42:26.426745Z",
+  "configPath": "C:\\GitKrakenStuff\\ScoutingApp2026\\data-analysis\\pipeline_config.json",
+  "sourceMode": "mongo",
+  "runStage07": false,
+  "executedStages": [
+    "01_extract_source.py",
+    "02_clean_normalize.py",
+    "03_feature_engineering.py",
+    "04_team_aggregation.py",
+    "05_picklist_scores.py",
+    "06_export_app_payloads.py"
+  ]
+}

data-analysis/output/01_match_raw.csv

@@ -0,0 +1,2 @@
+batteryCount,drivebase,intakeDepot,intakeFloorNeutral,intakeOutpostCorral,maxFuelStorageEstimate,notes,preferredScoringSpot,scoringMethod,scouterName,teamNumber,towerCapabilityClaimed
+1,tank,True,False,False,1,Worst robot at event fr fr,nearHub,other,Phong,4201,level3

data-analysis/output/01_pit_raw.csv

@@ -0,0 +1,9 @@
+{
+  "stage": "01_extract_source",
+  "sourceMode": "mongo",
+  "counts": {
+    "match": 210,
+    "pit": 1
+  },
+  "configPath": "C:\\GitKrakenStuff\\ScoutingApp2026\\data-analysis\\pipeline_config.json"
+}

data-analysis/output/01_raw_snapshot.json

@@ -0,0 +1,2 @@
+batteryCount,drivebase,intakeDepot,intakeFloorNeutral,intakeOutpostCorral,maxFuelStorageEstimate,notes,preferredScoringSpot,scoringMethod,scouterName,teamNumber,towerCapabilityClaimed
+1,tank,True,False,False,1.0,Worst robot at event fr fr,nearHub,other,Phong,4201,level3

data-analysis/output/02_match_clean.csv

@@ -0,0 +1 @@
+dataset,rowNumber,severity,issues