docs: update readme

ruckerf22 · ruckerf22 · commit bfcbeb774ba3 · 2025-10-28T10:29:00.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -49,3 +49,6 @@ migrations/MIGRATION_SUMMARY.md
 migrations/KENTUCKY_INSTITUTION_IDS.md
 check_env.py
 migrations/REGENERATION_README.md
+testscripts/test_new_endpoints.py
+testscripts/check_llm_structure.py
+NEW_ENDPOINTS_SUMMARY.md
diff --git a/README.md b/README.md
@@ -8,6 +8,8 @@ A RESTful API for accessing educational institution data across multiple schools
 - **Standardized Endpoints**: Consistent API structure across all institutions
 - **Pagination**: Built-in support for large datasets
 - **Filtering**: Query specific data subsets using query parameters
+ - **Data Uploads**: Upload CSV/Excel to append data with dynamic column mapping (see Data Upload section)
+ - **New Datasets**: Access LLM recommendations and analysis-ready tables across all databases
 
 ## Prerequisites
 
@@ -48,7 +50,7 @@ A RESTful API for accessing educational institution data across multiple schools
 
 Start the development server:
 ```bash
-uvicorn main:app --reload
+uvicorn api.main:app --reload
 ```
 
 The API will be available at `http://localhost:8000`
@@ -92,6 +94,20 @@ For each institution, the following endpoints are available:
 - `GET /{institution_code}/financial-aid` - List financial aid records
 - `GET /{institution_code}/financial_aid/count` - Get count of financial aid records
 
+#### LLM Recommendations
+- `GET /{institution_code}/llm-recommendations` - List LLM recommendation records
+- `GET /{institution_code}/llm_recommendations/count` - Get count of LLM recommendation records
+
+#### Analysis-Ready Data
+- `GET /{institution_code}/analysis-ready` - List analysis-ready records
+- `GET /{institution_code}/ar_{institution_code}/count` - Get count of analysis-ready records
+
+## Data Upload Feature
+
+- Quick start: see `QUICKSTART_UPLOAD.md`
+- Full documentation: see `UPLOAD_FEATURE_README.md`
+- Endpoints are available under `/upload` in Swagger UI.
+
 ## Query Parameters
 
 ### Pagination
@@ -217,7 +233,7 @@ The workflow runs automatically on:
 - Pushes to `main` or `develop` branches
 - Pull requests to the `main` branch
 
-The Docker image will be tagged as `latest` and pushed to ECR.
+The Docker image will be tagged and pushed to Docker Hub (or your configured registry).
 
 **Install required packages:**
 ```bash
@@ -243,7 +259,7 @@ Each database in this project contains the following three tables:
 
 ### 1. Create Databases and Tables
 ```bash
-python db_setup.py
+python db_operations/db_setup.py
 ```
 
 This creates 5 databases with 3 tables each:
@@ -284,28 +300,34 @@ If Ollama is not available or fails, scripts automatically use rule-based synthe
 devcolor-backend/
 ├── api/
 │   ├── __init__.py
-│   ├── main.py                 # API routing setup
-│   ├── schemas.py              # Pydantic models
+│   ├── main.py                   # FastAPI app and router registration
+│   ├── schemas.py                # Pydantic models
 │   └── routers/
 │       ├── __init__.py
-│       ├── al.py               # Alabama institution endpoints
-│       ├── csusb.py            # CSUSB institution endpoints
-│       ├── kctcs.py            # KCTCS institution endpoints
-│       ├── ky.py               # Kentucky institution endpoints
-│       └── oh.py               # Ohio institution endpoints
+│       ├── al.py                 # AL endpoints (incl. new endpoints)
+│       ├── csusb.py              # CSUSB endpoints (incl. new endpoints)
+│       ├── kctcs.py              # KCTCS endpoints (incl. new endpoints)
+│       ├── ky.py                 # KY endpoints (incl. new endpoints)
+│       ├── oh.py                 # OH endpoints (incl. new endpoints)
+│       └── upload.py             # Data upload endpoints
 ├── db_operations/
 │   ├── __init__.py
-│   ├── connection.py           # Database connection utilities
-│   ├── db_setup.py             # Database setup and table creation
-│   ├── generate_db_summary.py  # Database summary generation
-│   └── test_db_connection.py   # Database connection testing
-├── .env                        # Environment variables (not version controlled)
-├── .gitignore
-├── check_databases.py          # Database availability checker
-├── check_tables.py             # Table structure inspector
-├── count_records.py            # Record counting utility
-├── main.py                     # FastAPI application entry point
-├── requirements.txt            # Python dependencies
-├── README.md                   # This file
-├── test_env.py                 # Environment variable testing
-└── test_ky_query.py            # KY database query testing
+│   ├── connection.py             # DB connection utilities
+│   ├── db_setup.py               # Database setup and table creation
+│   ├── add_dynamic_columns.py    # Migration for dynamic upload columns
+│   ├── upload_handler.py         # Upload processing logic
+│   └── generate_db_summary.py    # Database summary generation
+├── docker/
+│   ├── Dockerfile
+│   └── docker-compose.yml
+├── testscripts/
+│   ├── check_databases.py
+│   ├── check_schema.py
+│   ├── check_tables.py
+│   ├── count_records.py
+│   └── test_new_endpoints.py
+├── requirements.txt              # Python dependencies
+├── README.md                     # This file
+├── QUICKSTART_UPLOAD.md          # Upload quick start
+├── UPLOAD_FEATURE_README.md      # Upload feature docs
+└── NEW_ENDPOINTS_SUMMARY.md      # Summary of new endpoints
diff --git a/api/routers/al.py b/api/routers/al.py
@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query
 from typing import List
-from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, TableCount, DatabaseInfo
+from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, LlmRecommendationRecord, AnalysisReadyRecord, TableCount, DatabaseInfo
 from db_operations.connection import get_db_connection, format_records
 
 router = APIRouter()
@@ -69,10 +69,44 @@ async def get_financial_aid(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching financial aid: {str(e)}")
 
+@router.get("/llm-recommendations", response_model=List[LlmRecommendationRecord])
+async def get_llm_recommendations(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get LLM recommendation records from AL database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM llm_recommendations ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching LLM recommendations: {str(e)}")
+
+@router.get("/analysis-ready", response_model=List[AnalysisReadyRecord])
+async def get_analysis_ready(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get analysis ready records from AL database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM ar_al ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching analysis ready records: {str(e)}")
+
 @router.get("/{table_name}/count", response_model=TableCount)
 async def get_table_count(table_name: str):
     """Get the total count of records in a specific table."""
-    valid_tables = ["cohort", "course", "financial_aid"]
+    valid_tables = ["cohort", "course", "financial_aid", "llm_recommendations", "ar_al"]
     if table_name not in valid_tables:
         raise HTTPException(
             status_code=404,
diff --git a/api/routers/csusb.py b/api/routers/csusb.py
@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query
 from typing import List
-from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, TableCount, DatabaseInfo
+from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, LlmRecommendationRecord, AnalysisReadyRecord, TableCount, DatabaseInfo
 from db_operations.connection import get_db_connection, format_records
 
 router = APIRouter()
@@ -69,10 +69,44 @@ async def get_financial_aid(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching financial aid: {str(e)}")
 
+@router.get("/llm-recommendations", response_model=List[LlmRecommendationRecord])
+async def get_llm_recommendations(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get LLM recommendation records from CSUSB database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM llm_recommendations ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching LLM recommendations: {str(e)}")
+
+@router.get("/analysis-ready", response_model=List[AnalysisReadyRecord])
+async def get_analysis_ready(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get analysis ready records from CSUSB database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM ar_csusb ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching analysis ready records: {str(e)}")
+
 @router.get("/{table_name}/count", response_model=TableCount)
 async def get_table_count(table_name: str):
     """Get the total count of records in a specific table."""
-    valid_tables = ["cohort", "course", "financial_aid"]
+    valid_tables = ["cohort", "course", "financial_aid", "llm_recommendations", "ar_csusb"]
     if table_name not in valid_tables:
         raise HTTPException(
             status_code=404,
diff --git a/api/routers/kctcs.py b/api/routers/kctcs.py
@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query
 from typing import List
-from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, TableCount, DatabaseInfo
+from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, LlmRecommendationRecord, AnalysisReadyRecord, TableCount, DatabaseInfo
 from db_operations.connection import get_db_connection, format_records
 
 router = APIRouter()
@@ -69,10 +69,44 @@ async def get_financial_aid(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching financial aid: {str(e)}")
 
+@router.get("/llm-recommendations", response_model=List[LlmRecommendationRecord])
+async def get_llm_recommendations(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get LLM recommendation records from KCTCS database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM llm_recommendations ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching LLM recommendations: {str(e)}")
+
+@router.get("/analysis-ready", response_model=List[AnalysisReadyRecord])
+async def get_analysis_ready(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get analysis ready records from KCTCS database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM ar_kctcs ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching analysis ready records: {str(e)}")
+
 @router.get("/{table_name}/count", response_model=TableCount)
 async def get_table_count(table_name: str):
     """Get the total count of records in a specific table."""
-    valid_tables = ["cohort", "course", "financial_aid"]
+    valid_tables = ["cohort", "course", "financial_aid", "llm_recommendations", "ar_kctcs"]
     if table_name not in valid_tables:
         raise HTTPException(
             status_code=404,
diff --git a/api/routers/ky.py b/api/routers/ky.py
@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query
 from typing import List
-from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, TableCount, DatabaseInfo
+from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, LlmRecommendationRecord, AnalysisReadyRecord, TableCount, DatabaseInfo
 from db_operations.connection import get_db_connection, format_records
 
 router = APIRouter()
@@ -69,10 +69,44 @@ async def get_financial_aid(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching financial aid: {str(e)}")
 
+@router.get("/llm-recommendations", response_model=List[LlmRecommendationRecord])
+async def get_llm_recommendations(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get LLM recommendation records from KY database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM llm_recommendations ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching LLM recommendations: {str(e)}")
+
+@router.get("/analysis-ready", response_model=List[AnalysisReadyRecord])
+async def get_analysis_ready(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get analysis ready records from KY database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM ar_ky ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching analysis ready records: {str(e)}")
+
 @router.get("/{table_name}/count", response_model=TableCount)
 async def get_table_count(table_name: str):
     """Get the total count of records in a specific table."""
-    valid_tables = ["cohort", "course", "financial_aid"]
+    valid_tables = ["cohort", "course", "financial_aid", "llm_recommendations", "ar_ky"]
     if table_name not in valid_tables:
         raise HTTPException(
             status_code=404,
diff --git a/api/routers/oh.py b/api/routers/oh.py
@@ -1,6 +1,6 @@
 from fastapi import APIRouter, HTTPException, Query
 from typing import List
-from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, TableCount, DatabaseInfo
+from ..schemas import CohortRecord, CourseRecord, FinancialAidRecord, LlmRecommendationRecord, AnalysisReadyRecord, TableCount, DatabaseInfo
 from db_operations.connection import get_db_connection, format_records
 
 router = APIRouter()
@@ -69,10 +69,44 @@ async def get_financial_aid(
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error fetching financial aid: {str(e)}")
 
+@router.get("/llm-recommendations", response_model=List[LlmRecommendationRecord])
+async def get_llm_recommendations(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get LLM recommendation records from OH database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM llm_recommendations ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching LLM recommendations: {str(e)}")
+
+@router.get("/analysis-ready", response_model=List[AnalysisReadyRecord])
+async def get_analysis_ready(
+    limit: int = Query(100, ge=1, le=1000, description="Number of records to return"),
+    offset: int = Query(0, ge=0, description="Number of records to skip")
+):
+    """Get analysis ready records from OH database."""
+    try:
+        with get_db_connection(DATABASE_NAME) as connection:
+            cursor = connection.cursor(dictionary=True)
+            query = "SELECT * FROM ar_oh ORDER BY id LIMIT %s OFFSET %s"
+            cursor.execute(query, (limit, offset))
+            records = cursor.fetchall()
+            cursor.close()
+            return format_records(records)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Error fetching analysis ready records: {str(e)}")
+
 @router.get("/{table_name}/count", response_model=TableCount)
 async def get_table_count(table_name: str):
     """Get the total count of records in a specific table."""
-    valid_tables = ["cohort", "course", "financial_aid"]
+    valid_tables = ["cohort", "course", "financial_aid", "llm_recommendations", "ar_oh"]
     if table_name not in valid_tables:
         raise HTTPException(
             status_code=404,
diff --git a/api/schemas.py b/api/schemas.py
