clean up meeting data, add action to save meetings

Author: jdungan

.github/actions/setup-poetry-env/action.yml

@@ -0,0 +1,35 @@
+name: 'Setup Python with Poetry'
+description: 'Sets up Python environment with Poetry and installs dependencies'
+
+inputs:
+  python-version:
+    description: 'Python version to set up'
+    required: false
+    default: '3.11'
+
+runs:
+  using: "composite"
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Poetry
+      uses: abatilo/actions-poetry@v2
+
+    - name: Configure Poetry
+      run: poetry config virtualenvs.create true
+      shell: bash
+
+    - name: Load cached Poetry dependencies
+      uses: actions/cache@v3
+      with:
+        path: ~/.cache/pypoetry/virtualenvs
+        key: poetry-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+        restore-keys: |
+          poetry-${{ runner.os }}-
+
+    - name: Install dependencies
+      run: poetry install --no-interaction --no-root
+      shell: bash 

.github/workflows/run-tests.yml

@@ -13,27 +13,10 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-poetry-env
         with:
-          python-version: '3.11'  # Adjust as needed
-
-      - name: Install Poetry
-        uses: abatilo/actions-poetry@v2
-
-      - name: Configure Poetry
-        run: poetry config virtualenvs.create true
-
-      - name: Load cached Poetry dependencies
-        uses: actions/cache@v3
-        with:
-          path: ~/.cache/pypoetry/virtualenvs
-          key: poetry-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
-          restore-keys: |
-            poetry-${{ runner.os }}-
-
-      - name: Install dependencies
-        run: poetry install --no-interaction --no-root
+          python-version: '3.11'
 
       - name: Run tests
         run: poetry run pytest

.github/workflows/update_summary.yml

@@ -0,0 +1,30 @@
+name: Update Meetings Summary
+
+on:
+  schedule:
+    # Run every day at midnight
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+    # Allow manual trigger
+
+jobs:
+  update-summary:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        
+      - name: Setup Python environment
+        uses: ./.github/actions/setup-poetry-env
+        with:
+          python-version: '3.11'
+
+      - name: Run summary script
+        run: poetry run python scripts/summary.py
+
+      - name: Commit and push changes
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update meetings summary data"
+          file_pattern: data/meetings.jsonl 

.gitignore

@@ -5,6 +5,9 @@ data/audio/
 data/video/
 data/transcripts/
 models/
+# Include specific directories
+!src/models/
+
 notebooks/.ipynb_checkpoints/
 
 # Python

pyproject.toml

@@ -26,6 +26,7 @@ faster-whisper = "1.1.0"
 whisperx = {git = "https://github.com/m-bain/whisperx.git", rev = "main", python = ">=3.10,<3.13"}
 python-dotenv = "^1.0.1"
 aiofiles = "^24.1.0"
+pytz = "^2025.1"
 
 
 [tool.poetry.group.dev.dependencies]

scripts/summary.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""
+Summary generator for Tulsa Government Access Television meetings.
+
+This script retrieves the list of meetings from the TGOV website and
+saves them to a JSONL file for further processing or analysis.
+"""
+import os
+import sys
+
+
+import asyncio
+import json
+from pathlib import Path
+from typing import List, Dict, Any
+
+# Add the parent directory to the path so we can import from src
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+from src.meetings import get_meetings
+from src.models.meeting import Meeting
+
+
+async def generate_summary() -> List[Dict[str, Any]]:
+    """
+    Generate a summary of all meetings.
+
+    Returns:
+        List of meeting data as dictionaries
+    """
+    meetings: List[Meeting] = await get_meetings()
+
+    # Convert Pydantic models to dictionaries for JSON serialization
+    # Use model_dump with mode='json' to ensure all values are JSON serializable
+    return [meeting.model_dump(mode="json") for meeting in meetings]
+
+
+async def save_to_jsonl(meetings: List[Dict[str, Any]], file_path: Path) -> None:
+    """
+    Save meetings data to a JSONL file.
+
+    Args:
+        meetings: List of meeting data as dictionaries
+        file_path: Path to the output JSONL file
+    """
+    # Create directory if it doesn't exist
+    file_path.parent.mkdir(parents=True, exist_ok=True)
+
+    # Write each meeting as a JSON line
+    with file_path.open("w") as f:
+        for meeting in meetings:
+            f.write(json.dumps(meeting) + "\n")
+
+    print(f"Saved {len(meetings)} meetings to {file_path}")
+
+
+async def main() -> None:
+    """Main function to retrieve and save meeting data."""
+    output_path = Path("data/meetings.jsonl")
+
+    print("Retrieving meetings data...")
+    meetings = await generate_summary()
+    print(f"Found {len(meetings)} meetings")
+
+    await save_to_jsonl(meetings, output_path)
+    print("Summary generation complete")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

src/meetings.py

@@ -8,15 +8,19 @@
 
 import asyncio
 import json
-from typing import Dict, List, Optional, Any
+import re
+from datetime import datetime
+from typing import Dict, List, Optional, Any, Union
 from urllib.parse import urljoin
 
 import aiohttp
+import pytz
 from selectolax.parser import HTMLParser
 
 from .models.meeting import Meeting
 
 BASE_URL = "https://tulsa-ok.granicus.com/ViewPublisher.php?view_id=4"
+CENTRAL_TZ = pytz.timezone("America/Chicago")
 
 
 async def fetch_page(url: str, session: aiohttp.ClientSession) -> str:
@@ -36,7 +40,95 @@ async def fetch_page(url: str, session: aiohttp.ClientSession) -> str:
         return await response.text()
 
 
-async def parse_meetings(html: str) -> List[Dict[str, str]]:
+def parse_date_string(date_str: str) -> Optional[datetime]:
+    """
+    Parse the date string into a datetime object with Central timezone.
+
+    Args:
+        date_str: The raw date string from HTML
+
+    Returns:
+        A datetime object with Central timezone or None if parsing fails
+    """
+    # Replace non-breaking spaces with regular spaces
+    date_str = date_str.replace("\u00a0", " ")
+
+    # Replace multiple spaces with a single space
+    date_str = re.sub(r"\s+", " ", date_str)
+
+    # Find the month, day, year, and time parts
+    # Pattern typically looks like "March 12, 2025 - 5:00 PM"
+    match = re.search(
+        r"([A-Za-z]+)\s+(\d{1,2}),?\s+(\d{4}).*?(\d{1,2}):(\d{2})\s*([APM]{2})",
+        date_str,
+    )
+
+    if match:
+        month_str, day_str, year_str, hour_str, minute_str, am_pm = match.groups()
+
+        # Convert month name to number
+        try:
+            month_num = datetime.strptime(month_str, "%B").month
+        except ValueError:
+            # Try abbreviated month name
+            try:
+                month_num = datetime.strptime(month_str, "%b").month
+            except ValueError:
+                return None
+
+        # Convert to integers
+        day = int(day_str)
+        year = int(year_str)
+        hour = int(hour_str)
+        minute = int(minute_str)
+
+        # Adjust hour for PM
+        if am_pm.upper() == "PM" and hour < 12:
+            hour += 12
+        elif am_pm.upper() == "AM" and hour == 12:
+            hour = 0
+
+        # Create naive datetime
+        naive_dt = datetime(year, month_num, day, hour, minute)
+
+        # Localize to Central Time
+        return CENTRAL_TZ.localize(naive_dt)
+
+    return None
+
+
+def clean_date_string(date_str: str) -> str:
+    """
+    Clean up the date string by removing extra whitespace, newlines, and normalizing formats.
+
+    Args:
+        date_str: The raw date string from HTML
+
+    Returns:
+        A cleaned date string in the format "Month Day, Year - Time"
+    """
+    # Replace non-breaking spaces with regular spaces
+    date_str = date_str.replace("\u00a0", " ")
+
+    # Replace multiple spaces with a single space
+    date_str = re.sub(r"\s+", " ", date_str)
+
+    # Find the month, day, year, and time parts
+    # Pattern typically looks like "March 12, 2025 - 5:00 PM"
+    match = re.search(
+        r"([A-Za-z]+)\s+(\d{1,2}),?\s+(\d{4}).*?(\d{1,2}:\d{2}\s*[APM]{2})", date_str
+    )
+
+    if match:
+        month, day, year, time = match.groups()
+        # Format consistently
+        return f"{month} {day}, {year} - {time}"
+
+    # If the regex doesn't match, do basic cleanup
+    return date_str.strip()
+
+
+async def parse_meetings(html: str) -> List[Dict[str, Any]]:
     """
     Parse the meeting data from the HTML content.
 
@@ -68,9 +160,17 @@ async def parse_meetings(html: str) -> List[Dict[str, str]]:
             if len(cells) < 5:
                 continue
 
+            # Parse the date string into a datetime object
+            date_text = cells[1].text()
+            date_obj = parse_date_string(date_text)
+
+            # Get a cleaned date string as a fallback
+            date_str = clean_date_string(date_text)
+
             meeting_data = {
                 "meeting": cells[0].text().strip(),
-                "date": cells[1].text().strip(),
+                "date": date_obj.isoformat() if date_obj else date_str,
+                "date_display": date_str,  # Keep a human-readable version
                 "duration": cells[2].text().strip(),
                 "agenda": None,
                 "video": None,

src/models/meeting.py

@@ -3,9 +3,9 @@
 """
 
 from datetime import datetime
-from typing import Optional
+from typing import Optional, Union
 
-from pydantic import BaseModel, Field, HttpUrl
+from pydantic import BaseModel, Field, HttpUrl, validator
 
 
 class Meeting(BaseModel):
@@ -14,14 +14,32 @@ class Meeting(BaseModel):
     """
 
     meeting: str = Field(description="Name of the meeting")
-    date: str = Field(description="Date and time of the meeting")
+    date: str = Field(
+        description="ISO-formatted date and time of the meeting with timezone"
+    )
+    date_display: Optional[str] = Field(
+        None, description="Human-readable date and time format"
+    )
     duration: str = Field(description="Duration of the meeting")
     agenda: Optional[HttpUrl] = Field(None, description="URL to the meeting agenda")
     video: Optional[HttpUrl] = Field(None, description="URL to the meeting video")
 
+    @validator("date_display", pre=True, always=True)
+    def set_date_display(cls, v, values):
+        """Set date_display to a readable format if not provided"""
+        if v is None and "date" in values:
+            # If the date is in ISO format, try to make it more readable
+            try:
+                dt = datetime.fromisoformat(values["date"])
+                return dt.strftime("%B %d, %Y - %I:%M %p")
+            except (ValueError, TypeError):
+                return values["date"]
+        return v
+
     def __str__(self) -> str:
         """String representation of the meeting"""
-        return f"{self.meeting} - {self.date} ({self.duration})"
+        display_date = self.date_display or self.date
+        return f"{self.meeting} - {display_date} ({self.duration})"
 
 
 class GranicusPlayerPage(BaseModel):