Modernize RSS Bridge with comprehensive tests including E2E

.github/workflows/test-rss.yml

@@ -0,0 +1,74 @@
+name: RSS Bridge Tests
+
+on:
+  push:
+    branches: [ "rss" ]
+    paths:
+      - 'rss/**'
+      - '.github/workflows/test-rss.yml'
+  pull_request:
+    branches: [ "rss" ]
+    paths:
+      - 'rss/**'
+      - '.github/workflows/test-rss.yml'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: ./rss
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          echo "$HOME/.local/bin" >> $GITHUB_PATH
+
+      - name: Configure Poetry
+        run: |
+          poetry config virtualenvs.create true
+          poetry config virtualenvs.in-project true
+
+      - name: Cache Poetry dependencies
+        uses: actions/cache@v4
+        with:
+          path: ./rss/.venv
+          key: venv-${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+          restore-keys: |
+            venv-${{ runner.os }}-
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-ansi
+
+      - name: Run unit tests
+        run: poetry run pytest tests/test_rssbridge_unit.py -v --tb=short
+
+      - name: Run integration tests
+        run: poetry run pytest tests/test_rssbridge_integration.py -v --tb=short
+
+      - name: Run e2e tests
+        run: poetry run pytest tests/test_rssbridge_e2e.py -v --tb=short
+
+      - name: Run all tests with coverage
+        run: poetry run pytest tests/ -v --cov=rssbridge --cov-report=term-missing --cov-report=xml --cov-fail-under=30
+
+      - name: Upload coverage reports
+        uses: codecov/codecov-action@v4
+        if: always()
+        with:
+          file: ./rss/coverage.xml
+          flags: rss
+          name: rss-coverage
+          fail_ci_if_error: false
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

rss/Dockerfile

@@ -1,11 +1,11 @@
 # Use an official Python runtime as a parent image
 FROM python:3.10-slim
 
-LABEL org.opencontainers.image.source = "https://github.com/clemensv/real-time-sources/tree/main/rss"
-LABEL org.opencontainers.image.title = "RSS/Atom bridge to Kafka endpoints"
-LABEL org.opencontainers.image.description = "This container is a bridge between RSS and Atom feeds and Kafka endpoints. It fetches entries from feeds and forwards them to the configured Kafka endpoints."
-LABEL org.opencontainers.image.documentation = "https://github.com/clemensv/real-time-sources/blob/main/rss/CONTAINER.md"
-LABEL org.opencontainers.image.license = "MIT"
+LABEL org.opencontainers.image.source="https://github.com/clemensv/real-time-sources/tree/main/rss"
+LABEL org.opencontainers.image.title="RSS/Atom bridge to Kafka endpoints"
+LABEL org.opencontainers.image.description="This container is a bridge between RSS and Atom feeds and Kafka endpoints. It fetches entries from feeds and forwards them to the configured Kafka endpoints."
+LABEL org.opencontainers.image.documentation="https://github.com/clemensv/real-time-sources/blob/main/rss/CONTAINER.md"
+LABEL org.opencontainers.image.license="MIT"
 
 # Set the working directory in the container
 WORKDIR /app

rss/pyproject.toml

@@ -12,7 +12,7 @@ authors = ["Clemens Vasters <clemensv@microsoft.com>"]
 [tool.poetry.dependencies]
 rssbridge_producer_data = {path = "rssbridge_producer/rssbridge_producer_data"}
 rssbridge_producer_kafka_producer = {path = "rssbridge_producer/rssbridge_producer_kafka_producer"}
-python = ">=3.8"
+python = ">=3.10,<4.0"
 requests = ">=2.32.3"
 confluent-kafka = ">=2.5.3"
 cloudevents = ">=1.11.0"

rss/pytest.ini

@@ -0,0 +1,16 @@
+[pytest]
+testpaths = tests
+python_files = test_*.py
+python_classes = Test*
+python_functions = test_*
+addopts = 
+    -v
+    --tb=short
+    --cov=rssbridge
+    --cov-report=html
+    --cov-report=term-missing
+
+markers =
+    unit: Unit tests without external dependencies
+    integration: Integration tests with mocked external services
+    e2e: End-to-end tests requiring real external services

rss/rssbridge/rssbridge.py

@@ -527,10 +527,10 @@ def parse_connection_string(connection_string: str) -> Dict[str, str]:
     try:
         for part in connection_string.split(';'):
             if 'Endpoint' in part:
-                config_dict['bootstrap.servers'] = part.split('=')[1].strip(
-                    '"').replace('sb://', '').replace('/', '')+':9093'
+                endpoint = part.split('=', 1)[1].strip().strip('"').strip()
+                config_dict['bootstrap.servers'] = endpoint.replace('sb://', '').replace('/', '')+':9093'
             elif 'EntityPath' in part:
-                config_dict['kafka_topic'] = part.split('=')[1].strip('"')
+                config_dict['kafka_topic'] = part.split('=', 1)[1].strip().strip('"').strip()
     except IndexError as e:
         raise ValueError("Invalid connection string format") from e
     return config_dict

rss/tests/README.md

@@ -0,0 +1,111 @@
+# RSS Bridge Test Suite
+
+This directory contains comprehensive tests for the RSS Bridge project.
+
+## Test Structure
+
+The test suite is organized into three categories:
+
+### 1. Unit Tests (`test_rssbridge_unit.py`)
+
+Tests individual functions and classes without external dependencies:
+- Connection string parsing (Event Hubs format)
+- Feed URL extraction from OPML
+- State file management
+- Feed metadata extraction
+- Helper functions
+
+**Run unit tests only:**
+
+```bash
+pytest -m unit
+```
+
+### 2. Integration Tests (`test_rssbridge_integration.py`)
+
+Tests interactions with external services using mocked responses:
+- RSS/Atom feed parsing with mocked HTTP responses
+- Feed discovery from web pages
+- OPML file processing
+- Error handling for malformed feeds
+- Producer client integration
+
+**Run integration tests only:**
+
+```bash
+pytest -m integration
+```
+
+### 3. End-to-End Tests (`test_rssbridge_e2e.py`)
+
+Tests complete workflows with real feeds and services:
+- Parsing real public RSS/Atom feeds (W3C, IETF)
+- Feed discovery from real web pages
+- State and feedstore persistence workflows
+- OPML file parsing with realistic structure
+- Event Hubs connection string parsing
+- Producer client initialization
+
+**Note:** E2E tests require internet connectivity and may be slower.
+
+**Run e2e tests only:**
+
+```bash
+pytest -m e2e
+```
+
+## Running Tests
+
+**Run all tests except E2E:**
+
+```bash
+pytest -m "not e2e"
+```
+
+**Run all tests:**
+
+```bash
+pytest
+```
+
+**Run with coverage:**
+
+```bash
+pytest --cov=rssbridge --cov-report=html
+```
+
+**Run specific test file:**
+
+```bash
+pytest tests/test_rssbridge_unit.py -v
+```
+
+## Test Requirements
+
+The following test dependencies are required:
+- `pytest` >= 8.3.3
+- `pytest-cov` >= 5.0.0
+- `requests-mock` >= 1.12.1
+
+Install with: `poetry install`
+
+## Writing Tests
+
+Follow these guidelines when adding new tests:
+
+1. **Mark tests appropriately:**
+   - `@pytest.mark.unit` for unit tests
+   - `@pytest.mark.integration` for integration tests
+   - `@pytest.mark.e2e` for end-to-end tests
+
+2. **Use descriptive names:**
+   - Test class: `TestConnectionStringParsing`
+   - Test method: `test_parse_connection_string_with_all_components`
+
+3. **Keep tests focused:**
+   - Each test should verify one specific behavior
+   - Use fixtures for common setup
+
+4. **Mock external dependencies:**
+   - Use `requests_mock` for HTTP calls
+   - Use unittest.mock for other external dependencies

rss/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests package initialization"""