Integration of GitHub Actions workflows (#2)

Author: JanssenBrm

.flake8

@@ -0,0 +1,10 @@
+[flake8]
+max-line-length = 100
+extend-ignore = E203  
+exclude =
+    .git,
+    __pycache__,
+    venv,
+    .venv,
+    .mypy_cache,
+    .pytest_cache

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+jobs:
+  lint-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Create .env file
+        run: |
+          echo "DATABASE_URL=sqlite:///:memory:" >> .env
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install pytest pytest-cov flake8 mypy
+
+      - name: Lint with flake8
+        run: |
+          flake8 app tests
+
+      - name: Type check with mypy
+        run: |
+          mypy app
+
+      - name: Run tests
+        run: |
+          pytest --cov=app --cov-report=term-missing

.github/workflows/release.yml

@@ -0,0 +1,80 @@
+name: Release
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version number (e.g., 1.2.3)"
+        required: true
+
+
+env:
+  DOCKER_IMAGE_NAME: apex-dispatcher-api
+  DOCKER_REGISTRY_REGION: eu-nl
+  DOCKER_REGISTRY_URL: swr.eu-nl.otc.t-systems.com
+  DOCKER_REGISTRY_BUCKET: apex
+
+jobs:
+  publish-docker:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Get release version
+        env:
+          TAG_VERSION: ${{ inputs.version }}
+        run: |
+          echo "RELEASE_VERSION=${TAG_VERSION#v*}" >> $GITHUB_ENV
+          echo $RELEASE_VERSION
+      - name: Prepare envs
+        run: |
+          echo "IMAGE_NAME=${DOCKER_IMAGE_NAME}:${RELEASE_VERSION}" >> $GITHUB_ENV
+          echo "IMAGE_TAG_RELEASE=${DOCKER_REGISTRY_URL}/${DOCKER_REGISTRY_BUCKET}/${DOCKER_IMAGE_NAME}:${RELEASE_VERSION}" >> $GITHUB_ENV
+          echo "IMAGE_TAG_LATEST=${DOCKER_REGISTRY_URL}/${DOCKER_REGISTRY_BUCKET}/${DOCKER_IMAGE_NAME}:latest" >> $GITHUB_ENV
+      - name: Build docker image
+        run: |
+          docker build . --tag $IMAGE_NAME
+      - name: Tag the image
+        run: |
+          docker tag $IMAGE_NAME $IMAGE_TAG_RELEASE
+          docker tag $IMAGE_NAME $IMAGE_TAG_LATEST
+      - name: Docker login
+        env:
+          OS_ACCESS_KEY: ${{ secrets.OS_ACCESS_KEY }}
+          OS_SECRET_KEY: ${{ secrets.OS_SECRET_KEY }}
+        run: |
+          LOGIN_KEY=`printf "$OS_ACCESS_KEY" | openssl dgst -binary -sha256 -hmac "$OS_SECRET_KEY" | od -An -vtx1 | sed 's/[ \n]//g' | sed 'N;s/\n//'`
+          docker login -u ${DOCKER_REGISTRY_REGION}@${OS_ACCESS_KEY} -p ${LOGIN_KEY} ${DOCKER_REGISTRY_URL}
+      - name: Push image to registry
+        run: |
+          docker push $IMAGE_TAG_RELEASE
+          docker push $IMAGE_TAG_LATEST
+      - name: Clean up the local images
+        run: |
+          docker rmi $IMAGE_TAG_RELEASE
+          docker rmi $IMAGE_TAG_LATEST
+          docker rmi $IMAGE_NAME
+
+  github-release:
+    runs-on: ubuntu-latest
+    needs: [publish-docker]
+    permissions:
+      contents: write
+      packages: write 
+    steps:
+      - name: Create Git Tag
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+          git tag -a v${{ github.event.inputs.version }} -m "Release v${{ github.event.inputs.version }}"
+          git push origin v${{ github.event.inputs.version }}
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v2
+        with:
+          tag_name: v${{ github.event.inputs.version }}
+          name: Release v${{ github.event.inputs.version }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    
+

app/config/logger.py

@@ -25,11 +25,12 @@
         "uvicorn.error": {"level": "INFO"},
         "uvicorn.access": {"level": "INFO"},
         # custom API loggers
-        "app.routers": {"level": "DEBUG"},      # all your routers
-        "app.services": {"level": "DEBUG"},     # all your services
-        "app.platforms": {"level": "DEBUG"},     # all platform implementations
+        "app.routers": {"level": "DEBUG"},  # all your routers
+        "app.services": {"level": "DEBUG"},  # all your services
+        "app.platforms": {"level": "DEBUG"},  # all platform implementations
     },
 }
 
+
 def setup_logging():
-    logging.config.dictConfig(LOGGING_CONFIG)
+    logging.config.dictConfig(LOGGING_CONFIG)

app/config/settings.py

@@ -1,19 +1,31 @@
-from pydantic import AnyHttpUrl, ConfigDict, Field
-from pydantic_settings import BaseSettings
+from pydantic import AnyHttpUrl, Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
 
 class Settings(BaseSettings):
-    app_name: str = Field("", json_schema_extra={"env": "APP_NAME"})
-    app_description: str = Field("", json_schema_extra={"env":"APP_DESCRIPTION"})
-    env: str = Field("development", json_schema_extra={"env":"APP_ENV"})
+    app_name: str = Field(
+        default="APEx Disatpcher API", json_schema_extra={"env": "APP_NAME"}
+    )
+    app_description: str = Field(
+        default="API description for the APEx Dispatcher",
+        json_schema_extra={"env": "APP_DESCRIPTION"},
+    )
+    env: str = Field(default="development", json_schema_extra={"env": "APP_ENV"})
 
     # Keycloak / OIDC
-    keycloak_server_url: AnyHttpUrl = Field(None, json_schema_extra={"env":"KEYCLOAK_SERVER_URL"})
-    keycloak_realm: str = Field(None, json_schema_extra={"env":"KEYCLOAK_REALM"})
-    keycloak_client_id: str = Field(None, json_schema_extra={"env":"KEYCLOAK_CLIENT_ID"})
-    keycloak_client_secret: str | None = Field(None, json_schema_extra={"env":"KEYCLOAK_CLIENT_SECRET"})
-
+    keycloak_server_url: AnyHttpUrl = Field(
+        default=AnyHttpUrl("https://localhost"),
+        json_schema_extra={"env": "KEYCLOAK_SERVER_URL"},
+    )
+    keycloak_realm: str = Field(default="", json_schema_extra={"env": "KEYCLOAK_REALM"})
+    keycloak_client_id: str = Field(
+        default="", json_schema_extra={"env": "KEYCLOAK_CLIENT_ID"}
+    )
+    keycloak_client_secret: str | None = Field(
+        default="", json_schema_extra={"env": "KEYCLOAK_CLIENT_SECRET"}
+    )
 
-    model_config = ConfigDict(
+    model_config = SettingsConfigDict(
         env_file=".env",
         env_file_encoding="utf-8",
         extra="allow",

app/database/db.py

@@ -39,4 +39,4 @@ def get_db():
         db.rollback()
         raise
     finally:
-        db.close()
+        db.close()

app/database/models/processing_job.py

@@ -1,51 +1,76 @@
 import datetime
 import logging
-from typing import List
-from sqlalchemy import Column, DateTime, Enum, Integer, String
+from typing import List, Optional
+from sqlalchemy import DateTime, Enum, Integer, String
 from app.database.db import Base
-from sqlalchemy.orm import Session
+from sqlalchemy.orm import Session, Mapped, mapped_column
 
-from app.schemas import ProcessingStatusEnum
+from app.schemas import ProcessTypeEnum, ProcessingStatusEnum
 
 
 logger = logging.getLogger(__name__)
 
 
 class ProcessingJobRecord(Base):
-    __tablename__ = 'processing_jobs'
-    
-    id = Column(Integer, primary_key=True, index=True, autoincrement=True)
-    title = Column(String, index=True)
-    label = Column(String, index=True)
-    status = Column(Enum(ProcessingStatusEnum), index=True)
-    user_id = Column(String, index=True)
-    platform_job_id = Column(String, index=True)
-    parameters = Column(String, index=False)
-    result_link = Column(String, index=False)
-    service_record = Column(String, index=True)
-    created = Column(DateTime, default=datetime.datetime.utcnow, index=True)
-    updated = Column(DateTime, default=datetime.datetime.utcnow, onupdate=datetime.datetime.utcnow, index=True)
-
-        
-    
-def save_job_to_db(db_session: Session, job: ProcessingJobRecord) -> ProcessingJobRecord:
+    __tablename__ = "processing_jobs"
+
+    id: Mapped[int] = mapped_column(
+        Integer, primary_key=True, index=True, autoincrement=True
+    )
+    title: Mapped[str] = mapped_column(String, index=True)
+    label: Mapped[ProcessTypeEnum] = mapped_column(Enum(ProcessTypeEnum), index=True)
+    status: Mapped[ProcessingStatusEnum] = mapped_column(
+        Enum(ProcessingStatusEnum), index=True
+    )
+    user_id: Mapped[str] = mapped_column(String, index=True)
+    platform_job_id: Mapped[Optional[str]] = mapped_column(String, index=True)
+    parameters: Mapped[Optional[str]] = mapped_column(String, index=False)
+    result_link: Mapped[Optional[str]] = mapped_column(String, index=False)
+    service_record: Mapped[Optional[str]] = mapped_column(String, index=True)
+    created: Mapped[datetime.datetime] = mapped_column(
+        DateTime, default=datetime.datetime.utcnow, index=True
+    )
+    updated: Mapped[datetime.datetime] = mapped_column(
+        DateTime,
+        default=datetime.datetime.utcnow,
+        onupdate=datetime.datetime.utcnow,
+        index=True,
+    )
+
+
+def save_job_to_db(
+    db_session: Session, job: ProcessingJobRecord
+) -> ProcessingJobRecord:
     """
     Save a processing job record to the database and update the ID of the job.
-    
+
     :param db_session: The database session to use for saving the job.
     :param job: The ProcessingJobRecord instance to save.
-    """ 
+    """
     db_session.add(job)
     db_session.commit()
     db_session.refresh(job)  # Refresh to get the ID after commit
-    logger.debug("Processing job saved with ID: {job.id}") 
+    logger.debug("Processing job saved with ID: {job.id}")
     return job
-    
+
 
 def get_jobs_by_user_id(database: Session, user_id: str) -> List[ProcessingJobRecord]:
     logger.info(f"Retrieving all processing jobs for user {user_id}")
-    return database.query(ProcessingJobRecord).filter(ProcessingJobRecord.user_id == user_id).all()
+    return (
+        database.query(ProcessingJobRecord)
+        .filter(ProcessingJobRecord.user_id == user_id)
+        .all()
+    )
+
 
-def get_job_by_user_id(database: Session, job_id: int, user_id: str) -> ProcessingJobRecord:
+def get_job_by_user_id(
+    database: Session, job_id: int, user_id: str
+) -> Optional[ProcessingJobRecord]:
     logger.info(f"Retrieving processing job with ID {job_id} for user {user_id}")
-    return database.query(ProcessingJobRecord).filter(ProcessingJobRecord.id == job_id, ProcessingJobRecord.user_id == user_id).first()
+    return (
+        database.query(ProcessingJobRecord)
+        .filter(
+            ProcessingJobRecord.id == job_id, ProcessingJobRecord.user_id == user_id
+        )
+        .first()
+    )

app/main.py

@@ -12,7 +12,7 @@
 app = FastAPI(
     title=settings.app_name,
     description=settings.app_description,
-    version="1.0.0", 
+    version="1.0.0",
 )
 
 # Register Keycloak - must be done after FastAPI app creation
@@ -21,4 +21,4 @@
 # include routers
 app.include_router(jobs_status.router)
 app.include_router(unit_jobs.router)
-app.include_router(health.router)
+app.include_router(health.router)

app/platforms/base.py

@@ -19,4 +19,4 @@ def execute_job(self, title: str, details: ServiceDetails, parameters: dict) ->
         :param parameters: The parameters required for the job execution.
         :return: Return the ID of the job that was created
         """
-        pass
+        pass

app/platforms/dispatcher.py

@@ -1,7 +1,6 @@
-
 import importlib
 import logging
-import app.platforms.implementations 
+import app.platforms.implementations
 import pkgutil
 from typing import Dict, Type
 from app.platforms.base import BaseProcessingPlatform
@@ -11,28 +10,37 @@
 
 logger = logging.getLogger(__name__)
 
+
 def load_processing_platforms():
     """Dynamically load all processing platform implementations."""
-    for _, module_name, _ in pkgutil.iter_modules(app.platforms.implementations.__path__):
+    for _, module_name, _ in pkgutil.iter_modules(
+        app.platforms.implementations.__path__
+    ):
         importlib.import_module(f"app.platforms.implementations.{module_name}")
 
-def register_processing_platform(service_type: ProcessTypeEnum, cls: Type[BaseProcessingPlatform]):
-    """"Register a new processing platform class for a specific service type.
-    
+
+def register_processing_platform(
+    service_type: ProcessTypeEnum, cls: Type[BaseProcessingPlatform]
+):
+    """ "Register a new processing platform class for a specific service type.
+
     :param service_type: The type of service for which to register the platform.
     :param cls: The class that implements BaseProcessingPlatform.
     """
-    logger.debug(f"Registering processing platform with class {cls} for service type: {service_type}")
+    logger.debug(
+        f"Registering processing platform with class {cls} for service type: {service_type}"
+    )
     PROCESSING_PLATFORMS[service_type] = cls
 
+
 def get_processing_platform(service_type: ProcessTypeEnum) -> BaseProcessingPlatform:
     """
     Factory function to get the appropriate processing platform based on the service type.
-    
+
     :param service_type: The type of service for which to get the processing platform.
     :return: An instance of a class that implements BaseProcessingPlatform.
     """
     try:
         return PROCESSING_PLATFORMS[service_type]()
     except KeyError:
-        raise ValueError(f"Unsupported service type: {service_type}")
+        raise ValueError(f"Unsupported service type: {service_type}")