Merging 'develop' into master (#16)

* chore: clean up [#15]

* chore: version bump [#15]

* ops: fix GHA config[#15]

Author: nickatnight

.env_example

@@ -3,6 +3,7 @@ SECRET_KEY=test-key
 ENV=dev
 PROJECT_NAME=fastapi_backend_base
 VERSION=v1
+ENABLE_BUILD=false
 
 # Database
 POSTGRES_USER=fastapibackendbase

.github/workflows/main.yml

@@ -1,43 +1,47 @@
-name: lint and test
+---
+
+name: build, test and deploy
 
 on:
   push:
     tags:
       - '*'
-  pull_request:
     branches:
       - master
       - develop
 
+env:
+  DOCTL_VERSION: 1.84.1
+
 jobs:
-  linter:
-    runs-on: ubuntu-20.04
+  flake8:
+    runs-on: ubuntu-latest
     name: Check python linting
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Start linter
         run: |
-          docker run --rm -w="/code/backend" -v $(pwd):/code alpine/flake8:3.9.2 .
+          docker run --rm -w="/code/backend" -v $(pwd):/code alpine/flake8:5.0.4 .
 
   black:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: Black code formatting
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Python
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v4
         with:
           python-version: '3.9'
       - name: run black
         working-directory: ./backend
         run: |
-          pip install black==22.6.0
+          pip install black==21.5b1
           black --check .
 
   isort:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     name: isort imports
     steps:
       - name: Checkout
@@ -52,12 +56,72 @@ jobs:
           pip install isort==5.10.1
           isort src
 
+  build_backend:
+    if: ${{ secrets.ENABLE_BUILD }} == 'true'
+    runs-on: ubuntu-latest
+    needs: [flake8,black,isort]
+    name: Build backend
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_TOKEN }}
+      - name: Build backend
+        run: |
+          docker build --build-arg env=staging -t ${{ secrets.REGISTRY }}/backend:${{ github.sha }} ./backend
+      - name: Log in to DigitalOcean Container Registry with short-lived credentials
+        run: doctl registry login --expiry-seconds 600
+      - name: Push image to DigitalOcean Container Registry
+        run: docker push ${{ secrets.REGISTRY }}/backend:${{ github.sha }}
+
+  build_proxy:
+    if: ${{ secrets.ENABLE_BUILD }} == 'true'
+    runs-on: ubuntu-latest
+    needs: [flake8,black]
+    name: Build Nginx proxy
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Install doctl
+        uses: digitalocean/action-doctl@v2
+        with:
+          token: ${{ secrets.DIGITALOCEAN_TOKEN }}
+      - name: Build proxy
+        run: |
+          docker build --build-arg env=staging -t ${{ secrets.REGISTRY }}/proxy:${{ github.sha }} ./proxy
+      - name: Log in to DigitalOcean Container Registry with short-lived credentials
+        run: doctl registry login --expiry-seconds 600
+      - name: Push image to DigitalOcean Container Registry
+        run: docker push ${{ secrets.REGISTRY }}/proxy:${{ github.sha }}
+
+  unit_tests:
+    runs-on: ubuntu-latest
+    needs: [build_backend, build_proxy]
+    name: Run unit tests
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Run unit tests
+        run: |
+          docker-compose -f ops/docker-compose.test.yml up --exit-code-from backend
+      - name: Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          file: /data/coverage.xml
+          flags: unittests
+      - name: Clean-up
+        if: always()
+        run: |
+          docker-compose -f ops/docker-compose.test.yml down -v
+
   create-release:
-    needs: [linter, black, isort]
-    runs-on: ubuntu-20.04
+    needs: [flake8,black,isort]
+    runs-on: ubuntu-latest
     steps:
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0  # need this for all history for all branches and tags
       - name: Create Release
@@ -69,3 +133,55 @@ jobs:
 
     outputs:
       ReleaseTag: ${{ steps.create_release.outputs.release_tag }}
+
+  deploy:
+    runs-on: ubuntu-latest
+    name: Deploy
+    needs: [unit_tests]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Deploy staging
+        uses: ironhalik/docker-over-ssh-action@v6
+        if: github.ref == 'refs/heads/develop'
+        env:
+          COMPOSE_FILE: ops/docker-compose.staging.yml
+          STACK_NAME: tagyoureitbot-staging
+          DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          REGISTRY: ${{ secrets.REGISTRY }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          PASSWORD: ${{ secrets.PASSWORD }}
+          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+        with:
+          user: ubuntu
+          host: ${{ secrets.STAGING_HOST_IP }}
+          key: ${{ secrets.SSH_KEY }}
+          script: |
+            wget https://github.com/digitalocean/doctl/releases/download/v${{ env.DOCTL_VERSION }}/doctl-${{ env.DOCTL_VERSION }}-linux-amd64.tar.gz
+            tar xf ./doctl-${{ env.DOCTL_VERSION }}-linux-amd64.tar.gz
+            mv ./doctl /usr/local/bin
+            doctl registry login
+            docker stack deploy --compose-file ${COMPOSE_FILE} --with-registry-auth --prune ${STACK_NAME}
+
+      - name: Deploy prod
+        uses: ironhalik/docker-over-ssh-action@v6
+        if: github.ref == 'refs/heads/master'
+        env:
+          COMPOSE_FILE: ops/docker-compose.prod.yml
+          STACK_NAME: tagyoureit-prod
+          DIGITALOCEAN_ACCESS_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          REGISTRY: ${{ secrets.REGISTRY }}
+          POSTGRES_PASSWORD: ${{ secrets.POSTGRES_PASSWORD }}
+          PASSWORD: ${{ secrets.PASSWORD }}
+          CLIENT_SECRET: ${{ secrets.CLIENT_SECRET }}
+        with:
+          user: ubuntu
+          host: ${{ secrets.PROD_HOST_IP }}
+          key: ${{ secrets.SSH_KEY }}
+          script: |
+            wget https://github.com/digitalocean/doctl/releases/download/v${{ env.DOCTL_VERSION }}/doctl-${{ env.DOCTL_VERSION }}-linux-amd64.tar.gz
+            tar xf ./doctl-${{ env.DOCTL_VERSION }}-linux-amd64.tar.gz
+            mv ./doctl /usr/local/bin
+            doctl registry login
+            docker stack deploy --compose-file ${COMPOSE_FILE} --with-registry-auth --prune ${STACK_NAME}

backend/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "fastapi-backend-base"
-version = "3.0.0"
+version = "3.0.1"
 description = "Base project for building fastapi backends"
 authors = ["nickatnight <[email protected]>"]
 

backend/src/core/config.py

@@ -1,31 +1,52 @@
-from typing import List, Optional
+from typing import Any, Dict, List, Optional
 
-from pydantic import AnyHttpUrl, BaseSettings, Field
+from pydantic import AnyHttpUrl, BaseSettings, Field, PostgresDsn, validator
 
 
 class Settings(BaseSettings):
-    SECRET_KEY: str = Field(..., env="SECRET_KEY")
     # 60 minutes * 24 hours * 8 days = 8 days
     ACCESS_TOKEN_EXPIRE_MINUTES: int = 60 * 24 * 8
-    SERVER_NAME: Optional[str] = Field(..., env="NGINX_HOST")
+    # SERVER_NAME: Optional[str] = Field(..., env="NGINX_HOST")
     BACKEND_CORS_ORIGINS: List[AnyHttpUrl] = []
 
-    PROJECT_NAME: str = Field(..., env="PROJECT_NAME")
-
-    POSTGRES_HOST: str = Field(..., env="POSTGRES_HOST")
-    POSTGRES_USER: str = Field(..., env="POSTGRES_USER")
-    POSTGRES_PASSWORD: str = Field(..., env="POSTGRES_PASSWORD")
-    POSTGRES_DB: str = Field(..., env="POSTGRES_DB")
-    POSTGRES_PORT: str = Field(..., env="POSTGRES_PORT")
-
-    REDIS_HOST: str = Field(..., env="REDIS_HOST")
-    REDIS_PORT: str = Field(..., env="REDIS_PORT")
-    # REDIS_URL: str = f"redis://{REDIS_HOST}:{REDIS_PORT}/0"
-
-    CLIENT_ID: str = Field(..., env="CLIENT_ID")
-    CLIENT_SECRET: str = Field(..., env="CLIENT_SECRET")
-
-    VERSION: str = Field(..., env="VERSION")
+    CLIENT_ID: str = Field(default="", env="CLIENT_ID")
+    CLIENT_SECRET: str = Field(default="", env="CLIENT_SECRET")
+
+    VERSION: str = Field(default="", env="VERSION")
+    DEBUG: bool = Field(default=True, env="DEBUG")
+
+    POSTGRES_USER: str = Field(default="", env="POSTGRES_USER")
+    POSTGRES_PASSWORD: str = Field(default="", env="POSTGRES_PASSWORD")
+    POSTGRES_HOST: str = Field(default="", env="POSTGRES_HOST")
+    POSTGRES_PORT: str = Field(default="", env="POSTGRES_PORT")
+    POSTGRES_DB: str = Field(default="", env="POSTGRES_DB")
+
+    DB_POOL_SIZE: int = Field(default=83, env="DB_POOL_SIZE")
+    WEB_CONCURRENCY: int = Field(default=9, env="WEB_CONCURRENCY")
+    MAX_OVERFLOW: int = Field(default=64, env="MAX_OVERFLOW")
+    POOL_SIZE: Optional[int]
+    POSTGRES_URL: Optional[str]
+
+    @validator("POOL_SIZE", pre=True)
+    def build_pool(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
+        if isinstance(v, int):
+            return v
+
+        return max(values.get("DB_POOL_SIZE") // values.get("WEB_CONCURRENCY"), 5)  # type: ignore
+
+    @validator("POSTGRES_URL", pre=True)
+    def build_db_connection(cls, v: Optional[str], values: Dict[str, Any]) -> Any:
+        if isinstance(v, str):
+            return v
+
+        return PostgresDsn.build(
+            scheme="postgresql+asyncpg",
+            user=values.get("POSTGRES_USER"),
+            password=values.get("POSTGRES_PASSWORD"),
+            host=values.get("POSTGRES_HOST"),
+            port=str(values.get("POSTGRES_PORT")),
+            path=f"/{values.get('POSTGRES_DB') or ''}",
+        )
 
 
 settings = Settings()

backend/src/db/session.py

@@ -3,17 +3,18 @@
 from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine
 from sqlalchemy.orm import sessionmaker
 
-from src.db.utils import get_db_url
+from src.core.config import settings
 
 
 logger = logging.getLogger(__name__)
-DB_POOL_SIZE = 83
-WEB_CONCURRENCY = 9
-POOL_SIZE = max(DB_POOL_SIZE // WEB_CONCURRENCY, 5)
-POSTGRES_URL = get_db_url()
+
 
 engine = create_async_engine(
-    POSTGRES_URL, echo=True, future=True, pool_size=POOL_SIZE, max_overflow=64
+    settings.POSTGRES_URL,
+    echo=True,
+    future=True,
+    pool_size=settings.POOL_SIZE,
+    max_overflow=settings.MAX_OVERFLOW,
 )
 async_session = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False)
 

backend/src/db/utils.py

@@ -12,12 +12,10 @@
 
 sys.path.append(str(pathlib.Path(__file__).resolve().parents[1]))
 
-from src.db.utils import get_db_url  # noqa
+from src.core.config import settings  # noqa
 from src.models import *  # noqa
 
 
-POSTGRES_URL = get_db_url()
-
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
@@ -52,7 +50,7 @@ def run_migrations_offline() -> None:
 
     """
     context.configure(
-        url=POSTGRES_URL,
+        url=settings.POSTGRES_URL,
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
@@ -84,7 +82,7 @@ async def run_migrations_online() -> None:
     #         future=True,
     #     )
     # )
-    connectable = AsyncEngine(create_engine(POSTGRES_URL, echo=True, future=True))
+    connectable = AsyncEngine(create_engine(settings.POSTGRES_URL, echo=True, future=True))
 
     async with connectable.connect() as connection:
         await connection.run_sync(do_run_migrations)

backend/src/migrations/env.py

@@ -12,8 +12,7 @@ RUN apk add --no-cache\
     musl-dev &&\
     pip3 install --upgrade pip &&\
     pip3 install --no-cache-dir \
-    j2cli==0.3.10 \
-    certbot-nginx==1.20.0 &&\
+    j2cli==0.3.10 &&\
     rm -rf /etc/nginx/nginx.conf /etc/nginx/http.d/*
 
 COPY ./ /code/