Merge pull request #104 from febus982/src-layout

* Removes jaeger container and setup Opentelemetry collector to output to logs
* Move all the application sources in src directory for a cleaner layout
* Remove GRPC (The celery worker implementation is enough as example for another application)

Author: febus982

.codeclimate.yml

@@ -6,8 +6,7 @@ exclude_patterns:
   - "spec/"
   - "!spec/support/helpers"
   - "config/"
-  - "alembic/"
-  - "grpc_app/generated/"
+  - "src/alembic/"
   - "db/"
   - "dist/"
   - "features/"

.idea/bootstrap-fastapi-service.iml

@@ -5,6 +5,7 @@ ARG GID=2000
 RUN addgroup --gid $GID nonroot && \
     adduser --uid $UID --gid $GID --disabled-password --gecos "" nonroot
 WORKDIR /app
+RUN chown nonroot:nonroot /app
 
 # Creating a separate directory for venvs allows to easily
 # copy them from the builder and to mount the application
@@ -44,8 +45,8 @@ COPY --chown=nonroot:nonroot Makefile .
 
 # Test image, contains all files and dependencies
 FROM base_builder as dev
-RUN make dev-dependencies
 COPY --chown=nonroot:nonroot . .
+RUN make dev-dependencies
 # Note that opentelemetry doesn't play well together with uvicorn reloader
 # when signals are propagated, we disable it in dev image default CMD
 CMD ["uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "--factory", "--reload"]
@@ -58,40 +59,30 @@ RUN poetry install --no-root
 FROM base_builder as http_builder
 RUN poetry install --no-root --with http
 
-# Installs requirements to run production grpc application
-FROM base_builder as grpc_builder
-RUN poetry install --no-root --with grpc
-
 # Copy the shared python packages
 FROM base as base_app
 USER nonroot
 RUN poetry config virtualenvs.path /poetryvenvs
 COPY --chown=nonroot:nonroot pyproject.toml .
 COPY --chown=nonroot:nonroot poetry.lock .
-COPY --chown=nonroot:nonroot alembic ./alembic
-COPY --chown=nonroot:nonroot domains ./domains
-COPY --chown=nonroot:nonroot gateways ./gateways
-COPY --chown=nonroot:nonroot common ./common
-COPY --chown=nonroot:nonroot alembic.ini .
+COPY --chown=nonroot:nonroot src/alembic ./alembic
+COPY --chown=nonroot:nonroot src/domains ./domains
+COPY --chown=nonroot:nonroot src/gateways ./gateways
+COPY --chown=nonroot:nonroot src/common ./common
+COPY --chown=nonroot:nonroot src/alembic.ini .
 COPY --chown=nonroot:nonroot Makefile .
 
 # Copy the http python package and requirements from relevant builder
 FROM base_app as http_app
 COPY --from=http_builder /poetryvenvs /poetryvenvs
-COPY --chown=nonroot:nonroot http_app ./http_app
+COPY --chown=nonroot:nonroot src/http_app ./http_app
 # Run CMD using array syntax, so it's uses `exec` and runs as PID1
 CMD ["opentelemetry-instrument", "uvicorn", "http_app:create_app", "--host", "0.0.0.0", "--port", "8000", "--factory"]
 
-# Copy the grpc python package and requirements from relevant builder
-FROM base_app as grpc_app
-COPY --from=grpc_builder /poetryvenvs /poetryvenvs
-COPY --chown=nonroot:nonroot grpc_app ./grpc_app
-# Run CMD using array syntax, so it's uses `exec` and runs as PID1
-CMD ["opentelemetry-instrument", "python3", "-m", "grpc_app"]
-
 # Copy the celery python package and requirements from relevant builder
 FROM base_app as celery_app
 COPY --from=celery_builder /poetryvenvs /poetryvenvs
-COPY --chown=nonroot:nonroot celery_worker ./celery_worker
+COPY --chown=nonroot:nonroot src/celery_worker ./celery_worker
+RUN ls
 # Run CMD using array syntax, so it's uses `exec` and runs as PID1
 CMD ["opentelemetry-instrument", "celery", "-A", "celery_worker:app", "worker", "-l", "INFO"]

Dockerfile

@@ -17,9 +17,6 @@ otel:
 run:
 	poetry run uvicorn http_app:create_app --host 0.0.0.0 --port 8000 --factory
 
-grpc:
-	poetry run python3 -m grpc_app
-
 test:
 	poetry run pytest -n auto --cov
 
@@ -33,13 +30,13 @@ typing:
 	poetry run mypy
 
 install-dependencies:
-	poetry install --no-root --with http,grpc
+	poetry install --no-root --with http
 
 dev-dependencies:
-	poetry install --no-root --with http,grpc,dev
+	poetry install --with http,dev
 
 update-dependencies:
-	poetry update --with http,grpc,dev
+	poetry update --with http,dev
 
 migrate:
 	poetry run alembic upgrade heads
@@ -50,23 +47,6 @@ format:
 lint:
 	poetry run ruff .
 
-# There are issues on how python imports are generated when using nested
-# packages. The following setup appears to work, however it might need
-# to be reviewed. https://github.com/protocolbuffers/protobuf/issues/1491
-generate-proto:
-	rm -rf ./grpc_app/generated/*.p*
-	touch ./grpc_app/generated/__init__.py
-	poetry run python -m grpc_tools.protoc \
-	-I grpc_app.generated=./grpc_app/proto/ \
-	--python_out=. \
-	--mypy_out=. \
-	grpc_app/proto/*.proto
-	poetry run python -m grpc_tools.protoc \
-	-I grpc_app/generated=./grpc_app/proto/ \
-	--grpc_python_out=. \
-	grpc_app/proto/*.proto
-	git add ./grpc_app/generated
-
 fix:
 	poetry run ruff . --fix
 	poetry run ruff format .

Makefile

@@ -45,9 +45,10 @@ Packages are ordered from the highest level to the lowest one.
 
 ------
 
+* `alembic` (database migration manager)
 * `http_app` (http presentation layer)
-* `grpc_app` (grpc presentation layer)
-* `storage` (database connection manager, repository implementation)
+* `celery_worker` (async tasks runner)
+* `gateways` (database connection manager, repository implementation, event emitter, etc.)
 
 ------
 
@@ -70,7 +71,7 @@ Using Docker:
 * `make containers`: Build containers
 * `docker compose run --rm dev make migrate`: Run database migrations
 * `docker compose up dev`: Run HTTP application with hot reload
-* `docker compose up grpc`: Run GRPC application
+* `docker compose up celery-worker`: Run the celery worker
 * `docker compose run --rm test`: Run test suite
 
 Locally:
@@ -81,12 +82,10 @@ Locally:
 * `make update-dependencies`: Updates requirements
 * `make migrate`: Run database migrations
 * `make dev`: Run HTTP application with hot reload
-* `make grpc`: Run GRPC application
 * `make test`: Run test suite
 
 ## Other commands for development
 
-* `make generate-proto`: Generates grpcio python stubs from `.proto` files in `grpc_app/proto` directory
 * `make check`: Run tests, code style and lint checks
 * `make fix`: Run tests, code style and lint checks with automatic fixes (where possible)
 

README.md

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = alembic
+script_location = src/alembic
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
@@ -12,7 +12,7 @@ file_template = %%(year)d-%%(month).2d-%%(day).2d-%%(hour).2d%%(minute).2d%%(sec
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.
-prepend_sys_path = .
+#prepend_sys_path = .
 
 # timezone to use when rendering the date within the migration file
 # as well as the filename.
@@ -56,13 +56,6 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 # output_encoding = utf-8
 
 # We inject db names and config using env.py in alembic directory
-#databases = engine1, engine2
-
-#[engine1]
-#sqlalchemy.url = driver://user:pass@localhost/dbname
-
-#[engine2]
-#sqlalchemy.url = driver://user:pass@localhost/dbname2
 
 [post_write_hooks]
 # post_write_hooks defines scripts or Python functions that are run

alembic.ini

@@ -8,10 +8,6 @@ package http_app #fff {
     package routes #DDDDDD
 }
 
-package grpc_app #fff {
-    package servicers #DDDDDD
-}
-
 package domains #DDDDDD {
     package books {
         class Book
@@ -42,8 +38,6 @@ package gateways #DDDDDD {
 'links framework - domains
 routes --> BookService
 routes --> Book
-servicers --> BookService
-servicers --> Book
 celery_worker ---> tasks
 tasks <-u- BookService
 tasks -u-> Book

architecture.png

@@ -1,15 +1,8 @@
 version: '3.8'
 services:
 
-  jaeger:
-    image: jaegertracing/all-in-one
-    ports:
-      - "16686:16686"
-
   otel-collector:
     image: otel/opentelemetry-collector-contrib
-    depends_on:
-      - jaeger
     volumes:
       - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml
 
@@ -25,7 +18,24 @@ services:
       OTEL_SERVICE_NAME: "bootstrap-fastapi-worker"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
     volumes:
-      - '.:/app'
+      - './src:/app'
+      - './pyproject.toml:/app/pyproject.toml'
+    depends_on:
+      - redis
+      - otel-collector
+      - celery-beat
+
+  celery-beat:
+    build:
+      dockerfile: Dockerfile
+      context: .
+      target: celery_app
+    environment:
+      OTEL_SERVICE_NAME: "bootstrap-fastapi-worker"
+      OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
+    volumes:
+      - './src:/app'
+      - './pyproject.toml:/app/pyproject.toml'
     depends_on:
       - redis
       - otel-collector
@@ -34,8 +44,7 @@ services:
       - celery
       - -A
       - celery_worker:app
-      - worker
-      - --beat
+      - beat
       - -l
       - INFO
 
@@ -50,10 +59,11 @@ services:
     ports:
       - '8000:8000'
     volumes:
-      - '.:/app'
+      - './src:/app'
+      - './pyproject.toml:/app/pyproject.toml'
     depends_on:
       - redis
-      - celery-worker
+      - otel-collector
     command:
       - opentelemetry-instrument
       - uvicorn
@@ -72,7 +82,6 @@ services:
       target: http_app
     depends_on:
       - redis
-      - celery-worker
       - otel-collector
     environment:
       OTEL_SERVICE_NAME: "bootstrap-fastapi-http"
@@ -82,35 +91,7 @@ services:
     volumes:
       - './sqlite.db:/app/sqlite.db'
 
-  grpc:
-    build:
-      dockerfile: Dockerfile
-      context: .
-      target: grpc_app
-    depends_on:
-      - redis
-      - celery-worker
-      - otel-collector
-    environment:
-      OTEL_SERVICE_NAME: "bootstrap-fastapi-grpc"
-      OTEL_EXPORTER_OTLP_ENDPOINT: "http://otel-collector:4317"
-    ports:
-      - "9999:9999"
-    volumes:
-      - '.:/app'
-
   # Starting from here there are only single-run commands, we can use `make` here
-  generate-proto:
-    build:
-      dockerfile: Dockerfile
-      context: .
-      target: grpc_builder
-    volumes:
-      - '.:/app'
-    command:
-      - "make"
-      - "generate-proto"
-
   test:
     build:
       dockerfile: Dockerfile