[rel-db] Initial migration

.github/docker-compose/docker-compose.prod.yml

@@ -1,7 +1,10 @@
-version: "3"
 services:
     backend-action:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9002:9002"
@@ -14,7 +17,11 @@ services:
         secrets:
             - postgres_password
     backend-presenter:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9003:9003"

.github/workflows/continuous_integration.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v6
+      with:
+        submodules: recursive
 
     - name: Create secrets for datastore
       run: mkdir secrets && echo -n "openslides" > secrets/postgres_password
@@ -64,6 +66,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         path: openslides-backend
+        submodules: recursive
 
     - name: Set env vars
       working-directory: openslides-backend/requirements

Makefile

@@ -34,12 +34,22 @@ coverage:
 	pytest --cov --cov-report html
 
 test-file:
-	python -m debugpy --listen 0.0.0.0:5678 --wait-for-client /usr/local/bin/pytest $f
+# f= to pass the filename
+# k= to pass a test name
+# v=1 to run verbose test output
+# cap=1 to capture print to system out
+# cov=1 to run coverage report
+	python -m debugpy --listen 0.0.0.0:5678 --wait-for-client /usr/local/bin/pytest $f $(if $(k),-k $k) $(if $(v),-vv) $(if $(cap),--capture=no) $(if $(cov),--cov --cov-report term-missing:skip-covered)
 
 check-all: validate-models-yml check-models check-initial-data-json check-example-data-json check-permissions
 
 # Models
 
+generate-schema:
+	make -C meta/dev generate-relational-schema
+
+generate-db: | generate-schema create-database-with-schema
+
 generate-models:
 	python cli/generate_models.py $(MODELS_PATH)
 	black openslides_backend/models/models.py

data/example-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

data/initial-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

dev/entrypoint.sh

@@ -3,20 +3,19 @@
 printf "enter entrypoint.sh"
 set -e
 
-printf "\nOpenslides Database:\n"
+printf "\nOpenslides DBMS:\n"
 printf "Export env variables for database.\n"
 source scripts/export_database_variables.sh
 
 meta/dev/scripts/wait-for-database.sh
-printf "Database is started.\n"
+printf "DBMS is started.\n"
 
 printf "Create schema.\n"
 python cli/create_schema.py
 printf "\n"
 
-# TODO: Re-add this code
-# printf "\nMigrations:\n"
-# python openslides_backend/migrations/migrate.py finalize
-# printf "\n"
+printf "\nMigrations:\n"
+python openslides_backend/migrations/migrate.py finalize
+printf "\n"
 
 exec "$@"

docs/migration_route.md

@@ -3,20 +3,23 @@
 Migrations are available via the internal route `/internal/migrations/` with the following payload:
 ```js
 {
-    "cmd": "migrate" | "finalize" | "reset" | "clear-collectionfield-tables" | "stats" | "progress",
+    "cmd": "migrate" | "finalize" | "reset" | "stats",
     "verbose": bool
 }
 ```
 
-All commands except `progress` are directly translated the the respective method calls to the datastore module as well as the `verbose` flag. While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `progress`. The `progress` command is an additional command only available in the backend which reports the progress of a long-running migration (which is executed in a thread).
+While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `stats`. The `stats` command reports the progress of a long-running migration (which is executed in a thread).
 
 The output of all commands except `stats` is the following (for a successful request):
 ```js
 enum MigrationState {
-    MIGRATION_RUNNING = "migration_running"
     MIGRATION_REQUIRED = "migration_required"
+    MIGRATION_RUNNING = "migration_running"
+    MIGRATION_FAILED = "migration_failed"
     FINALIZATION_REQUIRED = "finalization_required"
-    NO_MIGRATION_REQUIRED = "no_migration_required"
+    FINALIZATION_RUNNING = "finalization_running"
+    FINALIZED = "finalized"
+    FINALIZATION_FAILED = "finalization_failed"
 }
 
 {
@@ -30,20 +33,20 @@ enum MigrationState {
 ```
 `output` always contains the full output of the migration command up to this point. `exception` contains the thrown exception, if any, which can only be the case if the command is finished (meaning `status != "migration_running"`). After issuing a migration command, it is waited a short period of time for the thread to finish, so the status can be all of these things for any command (e.g. after calling `migrate`, the returned status can be either `MIGRATION_RUNNING` if the migrations did not finish directly or `FINALIZATION_REQUIRED` if the migration is already done).
 
-The output of migration commands is stored until a new migration command is issued, meaning repeated `progress` requests after a finished command will always return the same result with the full output.
-
 The `stats` return value is the following:
 ```js
 {
     "success": True,
     "stats": {
         "status": MigrationState,
+        "output": str, // Optional
+        "exception": str, // Optional
         "current_migration_index": int,
         "target_migration_index": int,
-        "positions": int,
-        "events": int,
-        "partially_migrated_positions": int,
-        "fully_migrated_positions": int
+        "migratable_models": {
+            "count": int,
+            "migrated": int
+        }
     }
 }
 ```

entrypoint.sh

@@ -1,9 +1,15 @@
 #!/bin/bash
 
+printf "enter entrypoint.sh"
+set -e
+
+printf "\nOpenslides DBMS:\n"
+printf "Export env variables for database.\n"
 source scripts/export_database_variables.sh
 
 if [ ! $ANONYMOUS_ONLY ]; then
   meta/dev/scripts/wait-for-database.sh
+  printf "DBMS is started.\n"
 fi
 
 printf "Create schema.\n"

openslides_backend/action/actions/meeting/import_.py

@@ -5,7 +5,7 @@
 from typing import Any
 
 from openslides_backend.action.actions.meeting.mixins import MeetingPermissionMixin
-from openslides_backend.migrations import get_backend_migration_index
+from openslides_backend.migrations.migration_helper import MigrationHelper
 from openslides_backend.models.base import model_registry
 from openslides_backend.models.checker import Checker, CheckException
 from openslides_backend.models.fields import (
@@ -753,7 +753,7 @@ def check_data_migration_index(self, instance: dict[str, Any]) -> None:
         Check for valid migration index.
         """
         start_migration_index = instance["meeting"].pop("_migration_index")
-        backend_migration_index = get_backend_migration_index()
+        backend_migration_index = MigrationHelper.get_backend_migration_index()
         if backend_migration_index < start_migration_index:
             raise ActionException(
                 f"Your data migration index '{start_migration_index}' is higher than the migration index of this backend '{backend_migration_index}'! Please, update your backend!"

openslides_backend/action/actions/organization/initial_import.py

@@ -1,6 +1,8 @@
 from collections.abc import Iterable
 from typing import Any
 
+from openslides_backend.migrations.migration_helper import MigrationHelper
+
 from ....i18n.translator import Translator
 from ....i18n.translator import translate as _
 from ....models.base import json_dict_to_non_json_data_types
@@ -142,10 +144,7 @@ def build_write_request(
     def create_action_result_element(
         self, instance: dict[str, Any]
     ) -> ActionResultElement | None:
-        backend_migration_index = 1
-        # TODO set to fixed value because of lacking migrations
-        # needs to be readded in some shape or form
-        # backend_migration_index = get_backend_migration_index()
+        backend_migration_index = MigrationHelper.get_backend_migration_index()
         result = {
             "data_migration_index": self.data_migration_index,
             "backend_migration_index": backend_migration_index,

openslides_backend/http/views/action_view.py

@@ -5,9 +5,10 @@
 from ...action.action_handler import ActionHandler
 from ...action.action_worker import handle_action_in_worker_thread
 from ...i18n.translator import Translator
-from ...migrations import assert_migration_index
-from ...migrations.migration_handler import MigrationHandler
+from ...migrations.migration_helper import MigrationHelper
+from ...migrations.migration_manager import MigrationManager
 from ...services.auth.interface import AUTHENTICATION_HEADER, COOKIE_NAME
+from ...services.postgresql.db_connection_handling import get_new_os_conn
 from ...shared.env import DEV_PASSWORD
 from ...shared.exceptions import AuthenticationException, ServerError
 from ...shared.interfaces.wsgi import RouteResponse
@@ -31,8 +32,9 @@ class ActionView(BaseView):
     def action_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching action request.")
 
-        assert_migration_index()
-
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
         # Get user id.
         user_id, access_token = self.get_user_id_from_headers(
             request.headers, request.cookies
@@ -56,7 +58,9 @@ def action_route(self, request: Request) -> RouteResponse:
     def internal_action_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching internal action request.")
 
-        assert_migration_index()
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
         self.check_internal_auth_password(request)
 
         handler = ActionHandler(self.env, self.services, self.logging)
@@ -71,8 +75,8 @@ def internal_action_route(self, request: Request) -> RouteResponse:
     def migrations_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start executing migrations request.")
         self.check_internal_auth_password(request)
-        handler = MigrationHandler(self.env, self.services, self.logging)
-        response = handler.handle_request(request.json)
+        manager = MigrationManager(self.env, self.services, self.logging)
+        response = manager.handle_request(request.json)
         self.logger.debug("Migrations request finished successfully.")
         return {"success": True, **response}, None
 

openslides_backend/http/views/presenter_view.py

@@ -1,5 +1,6 @@
-from ...migrations import assert_migration_index
+from ...migrations.migration_helper import MigrationHelper
 from ...presenter.presenter import PresenterHandler
+from ...services.postgresql.db_connection_handling import get_new_os_conn
 from ...shared.interfaces.wsgi import RouteResponse
 from ..request import Request
 from .base_view import BaseView, route
@@ -17,7 +18,9 @@ class PresenterView(BaseView):
     def presenter_route(self, request: Request) -> RouteResponse:
         self.logger.debug("Start dispatching presenter request.")
 
-        assert_migration_index()
+        with get_new_os_conn() as conn:
+            with conn.cursor() as curs:
+                MigrationHelper.assert_migration_index(curs)
 
         # Handle request.
         handler = PresenterHandler(

openslides_backend/migrations/README.md

@@ -1,32 +1,21 @@
-# Migrations for the Datastore
+# TODO revise this file
+# Migrations for the Database
 
-The `migrate.py` script is the main entrypoint to execute migrations. The migrations itself are in the `migrations` folder. Each file can be arbitrarily named but must include a class called `Migration` that can be found by the loader. Note that all docker related files in this folder are only for **developing** migrations, not execute them. For execution in the dev mode, see the actual backend target for the dockerfile (`target dev`) and `dev/entrypoint.sh`.
+The migrations will create shadow copies with `{name}_mig` of all tables to be migrated and rereference all triggers and references to those. Migration will be done on the `_mig`-tables. In the finalization step the original tables are deleted and the shadow copies put into place.
+Collections in READ_MODELS will prevent writing on corresponding tables for all parrallel processes.
+Collections in WRITE_MODELS will additionally prevent reading on corresponding tables for all parrallel processes.
 
-There are three ways to execute migrations:
-
-## 1) Developing migrations
-This is done within the dockersetup in this folder. Note that the datastore needs to be checked out with the name `openslides-datastore-service` next to the backend repository.
-
-- checkout the current datastore (in ../../openslides-datastore-service) and the current backend.
-- `make dev`: Starts the compose setup -> A shell opens with the ability to run `migrate.py` and auxillary scripts.
-- You can exit from it with `exit` and shut down the docker setup with `make stop-dev`
-- You can write migrations in the backend and also adjust the Datastore at the same time since both are mounted into the container.
-
-The following scripts can be used to make snapshots and trying out new migrations
+The migrations itself are in the `migrations` folder. Each file must start with a four digit number and can include certain functions that can be found by the loader. A psycopg cursor object will be passed as function parameter.
+ * data_definition: should do necessary schema changes
+ * data_manipulation: should pass the altered original data from the table to table_mig
+ * cleanup: should do all cleanups that aren't done automatically like deleting additional temporary tables. This step happens during finalization before all automatic changes.
 
 #### Scripts for setting initial data
-Only the current dataset is exported, so after a (re-)import, only one position exists in the datastore. Also note that importing clears the old content
 
-- `export-data-only.sh` [to:export.json]
-- `import-data-only.sh` [from:export.json]
-
-#### Scripts for the full backup (Does a DB dump)
-- `export-events.sh` [to:export.sql]
-- `import-events.sh` [from:export.sql]
-
-#### Downloading example data
-- `fetch-example-data.sh` [to:example-data.json]
+There are three ways to execute migrations:
 
+## 1) The `migrate.py` script is the cli entrypoint to execute migrations.
+Use python `migrate.py -h` to see all available commands.
 
 ## 2) Migrations in dev mode
 

openslides_backend/migrations/__init__.py

@@ -14,51 +14,3 @@
     MigrationSetupException,
     MismatchingMigrationIndicesException,
 )
-from .migrate import MigrationWrapper
-
-
-def get_backend_migration_index() -> int:
-    migration_classes = MigrationWrapper.load_migrations()
-
-    backend_migration_index = 1
-    for migration_class in migration_classes:
-        backend_migration_index = max(
-            backend_migration_index, migration_class().target_migration_index
-        )
-    return backend_migration_index
-
-
-def get_datastore_migration_index() -> int:
-    # TODO: Re-implement
-    # read_db = injector.get(ReadDatabase)
-    # with read_db.get_context():
-    #     datastore_migration_index = read_db.get_current_migration_index()
-    # return datastore_migration_index
-
-    return 1
-
-
-def assert_migration_index() -> None:
-    # TODO: Re-implement
-    # connection = injector.get(ConnectionHandler)
-    # with connection.get_connection_context():
-    #     if connection.query_single_value("select count(*) from positions", []) == 0:
-    #         return  # Datastore is empty; nothing to check.
-
-    # datastore_migration_index = get_datastore_migration_index()
-
-    # if datastore_migration_index == -1:
-    #     return  # Datastore is up-to-date; nothing to do.
-
-    # backend_migration_index = get_backend_migration_index()
-
-    # if backend_migration_index > datastore_migration_index:
-    #     raise ActionException(
-    #         f"Missing {backend_migration_index-datastore_migration_index} migrations to apply."
-    #     )
-
-    # if backend_migration_index < datastore_migration_index:
-    #     raise ActionException(
-    #         f"Migration indices do not match: Datastore has {datastore_migration_index} and the backend has {backend_migration_index}"
-    #     )
-    pass

openslides_backend/migrations/core/exceptions.py

@@ -8,3 +8,8 @@ class MigrationSetupException(MigrationException):
 
 class MismatchingMigrationIndicesException(MigrationException):
     pass
+
+
+class InvalidMigrationCommand(MigrationException):
+    def __init__(self, command: str) -> None:
+        super().__init__(f"Invalid migration command: {command}")