[rel-db] Initial migration (#3152)

* Add initial migration
* Add new migration system and migration states in version table
* deprecate 'progress' and remove 'clear-collectionfield-tables' routes
* set decimals to minimum if below minimum
* use migration tables for initial migration
* centralize MODELS in InternalHelper
* hard coded ORIGIN_COLLECTIONS
* unskip initial import tests
* delete migrations core
* use advisory locks instead in schema creation
* source environment variables in interactive shell
---------

Co-authored-by: Danny <dreichelt@intevation.de>
Co-authored-by: reicda <165993199+reicda@users.noreply.github.com>
Co-authored-by: Luisa <luisa.beerboom@intevation.de>
Co-authored-by: Alexej <33332102+4echow@users.noreply.github.com>
Co-authored-by: luisa-beerboom <101706784+luisa-beerboom@users.noreply.github.com>
Co-authored-by: peb-adr <adrichter97@gmail.com>

Author: 5 people

.github/docker-compose/docker-compose.prod.yml

@@ -1,7 +1,10 @@
-version: "3"
 services:
     backend-action:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9002:9002"
@@ -14,7 +17,11 @@ services:
         secrets:
             - postgres_password
     backend-presenter:
-        build: ../..
+        build: 
+            context: ../..
+            target: "prod"
+            args:
+                CONTEXT: "prod"
         image: openslides-backend
         ports:
             - "9003:9003"

.github/workflows/continuous_integration.yml

@@ -23,6 +23,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v6
+      with:
+        submodules: recursive
 
     - name: Create secrets for datastore
       run: mkdir secrets && echo -n "openslides" > secrets/postgres_password
@@ -64,6 +66,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         path: openslides-backend
+        submodules: recursive
 
     - name: Set env vars
       working-directory: openslides-backend/requirements

Makefile

@@ -45,6 +45,11 @@ check-all: validate-models-yml check-models check-initial-data-json check-exampl
 
 # Models
 
+generate-schema:
+	make -C meta/dev generate-relational-schema
+
+generate-db: | generate-schema create-database-with-schema
+
 generate-models:
 	python cli/generate_models.py $(MODELS_PATH)
 	black openslides_backend/models/models.py

cli/create_schema.py

@@ -1,8 +1,37 @@
-from openslides_backend.services.postgresql.create_schema import create_schema
+from psycopg import Connection, rows, sql
+
+from openslides_backend.services.postgresql.create_schema import (
+    create_db,
+    create_schema,
+)
+from openslides_backend.services.postgresql.db_connection_handling import (
+    env,
+    get_unpooled_db_connection,
+)
+from openslides_backend.shared.exceptions import DatabaseException
 
 
 def main() -> None:
-    create_schema()
+    connection: Connection[rows.DictRow]
+    try:
+        connection = get_unpooled_db_connection(env.DATABASE_NAME, False)
+    except DatabaseException:
+        create_db()
+        connection = get_unpooled_db_connection(env.DATABASE_NAME, False)
+    with connection:
+        with connection.cursor() as cursor:
+            lock_int = 13371234564223
+            cursor.execute(
+                sql.SQL("SELECT pg_advisory_lock({lock_int});").format(
+                    lock_int=lock_int
+                )
+            )
+            create_schema()
+            cursor.execute(
+                sql.SQL("SELECT pg_advisory_unlock({lock_int});").format(
+                    lock_int=lock_int
+                )
+            )
 
 
 if __name__ == "__main__":

cli/generate_models.py

@@ -4,13 +4,7 @@
 from textwrap import dedent
 from typing import Any, Optional
 
-from cli.util.util import (
-    ROOT,
-    assert_equal,
-    open_output,
-    open_yml_file,
-    parse_arguments,
-)
+from cli.util.util import ROOT, assert_equal, open_output, parse_arguments
 from meta.dev.src.helper_get_names import (
     FieldSqlErrorType,
     HelperGetNames,
@@ -76,8 +70,6 @@
     """
 )
 
-MODELS: dict[str, dict[str, Any]] = {}
-
 
 def main() -> None:
     """
@@ -101,10 +93,8 @@ def main() -> None:
             to: some_model/some_attribute_id
     """
     args = parse_arguments(SOURCE)
-    global MODELS
-    MODELS = open_yml_file(args.filename)
 
-    InternalHelper.MODELS = MODELS
+    InternalHelper.read_models_yml(SOURCE)
 
     # Load and parse models.yml
     with open_output(DESTINATION, args.check) as dest:
@@ -114,7 +104,7 @@ def main() -> None:
             + ", ".join(mixin.__name__ for mixin in MODEL_MIXINS.values())
             + "\n"
         )
-        for collection, fields in MODELS.items():
+        for collection, fields in InternalHelper.MODELS.items():
             if collection.startswith("_"):
                 continue
             model = Model(collection, fields)
@@ -132,7 +122,7 @@ def get_model_field(collection: str, field_name: str) -> str | dict:
     Helper function the get a specific model field. Used to create generic relations.
     """
 
-    model = MODELS.get(collection)
+    model = InternalHelper.MODELS.get(collection)
     if model is None:
         raise ValueError(f"Collection {collection} does not exist.")
     value = model.get(field_name)
@@ -376,16 +366,21 @@ def get_view_field_state_write_fields(
                 else:
                     write_fields = (table_name, field1, field2, [])
             elif "generic-relation-list" in (field_type, foreign_type):
-                write_fields = self.get_write_fields_for_generic(own, foreign_fields)
+                write_fields = self.get_write_fields_for_generic(
+                    own, foreign_fields, primary
+                )
 
         assert error == "", error
 
         return is_view_field, primary, write_fields
 
     def get_write_fields_for_generic(
-        self, own: TableFieldType, foreign_fields: list[TableFieldType]
+        self, own: TableFieldType, foreign_fields: list[TableFieldType], primary: bool
     ) -> tuple[str, str, str, list[str]] | None:
-        table_name = HelperGetNames.get_gm_table_name(own)
+        if primary:
+            table_name = HelperGetNames.get_gm_table_name(own)
+        else:
+            table_name = HelperGetNames.get_gm_table_name(foreign_fields[0])
         field1 = f"{own.table}_{own.ref_column}"
         field2 = own.intermediate_column
         return (

data/example-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

data/initial-data.json

@@ -1,5 +1,5 @@
 {
-    "_migration_index": 74,
+    "_migration_index": 100,
     "gender":{
         "1":{
             "id": 1,

dev/.bashrc

@@ -1,3 +1,5 @@
 if [ -f /etc/bash_completion ]; then source /etc/bash_completion; fi
 alias ls='ls --color=auto'
 alias l='ls -CF'
+
+source scripts/export_database_variables.sh

dev/entrypoint.sh

@@ -3,20 +3,19 @@
 printf "enter entrypoint.sh"
 set -e
 
-printf "\nOpenslides Database:\n"
+printf "\nOpenslides DBMS:\n"
 printf "Export env variables for database.\n"
 source scripts/export_database_variables.sh
 
 meta/dev/scripts/wait-for-database.sh
-printf "Database is started.\n"
+printf "DBMS is started.\n"
 
-printf "Create schema.\n"
+printf "Calling cli/create_schema.py ...\n"
 python cli/create_schema.py
 printf "\n"
 
-# TODO: Re-add this code
-# printf "\nMigrations:\n"
-# python openslides_backend/migrations/migrate.py finalize
-# printf "\n"
+printf "\nMigrations:\n"
+python openslides_backend/migrations/migrate.py finalize
+printf "\n"
 
 exec "$@"

docs/migration_route.md

@@ -3,20 +3,23 @@
 Migrations are available via the internal route `/internal/migrations/` with the following payload:
 ```js
 {
-    "cmd": "migrate" | "finalize" | "reset" | "clear-collectionfield-tables" | "stats" | "progress",
+    "cmd": "migrate" | "finalize" | "reset" | "stats",
     "verbose": bool
 }
 ```
 
-All commands except `progress` are directly translated the the respective method calls to the datastore module as well as the `verbose` flag. While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `progress`. The `progress` command is an additional command only available in the backend which reports the progress of a long-running migration (which is executed in a thread).
+While a migration is running (either via `migrate` or `finalize`), no other commands are permitted except `stats`. The `stats` command reports the progress of a long-running migration (which is executed in a thread).
 
 The output of all commands except `stats` is the following (for a successful request):
 ```js
 enum MigrationState {
-    MIGRATION_RUNNING = "migration_running"
     MIGRATION_REQUIRED = "migration_required"
+    MIGRATION_RUNNING = "migration_running"
+    MIGRATION_FAILED = "migration_failed"
     FINALIZATION_REQUIRED = "finalization_required"
-    NO_MIGRATION_REQUIRED = "no_migration_required"
+    FINALIZATION_RUNNING = "finalization_running"
+    FINALIZED = "finalized"
+    FINALIZATION_FAILED = "finalization_failed"
 }
 
 {
@@ -30,20 +33,20 @@ enum MigrationState {
 ```
 `output` always contains the full output of the migration command up to this point. `exception` contains the thrown exception, if any, which can only be the case if the command is finished (meaning `status != "migration_running"`). After issuing a migration command, it is waited a short period of time for the thread to finish, so the status can be all of these things for any command (e.g. after calling `migrate`, the returned status can be either `MIGRATION_RUNNING` if the migrations did not finish directly or `FINALIZATION_REQUIRED` if the migration is already done).
 
-The output of migration commands is stored until a new migration command is issued, meaning repeated `progress` requests after a finished command will always return the same result with the full output.
-
 The `stats` return value is the following:
 ```js
 {
     "success": True,
     "stats": {
         "status": MigrationState,
+        "output": str, // Optional
+        "exception": str, // Optional
         "current_migration_index": int,
         "target_migration_index": int,
-        "positions": int,
-        "events": int,
-        "partially_migrated_positions": int,
-        "fully_migrated_positions": int
+        "migratable_models": {
+            "count": int,
+            "migrated": int
+        }
     }
 }
 ```