Chore: Upgrade Flask to 3.x and Sqlalchemy to 2.x

Minimal changes to get server working without any migrations to alchemy new API

Author: varmar05

server/Pipfile

@@ -4,25 +4,25 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-connexion = {extras = ["swagger-ui"],version = "==2.14.1"}
-flask = "==2.2.5"
+connexion = {extras = ["swagger-ui"],version = "==2.15.1"}
+flask = "==3.1.2"
 python-dateutil = "==2.8.2"
-marshmallow = "==3.20.1"
-flask-marshmallow = "==0.14.0"
-marshmallow-sqlalchemy = "==1.1.0"
+marshmallow = "==3.26.1"
+flask-marshmallow = "==0.15.0"
+marshmallow-sqlalchemy = "==1.4.1"
 psycopg2-binary = "==2.9.9"
 itsdangerous = "==2.2.0"
-Flask-SQLAlchemy = "==2.5.1"
-sqlalchemy = "==1.4.53"
-gunicorn = {extras = ["gevent"],version = "==19.9"}
+Flask-SQLAlchemy = "==3.1.1"
+sqlalchemy = "==2.0.44"
+gunicorn = {extras = ["gevent"],version = "==23.0"}
 python-dotenv = "==0.20.0"
-flask-login = "==0.6.2"
+flask-login = "==0.6.3"
 bcrypt = "==4.2.0"
-wtforms = {extras = ["email"],version = "==3.1.2"}
-flask-wtf = "==1.0.1"
+wtforms = {extras = ["email"],version = "==3.2.1"}
+flask-wtf = "==1.2.2"
 flask-mail = "==0.10.0"
 safe = "==0.4"
-flask-migrate = "==2.6.0" # 3.1.0
+flask-migrate = "==3.1.0"
 wtforms-json = "==0.3.5"
 pytz = "==2022.2.1"
 scikit-build = "==0.18.1"

server/Pipfile.lock

@@ -10,6 +10,7 @@
 import gevent
 from marshmallow import fields
 from sqlalchemy.schema import MetaData
+from sqlalchemy import text
 from flask_sqlalchemy import SQLAlchemy
 from flask_marshmallow import Marshmallow
 from flask import (
@@ -27,7 +28,6 @@
 from flask_wtf.csrf import generate_csrf, CSRFProtect
 from flask_migrate import Migrate
 from flask_mail import Mail
-from connexion.apps.flask_app import FlaskJSONEncoder
 from flask_wtf import FlaskForm
 from wtforms import StringField
 from pathlib import Path
@@ -37,7 +37,6 @@
 from werkzeug.exceptions import HTTPException
 from typing import List, Dict, Optional, Tuple
 
-from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
 from .config import Configuration
 from .commands import add_commands as server_commands
 
@@ -139,7 +138,6 @@ def create_simple_app() -> Flask:
     app = connexion.FlaskApp(__name__, specification_dir=os.path.join(this_dir))
     flask_app = app.app
 
-    flask_app.json_encoder = FlaskJSONEncoder
     flask_app.config.from_object(Configuration)
     db.init_app(flask_app)
     ma.init_app(flask_app)
@@ -155,54 +153,36 @@ def create_simple_app() -> Flask:
 def create_app(public_keys: List[str] = None) -> Flask:
     """Factory function to create Flask app instance"""
     from itsdangerous import BadTimeSignature, BadSignature
-    from .auth import auth_required, decode_token
-    from .auth.models import User
 
-    # from .celery import celery
-    from .sync.db_events import register_events
-    from .sync.workspace import GlobalWorkspaceHandler
-    from .sync.config import Configuration as SyncConfig
-    from .sync.commands import add_commands
-    from .auth import register as register_auth
+    from .auth import auth_required, decode_token, register as register_auth
+    from .auth.models import User
+    from .sync.app import register as register_sync
     from .sync.project_handler import ProjectHandler
+    from .sync.utils import get_blacklisted_dirs, get_blacklisted_files
+    from .sync.workspace import GlobalWorkspaceHandler
 
     app = create_simple_app().connexion_app
 
-    app.add_api(
-        "sync/public_api.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/public_api_v2.yaml",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": Configuration.SWAGGER_UI},
-        validate_responses=True,
-    )
-    app.add_api(
-        "sync/private_api.yaml",
-        base_path="/app",
-        arguments={"title": "Mergin"},
-        options={"swagger_ui": False, "serve_spec": False},
-        validate_responses=True,
-    )
     app.add_api(
         "api.yaml",
         arguments={"title": "Mergin"},
         options={"swagger_ui": False, "serve_spec": False},
         validate_responses=True,
     )
+    app.app.blueprints["/"].name = "main"
+    app.app.blueprints["main"] = app.app.blueprints.pop("/")
 
-    app.app.config.from_object(SyncConfig)
-    app.app.connexion_app = app
+    # register sync module
+    register_sync(app.app)
 
+    # initialize extensions
     mail.init_app(app.app)
-    app.mail = mail
     csrf.init_app(app.app)
     login_manager.init_app(app.app)
+
     # register auth blueprint
     register_auth(app.app)
+
     server_commands(app.app)
 
     # adjust login manager
@@ -228,8 +208,6 @@ def load_user_from_header(header_val):  # pylint: disable=W0613,W0612
             except (BadSignature, BadTimeSignature, KeyError):
                 pass
 
-    # csrf = app.app.extensions['csrf']
-
     @app.app.before_request
     def check_maintenance():
         allowed_endpoints = ["/project/by_names", "/auth/login", "/alive"]
@@ -275,9 +253,6 @@ def get_startup_data():
         }
         return data
 
-    # update celery config with flask app config
-    # celery.conf.update(app.app.config)
-
     @app.route("/alive", methods=["POST"])
     @csrf.exempt
     def alive():  # pylint: disable=E0722
@@ -287,7 +262,7 @@ def alive():  # pylint: disable=E0722
         start_time = time.time()
         try:
             with db.engine.connect() as con:
-                rs = con.execute("SELECT 2 * 2")
+                rs = con.execute(text("SELECT 2 * 2"))
                 assert rs.fetchone()[0] == 4
         except:
             """Although bad form, we have deliberate left this except broad. When we have an uncaught exception in
@@ -388,7 +363,6 @@ def init():  # pylint: disable=W0612
         response.headers.set("X-CSRF-Token", generate_csrf())
         return response
 
-    register_events()
     application = app.app
 
     @application.errorhandler(Exception)
@@ -467,8 +441,6 @@ def config():
             cfg["build_hash"] = application.config["BUILD_HASH"]
             return jsonify(cfg), 200
 
-    # append project commands (from default sync module)
-    add_commands(application)
     return application
 
 

server/mergin/app.py

@@ -396,7 +396,7 @@ def register_user():  # pylint: disable=W0613,W0612
 
     form = UserRegistrationForm()
     form.username.data = User.generate_username(form.email.data)
-    if form.validate_on_submit():
+    if form.is_submitted() and form.validate():
         user = User.create(form.username.data, form.email.data, form.password.data)
         user_created.send(user, source="admin")
         token = generate_confirmation_token(
@@ -497,8 +497,9 @@ def get_paginated_users(
     elif not descending and order_by:
         users = users.order_by(asc(User.__table__.c[order_by]))
 
-    result = users.paginate(page, per_page).items
-    total = users.paginate(page, per_page).total
+    paginate = users.paginate(page=page, per_page=per_page)
+    result = paginate.items
+    total = paginate.total
 
     result_users = UserSchema(many=True).dump(result)
 

server/mergin/auth/controller.py

@@ -14,7 +14,7 @@
 @celery.task
 def anonymize_removed_users():
     """Permanently 'delete' users marked for removal by removing personal information"""
-    db.session.info = {"msg": "anonymize_removed_users"}
+    db.session.info["msg"] = "anonymize_removed_users"
     before_expiration = datetime.today() - timedelta(Configuration.ACCOUNT_EXPIRATION)
     users = User.query.filter(
         isnot(User.active, True),

server/mergin/auth/tasks.py

@@ -7,6 +7,7 @@
 import random
 import string
 import os
+from sqlalchemy import inspect
 
 
 def _echo_title(title):
@@ -136,7 +137,8 @@ def _check_server(app: Flask):  # pylint: disable=W0612
     else:
         _echo_error("No service ID set.")
 
-    tables = db.engine.table_names()
+    inspect_engine = inspect(db.engine)
+    tables = inspect_engine.get_table_names()
     if not tables:
         _echo_error("Database not initialized. Run flask init-db command")
     else:
@@ -157,9 +159,9 @@ def _init_db(app: Flask):
         label="Creating database", length=4, show_eta=False
     ) as progress_bar:
         progress_bar.update(0)
-        db.drop_all(bind=None)
+        db.drop_all(bind_key=None)
         progress_bar.update(1)
-        db.create_all(bind=None)
+        db.create_all(bind_key=None)
         progress_bar.update(2)
         db.session.commit()
         progress_bar.update(3)
@@ -202,7 +204,8 @@ def init(email: str, recreate: bool):
         """Initialize database if does not exist or -r is provided. Perform check of server configuration. Send statistics, respecting your setup."""
         from .auth.models import User, UserProfile
 
-        tables = db.engine.table_names()
+        inspect_engine = inspect(db.engine)
+        tables = inspect_engine.get_table_names()
         if recreate and tables:
             click.confirm(
                 "Are you sure you want to recreate database and admin user? This will remove all data!",

server/mergin/commands.py

@@ -1,15 +1,14 @@
-import json
-import logging
-import os
+# Copyright (C) Lutra Consulting Limited
+#
+# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
 from flask import abort, current_app, request
 from flask_login import current_user
 from magic import from_buffer
-import time
-
 import requests
 
 from .utils import save_diagnostic_log_file
-from .app import parse_version_string, db
+from .app import parse_version_string
 
 
 def get_latest_version():

server/mergin/controller.py

@@ -4,11 +4,12 @@
 
 from dataclasses import asdict
 import requests
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 import json
 import logging
 from flask import current_app
 from sqlalchemy.sql.operators import is_
+from sqlalchemy import inspect
 
 from .models import MerginInfo, MerginStatistics, ServerCallhomeData
 from ..celery import celery
@@ -71,7 +72,8 @@ def send_statistics():
     if not current_app.config["COLLECT_STATISTICS"]:
         return
 
-    if not db.engine.has_table("mergin_info"):
+    inspect_engine = inspect(db.engine)
+    if not inspect_engine.has_table("mergin_info"):
         logging.warning("Database not initialized")
         return
 

server/mergin/stats/tasks.py

@@ -0,0 +1,42 @@
+# Copyright (C) Lutra Consulting Limited
+#
+# SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-MerginMaps-Commercial
+
+from flask import Flask
+
+from .commands import add_commands
+from .config import Configuration
+from .db_events import register_events
+
+
+def register(app: Flask):
+    """Register mergin sync module in Flask app
+    Adds Flask blueprint with autogenerated Flask/Connexion routes from openAPI definition.
+    Register db events/hooks.
+    """
+    app.config.from_object(Configuration)
+
+    app.connexion_app.add_api(
+        "sync/public_api.yaml",
+        arguments={"title": "Mergin"},
+        options={"swagger_ui": False},
+        validate_responses=True,
+    )
+
+    app.connexion_app.add_api(
+        "sync/public_api_v2.yaml",
+        arguments={"title": "Mergin"},
+        options={"swagger_ui": False},
+        validate_responses=True,
+    )
+
+    app.connexion_app.add_api(
+        "sync/private_api.yaml",
+        base_path="/app",
+        arguments={"title": "Mergin"},
+        options={"swagger_ui": False, "serve_spec": False},
+        validate_responses=True,
+    )
+
+    add_commands(app)
+    register_events()

server/mergin/sync/app.py

@@ -167,7 +167,7 @@ def cache_latest_files(self) -> None:
             WHERE a.project_id = pf.project_id;
         """
         params = {"project_id": self.id, "latest_version": self.latest_version}
-        db.session.execute(query, params)
+        db.session.execute(text(query), params)
         db.session.commit()
 
     @property
@@ -222,7 +222,7 @@ def files(self) -> List[ProjectFile]:
                     else None
                 ),
             )
-            for row in db.session.execute(query, params).fetchall()
+            for row in db.session.execute(text(query), params).fetchall()
         ]
         return files
 
@@ -1506,7 +1506,7 @@ def _files_from_start(self):
             WHERE fh.change != 'delete';
         """
         params = {"project_id": self.project_id, "version": self.name}
-        return db.session.execute(query, params).fetchall()
+        return db.session.execute(text(query), params).fetchall()
 
     def _files_from_end(self):
         """Calculate version files using lookup from the last version
@@ -1567,7 +1567,7 @@ def _files_from_end(self):
             ORDER BY fp.path;
         """
         params = {"project_id": self.project_id, "version": self.name}
-        return db.session.execute(query, params).fetchall()
+        return db.session.execute(text(query), params).fetchall()
 
     @property
     def files(self) -> List[ProjectFile]:
@@ -1673,7 +1673,7 @@ def changes_count(self) -> Dict:
         """Return number of changes by type"""
         query = f"SELECT change, COUNT(change) FROM file_history WHERE version_id = :version_id GROUP BY change;"
         params = {"version_id": self.id}
-        result = db.session.execute(query, params).fetchall()
+        result = db.session.execute(text(query), params).fetchall()
         return {row[0]: row[1] for row in result}
 
     @property