ITEP-88294: Fix database migration flow

.gitignore

@@ -22,12 +22,14 @@ manager/src/static/assets
 manager/src/static/bootstrap*
 manager/src/static/examples
 manager/secrets
-manager/src/django/secrets.py
+manager/src/manager/secrets.py
 docker-compose.yml
 kubernetes/setup/
 kubernetes/manager
 media
-migrations
+# Ignore runtime migrations directory (not Django source migrations)
+/migrations
+**/workspace/migrations
 pvd-mqtt/detect
 sample_data/xREF_*
 test_data

REUSE.toml

@@ -7,7 +7,7 @@ version = 1
 path = [
   "docs/doxygen/Doxyfile.common",
   "docs/doxygen/api_docs",
-  "manager/src/static/site.webmanifest",
+  "manager/src/manager/static/site.webmanifest",
   "tests/security/fuzzing/token",
   "**/*.env",
   "**/mp4",

docs/development/vscode-setup.md

@@ -91,7 +91,7 @@ Create `.vscode/settings.json` in the project root, or press `Ctrl+Shift+P` →
   "python.terminal.activateEnvironment": true,
   "python.analysis.extraPaths": [
     "${workspaceFolder}",
-    "${workspaceFolder}/manager/src/django",
+    "${workspaceFolder}/manager/src/manager",
     "${workspaceFolder}/tests",
     "${workspaceFolder}/scene_common/src",
     "${workspaceFolder}/controller/src",

exclude_for_release.txt

@@ -17,7 +17,7 @@ docs/doxygen/html
 exclude_for_release.txt
 model_installer/models
 venv
-manager/src/django/settings_local.py
+manager/src/manager/settings_local.py
 secrets
 SECURITY.md
 *.bundle

manager/Agents.md

@@ -5,6 +5,10 @@ SPDX-License-Identifier: Apache-2.0
 
 # Manager Service - AI Agent Guide
 
+## Documentation
+
+- [MIGRATIONS.md](MIGRATIONS.md) - Django migrations workflow and best practices
+
 ## Service Overview
 
 The **Manager** service is the Django-based web UI and REST API gateway for Intel® SceneScape. It provides user-facing interfaces for system configuration, scene management, camera setup, and PostgreSQL-backed persistence for metadata and configuration.
@@ -15,13 +19,13 @@ The **Manager** service is the Django-based web UI and REST API gateway for Inte
 
 ### Core Modules
 
-1. **Django Application** (`src/django/`):
+1. **Django Application** (`src/manager/`):
    - Scene management views and APIs
    - Camera configuration interfaces
    - User authentication and authorization
    - PostgreSQL ORM models
 
-2. **REST API** (`src/django/api/`):
+2. **REST API** (`src/manager/api/`):
    - RESTful endpoints for external integrations
    - Scene CRUD operations
    - Camera calibration triggers
@@ -225,11 +229,12 @@ class Camera(models.Model):
 
 ### Adding New Database Model
 
-1. Define model in `src/django/scenescape/models.py`
+1. Define model in `src/manager/models.py`
 2. Create migration: `docker compose exec manager python manage.py makemigrations`
-3. Review migration file in `src/django/scenescape/migrations/`
-4. Apply: `docker compose exec manager python manage.py migrate`
-5. Update admin interface if needed: `src/django/scenescape/admin.py`
+3. Review migration file in `src/manager/migrations/`
+4. Commit migration file to version control
+5. Apply: `docker compose exec manager python manage.py migrate`
+6. Update admin interface if needed: `src/manager/admin.py`
 
 ### Modifying Web UI
 

manager/Dockerfile

@@ -144,15 +144,9 @@ RUN : \
 
 # Copy application code
 RUN mkdir -p $SCENESCAPE_HOME/manager
-COPY ./manager/src/django/*.py \
-    ./manager/src/setup.py \
-    ./version.txt $SCENESCAPE_HOME/manager/
-COPY ./manager/src/django/ppl_generator/* $SCENESCAPE_HOME/manager/ppl_generator/
-COPY ./manager/src/management/ $SCENESCAPE_HOME/manager/management
-COPY ./manager/src/templates/ $SCENESCAPE_HOME/manager/templates
-COPY ./manager/src/static/ $SCENESCAPE_HOME/manager/static
-COPY ./manager/src/templatetags/ $SCENESCAPE_HOME/manager/templatetags
-COPY ./manager/src/django/manage.py $SCENESCAPE_HOME
+COPY ./manager/src/manager $SCENESCAPE_HOME/manager
+COPY ./manager/src/manage.py $SCENESCAPE_HOME/manage.py
+COPY ./version.txt $SCENESCAPE_HOME/manager/
 COPY --from=scenescape-manager-builder /usr/local/lib/python3.11/site-packages/fast_geometry /usr/local/lib/python3.11/site-packages/fast_geometry
 COPY --from=scenescape-manager-builder /usr/local/lib/python3.11/site-packages/scene_common /usr/local/lib/python3.11/site-packages/scene_common
 

manager/MIGRATIONS.md

@@ -0,0 +1,180 @@
+# SPDX-FileCopyrightText: (C) 2025 Intel Corporation
+
+# SPDX-License-Identifier: Apache-2.0
+
+# Django Migrations Guide
+
+## Overview
+
+SceneScape uses Django's migration system to manage database schema changes. Migration files are version-controlled and applied at runtime to ensure consistent database upgrades across releases.
+
+## Important: Proper Migration Usage
+
+**DO NOT** run `makemigrations` at runtime or in production. Migrations should be:
+
+1. Generated during development
+2. Reviewed and tested
+3. Committed to version control
+4. Built into the Docker image
+5. Applied at runtime via `migrate` only
+
+## Creating Migrations
+
+### For Local Development
+
+When you modify Django models in `src/manager/models.py`, follow these steps:
+
+1. **Make your model changes** in `src/manager/models.py`
+
+2. **Generate migration file** using the generate_migrations.sh script:
+
+   ```bash
+   bash manager/tools/generate_migrations.sh
+   ```
+
+3. **Review the generated migration** in `src/manager/migrations/`:
+
+   ```bash
+   ls -la manager/src/manager/migrations/
+   ```
+
+4. ** Re-build manager**
+
+   ```bash
+   make manager
+   ```
+
+5. **Verify the migration is being applied **:
+
+   ```bash
+   docker compose down web && docker compose up web
+   ```
+
+6. **Check migration status**:
+
+   ```bash
+   bash manager/tools/generate_migrations.sh --show --network scenescape_scenescape --dbhost pgserver --dbport 5432
+   ```
+
+7. **Commit the migration file** to version control:
+   ```bash
+   git add manager/src/manager/migrations/XXXX_*.py
+   git commit -m "Add migration for [describe changes]"
+   ```
+
+### For CI/CD and Release Process
+
+Migration generation should be automated as part of the release build:
+
+1. **In CI/CD pipeline**, after model changes are merged:
+
+   ```bash
+   # If migrations are needed, generate them
+   bash manager/tools/generate_migrations.sh
+
+   # Build the manager image
+   make manager
+
+   # Review migrations
+   manager/src/manager/migrations
+
+   # Commit and push the generated migrations
+   git add manager/src/manager/migrations/
+   git commit -m "Generate migrations for release"
+   git push
+   ```
+
+2. **Rebuild the image** with the new migrations included
+
+## Migration Naming Convention
+
+Django automatically names migrations as:
+
+- `0001_initial.py` - Initial schema
+- `0002_<description>.py` - Subsequent changes
+- `0003_<description>.py` - More changes
+- etc.
+
+For releases, you may want to include version information in the description:
+
+```bash
+docker compose exec manager python manage.py makemigrations manager --name release_2026_1_0
+```
+
+This creates: `0002_release_2026_1_0.py`
+
+## Applying Migrations
+
+Migrations are applied automatically at container startup via the `migrate` command in `config/scenescape-init`.
+
+To manually apply migrations:
+
+```bash
+docker exec -it -w /home/scenescape/SceneScape scenescape-web-1 \
+python manage.py migrate
+```
+
+## Checking Migration Status
+
+View all migrations and their application status:
+
+```bash
+docker exec -it -w /home/scenescape/SceneScape scenescape-web-1 \
+python manage.py showmigrations
+```
+
+Example output:
+
+```
+manager
+ [X] 0001_initial
+ [X] 0002_release_2026_1_0
+ [ ] 0003_add_new_field
+```
+
+## Troubleshooting
+
+### Migration conflicts
+
+If multiple developers create migrations in parallel:
+
+1. Merge the code
+2. Run `makemigrations --merge` to create a merge migration
+3. Test the merge migration
+4. Commit the merge migration file
+
+### Reverting migrations
+
+```bash
+# Revert to a specific migration
+docker exec -it -w /home/scenescape/SceneScape scenescape-web-1 \
+python manage.py migrate manager 0001_initial
+
+# Revert all migrations for an app
+docker exec -it -w /home/scenescape/SceneScape scenescape-web-1 \
+python manage.py migrate manager zero
+```
+
+### Fake migrations
+
+In rare cases (like database schema already matches):
+
+```bash
+docker exec -it -w /home/scenescape/SceneScape scenescape-web-1 \
+python manage.py migrate manager --fake
+```
+
+## Best Practices
+
+1. **Always review generated migrations** before committing
+2. **Test migrations** on a copy of production data
+3. **Keep migrations small** - one logical change per migration
+4. **Never edit applied migrations** - create a new migration instead
+5. **Document complex migrations** with comments in the migration file
+6. **Backup database** before applying migrations in production
+
+## References
+
+- [Django Migrations Documentation](https://docs.djangoproject.com/en/5.2/topics/migrations/)
+- [Django makemigrations Command](https://docs.djangoproject.com/en/5.2/ref/django-admin/#makemigrations)
+- [Django migrate Command](https://docs.djangoproject.com/en/5.2/ref/django-admin/#migrate)

manager/Makefile

@@ -6,7 +6,7 @@ RUNTIME_OS_IMAGE = python:3.11-slim-bookworm
 TARGET = scenescape-manager-runtime
 SHELL = /bin/bash
 CERTDOMAIN = scenescape.intel.com
-JSLIBDIR = src/static
+JSLIBDIR = src/manager/static
 CURL_FLAGS = --connect-timeout 5 --max-time 120 --retry 5 --retry-delay 0
 FORCE_VAAPI ?= 0
 THREEJS_VERSION = 168

manager/config/scenescape-init

@@ -244,21 +244,26 @@ fi
 export -n DBPASS SUPASS
 
 if [ -n "$DBTYPE" ]; then
-    cd ${MANAGERDIR}
+    # Determine "first run" using the marker (or another check)
     CREATEDB=0
-    MIGRATIONS=${DBROOT}/migrations/__init__.py
-    if [ ! -e ${MIGRATIONS} ] ; then
-        CREATEDB=1
+    MIGRATIONS_MARKER="${DBROOT}/migrations/__init__.py"
+    if [ ! -e "${MIGRATIONS_MARKER}" ]; then
+    CREATEDB=1
+    mkdir -p "$(dirname "${MIGRATIONS_MARKER}")"
+    touch "${MIGRATIONS_MARKER}"
     fi
-    if [ ${CREATEDB} = 1 ] ; then
-        mkdir -p ${DBROOT}/migrations
-        touch ${MIGRATIONS}
-        echo "Checking if database is empty..."
-        if ! (./manage.py showmigrations | grep -q '\[X\]'); then
-            echo "Database is empty — running initial migrations..."
-            (./manage.py makemigrations manager)
-            (./manage.py migrate)
-        fi
+
+    # ALWAYS apply migrations
+    cd ${MANAGERDIR}
+    echo "==> Applying pending migrations..."
+    ./manage.py migrate --noinput
+
+    if [ "${CREATEDB}" -eq 1 ] && [ -n "${EXAMPLEDB:-}" ]; then
+    echo "==> Preloading example DB..."
+    mkdir -p "${DBROOT}/media"
+    tar -C "${DBROOT}/media" -xf "${EXAMPLEDB}"
+    ./manage.py loaddata "${DBROOT}/media/data.json"
+    rm -f "${DBROOT}/media/data.json" "${DBROOT}/media/meta.json"
     fi
 
     ./manage.py updatedbstatus --not-ready