refactor: improve Traefik v3 migration with async processing

Pascal Klesse · Pascal Klesse · commit d3536bb6ca36 · 2025-11-19T11:27:47.000+01:00
- Run container migration in background to prevent HTTP timeouts
- Add detailed progress logging with container-by-container status
- Load containers with full relations before migration
- Add 2s delay between migrations to avoid overwhelming Docker
- Create dynamic directory for Traefik file provider
- Move deploy.party update before migration (required for endpoint)
- Improve user feedback with background process monitoring tips
- Export env vars when deploying Traefik stack
- Replace read -sp with stty for /bin/sh compatibility
- Update documentation to reflect async migration behavior
diff --git a/MIGRATION.md b/MIGRATION.md
@@ -35,11 +35,13 @@ After upgrading Traefik to v3, all deployed containers must be migrated to use t
 ### Automatic Migration (Recommended)
 
 The `upgrade-traefik-v3.sh` script will prompt you for a deploy.party API token and automatically:
-1. Find all deployed containers
-2. Stop each container
-3. Regenerate docker-compose.yml with updated middleware syntax
-4. Redeploy the container
-5. Continue with the next container (rolling update)
+1. Start the migration in the background
+2. Each container is processed sequentially:
+   - Stop container
+   - Regenerate docker-compose.yml with updated middleware syntax
+   - Redeploy container
+3. Migration runs asynchronously to avoid HTTP timeouts
+4. Progress can be monitored via API logs
 
 ### Manual Migration via REST API
 
@@ -145,10 +147,10 @@ The script will:
 - ✅ Stop Traefik v2 safely
 - ✅ Download the correct v3 configuration
 - ✅ Start Traefik v3
+- ✅ Update deploy.party to latest version (required for migration endpoint)
 - ✅ Auto-detect API URL from APP_URL environment variable
 - ✅ Prompt for API token or accept via parameter
 - ✅ Migrate all containers with rolling update
-- ✅ Update deploy.party to latest version
 - ✅ Verify the upgrade was successful
 - ✅ Provide rollback instructions if anything goes wrong
 
diff --git a/projects/api/src/server/extern.controller.ts b/projects/api/src/server/extern.controller.ts
@@ -50,7 +50,7 @@ export class ExternController {
       return 'Invalid API Token';
     }
 
-    const containers = await this.containerService.findContainersForProject(projectId);
+    const containers = await this.containerService.findContainersForProject(projectId, { force: true });
 
     // loop through containers
     for (const container of containers) {
@@ -104,7 +104,7 @@ export class ExternController {
       return 'Invalid API Token';
     }
 
-    const containers = await this.containerService.findContainersForProject(projectId);
+    const containers = await this.containerService.findContainersForProject(projectId, { force: true });
     const dbContainer = containers.find((container) => container.kind === ContainerKind.DATABASE);
     if (!dbContainer) {
       throw new Error('No database container found');
@@ -268,11 +268,11 @@ export class ExternController {
   @Post('migrate/traefik-middleware')
   @ApiOperation({
     summary: 'Migrate Traefik middleware configuration',
-    description: 'Updates all deployed containers to use Traefik v3/v4 middleware syntax with @swarm provider suffix. Performs rolling update to minimize downtime.',
+    description: 'Updates all deployed containers to use Traefik v3/v4 middleware syntax with @swarm provider suffix. Performs rolling update to minimize downtime. This operation runs in the background and returns immediately.',
   })
   @ApiResponse({
     status: 200,
-    description: 'Migration completed with details about success and failures',
+    description: 'Migration started successfully. Check server logs for progress.',
   })
   async migrateTraefikMiddleware(@Headers('dp-api-token') apiToken: string) {
     if (!apiToken) {
@@ -284,29 +284,66 @@ export class ExternController {
       return 'Invalid API Token';
     }
 
-    const allDeployed = await this.containerService.findForce({ filterQuery: { status: ContainerStatus.DEPLOYED } });
+    const allDeployed = await this.containerService.findForce({
+      filterQuery: { status: ContainerStatus.DEPLOYED }
+    });
 
     // Filter out database containers (they don't use Traefik)
-    const deployedContainers = allDeployed.filter(
-      (c: Container) => c.kind !== ContainerKind.DATABASE
-    );
+    const deployedContainerIds = allDeployed
+      .filter((c: Container) => c.kind !== ContainerKind.DATABASE)
+      .map(c => c.id);
+
+    const totalContainers = deployedContainerIds.length;
+
+    console.debug(`[Traefik Migration] Starting background migration for ${totalContainers} containers...`);
+
+    // Run migration in background (don't await)
+    this.runMigrationInBackground(deployedContainerIds).catch(error => {
+      console.error('[Traefik Migration] Unexpected error in background migration:', error);
+    });
+
+    // Return immediately
+    return {
+      success: true,
+      message: 'Migration started in background',
+      total: totalContainers,
+      note: 'Check server logs for progress. Migration may take several minutes.',
+    };
+  }
 
+  private async runMigrationInBackground(containerIds: string[]) {
     const result = {
       success: true,
-      total: deployedContainers.length,
+      total: containerIds.length,
       migrated: 0,
       failed: 0,
       details: [],
     };
 
-    console.debug(`Starting Traefik middleware migration for ${deployedContainers.length} containers...`);
+    for (let i = 0; i < containerIds.length; i++) {
+      const containerId = containerIds[i];
+      const progress = `[${i + 1}/${containerIds.length}]`;
+      let container: Container;
 
-    for (const container of deployedContainers) {
       try {
-        console.debug(`Migrating container ${container.id} (${container.name})...`);
+        // Load container with all relations (registry, source, etc.)
+        console.debug(`${progress} Loading container ${containerId}...`);
+        container = await this.containerService.get(containerId, {
+          populate: [{ path: 'registry' }, { path: 'source' }, { path: 'project' }]
+        });
+
+        console.debug(`${progress} Migrating container: ${container.name} (${container.id})`);
 
+        // Stop container
+        console.debug(`${progress} - Stopping container...`);
         await this.dockerService.stop(container);
+
+        // Regenerate docker-compose with updated middleware syntax
+        console.debug(`${progress} - Regenerating docker-compose.yml...`);
         await this.dockerService.createDockerComposeFile(container);
+
+        // Deploy container
+        console.debug(`${progress} - Deploying container...`);
         await this.dockerService.deploy(container);
 
         result.migrated++;
@@ -316,23 +353,43 @@ export class ExternController {
           status: 'success',
         });
 
-        console.debug(`✓ Successfully migrated container ${container.id}`);
+        console.debug(`${progress} ✓ Successfully migrated: ${container.name}`);
+
+        // Small delay between containers to avoid overwhelming Docker
+        if (i < containerIds.length - 1) {
+          await new Promise(resolve => setTimeout(resolve, 2000));
+        }
       } catch (error) {
         result.failed++;
         result.success = false;
         result.details.push({
-          containerId: container.id,
-          name: container.name,
+          containerId: containerId,
+          name: container?.name || 'Unknown',
           status: 'failed',
           error: error.message,
+          stack: error.stack,
         });
 
-        console.error(`✗ Failed to migrate container ${container.id}:`, error.message);
+        console.error(`${progress} ✗ Failed to migrate: ${container?.name || containerId}`);
+        console.error(`${progress} Error:`, error.message);
+        console.error(`${progress} Stack:`, error.stack);
+
+        // Continue with next container even if this one failed
       }
     }
 
-    console.debug(`Migration completed: ${result.migrated} succeeded, ${result.failed} failed`);
-
-    return result;
+    console.debug(`[Traefik Migration] ========================================`);
+    console.debug(`[Traefik Migration] MIGRATION COMPLETED`);
+    console.debug(`[Traefik Migration] Total:    ${result.total}`);
+    console.debug(`[Traefik Migration] Migrated: ${result.migrated}`);
+    console.debug(`[Traefik Migration] Failed:   ${result.failed}`);
+    console.debug(`[Traefik Migration] ========================================`);
+
+    if (result.failed > 0) {
+      console.error('[Traefik Migration] Failed containers:');
+      result.details
+        .filter(d => d.status === 'failed')
+        .forEach(d => console.error(`  - ${d.name}: ${d.error}`));
+    }
   }
 }
diff --git a/upgrade-traefik-v3.sh b/upgrade-traefik-v3.sh
@@ -255,6 +255,17 @@ else
     exit 1
 fi
 
+# Create dynamic directory for Traefik file provider if it doesn't exist
+echo "--------------------------------------------------------------------------------"
+echo "Checking dynamic configuration directory..."
+if [ ! -d "dynamic" ]; then
+    echo "Creating dynamic directory..."
+    mkdir -p dynamic
+    echo -e "${GREEN}Dynamic directory created${NC}"
+else
+    echo "Dynamic directory already exists."
+fi
+
 # Ensure traefik-public network exists
 echo "--------------------------------------------------------------------------------"
 echo "Checking traefik-public network..."
@@ -268,7 +279,7 @@ fi
 # Start Traefik v3
 echo "--------------------------------------------------------------------------------"
 echo "Starting Traefik v3..."
-docker stack deploy -c docker-compose.traefik.yml traefik
+export $(cat /data/.env | grep -v '#' | awk '/=/ {print $1}') && docker stack deploy -c docker-compose.traefik.yml traefik
 
 # Wait for Traefik to start
 echo "--------------------------------------------------------------------------------"
@@ -309,7 +320,18 @@ while true; do
             echo "  docker logs \$(docker ps -q -f name=traefik)"
             echo ""
 
+            # Update deploy.party first (required for migration endpoint)
+            echo "--------------------------------------------------------------------------------"
+            echo "Updating deploy.party"
+            echo "--------------------------------------------------------------------------------"
+            echo ""
+            echo "Updating deploy.party to latest version (required for migration endpoint)..."
+            echo ""
+
+            update_deploy_party
+
             # Container Migration for Traefik v3 Middleware Syntax
+            echo ""
             echo "--------------------------------------------------------------------------------"
             echo "Container Migration for Traefik v3"
             echo "--------------------------------------------------------------------------------"
@@ -324,11 +346,6 @@ while true; do
                 echo ""
                 echo "To migrate containers later, run:"
                 echo "  curl -X POST -H \"dp-api-token: YOUR_API_TOKEN\" https://YOUR_API_URL/extern/migrate/traefik-middleware"
-                echo ""
-
-                # Update deploy.party before exit
-                update_deploy_party
-
                 echo ""
                 echo -e "${GREEN}Traefik v3 upgrade completed.${NC}"
                 exit 0
@@ -350,11 +367,6 @@ while true; do
                         echo ""
                         echo "To migrate containers later, run:"
                         echo "  curl -X POST -H \"dp-api-token: YOUR_API_TOKEN\" https://YOUR_API_URL/extern/migrate/traefik-middleware"
-                        echo ""
-
-                        # Update deploy.party before exit
-                        update_deploy_party
-
                         echo ""
                         echo -e "${GREEN}Traefik v3 upgrade completed.${NC}"
                         exit 0
@@ -368,19 +380,18 @@ while true; do
                 echo "API token required for automatic container migration."
                 echo "Get your token from: Settings → API Keys (starts with dp-)"
                 echo ""
-                read -sp "Enter your API token (or press Enter to skip): " API_TOKEN
+                # Use stty for compatibility with /bin/sh
+                printf "Enter your API token (or press Enter to skip): "
+                stty -echo
+                read API_TOKEN
+                stty echo
                 echo ""
 
                 if [ -z "$API_TOKEN" ]; then
                     echo -e "${YELLOW}No API token provided. Skipping automatic migration.${NC}"
                     echo ""
                     echo "To migrate containers later, run:"
                     echo "  curl -X POST -H \"dp-api-token: YOUR_API_TOKEN\" $API_URL/extern/migrate/traefik-middleware"
-                    echo ""
-
-                    # Update deploy.party before exit
-                    update_deploy_party
-
                     echo ""
                     echo -e "${GREEN}Traefik v3 upgrade completed.${NC}"
                     exit 0
@@ -399,42 +410,48 @@ while true; do
                 "$API_URL/extern/migrate/traefik-middleware")
 
             if echo "$MIGRATION_RESULT" | grep -q '"success":true'; then
-                echo -e "${GREEN}Container migration completed successfully!${NC}"
+                TOTAL_CONTAINERS=$(echo "$MIGRATION_RESULT" | grep -o '"total":[0-9]*' | grep -o '[0-9]*')
+                echo -e "${GREEN}Container migration started successfully!${NC}"
+                echo ""
+                echo "Migration running in background for $TOTAL_CONTAINERS containers."
+                echo ""
+                echo -e "${YELLOW}NOTE: The migration is running in the background and may take several minutes.${NC}"
+                echo "Each container will be stopped, regenerated, and redeployed one by one."
                 echo ""
-                echo "Migration details:"
-                echo "$MIGRATION_RESULT" | grep -o '"migrated":[0-9]*' | sed 's/"migrated":/  Migrated: /'
-                echo "$MIGRATION_RESULT" | grep -o '"failed":[0-9]*' | sed 's/"failed":/  Failed:   /'
+                echo "To monitor progress, check the API logs:"
+                echo "  docker service logs -f deploy-party_api | grep 'Traefik Migration'"
+                echo ""
+                echo "Or check running containers:"
+                echo "  watch -n 2 'docker ps --format \"table {{.Names}}\t{{.Status}}\" | grep -v traefik'"
             elif echo "$MIGRATION_RESULT" | grep -q "Invalid API Token"; then
                 echo -e "${RED}Error: Invalid API token provided.${NC}"
                 echo ""
                 echo "To migrate containers manually with correct token, run:"
                 echo "  curl -X POST -H \"dp-api-token: YOUR_API_TOKEN\" $API_URL/extern/migrate/traefik-middleware"
             else
-                echo -e "${YELLOW}Warning: Container migration encountered issues.${NC}"
+                echo -e "${YELLOW}Warning: Could not start container migration.${NC}"
                 echo "Response: $MIGRATION_RESULT"
                 echo ""
                 echo "To retry migration manually, run:"
                 echo "  curl -X POST -H \"dp-api-token: YOUR_API_TOKEN\" $API_URL/extern/migrate/traefik-middleware"
             fi
 
-            echo ""
-
-            # deploy.party Update Section
-            update_deploy_party
-
             echo ""
             echo "================================================================================"
             echo -e "${GREEN}Upgrade completed successfully!${NC}"
             echo "================================================================================"
             echo ""
-            echo "✅ Traefik upgraded from v2 to v3"
-            if [ "$SKIP_MIGRATION" != true ]; then
-                MIGRATED_COUNT=$(echo "$MIGRATION_RESULT" | grep -o '"migrated":[0-9]*' | grep -o '[0-9]*')
-                echo "✅ Container middleware migrated ($MIGRATED_COUNT containers)"
-            fi
             if [ "$SKIP_DEPLOY_PARTY_UPDATE" != true ]; then
                 echo "✅ deploy.party updated to latest version"
             fi
+            echo "✅ Traefik upgraded from v2 to v3"
+            if [ "$SKIP_MIGRATION" != true ]; then
+                if [ -n "$TOTAL_CONTAINERS" ]; then
+                    echo "✅ Container migration started for $TOTAL_CONTAINERS containers (running in background)"
+                else
+                    echo "⚠️  Container migration status unknown - check logs"
+                fi
+            fi
             echo ""
             echo "Dashboard: lb.$APP_URL"
             echo "deploy.party: $APP_URL"
