WIP get python running

.github/workflows/ci.yml

@@ -4,7 +4,8 @@ on:
   push:
     branches:
       - main
-  pull_request:
+  # CHRISM TEMP
+  # pull_request:
 
 # Cancel previous workflows on previous push
 concurrency:

.github/workflows/nightly-seed-grouping.yml

@@ -6,6 +6,10 @@ on:
     - cron: "0 5 * * *"
   # Runs when triggered manually to do so
   workflow_dispatch:
+  # CHRISM - temp
+  push:
+    branches:
+      - mallinson/remove-bloat
 
 # Cancel previous workflows when another is triggered
 concurrency:
@@ -20,30 +24,36 @@ jobs:
       fail-fast: false # Let all tests run and cascade errors, will only PR generator updates that passed
       matrix:
         sdk-name: [
-            ruby-model,
-            ruby-sdk,
-            ruby-sdk-v2,
-            pydantic,
-            # python-sdk, Turned off until Python is no longer hangings
-            fastapi,
-            openapi,
-            postman,
-            java-sdk,
-            java-model,
-            java-spring,
-            ts-sdk,
-            ts-express,
-            go-fiber,
-            go-model,
-            go-sdk,
-            csharp-model,
-            csharp-sdk,
-            php-model,
-            php-sdk,
-            swift-sdk,
-            rust-model,
-            rust-sdk
+            # CHRISM - temp
+            python-sdk
+            # ruby-model,
+            # ruby-sdk,
+            # ruby-sdk-v2,
+            # pydantic,
+            # python-sdk,
+            # fastapi,
+            # openapi,
+            # postman,
+            # java-sdk,
+            # java-model,
+            # java-spring,
+            # ts-sdk,
+            # ts-express,
+            # go-fiber,
+            # go-model,
+            # go-sdk,
+            # csharp-model,
+            # csharp-sdk,
+            # php-model,
+            # php-sdk,
+            # swift-sdk,
+            # rust-model,
+            # rust-sdk
           ]
+        include:
+          - concurrency-count-override: 8
+            sdk-name: python-sdk
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -71,6 +81,8 @@ jobs:
       - name: Save Seed Test Log to File
         id: save-seed-test-log-to-file
         run: |
+          # Default to 16 concurrent fixtures if not overridden. In the case of python-sdk,
+          # the virtual environments take up too much space so this needs to be limited
           pnpm seed:local test --generator ${{ matrix.sdk-name }} --parallel 16 --allow-unexpected-failures > ${{ matrix.sdk-name }}-seed-test-log.txt
 
       - name: Upload Seed Test Log as Artifact
@@ -94,36 +106,37 @@ jobs:
   build-seed-groups:
     runs-on: ubuntu-latest
     timeout-minutes: 3
-    if: always() && (needs.run-seed-test.result == 'success' || needs.run-seed-test.result == 'failure')
+    if: always() && !cancelled() && (needs.run-seed-test.result == 'success' || needs.run-seed-test.result == 'failure')
     needs: [run-seed-test]
     strategy:
       fail-fast: false
       matrix:
-        sdk-name:
-          [
-            ruby-model,
-            ruby-sdk,
-            ruby-sdk-v2,
-            pydantic,
-            python-sdk,
-            fastapi,
-            openapi,
-            postman,
-            java-sdk,
-            java-model,
-            java-spring,
-            ts-sdk,
-            ts-express,
-            go-fiber,
-            go-model,
-            go-sdk,
-            csharp-model,
-            csharp-sdk,
-            php-model,
-            php-sdk,
-            swift-sdk,
-            rust-model,
-            rust-sdk
+        sdk-name: [
+            # CHRISM - temp
+            python-sdk
+            # ruby-model,
+            # ruby-sdk,
+            # ruby-sdk-v2,
+            # pydantic,
+            # python-sdk,
+            # fastapi,
+            # openapi,
+            # postman,
+            # java-sdk,
+            # java-model,
+            # java-spring,
+            # ts-sdk,
+            # ts-express,
+            # go-fiber,
+            # go-model,
+            # go-sdk,
+            # csharp-model,
+            # csharp-sdk,
+            # php-model,
+            # php-sdk,
+            # swift-sdk,
+            # rust-model,
+            # rust-sdk
           ]
         include:
           - number-of-groups-override: 14 # java-sdk override since this one is taking a while
@@ -171,7 +184,7 @@ jobs:
   pr-seed-group-files:
     runs-on: ubuntu-latest
     timeout-minutes: 10
-    if: always() && (needs.build-seed-groups.result == 'success' || needs.build-seed-groups.result == 'failure')
+    if: always() && !cancelled() && (needs.build-seed-groups.result == 'success' || needs.build-seed-groups.result == 'failure')
     needs: [build-seed-groups]
     steps:
       - name: Checkout Repo
@@ -262,17 +275,19 @@ jobs:
           echo "PR was not created or updated, likely due to no diff for seed group files"
           echo "PR Operation: ${{ steps.cpr.outputs.pull-request-operation }}. Note: if operation is blank then there were no changes and create-pull-request GitHub Action exited silently and early."
 
-      - name: Enable Pull Request Automerge
-        if: steps.cpr.outputs.pull-request-operation == 'created'
-        uses: peter-evans/enable-pull-request-automerge@v3
-        with:
-          token: ${{ secrets.FERN_GITHUB_PAT }}
-          pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
-          merge-method: squash
+      # CHRISM - temp
+      # - name: Enable Pull Request Automerge
+      #   if: steps.cpr.outputs.pull-request-operation == 'created'
+      #   uses: peter-evans/enable-pull-request-automerge@v3
+      #   with:
+      #     token: ${{ secrets.FERN_GITHUB_PAT }}
+      #     pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+      #     merge-method: squash
 
-      - name: Approve PR
-        if: steps.cpr.outputs.pull-request-operation == 'created'
-        uses: ./.github/actions/auto-approve
-        with:
-          approver-gh-token: ${{ secrets.PR_BOT_GH_PAT }}
-          pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
+      # CHRISM - temp
+      # - name: Approve PR
+      #   if: steps.cpr.outputs.pull-request-operation == 'created'
+      #   uses: ./.github/actions/auto-approve
+      #   with:
+      #     approver-gh-token: ${{ secrets.PR_BOT_GH_PAT }}
+      #     pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}

.github/workflows/seed.yml

@@ -5,10 +5,11 @@ on:
     branches:
       - main
   # Note: pull request trigger will be removed once repository_dispatch trigger is verified
-  pull_request:
-    types: [opened, synchronize, reopened, ready_for_review]
-    branches:
-      - main
+  # CHRISM TEMP
+  # pull_request:
+  #   types: [opened, synchronize, reopened, ready_for_review]
+  #   branches:
+  #     - main
   repository_dispatch:
     types: [seed-test-metrics]
   workflow_call:

.github/workflows/update-seed.yml

@@ -1072,7 +1072,7 @@ jobs:
   commit-seed-changes-by-push:
     if: >-
       ${{
-        always() && needs.setup.outputs.update-by-push == 'true' &&
+        always() && !cancelled() && needs.setup.outputs.update-by-push == 'true' &&
         !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled')
       }}
     needs:
@@ -1147,7 +1147,7 @@ jobs:
   commit-seed-changes-by-pr:
     if: >-
       ${{
-        always() &&
+        always() && !cancelled() &&
         github.ref == 'refs/heads/main' &&
         needs.setup.outputs.update-by-pr == 'true' &&
         !contains(needs.*.result, 'failure') && 

packages/seed/fern/definition/config.yml

@@ -38,6 +38,7 @@ types:
       fixtures:
         type: optional<map<string, list<FixtureConfigurations>>>
       scripts: optional<list<DockerScriptConfig>>
+      postScripts: optional<list<DockerScriptConfig>>
       allowedFailures:
         type: optional<list<string>>
         docs: |

packages/seed/src/commands/test/script-runner/DockerScriptRunner.ts

@@ -43,7 +43,62 @@ export class DockerScriptRunner extends ScriptRunner {
         return { type: "success" };
     }
 
+    public async cleanup({
+        taskContext,
+        id,
+        outputDir
+    }: {
+        taskContext: TaskContext;
+        id: string;
+        outputDir?: AbsoluteFilePath;
+    }): Promise<void> {
+        if (this.skipScripts) {
+            return;
+        }
+
+        await this.startContainersFn;
+
+        const postScripts = this.workspace.workspaceConfig.postScripts ?? [];
+
+        if (postScripts.length > 0 && this.scripts.length > 0) {
+            // Run configured postScripts using existing script containers (avoids setup overhead)
+            for (let i = 0; i < postScripts.length && i < this.scripts.length; i++) {
+                const postScript = postScripts[i];
+                const containerToUse = this.scripts[i];
+
+                if (!postScript || !containerToUse) {
+                    taskContext.logger.warn(`Skipping postScript ${i}: missing postScript or container`);
+                    continue;
+                }
+
+                taskContext.logger.debug(
+                    `Running postScript cleanup for fixture ${id} in container ${containerToUse.containerId}`
+                );
+
+                // Execute postScript commands directly without file copying overhead
+                const postScriptCommands = postScript.commands.join(" && ");
+
+                const cleanupCommand = await loggingExeca(
+                    taskContext.logger,
+                    "docker",
+                    ["exec", containerToUse.containerId, "/bin/sh", "-c", postScriptCommands],
+                    {
+                        doNotPipeOutput: false,
+                        reject: false // Don't fail if cleanup has issues
+                    }
+                );
+
+                if (cleanupCommand.failed) {
+                    taskContext.logger.warn(`PostScript failed for fixture ${id}: ${cleanupCommand.stderr}`);
+                } else {
+                    taskContext.logger.debug(`PostScript completed for fixture ${id}`);
+                }
+            }
+        } 
+    }
+
     public async stop(): Promise<void> {
+        // Stop script containers (postScripts reuse these containers)
         for (const script of this.scripts) {
             await loggingExeca(this.context.logger, "docker", ["kill", script.containerId], {
                 doNotPipeOutput: false
@@ -75,10 +130,11 @@ export class DockerScriptRunner extends ScriptRunner {
         await writeFile(scriptFile.path, ["set -e", `cd /${workDir}/generated`, ...script.commands].join("\n"));
 
         // Move scripts and generated files into the container
+        // Use mkdir -p to avoid failing if directory already exists (e.g., during cleanup)
         const mkdirCommand = await loggingExeca(
             taskContext.logger,
             "docker",
-            ["exec", containerId, "mkdir", `/${workDir}`],
+            ["exec", containerId, "mkdir", "-p", `/${workDir}`],
             {
                 doNotPipeOutput: false,
                 reject: false
@@ -150,7 +206,9 @@ export class DockerScriptRunner extends ScriptRunner {
     private async startContainers(context: TaskContext): Promise<void> {
         const absoluteFilePathToFernCli = await this.buildFernCli(context);
         const cliVolumeBind = `${absoluteFilePathToFernCli}:/fern`;
+
         // Start running a docker container for each script instance
+        // Note: postScripts reuse existing script containers
         for (const script of this.workspace.workspaceConfig.scripts ?? []) {
             const startSeedCommand = await loggingExeca(
                 context.logger,

packages/seed/src/commands/test/script-runner/LocalScriptRunner.ts

@@ -40,6 +40,41 @@ export class LocalScriptRunner extends ScriptRunner {
         // No containers to stop for local execution
     }
 
+    public async cleanup({
+        taskContext,
+        id,
+        outputDir
+    }: {
+        taskContext: TaskContext;
+        id: string;
+        outputDir?: AbsoluteFilePath;
+    }): Promise<void> {
+        if (this.skipScripts) {
+            return;
+        }
+
+        taskContext.logger.debug(`Cleaning up fixture ${id} (local execution)`);
+
+        const postScripts = this.workspace.workspaceConfig.postScripts ?? [];
+
+        if (postScripts.length > 0) {
+            // Run configured postScripts
+            for (const script of postScripts) {
+                const result = await this.runScript({
+                    taskContext,
+                    script,
+                    id,
+                    outputDir: outputDir ?? AbsoluteFilePath.of(process.cwd())
+                });
+                if (result.type === "failure") {
+                    taskContext.logger.warn(
+                        `PostScript failed for fixture ${id}: ${result.message ?? "Unknown error"}`
+                    );
+                }
+            }
+        } 
+    }
+
     protected async initialize(): Promise<void> {
         // No initialization needed for local execution
     }

packages/seed/src/commands/test/script-runner/ScriptRunner.ts

@@ -37,6 +37,15 @@ export abstract class ScriptRunner {
 
     public abstract run({ taskContext, id, outputDir }: ScriptRunner.RunArgs): Promise<ScriptRunner.RunResponse>;
     public abstract stop(): Promise<void>;
+    public abstract cleanup({
+        taskContext,
+        id,
+        outputDir
+    }: {
+        taskContext: TaskContext;
+        id: string;
+        outputDir?: AbsoluteFilePath;
+    }): Promise<void>;
 
     protected abstract initialize(): Promise<void>;
 }

packages/seed/src/commands/test/test-runner/TestRunner.ts

@@ -263,6 +263,15 @@ export abstract class TestRunner {
             scriptStopwatch.stop();
             metrics.compileTime = scriptStopwatch.duration();
 
+            // CHRISM - is this already covered?
+            // // Clean up virtualenvs after script execution (success or failure)
+            // try {
+            //     await this.scriptRunner?.cleanup({ taskContext, id, outputDir });
+            // } catch (cleanupError) {
+            //     taskContext.logger.warn(`Cleanup failed for fixture ${id}: ${cleanupError}`);
+            //     // Don't fail the test due to cleanup issues
+            // }
+
             if (scriptResponse?.type === "failure") {
                 return {
                     type: "failure",