feat(ingestion): Make PySpark optional for S3, ABS, and Unity Catalog sources

datahub-actions/build.gradle

@@ -28,7 +28,7 @@ ext {
   docker_registry = 'acryldata'
   docker_repo = 'datahub-actions'
   docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
-  
+
   python_docker_version = project.getProperties().getOrDefault("pythonDockerVersion", "1!0.0.0+docker.${version}")
 }
 
@@ -143,11 +143,17 @@ docker {
 
   additionalTag("Debug", "${docker_registry}/${docker_repo}:debug")
 
-  defaultVariant = "slim"
+  defaultVariant = "full"
   variants = [
-    "slim": [suffix: "-slim", args: [APP_ENV: "slim", RELEASE_VERSION: python_docker_version, BUNDLED_CLI_VERSION: cliVersion]],
-    "full": [suffix: "", args: [APP_ENV: "full", RELEASE_VERSION: python_docker_version, BUNDLED_CLI_VERSION: cliVersion]]
+    "full": [suffix: "", args: [APP_ENV: "full", RELEASE_VERSION: python_docker_version, BUNDLED_CLI_VERSION: project.ext.cliVersion, BUNDLED_VENV_SLIM_MODE: "false"]],
+    "slim": [suffix: "-slim", args: [APP_ENV: "slim", RELEASE_VERSION: python_docker_version, BUNDLED_CLI_VERSION: project.ext.cliVersion, BUNDLED_VENV_SLIM_MODE: "true"]],
+    "locked": [suffix: "-locked", args: [APP_ENV: "locked", RELEASE_VERSION: python_docker_version, BUNDLED_CLI_VERSION: project.ext.cliVersion, BUNDLED_VENV_SLIM_MODE: "true"]]
   ]
+
+  // Set build args for non-bake builds based on dockerTarget property
+  def targetVariant = docker_target ?: defaultVariant.get()
+  def variantArgs = variants.get()[targetVariant]?.args ?: variants.get()[defaultVariant.get()].args
+  buildArgs(variantArgs)
 }
 
 build.dependsOn install

docker/build.gradle

@@ -26,7 +26,7 @@ ext {
     loadCommonEnvFile = {
         def envFile = System.getenv("DATAHUB_LOCAL_COMMON_ENV")
         def envVars = [:]
-        
+
         if (envFile && new File(envFile).exists()) {
             logger.lifecycle("Loading environment variables from: ${envFile}")
             new File(envFile).eachLine { line ->
@@ -39,14 +39,14 @@ ext {
                 }
             }
         }
-        
+
         // Also load any environment variables that start with DATAHUB_
         System.getenv().each { key, value ->
             if (key.startsWith("DATAHUB_")) {
                 envVars[key] = value
             }
         }
-        
+
         return envVars
     }
 
@@ -533,6 +533,49 @@ tasks.register('quickstartDown') {
     }
 }
 
+tasks.register('quickstartLocked') {
+    group = 'quickstart'
+    description = 'Build locked variants and run quickstart (PySpark-free, network-blocked images)'
+
+    // Build locked variants first
+    dependsOn ':datahub-actions:docker'
+    dependsOn ':docker:datahub-ingestion:docker'
+
+    // Then run quickstart
+    finalizedBy 'quickstart'
+
+    doFirst {
+        logger.lifecycle("")
+        logger.lifecycle("=" * 80)
+        logger.lifecycle("Building LOCKED variants (no PySpark, network blocked)...")
+        logger.lifecycle("=" * 80)
+
+        // Set dockerTarget property so the docker tasks build locked variants
+        rootProject.ext.dockerTarget = 'locked'
+        project.project(':datahub-actions').ext.dockerTarget = 'locked'
+        project.project(':docker:datahub-ingestion').ext.dockerTarget = 'locked'
+    }
+
+    doLast {
+        logger.lifecycle("")
+        logger.lifecycle("=" * 80)
+        logger.lifecycle("Locked images built successfully!")
+        logger.lifecycle("=" * 80)
+        logger.lifecycle("")
+        logger.lifecycle("Images:")
+        logger.lifecycle("  - datahub-actions:v${version} (locked variant)")
+        logger.lifecycle("  - datahub-ingestion:v${version} (locked variant)")
+        logger.lifecycle("")
+        logger.lifecycle("Features:")
+        logger.lifecycle("  ✓ No PySpark dependencies")
+        logger.lifecycle("  ✓ Network access to PyPI BLOCKED")
+        logger.lifecycle("  ✓ Only bundled venvs available (actions)")
+        logger.lifecycle("")
+        logger.lifecycle("Quickstart will start with these locked images...")
+        logger.lifecycle("=" * 80)
+    }
+}
+
 tasks.withType(ComposeUp).configureEach {
     shouldRunAfter('quickstartNuke')
     dependsOn tasks.named("minDockerCompose2.20")