Initial implementation of the local cache layer

acceptance/acceptance_test.go

@@ -628,6 +628,11 @@ func runTest(t *testing.T,
 		cmd.Env = append(cmd.Env, "GOCOVERDIR="+coverDir)
 	}
 
+	// Set unique cache folder for this test to avoid race conditions between parallel tests
+	// Use test temp directory to avoid polluting user's cache
+	uniqueCacheDir := filepath.Join(t.TempDir(), ".cache")
+	cmd.Env = append(cmd.Env, "DATABRICKS_CACHE_DIR="+uniqueCacheDir)
+
 	for _, key := range utils.SortedKeys(config.Env) {
 		if hasKey(customEnv, key) {
 			// We want EnvMatrix to take precedence.

acceptance/bundle/resource_deps/job_tasks/out.telemetry.direct.txt

@@ -2,6 +2,8 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute true
+local.cache.attempt true
+local.cache.miss true
 presets_name_prefix_is_set false
 python_wheel_wrapper_is_set false
 resref_jobs.tags.* true

acceptance/bundle/resource_deps/job_tasks/out.telemetry.terraform.txt

@@ -2,6 +2,8 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute true
+local.cache.attempt true
+local.cache.miss true
 presets_name_prefix_is_set false
 python_wheel_wrapper_is_set false
 resref_jobs.tags.* true

acceptance/bundle/resource_deps/resources_var/output.txt

@@ -40,6 +40,8 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute false
+local.cache.attempt true
+local.cache.hit true
 presets_name_prefix_is_set true
 python_wheel_wrapper_is_set false
 resref_volumes.catalog_name true

acceptance/bundle/run/scripts/databricks-cli/test.toml

@@ -1,6 +1,9 @@
 RecordRequests = true
 IncludeRequestHeaders = ["Authorization"]
 
+[Env]
+DATABRICKS_CACHE_ENABLED = 'false'
+
 # "client_id:client_secret" in base64 is Y2xpZW50X2lkOmNsaWVudF9zZWNyZXQ=, expect to
 # see this in Authorization header
 [[Repls]]

acceptance/bundle/telemetry/deploy-compute-type/output.txt

@@ -13,6 +13,14 @@ Deployment complete!
 
 >>> cat out.requests.txt
 [
+  {
+    "key": "local.cache.attempt",
+    "value": true
+  },
+  {
+    "key": "local.cache.miss",
+    "value": true
+  },
   {
     "key": "experimental.use_legacy_run_as",
     "value": false
@@ -47,6 +55,14 @@ Deployment complete!
   }
 ]
 [
+  {
+    "key": "local.cache.attempt",
+    "value": true
+  },
+  {
+    "key": "local.cache.hit",
+    "value": true
+  },
   {
     "key": "experimental.use_legacy_run_as",
     "value": false

acceptance/bundle/telemetry/deploy-experimental/output.txt

@@ -12,6 +12,14 @@ Deployment complete!
 >>> cat out.requests.txt
 {
   "bool_values": [
+    {
+      "key": "local.cache.attempt",
+      "value": true
+    },
+    {
+      "key": "local.cache.miss",
+      "value": true
+    },
     {
       "key": "experimental.use_legacy_run_as",
       "value": true

acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt

@@ -8,6 +8,14 @@ Deployment complete!
 >>> cat out.requests.txt
 {
   "bool_values": [
+    {
+      "key": "local.cache.attempt",
+      "value": true
+    },
+    {
+      "key": "local.cache.miss",
+      "value": true
+    },
     {
       "key": "experimental.use_legacy_run_as",
       "value": false

acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt

@@ -8,6 +8,14 @@ Deployment complete!
 >>> cat out.requests.txt
 {
   "bool_values": [
+    {
+      "key": "local.cache.attempt",
+      "value": true
+    },
+    {
+      "key": "local.cache.miss",
+      "value": true
+    },
     {
       "key": "experimental.use_legacy_run_as",
       "value": false

acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt

@@ -16,6 +16,14 @@ Deployment complete!
 >>> cat out.requests.txt
 {
   "bool_values": [
+    {
+      "key": "local.cache.attempt",
+      "value": true
+    },
+    {
+      "key": "local.cache.miss",
+      "value": true
+    },
     {
       "key": "artifact_build_command_is_set",
       "value": false
@@ -48,6 +56,14 @@ Deployment complete!
 }
 {
   "bool_values": [
+    {
+      "key": "local.cache.attempt",
+      "value": true
+    },
+    {
+      "key": "local.cache.hit",
+      "value": true
+    },
     {
       "key": "artifact_build_command_is_set",
       "value": true

acceptance/bundle/telemetry/deploy/out.telemetry.txt

@@ -42,6 +42,14 @@
           "lookup_variable_count": 0,
           "target_count": 1,
           "bool_values": [
+            {
+              "key": "local.cache.attempt",
+              "value": true
+            },
+            {
+              "key": "local.cache.miss",
+              "value": true
+            },
             {
               "key": "experimental.use_legacy_run_as",
               "value": false
@@ -76,7 +84,8 @@
             }
           ],
           "bundle_mode": "TYPE_UNSPECIFIED",
-          "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM"
+          "workspace_artifact_path_type": "WORKSPACE_FILE_SYSTEM",
+          "local_cache_measurements_ms": [...redacted...]
         }
       }
     }

acceptance/bundle/telemetry/test.toml

@@ -1,10 +1,17 @@
 RecordRequests = true
 IncludeRequestHeaders = ["User-Agent"]
 
+[Env]
+DATABRICKS_CACHE_ENABLED = 'false'
+
 [[Repls]]
 Old = '"execution_time_ms": \d{1,5},'
 New = '"execution_time_ms": SMALL_INT,'
 
 [[Repls]]
 Old = '(linux|darwin|windows)'
 New = '[OS]'
+
+[[Repls]]
+Old = '"local_cache_measurements_ms": \[[^\]]*\]'
+New = '"local_cache_measurements_ms": [...redacted...]'

acceptance/bundle/user_agent/test.toml

@@ -1,3 +1,6 @@
 RecordRequests = true
 Local = true
 IncludeRequestHeaders = ["User-Agent"]
+
+[Env]
+DATABRICKS_CACHE_ENABLED = 'false'

acceptance/cache/clear/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: cache-clear-test

acceptance/cache/clear/output.txt

@@ -0,0 +1,19 @@
+
+=== First call in a session is expected to be a cache miss:
+[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH]
+[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted)
+[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing
+[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result
+
+=== Second call in a session is expected to be a cache hit
+[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH]
+[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit
+
+>>> [CLI] cache clear
+Cache cleared successfully from [TEST_TMP_DIR]/.cache
+
+=== First call after a clear is expected to be a cache miss:
+[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH]
+[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted)
+[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing
+[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result

acceptance/cache/clear/script

@@ -0,0 +1,16 @@
+export DATABRICKS_CACHE_DIR=$(pwd)/.cache
+
+$CLI cache clear &> /dev/null
+
+title "First call in a session is expected to be a cache miss:\n"
+trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path"
+
+title "Second call in a session is expected to be a cache hit\n"
+trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path"
+
+trace $CLI cache clear
+
+title "First call after a clear is expected to be a cache miss:\n"
+trace $CLI bundle validate --debug 2>&1 | grep "Local Cache" | grep -v "cache path"
+
+rm -rf "${DATABRICKS_CACHE_DIR}"

acceptance/cache/clear/test.toml

@@ -0,0 +1,14 @@
+Cloud = false
+Local = true
+
+[Env]
+DATABRICKS_CACHE_ENABLED = 'true'
+
+# Redact structured logging fields from debug output
+[[Repls]]
+Old = ' pid=[0-9]+'
+New = ''
+
+[[Repls]]
+Old = ' mutator=[A-Za-z]+'
+New = ''

acceptance/cache/simple/databricks.yml

@@ -0,0 +1,2 @@
+bundle:
+  name: exploratory-cache-test

acceptance/cache/simple/output.txt

@@ -0,0 +1,21 @@
+
+=== First call in a session is expected to be a cache miss:
+[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH]
+[DEBUG_TIMESTAMP] Debug: [Local Cache] failed to stat cache file: (redacted)
+[DEBUG_TIMESTAMP] Debug: [Local Cache] cache miss, computing
+[DEBUG_TIMESTAMP] Debug: [Local Cache] computed and stored result
+
+=== Second call in a session is expected to be a cache hit
+[DEBUG_TIMESTAMP] Debug: [Local Cache] using cache key: [SHA256_HASH]
+[DEBUG_TIMESTAMP] Debug: [Local Cache] cache hit
+
+=== Bundle deploy should send telemetry values
+
+>>> [CLI] bundle deploy -p dogfood
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/exploratory-cache-test/default/files...
+Deploying resources...
+Deployment complete!
+
+>>> print_telemetry_bool_values
+local.cache.attempt true
+local.cache.hit true

acceptance/cache/simple/script

@@ -0,0 +1,16 @@
+export DATABRICKS_CACHE_DIR=$(pwd)/.cache
+
+$CLI cache clear &> /dev/null
+
+title "First call in a session is expected to be a cache miss:\n"
+trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path"
+
+title "Second call in a session is expected to be a cache hit\n"
+trace $CLI bundle validate -p dogfood --debug 2>&1 | grep "Local Cache" | grep -v "cache path"
+
+title "Bundle deploy should send telemetry values\n"
+trace $CLI bundle deploy -p dogfood
+
+trace print_telemetry_bool_values | grep "local.cache"
+rm out.requests.txt
+rm -rf "${DATABRICKS_CACHE_DIR}"

acceptance/cache/simple/test.toml

@@ -0,0 +1,16 @@
+Cloud = false
+Local = true
+
+RecordRequests = true
+
+[Env]
+DATABRICKS_CACHE_ENABLED = 'true'
+
+# Redact structured logging fields from debug output
+[[Repls]]
+Old = ' pid=[0-9]+'
+New = ''
+
+[[Repls]]
+Old = ' mutator=[A-Za-z]+'
+New = ''

acceptance/cache/test.toml

@@ -0,0 +1,15 @@
+[[Repls]]
+Old = '\d\d:\d\d:\d\d'
+New = "[DEBUG_TIMESTAMP]"
+
+[[Repls]]
+Old = '[a-f0-9]{64}'
+New = "[SHA256_HASH]"
+
+[[Repls]]
+Old = '[a-f0-9]{12}'
+New = "[CACHE_KEY]"
+
+[[Repls]]
+Old = 'failed to stat cache file: .*'
+New = "failed to stat cache file: (redacted)"

acceptance/help/output.txt

@@ -158,6 +158,7 @@ Additional Commands:
   account                                Databricks Account Commands
   api                                    Perform Databricks API call
   auth                                   Authentication related commands
+  cache                                  Local cache related commands
   completion                             Generate the autocompletion script for the specified shell
   configure                              Configure authentication
   help                                   Help about any command

acceptance/internal/config.go

@@ -213,6 +213,8 @@ func LoadConfig(t *testing.T, dir string) (TestConfig, string) {
 		}
 	}
 
+	// Always ignore .cache directory (used by local cache)
+	result.Ignore = append(result.Ignore, ".cache")
 	result.CompiledIgnoreObject = ignore.CompileIgnoreLines(result.Ignore...)
 
 	return result, strings.Join(configs, ", ")