Add arca friendly make rule for generating CLI (#3297)

## Changes

1. Users can now run `make generate` from their arca instance to
generate the CLI code.
2. It also moves CLI steps like tagging and generating pydabs code to
the post_generate step in .codegen.json rather than having it live in
Makefile.
3. The generate rule also enforces that universe be checked out to the
same version as the openapi sha.

## Why
This PR makes it easy to generate CLI both from arca and your local
machine in a consistent and reproducible manner.

## Tests
No diff when regenerating the CLI on the current commit.
Generating with a new openapi spec correctly generates different files
like the JSON schema, CLI files or PyDABs files.

Author: shreyas-goenka

.codegen.json

@@ -14,9 +14,7 @@
             "go"
         ],
         "post_generate": [
-            "go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build",
-            "make schema",
-            "make generate-validation"
+            "./tools/post-generate.sh"
         ]
     }
 }

Makefile

@@ -101,13 +101,27 @@ integration-short:
 generate-validation:
 	go run ./bundle/internal/validation/.
 
+# Rule to generate the CLI from a new version of the OpenAPI spec.
+# I recommend running this rule from Arca because of faster build times
+# because of better caching and beefier machines, but it should also work
+# fine from your local mac.
+#
+# By default, this rule will use the universe directory in your home
+# directory. You can override this by setting the UNIVERSE_DIR
+# environment variable.
+#
+# Example:
+# UNIVERSE_DIR=/Users/shreyas.goenka/universe make generate
+UNIVERSE_DIR ?= $(HOME)/universe
+GENKIT_BINARY := $(UNIVERSE_DIR)/bazel-bin/openapi/genkit/genkit_/genkit
+
 generate:
-	genkit update-sdk
-	[ ! -f tagging.py ] || mv tagging.py internal/genkit/tagging.py
-# tagging.yml is automatically synced by update-sdk command and contains a reference to tagging.py in root
-# since we move tagging.py to different folder, we need to update this reference here as well
-	[ ! -f .github/workflows/tagging.yml ] || sed -i '' 's/python tagging.py/python internal\/genkit\/tagging.py/g' .github/workflows/tagging.yml
-	[ ! -f .github/workflows/next-changelog.yml ] || rm .github/workflows/next-changelog.yml
-	pushd experimental/python && make codegen
+	@echo "Checking out universe at SHA: $$(cat .codegen/_openapi_sha)"
+	cd $(UNIVERSE_DIR) && git checkout $$(cat $(PWD)/.codegen/_openapi_sha)
+	@echo "Building genkit..."
+	cd $(UNIVERSE_DIR) && bazel build //openapi/genkit
+	@echo "Generating CLI code..."
+	$(GENKIT_BINARY) update-sdk
+
 
 .PHONY: lint lintfull tidy lintcheck fmt fmtfull test cover showcover build snapshot schema integration integration-short acc-cover acc-showcover docs ws links checks test-update test-update-aws test-update-all generate-validation

tools/post-generate.sh

@@ -0,0 +1,28 @@
+# Ensure the SDK version is consistent with the OpenAPI SHA the CLI is generated from.
+go test -timeout 240s -run TestConsistentDatabricksSdkVersion github.com/databricks/cli/internal/build
+
+# Generate the bundle JSON schema.
+make schema
+
+# Generate bundle validation code for enuma and required fields.
+make generate-validation
+
+# Remove the next-changelog.yml workflow.
+rm .github/workflows/next-changelog.yml
+
+# Move the tagging.py file to the internal/genkit/tagging.py file. We do this to avoid
+# cluttering the root directory.
+mv tagging.py internal/genkit/tagging.py
+
+# Update the tagging.yml workflow to use the new tagging.py file location.
+if [[ "$(uname)" == "Darwin" ]]; then
+    # macOS (BSD sed) requires empty string after -i
+    sed -i '' 's|python tagging.py|python internal/genkit/tagging.py|g' .github/workflows/tagging.yml
+else
+    # Linux (GNU sed)
+    sed -i 's|python tagging.py|python internal/genkit/tagging.py|g' .github/workflows/tagging.yml
+fi
+./tools/yamlfmt .github/workflows/tagging.yml
+
+# Generate PyDABs code.
+make -C experimental/python codegen