Upgrade Go SDK to 0.75.0 (#3256)

## Changes
Upgrade Go SDK to 0.75.0

The biggest difference is that `genkit` does not infer field description
from the custom types description anymore (for example fields like
`data_security_mode`), thus we do it for annotations ourselves.

The result is that we are now missing descriptions for some of the enum
flags.

## Tests
Existing tests pass

<!-- If your PR needs to be included in the release notes for next
release,
add a separate entry in NEXT_CHANGELOG.md as part of your PR. -->

Author: andrewnester

.codegen/_openapi_sha

@@ -1 +1 @@
-033bcb9242b006001e2cf3956896711681de1a8c
+90fefb5618fdecf7dfdf6be7d56c2213d24ad944

.codegen/service.go.tmpl

@@ -161,7 +161,7 @@ func new{{.PascalName}}() *cobra.Command {
 			{{else if .Entity.IsEmpty }}// TODO: output-only field
 			{{else if .IsComputed -}}
 			{{else if .IsOutputOnly -}}
-			{{else if .Entity.Enum }}cmd.Flags().Var(&{{- template "request-body-obj" (dict "Method" $method "Field" .)}}, "{{.KebabName}}", `{{.Summary | without "`" | trimSuffix "."}}. Supported values: {{template "printArray" .Entity.Enum}}`)
+			{{else if .Entity.Enum }}cmd.Flags().Var(&{{- template "request-body-obj" (dict "Method" $method "Field" .)}}, "{{.KebabName}}", `{{.Summary | without "`" | trimSuffix "."}}{{if .Summary}}. {{end}}Supported values: {{template "printArray" .Entity.Enum}}`)
 			{{else}}cmd.Flags().{{template "arg-type" .Entity}}(&{{- template "request-body-obj" (dict "Method" $method "Field" .)}}, "{{.KebabName}}", {{- template "request-body-obj" (dict "Method" $method "Field" .)}}, `{{.Summary | without "`"}}`)
 			{{end}}
 		{{- end -}}

.gitattributes

@@ -70,14 +70,17 @@ cmd/workspace/dashboards/dashboards.go linguist-generated=true
 cmd/workspace/data-sources/data-sources.go linguist-generated=true
 cmd/workspace/database/database.go linguist-generated=true
 cmd/workspace/default-namespace/default-namespace.go linguist-generated=true
+cmd/workspace/default-warehouse-id/default-warehouse-id.go linguist-generated=true
 cmd/workspace/disable-legacy-access/disable-legacy-access.go linguist-generated=true
 cmd/workspace/disable-legacy-dbfs/disable-legacy-dbfs.go linguist-generated=true
 cmd/workspace/enable-export-notebook/enable-export-notebook.go linguist-generated=true
 cmd/workspace/enable-notebook-table-clipboard/enable-notebook-table-clipboard.go linguist-generated=true
 cmd/workspace/enable-results-downloading/enable-results-downloading.go linguist-generated=true
 cmd/workspace/enhanced-security-monitoring/enhanced-security-monitoring.go linguist-generated=true
 cmd/workspace/experiments/experiments.go linguist-generated=true
+cmd/workspace/external-lineage/external-lineage.go linguist-generated=true
 cmd/workspace/external-locations/external-locations.go linguist-generated=true
+cmd/workspace/external-metadata/external-metadata.go linguist-generated=true
 cmd/workspace/feature-store/feature-store.go linguist-generated=true
 cmd/workspace/forecasting/forecasting.go linguist-generated=true
 cmd/workspace/functions/functions.go linguist-generated=true
@@ -94,6 +97,7 @@ cmd/workspace/lakeview-embedded/lakeview-embedded.go linguist-generated=true
 cmd/workspace/lakeview/lakeview.go linguist-generated=true
 cmd/workspace/libraries/libraries.go linguist-generated=true
 cmd/workspace/llm-proxy-partner-powered-workspace/llm-proxy-partner-powered-workspace.go linguist-generated=true
+cmd/workspace/materialized-features/materialized-features.go linguist-generated=true
 cmd/workspace/metastores/metastores.go linguist-generated=true
 cmd/workspace/model-registry/model-registry.go linguist-generated=true
 cmd/workspace/model-versions/model-versions.go linguist-generated=true
@@ -130,6 +134,7 @@ cmd/workspace/resource-quotas/resource-quotas.go linguist-generated=true
 cmd/workspace/restrict-workspace-admins/restrict-workspace-admins.go linguist-generated=true
 cmd/workspace/schemas/schemas.go linguist-generated=true
 cmd/workspace/secrets/secrets.go linguist-generated=true
+cmd/workspace/service-principal-secrets-proxy/service-principal-secrets-proxy.go linguist-generated=true
 cmd/workspace/service-principals/service-principals.go linguist-generated=true
 cmd/workspace/serving-endpoints/serving-endpoints.go linguist-generated=true
 cmd/workspace/settings/settings.go linguist-generated=true

NEXT_CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Notable Changes
 
 ### Dependency updates
+* Upgrade Go SDK to 0.75.0 ([#3256](https://github.com/databricks/cli/pull/3256))
 
 ### CLI
 

acceptance/help/output.txt

@@ -57,7 +57,9 @@ Unity Catalog
   catalogs                               A catalog is the first layer of Unity Catalog’s three-level namespace.
   connections                            Connections allow for creating a connection to an external data source.
   credentials                            A credential represents an authentication and authorization mechanism for accessing services on your cloud tenant.
+  external-lineage                       External Lineage APIs enable defining and managing lineage relationships between Databricks objects and external systems.
   external-locations                     An external location is an object that combines a cloud storage path with a storage credential that authorizes access to the cloud storage path.
+  external-metadata                      External Metadata objects enable customers to register and manage metadata about external systems within Unity Catalog.
   functions                              Functions implement User-Defined Functions (UDFs) in Unity Catalog.
   grants                                 In Unity Catalog, data is secure by default.
   metastores                             A metastore is the top-level container of objects in Unity Catalog.
@@ -122,14 +124,17 @@ Apps
 Clean Rooms
   clean-room-assets                      Clean room assets are data and code objects — Tables, volumes, and notebooks that are shared with the clean room.
   clean-room-task-runs                   Clean room task runs are the executions of notebooks in a clean room.
-  clean-rooms                            A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other’s data.
+  clean-rooms                            A clean room uses Delta Sharing and serverless compute to provide a secure and privacy-protecting environment where multiple parties can work together on sensitive enterprise data without direct access to each other's data.
 
 Database
   database                               Database Instances provide access to a database via REST API or direct SQL.
 
 Quality Monitor v2
   quality-monitor-v2                     Manage data quality of UC objects (currently support schema).
 
+OAuth
+  service-principal-secrets-proxy        These APIs enable administrators to manage service principal secrets at the workspace level.
+
 Additional Commands:
   account                                Databricks Account Commands
   api                                    Perform Databricks API call

bundle/config/variable/resolve_dashboard.go

@@ -2,20 +2,37 @@ package variable
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/sql"
 )
 
 type resolveDashboard struct {
 	name string
 }
 
 func (l resolveDashboard) Resolve(ctx context.Context, w *databricks.WorkspaceClient) (string, error) {
-	entity, err := w.Dashboards.GetByName(ctx, l.name)
+	// List dashboards and find the one with the given name
+	// If there are multiple dashboards with the same name, return an error
+	dashboards, err := w.Dashboards.ListAll(ctx, sql.ListDashboardsRequest{})
 	if err != nil {
 		return "", err
 	}
-	return entity.Id, nil
+
+	dashboardMap := make(map[string][]sql.Dashboard)
+	for _, dashboard := range dashboards {
+		dashboardMap[dashboard.Name] = append(dashboardMap[dashboard.Name], dashboard)
+	}
+
+	alternatives, ok := dashboardMap[l.name]
+	if !ok || len(alternatives) == 0 {
+		return "", fmt.Errorf("dashboard name '%s' does not exist", l.name)
+	}
+	if len(alternatives) > 1 {
+		return "", fmt.Errorf("there are %d instances of dashboards named '%s'", len(alternatives), l.name)
+	}
+	return alternatives[0].Id, nil
 }
 
 func (l resolveDashboard) String() string {

bundle/config/variable/resolve_dashboard_test.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/databricks/databricks-sdk-go/apierr"
 	"github.com/databricks/databricks-sdk-go/experimental/mocks"
 	"github.com/databricks/databricks-sdk-go/service/sql"
 	"github.com/stretchr/testify/assert"
@@ -17,9 +16,10 @@ func TestResolveDashboard_ResolveSuccess(t *testing.T) {
 
 	api := m.GetMockDashboardsAPI()
 	api.EXPECT().
-		GetByName(mock.Anything, "dashboard").
-		Return(&sql.Dashboard{
-			Id: "1234",
+		ListAll(mock.Anything, mock.Anything).
+		Return([]sql.Dashboard{
+			{Id: "1234", Name: "dashboard"},
+			{Id: "5678", Name: "dashboard2"},
 		}, nil)
 
 	ctx := context.Background()
@@ -34,13 +34,16 @@ func TestResolveDashboard_ResolveNotFound(t *testing.T) {
 
 	api := m.GetMockDashboardsAPI()
 	api.EXPECT().
-		GetByName(mock.Anything, "dashboard").
-		Return(nil, &apierr.APIError{StatusCode: 404})
+		ListAll(mock.Anything, mock.Anything).
+		Return([]sql.Dashboard{
+			{Id: "1234", Name: "dashboard1"},
+			{Id: "5678", Name: "dashboard2"},
+		}, nil)
 
 	ctx := context.Background()
 	l := resolveDashboard{name: "dashboard"}
 	_, err := l.Resolve(ctx, m.WorkspaceClient)
-	require.ErrorIs(t, err, apierr.ErrNotFound)
+	require.ErrorContains(t, err, "dashboard name 'dashboard' does not exist")
 }
 
 func TestResolveDashboard_String(t *testing.T) {

bundle/internal/schema/annotations_openapi.yml

@@ -365,9 +365,9 @@ github.com/databricks/cli/bundle/config/resources.Job:
       The queue settings of the job.
   "run_as":
     "description": |-
-      Write-only setting. Specifies the user or service principal that the job runs as. If not specified, the job runs as the user who created the job.
-
-      Either `user_name` or `service_principal_name` should be specified. If not, an error is thrown.
+      The user or service principal that the job runs as, if specified in the request.
+      This field indicates the explicit configuration of `run_as` for the job.
+      To find the value in all cases, explicit or implicit, use `run_as_user_name`.
   "schedule":
     "description": |-
       An optional periodic schedule for this job. The default behavior is that the job only runs when triggered by clicking “Run Now” in the Jobs UI or sending an API request to `runNow`.
@@ -434,6 +434,7 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
   "config":
     "description": |-
       The core config of the serving endpoint.
+  "description": {}
   "name":
     "description": |-
       The name of the serving endpoint. This field is required and must be unique across a Databricks workspace.
@@ -486,8 +487,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "environment":
     "description": |-
       Environment specification for this pipeline used to install dependencies.
-    "x-databricks-preview": |-
-      PRIVATE
   "event_log":
     "description": |-
       Event log configuration for this pipeline
@@ -527,8 +526,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
       Root path for this pipeline.
       This is used as the root directory when editing the pipeline in the Databricks user interface and it is
       added to sys.path when executing Python sources during pipeline execution.
-    "x-databricks-preview": |-
-      PRIVATE
   "run_as":
     "description": |-
       Write-only setting, available only in Create/Update calls. Specifies the user or service principal that the pipeline runs as. If not specified, the pipeline runs as the user who created the pipeline.
@@ -718,6 +715,7 @@ github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
     "description": |-
       State of the deployment.
 github.com/databricks/databricks-sdk-go/service/apps.AppResource:
+  "database": {}
   "description":
     "description": |-
       Description of the App Resource.
@@ -729,6 +727,15 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "serving_endpoint": {}
   "sql_warehouse": {}
   "uc_securable": {}
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceDatabase:
+  "database_name": {}
+  "instance_name": {}
+  "permission": {}
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceDatabaseDatabasePermission:
+  "_":
+    "enum":
+      - |-
+        CAN_CONNECT_AND_CREATE
 github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
   "id":
     "description": |-
@@ -2790,6 +2797,9 @@ github.com/databricks/databricks-sdk-go/service/ml.ExperimentTag:
     "description": |-
       The tag value.
 github.com/databricks/databricks-sdk-go/service/ml.ModelTag:
+  "_":
+    "description": |-
+      Tag for a registered model
   "key":
     "description": |-
       The tag key.
@@ -2910,6 +2920,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionSourceType:
         SQLSERVER
       - |-
         SALESFORCE
+      - |-
+        BIGQUERY
       - |-
         NETSUITE
       - |-
@@ -2928,6 +2940,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.IngestionSourceType:
         SHAREPOINT
       - |-
         DYNAMICS365
+      - |-
+        CONFLUENCE
 github.com/databricks/databricks-sdk-go/service/pipelines.ManualTrigger: {}
 github.com/databricks/databricks-sdk-go/service/pipelines.NotebookLibrary:
   "path":
@@ -3092,8 +3106,6 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary:
       The unified field to include source codes.
       Each entry can be a notebook path, a file path, or a folder path that ends `/**`.
       This field cannot be used together with `notebook` or `file`.
-    "x-databricks-preview": |-
-      PRIVATE
   "jar":
     "description": |-
       URI of the jar to be installed. Currently only DBFS is supported.
@@ -3243,6 +3255,8 @@ github.com/databricks/databricks-sdk-go/service/pipelines.TableSpecificConfigScd
         SCD_TYPE_1
       - |-
         SCD_TYPE_2
+      - |-
+        APPEND_ONLY
 github.com/databricks/databricks-sdk-go/service/serving.Ai21LabsConfig:
   "ai21labs_api_key":
     "description": |-
@@ -3280,6 +3294,8 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame
     "description": |-
       List of invalid keywords.
       AI guardrail uses keyword or string matching to decide if the keyword exists in the request or response content.
+    "deprecation_message": |-
+      This field is deprecated
   "pii":
     "description": |-
       Configuration for guardrail PII filter.
@@ -3290,6 +3306,8 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParame
     "description": |-
       The list of allowed topics.
       Given a chat request, this guardrail flags the request if its topic is not in the allowed topics.
+    "deprecation_message": |-
+      This field is deprecated
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailPiiBehavior:
   "behavior":
     "description": |-
@@ -3330,8 +3348,11 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimit:
       Used to specify how many calls are allowed for a key within the renewal_period.
   "key":
     "description": |-
-      Key field for a rate limit. Currently, only 'user' and 'endpoint' are supported,
+      Key field for a rate limit. Currently, 'user', 'user_group, 'service_principal', and 'endpoint' are supported,
       with 'endpoint' being the default if not specified.
+  "principal":
+    "description": |-
+      Principal field for a user, user group, or service principal to apply rate limiting to. Accepts a user email, group name, or service principal application ID.
   "renewal_period":
     "description": |-
       Renewal period field for a rate limit. Currently, only 'minute' is supported.
@@ -3342,6 +3363,10 @@ github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitKey:
         user
       - |-
         endpoint
+      - |-
+        user_group
+      - |-
+        service_principal
 github.com/databricks/databricks-sdk-go/service/serving.AiGatewayRateLimitRenewalPeriod:
   "_":
     "enum":
@@ -3728,6 +3753,7 @@ github.com/databricks/databricks-sdk-go/service/serving.RateLimitRenewalPeriod:
       - |-
         minute
 github.com/databricks/databricks-sdk-go/service/serving.Route:
+  "served_entity_name": {}
   "served_model_name":
     "description": |-
       The name of the served model this route configures traffic for.
@@ -3846,6 +3872,9 @@ github.com/databricks/databricks-sdk-go/service/serving.TrafficConfig:
     "description": |-
       The list of routes that define traffic to each served entity.
 github.com/databricks/databricks-sdk-go/service/workspace.AzureKeyVaultSecretScopeMetadata:
+  "_":
+    "description": |-
+      The metadata of the Azure KeyVault for a secret scope of type `AZURE_KEYVAULT`
   "dns_name":
     "description": |-
       The DNS of the KeyVault
@@ -3854,6 +3883,9 @@ github.com/databricks/databricks-sdk-go/service/workspace.AzureKeyVaultSecretSco
       The resource id of the azure KeyVault that user wants to associate the scope with.
 github.com/databricks/databricks-sdk-go/service/workspace.ScopeBackendType:
   "_":
+    "description": |-
+      The types of secret scope backends in the Secret Manager. Azure KeyVault backed secret scopes
+      will be supported in a later release.
     "enum":
       - |-
         DATABRICKS

bundle/internal/schema/annotations_openapi_overrides.yml

@@ -254,6 +254,9 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
             - key: "team"
               value: "data science"
       ```
+  "description":
+    "description": |-
+      PLACEHOLDER
   "permissions":
     "description": |-
       PLACEHOLDER
@@ -493,6 +496,9 @@ github.com/databricks/databricks-sdk-go/service/apps.AppDeploymentStatus:
     "description": |-
       PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/apps.AppResource:
+  "database":
+    "description": |-
+      PLACEHOLDER
   "job":
     "description": |-
       PLACEHOLDER
@@ -508,6 +514,16 @@ github.com/databricks/databricks-sdk-go/service/apps.AppResource:
   "uc_securable":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/apps.AppResourceDatabase:
+  "database_name":
+    "description": |-
+      PLACEHOLDER
+  "instance_name":
+    "description": |-
+      PLACEHOLDER
+  "permission":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/apps.AppResourceJob:
   "id":
     "description": |-
@@ -700,6 +716,10 @@ github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "manual":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.Route:
+  "served_entity_name":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/serving.ServedEntityInput:
   "entity_version":
     "description": |-