Add all deprecated fields except private ones

Author: kanterov

.github/workflows/push.yml

@@ -180,7 +180,6 @@ jobs:
             a.addKeyword('doNotSuggest');
             a.addKeyword('markdownDescription');
             a.addKeyword('x-databricks-preview');
-            a.addKeyword('x-databricks-python-keep-deprecated');
           }" >> keywords.js
 
           for file in ./bundle/internal/schema/testdata/pass/*.yml; do

bundle/docsgen/main.go

@@ -137,10 +137,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Deprecated = true
 		s.DeprecationMessage = a.DeprecationMessage
 	}
-	if a.ForceNotDeprecated {
-		s.Deprecated = false
-		s.DeprecationMessage = ""
-	}
 	if a.Preview == "PRIVATE" {
 		s.DoNotSuggest = true
 		s.Preview = a.Preview

bundle/internal/annotation/descriptor.go

@@ -9,9 +9,6 @@ type Descriptor struct {
 	MarkdownExamples    string `json:"markdown_examples,omitempty"`
 	DeprecationMessage  string `json:"deprecation_message,omitempty"`
 	Preview             string `json:"x-databricks-preview,omitempty"`
-
-	// If true, takes priority over 'DeprecationMessage'
-	ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"`
 }
 
 const Placeholder = "PLACEHOLDER"

bundle/internal/schema/annotations.go

@@ -138,11 +138,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Preview = a.Preview
 	}
 
-	if a.ForceNotDeprecated {
-		s.Deprecated = false
-		s.DeprecationMessage = ""
-	}
-
 	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
 	s.Title = a.Title
 	s.Enum = a.Enum

bundle/internal/schema/annotations_openapi_overrides.yml

@@ -353,9 +353,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "run_as":
     "description": |-
       PLACEHOLDER
-  "target":
-    "force_not_deprecated": |-
-      true
   "trigger":
     "deprecation_message": |-
       Use continuous instead

bundle/schema/jsonschema.json

@@ -16,7 +16,6 @@ class Property:
     ref: str
     description: Optional[str] = None
     deprecated: Optional[bool] = None
-    keep_deprecated: Optional[bool] = None
     stage: Optional[str] = None
 
 
@@ -102,7 +101,6 @@ def _parse_bool(value) -> Optional[bool]:
             ref=v["$ref"],
             description=v.get("description"),
             deprecated=_parse_bool(v.get("deprecated")),
-            keep_deprecated=_parse_bool(v.get("x-databricks-python-keep-deprecated")),
             stage=v.get("x-databricks-preview"),
         )
 

experimental/python/codegen/codegen/jsonschema.py

@@ -7,6 +7,15 @@
         # doesn't work, openapi schema needs to be updated to be enum
         "kind",
     },
+    # fields that were deprecated a long time ago
+    "resources.Pipeline": {
+        # 'trigger' is deprecated, use 'continuous' or schedule pipeline refresh using job instead
+        "trigger",
+    },
+    "pipelines.PipelineLibrary": [
+        # 'whl' is deprecated, install libraries through notebooks and %pip command
+        "whl",
+    ],
 }
 
 EXTRA_REQUIRED_FIELDS: dict[str, list[str]] = {

experimental/python/codegen/codegen/jsonschema_patch.py

@@ -77,13 +77,17 @@ def _transitively_mark_deprecated_and_private(
 def _remove_deprecated_fields(
     schemas: dict[str, openapi.Schema],
 ) -> dict[str, openapi.Schema]:
+    """
+    Remove fields that were deprecated during Private Preview.
+    """
+
     new_schemas = {}
 
     for name, schema in schemas.items():
         if schema.type == openapi.SchemaType.OBJECT:
             new_properties = {}
             for field_name, field in schema.properties.items():
-                if field.deprecated and not field.keep_deprecated:
+                if field.deprecated and field.stage == openapi.Stage.PRIVATE:
                     continue
 
                 new_properties[field_name] = field
@@ -246,9 +250,6 @@ def _collect_reachable_schemas(
                     if (
                         not include_deprecated
                         and field.deprecated
-                        # we don't remove keep_deprecated fields so they should be considered
-                        # reachable
-                        and not field.keep_deprecated
                     ):
                         continue
 

experimental/python/codegen/codegen/main.py

@@ -53,6 +53,13 @@ class GcpAttributes:
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool] = None
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str] = None
     """
     Identifier for the availability zone in which the cluster resides.
@@ -108,6 +115,13 @@ class GcpAttributesDict(TypedDict, total=False):
     for the supported number of local SSDs for each instance type.
     """
 
+    use_preemptible_executors: VariableOrOptional[bool]
+    """
+    [DEPRECATED] This field determines whether the spark executors will be scheduled to run on preemptible
+    VMs (when set to true) versus standard compute engine VMs (when set to false; default).
+    Note: Soon to be deprecated, use the 'availability' field instead.
+    """
+
     zone_id: VariableOrOptional[str]
     """
     Identifier for the availability zone in which the cluster resides.