[Python] Keep deprecated fields in Python code (#3384)

## Changes
Python code generation doesn't remove deprecated fields anymore unless
they were deprecated during Private Preview. A few fields are manually
excluded because they have been deprecated for many years and have never
functioned.

Remove `ForceNotDeprecated` annotation because it isn't needed anymore.

## Why
During updates, removing deprecated fields immediately can be surprising
because there is no prior notice. There are also bundles that rely on
the usage of deprecated fields.

## Tests
By looking at diffs in the generated code

Author: kanterov

bundle/docsgen/main.go

@@ -137,10 +137,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Deprecated = true
 		s.DeprecationMessage = a.DeprecationMessage
 	}
-	if a.ForceNotDeprecated {
-		s.Deprecated = false
-		s.DeprecationMessage = ""
-	}
 	if a.Preview == "PRIVATE" {
 		s.DoNotSuggest = true
 		s.Preview = a.Preview

bundle/internal/annotation/descriptor.go

@@ -9,9 +9,6 @@ type Descriptor struct {
 	MarkdownExamples    string `json:"markdown_examples,omitempty"`
 	DeprecationMessage  string `json:"deprecation_message,omitempty"`
 	Preview             string `json:"x-databricks-preview,omitempty"`
-
-	// If true, takes priority over 'DeprecationMessage'
-	ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"`
 }
 
 const Placeholder = "PLACEHOLDER"

bundle/internal/schema/annotations.go

@@ -138,11 +138,6 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Preview = a.Preview
 	}
 
-	if a.ForceNotDeprecated {
-		s.Deprecated = false
-		s.DeprecationMessage = ""
-	}
-
 	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)
 	s.Title = a.Title
 	s.Enum = a.Enum

bundle/internal/schema/annotations_openapi_overrides.yml

@@ -353,9 +353,6 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
   "run_as":
     "description": |-
       PLACEHOLDER
-  "target":
-    "force_not_deprecated": |-
-      true
   "trigger":
     "deprecation_message": |-
       Use continuous instead

bundle/schema/jsonschema.json

@@ -102,6 +102,11 @@ class GeneratedField:
     be marked as experimental in docstring.
     """
 
+    deprecated: bool
+    """
+    If true, the field is deprecated and should be marked as deprecated in docstring.
+    """
+
     def __post_init__(self):
         if self.default_factory is not None and self.default is not None:
             raise ValueError("Can't have both default and default_factory", self)
@@ -131,6 +136,7 @@ class GeneratedDataclass:
     fields: list[GeneratedField]
     extends: list[GeneratedType]
     experimental: bool
+    deprecated: bool
 
 
 def generate_field(
@@ -156,6 +162,7 @@ def generate_field(
             default_factory="dict",
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     elif field_type.name == "VariableOrList":
         return GeneratedField(
@@ -168,6 +175,7 @@ def generate_field(
             default_factory="list",
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     elif is_required:
         return GeneratedField(
@@ -180,6 +188,7 @@ def generate_field(
             default_factory=None,
             create_func_default=None,
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     else:
         return GeneratedField(
@@ -192,6 +201,7 @@ def generate_field(
             default_factory=None,
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
 
 
@@ -326,6 +336,7 @@ def generate_dataclass(
         fields=fields,
         extends=extends,
         experimental=schema.stage == Stage.PRIVATE,
+        deprecated=schema.deprecated or False,
     )
 
 
@@ -365,10 +376,19 @@ def _append_dataclass(b: CodeBuilder, generated: GeneratedDataclass):
     b.append(":").newline()
 
     # FIXME should contain class docstring
-    if not generated.description and not generated.experimental:
+    if (
+        not generated.description
+        and not generated.experimental
+        and not generated.deprecated
+    ):
         b.indent().append_triple_quote().append_triple_quote().newline().newline()
     else:
-        _append_description(b, generated.description, generated.experimental)
+        _append_description(
+            b,
+            generated.description,
+            experimental=generated.experimental,
+            deprecated=generated.deprecated,
+        )
 
 
 def _append_field(b: CodeBuilder, field: GeneratedField):
@@ -446,7 +466,12 @@ def _append_typed_dict(b: CodeBuilder, generated: GeneratedDataclass):
     b.indent().append_triple_quote().append_triple_quote().newline().newline()
 
 
-def _append_description(b: CodeBuilder, description: Optional[str], experimental: bool):
+def _append_description(
+    b: CodeBuilder, description: Optional[str], *, experimental: bool, deprecated: bool
+):
+    if deprecated:
+        description = "[DEPRECATED] " + (description or "")
+
     if description or experimental:
         b.indent().append_triple_quote().newline()
         if experimental:
@@ -472,7 +497,12 @@ def get_code(generated: GeneratedDataclass) -> str:
 
     for field in generated.fields:
         _append_field(b, field)
-        _append_description(b, field.description, field.experimental)
+        _append_description(
+            b,
+            field.description,
+            experimental=field.experimental,
+            deprecated=field.deprecated,
+        )
 
         b.newline()
 
@@ -485,7 +515,12 @@ def get_code(generated: GeneratedDataclass) -> str:
 
     for field in generated.fields:
         _append_typed_dict_field(b, field)
-        _append_description(b, field.description, field.experimental)
+        _append_description(
+            b,
+            field.description,
+            experimental=field.experimental,
+            deprecated=field.deprecated,
+        )
 
         b.newline()
 

experimental/python/codegen/codegen/generated_dataclass.py

@@ -15,6 +15,7 @@ class GeneratedEnum:
     values: dict[str, str]
     description: Optional[str]
     experimental: bool
+    deprecated: bool
 
 
 def generate_enum(namespace: str, schema_name: str, schema: Schema) -> GeneratedEnum:
@@ -35,6 +36,7 @@ def generate_enum(namespace: str, schema_name: str, schema: Schema) -> Generated
         values=values,
         description=schema.description,
         experimental=schema.stage == Stage.PRIVATE,
+        deprecated=schema.deprecated or False,
     )
 
 
@@ -48,7 +50,12 @@ def get_code(generated: GeneratedEnum) -> str:
     b.append(f"class {generated.class_name}(Enum):")
     b.newline()
 
-    _append_description(b, generated.description, generated.experimental)
+    _append_description(
+        b,
+        generated.description,
+        experimental=generated.experimental,
+        deprecated=generated.deprecated,
+    )
 
     # Example:
     #

experimental/python/codegen/codegen/generated_enum.py

@@ -7,6 +7,15 @@
         # doesn't work, openapi schema needs to be updated to be enum
         "kind",
     },
+    # fields that were deprecated a long time ago
+    "resources.Pipeline": {
+        # 'trigger' is deprecated, use 'continuous' or schedule pipeline refresh using job instead
+        "trigger",
+    },
+    "pipelines.PipelineLibrary": [
+        # 'whl' is deprecated, install libraries through notebooks and %pip command
+        "whl",
+    ],
 }
 
 EXTRA_REQUIRED_FIELDS: dict[str, list[str]] = {

experimental/python/codegen/codegen/jsonschema_patch.py

@@ -77,13 +77,17 @@ def _transitively_mark_deprecated_and_private(
 def _remove_deprecated_fields(
     schemas: dict[str, openapi.Schema],
 ) -> dict[str, openapi.Schema]:
+    """
+    Remove fields that were deprecated during Private Preview.
+    """
+
     new_schemas = {}
 
     for name, schema in schemas.items():
         if schema.type == openapi.SchemaType.OBJECT:
             new_properties = {}
             for field_name, field in schema.properties.items():
-                if field.deprecated:
+                if field.deprecated and field.stage == openapi.Stage.PRIVATE:
                     continue
 
                 new_properties[field_name] = field

experimental/python/codegen/codegen/main.py

@@ -69,9 +69,11 @@ def test_generate_dataclass():
                 param_type_name=variable_or_type(str_type(), is_required=True),
                 type_name=variable_or_type(str_type(), is_required=True),
                 experimental=False,
+                deprecated=False,
             ),
         ],
         experimental=False,
+        deprecated=False,
     )