[Python] Keep newly deprecated fields in Python code

Author: kanterov

bundle/docsgen/main.go

@@ -137,7 +137,7 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Deprecated = true
 		s.DeprecationMessage = a.DeprecationMessage
 	}
-	if a.ForceNotDeprecated {
+	if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue {
 		s.Deprecated = false
 		s.DeprecationMessage = ""
 	}

bundle/internal/annotation/descriptor.go

@@ -1,5 +1,12 @@
 package annotation
 
+type PythonKeepDeprecated string
+
+const (
+	PythonKeepDeprecatedTrue  PythonKeepDeprecated = "true"
+	PythonKeepDeprecatedUnset PythonKeepDeprecated = ""
+)
+
 type Descriptor struct {
 	Description         string `json:"description,omitempty"`
 	MarkdownDescription string `json:"markdown_description,omitempty"`
@@ -10,8 +17,8 @@ type Descriptor struct {
 	DeprecationMessage  string `json:"deprecation_message,omitempty"`
 	Preview             string `json:"x-databricks-preview,omitempty"`
 
-	// If true, takes priority over 'DeprecationMessage'
-	ForceNotDeprecated bool `json:"force_not_deprecated,omitempty"`
+	// If true, keep the field in Python code even if it is deprecated.
+	PythonKeepDeprecated PythonKeepDeprecated `json:"x-databricks-python-keep-deprecated,omitempty"`
 }
 
 const Placeholder = "PLACEHOLDER"

bundle/internal/schema/annotations.go

@@ -138,9 +138,8 @@ func assignAnnotation(s *jsonschema.Schema, a annotation.Descriptor) {
 		s.Preview = a.Preview
 	}
 
-	if a.ForceNotDeprecated {
-		s.Deprecated = false
-		s.DeprecationMessage = ""
+	if a.PythonKeepDeprecated != annotation.PythonKeepDeprecatedUnset {
+		s.PythonKeepDeprecated = a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedTrue
 	}
 
 	s.MarkdownDescription = convertLinksToAbsoluteUrl(a.MarkdownDescription)

bundle/internal/schema/annotations_openapi_overrides.yml

@@ -210,6 +210,8 @@ github.com/databricks/cli/bundle/config/resources.Job:
       ```
 
       For information about defining job tasks and overriding job settings, see [_](/dev-tools/bundles/job-task-types.md), [_](/dev-tools/bundles/job-task-override.md), and [_](/dev-tools/bundles/cluster-override.md).
+  "format":
+    "x-databricks-python-keep-deprecated": "false"
   "health":
     "description": |-
       PLACEHOLDER
@@ -311,6 +313,8 @@ github.com/databricks/cli/bundle/config/resources.ModelServingEndpoint:
   "permissions":
     "description": |-
       PLACEHOLDER
+  "rate_limits":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/cli/bundle/config/resources.ModelServingEndpointPermissionLevel:
   "_":
     "enum":
@@ -354,11 +358,11 @@ github.com/databricks/cli/bundle/config/resources.Pipeline:
     "description": |-
       PLACEHOLDER
   "target":
-    "force_not_deprecated": |-
-      true
+    "x-databricks-python-keep-deprecated": "true"
   "trigger":
     "deprecation_message": |-
       Use continuous instead
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/cli/bundle/config/resources.PipelinePermissionLevel:
   "_":
     "enum":
@@ -710,17 +714,26 @@ github.com/databricks/databricks-sdk-go/service/compute.DockerImage:
     "description": |-
       PLACEHOLDER
 github.com/databricks/databricks-sdk-go/service/compute.Environment:
+  "client":
+    "x-databricks-python-keep-deprecated": "true"
   "dependencies":
     "description": |-
       List of pip dependencies, as supported by the version of pip in this environment.
 github.com/databricks/databricks-sdk-go/service/compute.GcpAttributes:
   "availability":
     "description": |-
       PLACEHOLDER
+  "use_preemptible_executors":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/compute.InitScriptInfo:
   "abfss":
     "description": |-
       Contains the Azure Data Lake Storage destination path
+  "dbfs":
+    "x-databricks-python-keep-deprecated": "false"
+github.com/databricks/databricks-sdk-go/service/compute.Library:
+  "egg":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/compute.LogAnalyticsInfo:
   "log_analytics_primary_key":
     "description": |-
@@ -743,6 +756,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.GitSource:
   "git_snapshot":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/jobs.JobEmailNotifications:
+  "no_alert_for_skipped_runs":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/jobs.JobEnvironment:
   "spec":
     "description": |-
@@ -762,6 +778,11 @@ github.com/databricks/databricks-sdk-go/service/jobs.RunJobTask:
   "python_named_params":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/jobs.SparkJarTask:
+  "jar_uri":
+    "x-databricks-python-keep-deprecated": "false"
+  "run_as_repl":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/jobs.Subscription:
   "subscribers":
     "description": |-
@@ -783,6 +804,9 @@ github.com/databricks/databricks-sdk-go/service/jobs.Task:
   "health":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/jobs.TaskEmailNotifications:
+  "no_alert_for_skipped_runs":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/jobs.TriggerSettings:
   "table_update":
     "description": |-
@@ -798,17 +822,26 @@ github.com/databricks/databricks-sdk-go/service/pipelines.CronTrigger:
   "timezone_id":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/pipelines.IngestionGatewayPipelineDefinition:
+  "connection_id":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/pipelines.PipelineLibrary:
   "whl":
     "deprecation_message": |-
       This field is deprecated
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/pipelines.PipelineTrigger:
   "cron":
     "description": |-
       PLACEHOLDER
   "manual":
     "description": |-
       PLACEHOLDER
+github.com/databricks/databricks-sdk-go/service/serving.AiGatewayGuardrailParameters:
+  "invalid_keywords":
+    "x-databricks-python-keep-deprecated": "false"
+  "valid_topics":
+    "x-databricks-python-keep-deprecated": "false"
 github.com/databricks/databricks-sdk-go/service/serving.Route:
   "served_entity_name":
     "description": |-

bundle/internal/schema/parser.go

@@ -196,6 +196,11 @@ func (p *openapiParser) extractAnnotations(typ reflect.Type, outputPath, overrid
 					if description == "" {
 						addEmptyOverride(k, basePath, overrides)
 					}
+
+					// if field got deprecated outside of preview it can't be just removed and we keep it
+					if refProp.Deprecated && refProp.Preview != "PRIVATE" {
+						addPythonKeepDeprecatedOverride(k, basePath, overrides)
+					}
 				} else {
 					addEmptyOverride(k, basePath, overrides)
 				}
@@ -241,22 +246,46 @@ func prependCommentToFile(outputPath, comment string) error {
 	return err
 }
 
-func addEmptyOverride(key, pkg string, overridesFile annotation.File) {
+// addPythonKeepDeprecatedOverride adds an override with "x-databricks-python-keep-deprecated: true" to
+// annotations_openapi_overrides.yaml unless it already specifies a value for "x-databricks-python-keep-deprecated".
+//
+// This way if field got deprecated, we add an override to keep it in Python code, until we revisit deprecation.
+func addPythonKeepDeprecatedOverride(key, pkg string, overridesFile annotation.File) {
 	if overridesFile[pkg] == nil {
 		overridesFile[pkg] = map[string]annotation.Descriptor{}
 	}
 
 	overrides := overridesFile[pkg]
-	if overrides[key].Description == "" {
-		overrides[key] = annotation.Descriptor{Description: annotation.Placeholder}
+
+	a, ok := overrides[key]
+	if !ok {
+		a = annotation.Descriptor{}
+	}
+
+	// if we don't have explicit value, it means that field just got deprecated,
+	// and we should keep it in Python code
+	if a.PythonKeepDeprecated == annotation.PythonKeepDeprecatedUnset {
+		a.PythonKeepDeprecated = annotation.PythonKeepDeprecatedTrue
+	}
+
+	overrides[key] = a
+}
+
+func addEmptyOverride(key, pkg string, overridesFile annotation.File) {
+	if overridesFile[pkg] == nil {
+		overridesFile[pkg] = map[string]annotation.Descriptor{}
 	}
 
+	overrides := overridesFile[pkg]
+
 	a, ok := overrides[key]
 	if !ok {
 		a = annotation.Descriptor{}
 	}
+
 	if a.Description == "" {
 		a.Description = annotation.Placeholder
 	}
+
 	overrides[key] = a
 }

bundle/schema/jsonschema.json

@@ -102,6 +102,11 @@ class GeneratedField:
     be marked as experimental in docstring.
     """
 
+    deprecated: bool
+    """
+    If true, the field is deprecated and should be marked as deprecated in docstring.
+    """
+
     def __post_init__(self):
         if self.default_factory is not None and self.default is not None:
             raise ValueError("Can't have both default and default_factory", self)
@@ -131,6 +136,7 @@ class GeneratedDataclass:
     fields: list[GeneratedField]
     extends: list[GeneratedType]
     experimental: bool
+    deprecated: bool
 
 
 def generate_field(
@@ -156,6 +162,7 @@ def generate_field(
             default_factory="dict",
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     elif field_type.name == "VariableOrList":
         return GeneratedField(
@@ -168,6 +175,7 @@ def generate_field(
             default_factory="list",
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     elif is_required:
         return GeneratedField(
@@ -180,6 +188,7 @@ def generate_field(
             default_factory=None,
             create_func_default=None,
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
     else:
         return GeneratedField(
@@ -192,6 +201,7 @@ def generate_field(
             default_factory=None,
             create_func_default="None",
             experimental=prop.stage == Stage.PRIVATE,
+            deprecated=prop.deprecated or False,
         )
 
 
@@ -326,6 +336,7 @@ def generate_dataclass(
         fields=fields,
         extends=extends,
         experimental=schema.stage == Stage.PRIVATE,
+        deprecated=schema.deprecated or False,
     )
 
 
@@ -365,10 +376,19 @@ def _append_dataclass(b: CodeBuilder, generated: GeneratedDataclass):
     b.append(":").newline()
 
     # FIXME should contain class docstring
-    if not generated.description and not generated.experimental:
+    if (
+        not generated.description
+        and not generated.experimental
+        and not generated.deprecated
+    ):
         b.indent().append_triple_quote().append_triple_quote().newline().newline()
     else:
-        _append_description(b, generated.description, generated.experimental)
+        _append_description(
+            b,
+            generated.description,
+            experimental=generated.experimental,
+            deprecated=generated.deprecated,
+        )
 
 
 def _append_field(b: CodeBuilder, field: GeneratedField):
@@ -446,7 +466,12 @@ def _append_typed_dict(b: CodeBuilder, generated: GeneratedDataclass):
     b.indent().append_triple_quote().append_triple_quote().newline().newline()
 
 
-def _append_description(b: CodeBuilder, description: Optional[str], experimental: bool):
+def _append_description(
+    b: CodeBuilder, description: Optional[str], *, experimental: bool, deprecated: bool
+):
+    if deprecated:
+        description = "[DEPRECATED] " + (description or "")
+
     if description or experimental:
         b.indent().append_triple_quote().newline()
         if experimental:
@@ -472,7 +497,12 @@ def get_code(generated: GeneratedDataclass) -> str:
 
     for field in generated.fields:
         _append_field(b, field)
-        _append_description(b, field.description, field.experimental)
+        _append_description(
+            b,
+            field.description,
+            experimental=field.experimental,
+            deprecated=field.deprecated,
+        )
 
         b.newline()
 
@@ -485,7 +515,12 @@ def get_code(generated: GeneratedDataclass) -> str:
 
     for field in generated.fields:
         _append_typed_dict_field(b, field)
-        _append_description(b, field.description, field.experimental)
+        _append_description(
+            b,
+            field.description,
+            experimental=field.experimental,
+            deprecated=field.deprecated,
+        )
 
         b.newline()
 

experimental/python/codegen/codegen/generated_dataclass.py

@@ -15,6 +15,7 @@ class GeneratedEnum:
     values: dict[str, str]
     description: Optional[str]
     experimental: bool
+    deprecated: bool
 
 
 def generate_enum(namespace: str, schema_name: str, schema: Schema) -> GeneratedEnum:
@@ -35,6 +36,7 @@ def generate_enum(namespace: str, schema_name: str, schema: Schema) -> Generated
         values=values,
         description=schema.description,
         experimental=schema.stage == Stage.PRIVATE,
+        deprecated=schema.deprecated or False,
     )
 
 
@@ -48,7 +50,12 @@ def get_code(generated: GeneratedEnum) -> str:
     b.append(f"class {generated.class_name}(Enum):")
     b.newline()
 
-    _append_description(b, generated.description, generated.experimental)
+    _append_description(
+        b,
+        generated.description,
+        experimental=generated.experimental,
+        deprecated=generated.deprecated,
+    )
 
     # Example:
     #