Deprecating data_frame_transforms roles

docs/changelog/117519.yaml

@@ -0,0 +1,20 @@
+pr: 117519
+summary: Deprecating `data_frame_transforms` roles
+area: Transform
+type: breaking
+issues: []
+breaking:
+  title: Remove `data_frame_transforms` roles
+  area: Machine Learning
+  details: >-
+    `data_frame_transforms_admin` and `data_frame_transforms_user` were deprecated in
+    Elasticsearch 7 and are being removed in Elasticsearch 9.
+    `data_frame_transforms_admin` is now `transform_admin`.
+    `data_frame_transforms_user` is now `transform_user`.
+    Users must call the `_update` API to replace the permissions on the Transform before the
+    Transform can be started.
+  impact: >-
+    Transforms created with either the `data_frame_transforms_admin` or the
+    `data_frame_transforms_user` role will fail to start. The Transform will remain
+    in a `stopped` state, and its health will be red while displaying permission failures.
+  notable: false

docs/reference/rest-api/security/bulk-create-roles.asciidoc

@@ -328,7 +328,7 @@ The result would then have the `errors` field set to `true` and hold the error f
         "details": {
             "my_admin_role": { <4>
                 "type": "action_request_validation_exception",
-                "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_stats,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_data_frame_transforms,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_data_frame_transforms,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;"
+                "reason": "Validation Failed: 1: unknown cluster privilege [bad_cluster_privilege]. a privilege must be either one of the predefined cluster privilege names [manage_own_api_key,manage_data_stream_global_retention,monitor_data_stream_global_retention,none,cancel_task,cross_cluster_replication,cross_cluster_search,delegate_pki,grant_api_key,manage_autoscaling,manage_index_templates,manage_logstash_pipelines,manage_oidc,manage_saml,manage_search_application,manage_search_query_rules,manage_search_synonyms,manage_service_account,manage_token,manage_user_profile,monitor_connector,monitor_enrich,monitor_inference,monitor_ml,monitor_rollup,monitor_snapshot,monitor_stats,monitor_text_structure,monitor_watcher,post_behavioral_analytics_event,read_ccr,read_connector_secrets,read_fleet_secrets,read_ilm,read_pipeline,read_security,read_slm,transport_client,write_connector_secrets,write_fleet_secrets,create_snapshot,manage_behavioral_analytics,manage_ccr,manage_connector,manage_enrich,manage_ilm,manage_inference,manage_ml,manage_rollup,manage_slm,manage_watcher,monitor_transform,manage_api_key,manage_ingest_pipelines,manage_pipeline,manage_transform,manage_security,monitor,manage,all] or a pattern over one of the available cluster actions;"
             }
         }
     }

docs/reference/rest-api/security/get-builtin-privileges.asciidoc

@@ -78,7 +78,6 @@ A successful call returns an object with "cluster", "index", and "remote_cluster
     "manage_behavioral_analytics",
     "manage_ccr",
     "manage_connector",
-    "manage_data_frame_transforms",
     "manage_data_stream_global_retention",
     "manage_enrich",
     "manage_ilm",
@@ -104,7 +103,6 @@ A successful call returns an object with "cluster", "index", and "remote_cluster
     "manage_watcher",
     "monitor",
     "monitor_connector",
-    "monitor_data_frame_transforms",
     "monitor_data_stream_global_retention",
     "monitor_enrich",
     "monitor_inference",

docs/reference/security/authorization/built-in-roles.asciidoc

@@ -33,18 +33,6 @@ suitable for writing beats output to {es}.
 
 --
 
-[[built-in-roles-data-frame-transforms-admin]] `data_frame_transforms_admin` ::
-Grants `manage_data_frame_transforms` cluster privileges, which enable you to
-manage {transforms}. This role also includes all
-{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}.
-deprecated:[7.5.0,"Replaced by <<built-in-roles-transform-admin,`transform_admin`>>"].
-
-[[built-in-roles-data-frame-transforms-user]] `data_frame_transforms_user` ::
-Grants `monitor_data_frame_transforms` cluster privileges, which enable you to
-use {transforms}. This role also includes all
-{kibana-ref}/kibana-privileges.html[Kibana privileges] for the {ml-features}.
-deprecated:[7.5.0,"Replaced by <<built-in-roles-transform-user,`transform_user`>>"].
-
 [[built-in-roles-editor]] `editor` ::
 
 Grants full access to all features in {kib} (including Solutions) and read-only access to data indices.

docs/reference/security/authorization/privileges.asciidoc

@@ -95,12 +95,6 @@ only on clusters that contain follower indices.
 +
 This privilege is not available in {serverless-full}.
 
-`manage_data_frame_transforms`::
-All operations related to managing {transforms}.
-deprecated[7.5] Use `manage_transform` instead.
-+
-This privilege is not available in {serverless-full}.
-
 `manage_data_stream_global_retention`::
 This privilege has no effect.deprecated[8.16]
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/privilege/ClusterPrivilegeResolver.java

@@ -236,7 +236,7 @@ public class ClusterPrivilegeResolver {
     public static final NamedClusterPrivilege MONITOR_ML = new ActionClusterPrivilege("monitor_ml", MONITOR_ML_PATTERN);
     public static final NamedClusterPrivilege MONITOR_TRANSFORM_DEPRECATED = new ActionClusterPrivilege(
         "monitor_data_frame_transforms",
-        MONITOR_TRANSFORM_PATTERN
+        Set.of()
     );
     public static final NamedClusterPrivilege MONITOR_TEXT_STRUCTURE = new ActionClusterPrivilege(
         "monitor_text_structure",
@@ -255,7 +255,7 @@ public class ClusterPrivilegeResolver {
     public static final NamedClusterPrivilege MANAGE_ML = new ActionClusterPrivilege("manage_ml", MANAGE_ML_PATTERN);
     public static final NamedClusterPrivilege MANAGE_TRANSFORM_DEPRECATED = new ActionClusterPrivilege(
         "manage_data_frame_transforms",
-        MANAGE_TRANSFORM_PATTERN
+        Set.of()
     );
     public static final NamedClusterPrivilege MANAGE_TRANSFORM = new ActionClusterPrivilege("manage_transform", MANAGE_TRANSFORM_PATTERN);
     public static final NamedClusterPrivilege MANAGE_TOKEN = new ActionClusterPrivilege("manage_token", MANAGE_TOKEN_PATTERN);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/authz/store/ReservedRolesStore.java

@@ -534,70 +534,6 @@ private static Map<String, RoleDescriptor> initializeReservedRoles() {
                         + "and roles that grant access to Kibana."
                 )
             ),
-            // DEPRECATED: to be removed in 9.0.0
-            entry(
-                "data_frame_transforms_admin",
-                new RoleDescriptor(
-                    "data_frame_transforms_admin",
-                    new String[] { "manage_data_frame_transforms" },
-                    new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder()
-                            .indices(
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN,
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED,
-                                TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS
-                            )
-                            .privileges("view_index_metadata", "read")
-                            .build() },
-                    new RoleDescriptor.ApplicationResourcePrivileges[] {
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                            .application("kibana-*")
-                            .resources("*")
-                            .privileges("reserved_ml_user")
-                            .build() },
-                    null,
-                    null,
-                    MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_admin] role instead"),
-                    null,
-                    null,
-                    null,
-                    null,
-                    "Grants manage_data_frame_transforms cluster privileges, which enable you to manage transforms. "
-                        + "This role also includes all Kibana privileges for the machine learning features."
-                )
-            ),
-            // DEPRECATED: to be removed in 9.0.0
-            entry(
-                "data_frame_transforms_user",
-                new RoleDescriptor(
-                    "data_frame_transforms_user",
-                    new String[] { "monitor_data_frame_transforms" },
-                    new RoleDescriptor.IndicesPrivileges[] {
-                        RoleDescriptor.IndicesPrivileges.builder()
-                            .indices(
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN,
-                                TransformInternalIndexConstants.AUDIT_INDEX_PATTERN_DEPRECATED,
-                                TransformInternalIndexConstants.AUDIT_INDEX_READ_ALIAS
-                            )
-                            .privileges("view_index_metadata", "read")
-                            .build() },
-                    new RoleDescriptor.ApplicationResourcePrivileges[] {
-                        RoleDescriptor.ApplicationResourcePrivileges.builder()
-                            .application("kibana-*")
-                            .resources("*")
-                            .privileges("reserved_ml_user")
-                            .build() },
-                    null,
-                    null,
-                    MetadataUtils.getDeprecatedReservedMetadata("Please use the [transform_user] role instead"),
-                    null,
-                    null,
-                    null,
-                    null,
-                    "Grants monitor_data_frame_transforms cluster privileges, which enable you to use transforms. "
-                        + "This role also includes all Kibana privileges for the machine learning features. "
-                )
-            ),
             entry(
                 "transform_admin",
                 new RoleDescriptor(

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/TransformDeprecations.java

@@ -27,12 +27,5 @@ public class TransformDeprecations {
 
     public static final String MAX_PAGE_SEARCH_SIZE_BREAKING_CHANGES_URL = "https://ela.st/es-deprecation-7-transform-max-page-search-size";
 
-    public static final String DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL =
-        "https://ela.st/es-deprecation-9-data-frame-transforms-roles";
-
-    public static final String DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED = "This transform configuration uses one or more obsolete roles "
-        + "prefixed with [data_frame_transformers_] which will be unsupported after the next upgrade. Switch to a user with the equivalent "
-        + "roles prefixed with [transform_] and use [/_transform/_upgrade] to upgrade all transforms to the latest roles.";;
-
     private TransformDeprecations() {}
 }

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/transform/transforms/TransformConfig.java

@@ -24,13 +24,11 @@
 import org.elasticsearch.xcontent.ToXContentObject;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
-import org.elasticsearch.xpack.core.ClientHelper;
 import org.elasticsearch.xpack.core.common.time.TimeUtils;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator;
 import org.elasticsearch.xpack.core.common.validation.SourceDestValidator.SourceDestValidation;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue.Level;
-import org.elasticsearch.xpack.core.security.authc.support.AuthenticationContextSerializer;
 import org.elasticsearch.xpack.core.security.xcontent.XContentUtils;
 import org.elasticsearch.xpack.core.transform.TransformConfigVersion;
 import org.elasticsearch.xpack.core.transform.TransformDeprecations;
@@ -43,7 +41,6 @@
 import java.io.IOException;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
@@ -52,7 +49,6 @@
 
 import static org.elasticsearch.xcontent.ConstructingObjectParser.constructorArg;
 import static org.elasticsearch.xcontent.ConstructingObjectParser.optionalConstructorArg;
-import static org.elasticsearch.xpack.core.security.authc.AuthenticationField.AUTHENTICATION_KEY;
 
 /**
  * This class holds the configuration details of a data frame transform
@@ -69,10 +65,6 @@ public final class TransformConfig implements SimpleDiffable<TransformConfig>, W
     public static final ParseField HEADERS = new ParseField("headers");
     /** Version in which {@code FieldCapabilitiesRequest.runtime_fields} field was introduced. */
     private static final TransportVersion FIELD_CAPS_RUNTIME_MAPPINGS_INTRODUCED_TRANSPORT_VERSION = TransportVersions.V_7_12_0;
-    private static final List<String> DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES = List.of(
-        "data_frame_transforms_admin",
-        "data_frame_transforms_user"
-    );
 
     /** Specifies all the possible transform functions. */
     public enum Function {
@@ -413,37 +405,9 @@ public List<DeprecationIssue> checkForDeprecations(NamedXContentRegistry namedXC
             retentionPolicyConfig.checkForDeprecations(getId(), namedXContentRegistry, deprecations::add);
         }
 
-        var deprecatedTransformRoles = getRolesFromHeaders().stream().filter(DEPRECATED_DATA_FRAME_TRANSFORMS_ROLES::contains).toList();
-        if (deprecatedTransformRoles.isEmpty() == false) {
-            deprecations.add(
-                new DeprecationIssue(
-                    Level.CRITICAL,
-                    "Transform [" + id + "] uses deprecated transform roles " + deprecatedTransformRoles,
-                    TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_BREAKING_CHANGES_URL,
-                    TransformDeprecations.DATA_FRAME_TRANSFORMS_ROLES_IS_DEPRECATED,
-                    false,
-                    null
-                )
-            );
-        }
-
         return deprecations;
     }
 
-    private List<String> getRolesFromHeaders() throws IOException {
-        if (headers == null) {
-            return Collections.emptyList();
-        }
-
-        var encodedAuthenticationHeader = ClientHelper.filterSecurityHeaders(headers).getOrDefault(AUTHENTICATION_KEY, "");
-        if (encodedAuthenticationHeader.isEmpty()) {
-            return Collections.emptyList();
-        }
-
-        var decodedAuthenticationHeader = AuthenticationContextSerializer.decode(encodedAuthenticationHeader);
-        return Arrays.asList(decodedAuthenticationHeader.getEffectiveSubject().getUser().roles());
-    }
-
     @Override
     public void writeTo(final StreamOutput out) throws IOException {
         out.writeString(id);