Fix ApiKeyBackwardsCompatibilityIT, fix update error message check

Author: gmjehovich

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/action/apikey/ApiKey.java

@@ -435,7 +435,7 @@ public String toString() {
             + roleDescriptors
             + ", limited_by="
             + limitedBy
-            + ", certificate_identity"
+            + ", certificate_identity="
             + certificateIdentity
             + "]";
     }

x-pack/plugin/security/qa/security-trial/src/javaRestTest/java/org/elasticsearch/xpack/security/apikey/ApiKeyRestIT.java

@@ -1695,7 +1695,10 @@ public void testUpdateFailureCases() throws IOException {
         updateRequest.setJsonEntity("{}");
         final ResponseException e2 = expectThrows(ResponseException.class, () -> client().performRequest(updateRequest));
         assertThat(e2.getResponse().getStatusLine().getStatusCode(), equalTo(400));
-        assertThat(e2.getMessage(), containsString("must update either [access] or [metadata] for cross-cluster API keys"));
+        assertThat(
+            e2.getMessage(),
+            containsString("must update [access] or [metadata] or [certificate_identity] for cross-cluster API keys")
+        );
 
         // Access cannot be empty
         updateRequest.setJsonEntity("{\"access\":{}}");

x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/authc/ApiKeyService.java

@@ -640,7 +640,7 @@ private String getCertificateIdentityFromRequest(AbstractCreateApiKeyRequest req
                 );
                 throw new ElasticsearchException(
                     "API key creation failed. The cluster is in a mixed-version state and does not yet "
-                        + "support the certificate_identity field. Please retry after the upgrade is complete."
+                        + "support the [certificate_identity] field. Please retry after the upgrade is complete."
                 );
             }
         }

x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/ApiKeyBackwardsCompatibilityIT.java

@@ -217,13 +217,35 @@ public void testCertificateIdentityBackwardsCompatibility() throws Exception {
                 );
             }
             case MIXED -> {
-                // Mixed cluster should reject certificate identity due to feature gating
-                var exception = expectThrows(Exception.class, () -> createCrossClusterApiKeyWithCertIdentity("CN=test-.*"));
-                assertThat(
-                    exception.getMessage(),
-                    containsString("cluster is in a mixed-version state and does not yet support the certificate_identity field")
-                );
+                try {
+                    this.createClientsByVersion();
+
+                    Exception oldNodeException = expectThrows(
+                        Exception.class,
+                        () -> createCrossClusterApiKeyWithCertIdentity(oldVersionClient, "CN=test-.*")
+                    );
+                    assertThat(
+                        oldNodeException.getMessage(),
+                        anyOf(containsString("unknown field [certificate_identity]"), containsString("certificate_identity not supported"))
+                    );
+
+                    // Test against new node - should get mixed-version error
+                    Exception newNodeException = expectThrows(
+                        Exception.class,
+                        () -> createCrossClusterApiKeyWithCertIdentity(newVersionClient, "CN=test-.*")
+                    );
+                    assertThat(
+                        newNodeException.getMessage(),
+                        containsString(
+                            "API key creation failed. The cluster is in a mixed-version state and does not yet "
+                                + "support the [certificate_identity] field. Please retry after the upgrade is complete."
+                        )
+                    );
+                } finally {
+                    this.closeClientsByVersion();
+                }
             }
+
             case UPGRADED -> {
                 // Fully upgraded cluster should support certificate identity
                 final Tuple<String, String> apiKey = createCrossClusterApiKeyWithCertIdentity("CN=test-.*");
@@ -463,6 +485,11 @@ private void assertQuery(RestClient restClient, String body, Consumer<List<Map<S
     }
 
     private Tuple<String, String> createCrossClusterApiKeyWithCertIdentity(String certificateIdentity) throws IOException {
+        return createCrossClusterApiKeyWithCertIdentity(client(), certificateIdentity);
+    }
+
+    private Tuple<String, String> createCrossClusterApiKeyWithCertIdentity(RestClient client, String certificateIdentity)
+        throws IOException {
         final String name = "test-cc-api-key-" + randomAlphaOfLengthBetween(3, 5);
         final Request createApiKeyRequest = new Request("POST", "/_security/cross_cluster/api_key");
         createApiKeyRequest.setJsonEntity(Strings.format("""
@@ -478,7 +505,7 @@ private Tuple<String, String> createCrossClusterApiKeyWithCertIdentity(String ce
                 }
             }""", name, certificateIdentity));
 
-        final Response createResponse = client().performRequest(createApiKeyRequest);
+        final Response createResponse = client.performRequest(createApiKeyRequest);
         assertOK(createResponse);
         final ObjectPath path = ObjectPath.createFromResponse(createResponse);
         final String id = path.evaluate("id");
@@ -487,5 +514,4 @@ private Tuple<String, String> createCrossClusterApiKeyWithCertIdentity(String ce
         assertThat(key, notNullValue());
         return Tuple.tuple(id, key);
     }
-
 }