[8.10] [DOCS] Move x-pack docs to docs/reference dir (#99209) (#99495)

Author: jrodewig

docs/build.gradle

@@ -24,6 +24,9 @@ ext.docsFileTree = fileTree(projectDir) {
   exclude 'README.asciidoc'
   // Broken code snippet tests
   exclude 'reference/graph/explore.asciidoc'
+  // These files simply don't pass yet. We should figure out how to fix them.
+  exclude 'reference/watcher/reference/actions.asciidoc'
+  exclude 'reference/rest-api/security/ssl.asciidoc'
   if (BuildParams.inFipsJvm) {
     // We don't support this component in FIPS 140
     exclude 'reference/ingest/processors/attachment.asciidoc'
@@ -48,7 +51,13 @@ tasks.named("buildRestTests").configure {
           'reference/ml/anomaly-detection/apis/post-data.asciidoc',
           'reference/ml/anomaly-detection/apis/revert-snapshot.asciidoc',
           'reference/ml/anomaly-detection/apis/update-snapshot.asciidoc',
-          'reference/ml/anomaly-detection/apis/update-job.asciidoc'
+          'reference/ml/anomaly-detection/apis/update-job.asciidoc',
+          'reference/security/authentication/user-cache.asciidoc',
+          'reference/security/authorization/run-as-privilege.asciidoc',
+          'reference/security/ccs-clients-integrations/http.asciidoc',
+          'reference/rest-api/watcher/put-watch.asciidoc',
+          'reference/rest-api/watcher/stats.asciidoc',
+          'reference/watcher/example-watches/watching-time-series-data.asciidoc'
   ]
 }
 
@@ -74,7 +83,6 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
 
   // enable regexes in painless so our tests don't complain about example snippets that use them
   setting 'script.painless.regex.enabled', 'true'
-  setting 'xpack.security.enabled', 'false'
   setting 'path.repo', "${buildDir}/cluster/shared/repo"
   Closure configFile = {
     extraConfigFile it, file("src/test/cluster/config/$it")
@@ -102,6 +110,42 @@ testClusters.matching { it.name == "yamlRestTest"}.configureEach {
   requiresFeature 'es.index_mode_feature_flag_registered', Version.fromString("8.0.0")
   requiresFeature 'es.dlm_feature_flag_enabled', Version.fromString("8.8.0")
 
+  extraConfigFile 'op-jwks.json', project(':x-pack:test:idp-fixture').file("oidc/op-jwks.json")
+  extraConfigFile 'idp-docs-metadata.xml', project(':x-pack:test:idp-fixture').file("idp/shibboleth-idp/metadata/idp-docs-metadata.xml")
+  extraConfigFile 'testClient.crt', project(':x-pack:plugin:security').file("src/test/resources/org/elasticsearch/xpack/security/action/pki_delegation/testClient.crt")
+  setting 'xpack.security.enabled', 'true'
+  setting 'xpack.security.authc.api_key.enabled', 'true'
+  setting 'xpack.security.authc.token.enabled', 'true'
+  // disable the ILM history for doc tests to avoid potential lingering tasks that'd cause test flakiness
+  setting 'indices.lifecycle.history_index_enabled', 'false'
+  setting 'xpack.license.self_generated.type', 'trial'
+  setting 'xpack.security.authc.realms.file.file.order', '0'
+  setting 'xpack.security.authc.realms.native.native.order', '1'
+  setting 'xpack.security.authc.realms.oidc.oidc1.order', '2'
+  setting 'xpack.security.authc.realms.oidc.oidc1.op.issuer', 'http://127.0.0.1:8080'
+  setting 'xpack.security.authc.realms.oidc.oidc1.op.authorization_endpoint', "http://127.0.0.1:8080/c2id-login"
+  setting 'xpack.security.authc.realms.oidc.oidc1.op.token_endpoint', "http://127.0.0.1:8080/c2id/token"
+  setting 'xpack.security.authc.realms.oidc.oidc1.op.jwkset_path', 'op-jwks.json'
+  setting 'xpack.security.authc.realms.oidc.oidc1.rp.redirect_uri', 'https://my.fantastic.rp/cb'
+  setting 'xpack.security.authc.realms.oidc.oidc1.rp.client_id', 'elasticsearch-rp'
+  keystore 'xpack.security.authc.realms.oidc.oidc1.rp.client_secret', 'b07efb7a1cf6ec9462afe7b6d3ab55c6c7880262aa61ac28dded292aca47c9a2'
+  setting 'xpack.security.authc.realms.oidc.oidc1.rp.response_type', 'id_token'
+  setting 'xpack.security.authc.realms.oidc.oidc1.claims.principal', 'sub'
+  setting 'xpack.security.authc.realms.pki.pki1.order', '3'
+  setting 'xpack.security.authc.realms.pki.pki1.certificate_authorities', '[ "testClient.crt" ]'
+  setting 'xpack.security.authc.realms.pki.pki1.delegation.enabled', 'true'
+  setting 'xpack.security.authc.realms.saml.saml1.order', '4'
+  setting 'xpack.security.authc.realms.saml.saml1.sp.logout', 'https://kibana.org/logout'
+  setting 'xpack.security.authc.realms.saml.saml1.idp.entity_id', 'https://my-idp.org'
+  setting 'xpack.security.authc.realms.saml.saml1.idp.metadata.path', 'idp-docs-metadata.xml'
+  setting 'xpack.security.authc.realms.saml.saml1.sp.entity_id', 'https://kibana.org'
+  setting 'xpack.security.authc.realms.saml.saml1.sp.acs', 'https://kibana.org/api/security/saml/callback'
+  setting 'xpack.security.authc.realms.saml.saml1.attributes.principal', 'uid'
+  setting 'xpack.security.authc.realms.saml.saml1.attributes.name', 'urn:oid:2.5.4.3'
+
+  user username: 'test_admin'
+  user username: 'test_user'
+
   // build the cluster with all plugins
   project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { subproj ->
     /* Skip repositories. We just aren't going to be able to test them so it
@@ -1657,6 +1701,210 @@ setups['setup-snapshots'] = setups['setup-repository'] + '''
           body: |
 #atomic_red_data#
 '''
+
+  setups['my_inactive_watch'] = '''
+  - do:
+      watcher.put_watch:
+        id: "my_watch"
+        active: false
+        body:  >
+          {
+            "trigger": {
+              "schedule": {
+                "hourly": {
+                  "minute": [ 0, 5 ]
+                }
+              }
+            },
+            "input": {
+              "simple": {
+                "payload": {
+                  "send": "yes"
+                }
+              }
+            },
+            "condition": {
+              "always": {}
+            },
+            "actions": {
+              "test_index": {
+                "index": {
+                  "index": "test"
+                }
+              }
+            }
+          }
+  - match: { _id: "my_watch" }
+'''
+
+  setups['my_active_watch'] = setups['my_inactive_watch'].replace(
+        'active: false', 'active: true')
+
+  setups['role_mapping'] = '''
+  - do:
+        security.put_role_mapping:
+         name: "mapping1"
+         body:  >
+            {
+               "enabled": true,
+               "roles": [ "user" ],
+               "rules": { "field": { "username": "*" } }
+            }
+'''
+
+  setups['admin_role'] = '''
+  - do:
+      security.put_role:
+        name: "my_admin_role"
+        body: >
+          {
+            "cluster": ["all"],
+            "indices": [
+              {"names": ["index1", "index2" ], "privileges": ["all"], "field_security" : {"grant" : [ "title", "body" ]}}
+              ],
+            "run_as": [ "other_user" ],
+            "metadata" : {"version": 1}
+          }
+'''
+  setups['jacknich_user'] = '''
+  - do:
+      security.put_user:
+        username: "jacknich"
+        body:  >
+          {
+            "password" : "l0ng-r4nd0m-p@ssw0rd",
+            "roles" : [ "admin", "other_role1" ],
+            "full_name" : "Jack Nicholson",
+            "email" : "[email protected]",
+            "metadata" : { "intelligence" : 7 }
+          }
+  - do:
+      security.activate_user_profile:
+        body: >
+          {
+            "grant_type": "password",
+            "username": "jacknich",
+            "password" : "l0ng-r4nd0m-p@ssw0rd"
+          }
+'''
+  setups['app0102_privileges'] = '''
+  - do:
+        security.put_privileges:
+          body: >
+            {
+              "myapp": {
+                "read": {
+                  "application": "myapp",
+                  "name": "read",
+                  "actions": [
+                    "data:read/*",
+                    "action:login" ],
+                  "metadata": {
+                    "description": "Read access to myapp"
+                  }
+                }
+              }
+            }
+'''
+  setups['service_token42'] = '''
+  - do:
+      security.create_service_token:
+        namespace: elastic
+        service: fleet-server
+        name: token42
+'''
+  setups['user_profiles'] = '''
+  - do:
+      security.put_user:
+        username: "jacknich"
+        body:  >
+          {
+            "password" : "l0ng-r4nd0m-p@ssw0rd",
+            "roles" : [ "admin", "other_role1" ],
+            "full_name" : "Jack Nicholson",
+            "email" : "[email protected]"
+          }
+  - do:
+      security.put_user:
+        username: "jackrea"
+        body:  >
+          {
+            "password" : "l0ng-r4nd0m-p@ssw0rd",
+            "roles" : [ "admin" ],
+            "full_name" : "Jack Reacher",
+            "email" : "[email protected]"
+          }
+  - do:
+      security.put_user:
+        username: "jackspa"
+        body:  >
+          {
+            "password" : "l0ng-r4nd0m-p@ssw0rd",
+            "roles" : [ "user" ],
+            "full_name" : "Jack Sparrow",
+            "email" : "[email protected]"
+          }
+  - do:
+      security.activate_user_profile:
+        body: >
+          {
+            "grant_type": "password",
+            "username": "jacknich",
+            "password" : "l0ng-r4nd0m-p@ssw0rd"
+          }
+  - do:
+      security.activate_user_profile:
+        body: >
+          {
+            "grant_type": "password",
+            "username": "jackrea",
+            "password" : "l0ng-r4nd0m-p@ssw0rd"
+          }
+  - do:
+      security.activate_user_profile:
+        body: >
+          {
+            "grant_type": "password",
+            "username": "jackspa",
+            "password" : "l0ng-r4nd0m-p@ssw0rd"
+          }
+  # jacknich
+  - do:
+      security.update_user_profile_data:
+        uid: "u_79HkWkwmnBH5gqFKwoxggWPjEBOur1zLPXQPEl1VBW0_0"
+        body: >
+          {
+            "labels": {
+              "direction": "north"
+            },
+            "data": {
+              "app1": {
+                "key1": "value1"
+              }
+            }
+          }
+  # jackrea
+  - do:
+      security.update_user_profile_data:
+        uid: "u_P_0BMHgaOK3p7k-PFWUCbw9dQ-UFjt01oWJ_Dp2PmPc_0"
+        body: >
+          {
+            "labels": {
+              "direction": "west"
+            }
+          }
+  # jackspa
+  - do:
+      security.update_user_profile_data:
+        uid: "u_8RKO7AKfEbSiIHZkZZ2LJy2MUSDPWDr3tMI_CkIGApU_0"
+        body: >
+          {
+            "labels": {
+              "direction": "south"
+            }
+          }
+'''
+
   // fake data used by the correlation bucket agg
   buildRestTests.setups['correlate_latency'] = '''
   - do:

docs/reference/ccr/getting-started.asciidoc

@@ -159,7 +159,7 @@ cluster with cluster alias `leader`.
     connected to.
 ====
 
-include::../../../x-pack/docs/en/security/authentication/remote-clusters-privileges-cert.asciidoc[tag=configure-ccr-privileges]
+include::{es-repo-dir}/security/authentication/remote-clusters-privileges-cert.asciidoc[tag=configure-ccr-privileges]
 
 [[ccr-getting-started-follower-index]]
 ==== Create a follower index to replicate a specific index

docs/reference/data-streams/set-up-a-data-stream.asciidoc

@@ -261,7 +261,7 @@ PUT _data_stream/my-data-stream
 [[secure-data-stream]]
 === Secure the data stream
 
-include::{xes-repo-dir}/security/authorization/alias-privileges.asciidoc[tag=data-stream-security]
+include::{es-repo-dir}/security/authorization/alias-privileges.asciidoc[tag=data-stream-security]
 
 For an example, see <<data-stream-privileges>>.
 

docs/reference/docs/reindex.asciidoc

@@ -984,9 +984,9 @@ POST _reindex
 --------------------------------------------------
 // TEST[setup:host]
 // TEST[s/^/PUT my-index-000001\n/]
-// TEST[s/otherhost:9200",/\${host}"/]
-// TEST[s/"username": "user",//]
-// TEST[s/"password": "pass"//]
+// TEST[s/otherhost:9200",/\${host}",/]
+// TEST[s/"username": "user",/"username": "test_admin",/]
+// TEST[s/"password": "pass"/"password": "x-pack-test-password"/]
 
 The `host` parameter must contain a scheme, host, port (e.g.
 `https://otherhost:9200`), and optional path (e.g. `https://otherhost:9200/proxy`).
@@ -1025,7 +1025,7 @@ POST _reindex
 // TEST[setup:host]
 // TEST[s/^/PUT my-index-000001\n/]
 // TEST[s/otherhost:9200",/\${host}",/]
-// TEST[s/API_KEY_VALUE" /\93116930-2ecb-4161-aa5e-4f3586c87ac6"/]
+// TEST[s/"headers": \{[^}]*\}/"username": "test_admin", "password": "x-pack-test-password"/]
 
 Remote hosts have to be explicitly allowed in `elasticsearch.yml` using the
 `reindex.remote.whitelist` property. It can be set to a comma delimited list
@@ -1064,7 +1064,8 @@ POST _reindex
 {
   "source": {
     "remote": {
-      "host": "http://otherhost:9200"
+      "host": "http://otherhost:9200",
+      ...
     },
     "index": "source",
     "size": 10,
@@ -1082,6 +1083,7 @@ POST _reindex
 // TEST[setup:host]
 // TEST[s/^/PUT source\n/]
 // TEST[s/otherhost:9200/\${host}/]
+// TEST[s/\.\.\./"username": "test_admin", "password": "x-pack-test-password"/]
 
 It is also possible to set the socket read timeout on the remote connection
 with the `socket_timeout` field and the connection timeout with the
@@ -1096,6 +1098,7 @@ POST _reindex
   "source": {
     "remote": {
       "host": "http://otherhost:9200",
+      ...,
       "socket_timeout": "1m",
       "connect_timeout": "10s"
     },
@@ -1114,6 +1117,7 @@ POST _reindex
 // TEST[setup:host]
 // TEST[s/^/PUT source\n/]
 // TEST[s/otherhost:9200/\${host}/]
+// TEST[s/\.\.\.,/"username": "test_admin", "password": "x-pack-test-password",/]
 
 [[reindex-ssl]]
 ===== Configuring SSL parameters

docs/reference/index.asciidoc

@@ -61,9 +61,9 @@ include::high-availability.asciidoc[]
 
 include::snapshot-restore/index.asciidoc[]
 
-include::{xes-repo-dir}/security/index.asciidoc[]
+include::security/index.asciidoc[]
 
-include::{xes-repo-dir}/watcher/index.asciidoc[]
+include::watcher/index.asciidoc[]
 
 include::commands/index.asciidoc[]
 

docs/reference/indices/shard-stores.asciidoc

@@ -192,8 +192,8 @@ The API returns the following response:
 // TESTRESPONSE[s/"attributes": \{[^}]*\}/"attributes": $body.$_path/]
 // TESTRESPONSE[s/"roles": \[[^]]*\]/"roles": $body.$_path/]
 // TESTRESPONSE[s/"8.10.0"/\$node_version/]
-// TESTRESPONSE[s/"7000099"/"\d+"/]
-// TESTRESPONSE[s/"8100099"/"\d+"/]
+// TESTRESPONSE[s/"min_index_version": 7000099/"min_index_version": $body.$_path/]
+// TESTRESPONSE[s/"max_index_version": 8100099/"max_index_version": $body.$_path/]
 
 
 

docs/reference/ml/anomaly-detection/apis/put-job.asciidoc

@@ -536,4 +536,4 @@ The API returns the following results:
 ----
 // TESTRESPONSE[s/"job_version" : "8.4.0"/"job_version" : $body.job_version/]
 // TESTRESPONSE[s/1656087283340/$body.$_path/]
-// TESTRESPONSE[s/"authorization" : \{[^}]*\},//]
+// TESTRESPONSE[s/"superuser"/"_es_test_root"/]