Merge remote-tracking branch 'upstream/main' into entitlements/missing-url-connection-5

Author: ldematte

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -133,7 +133,7 @@ develocity {
         }
       } else {
         tag 'LOCAL'
-        if (providers.systemProperty('idea.active').present) {
+        if (System.getProperty('idea.active') == 'true') {
           tag 'IDEA'
         }
       }

docs/README.md

@@ -17,7 +17,7 @@ Docs live in **three places**:
 
 1. **Reference content** lives in this repo. This covers low-level stuff like settings and configuration information that is tightly coupled to code.
 - 👩🏽‍💻 **Engineers** own the bulk of this content.
-2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification)
+2.  **API reference docs** live in the [Elasticsearch specification](https://github.com/elastic/elasticsearch-specification/blob/main/README.md#how-to-generate-the-openapi-representation)
     - This is where you need to update API docs published in the [new API docs system](https://www.elastic.co/docs/api/doc/elasticsearch/v8/)
 - 👩🏽‍💻 **Engineers** own this content.
 3. **Narrative, overview, and conceptual content** mostly lives in the [`docs-content`](https://github.com/elastic/docs-content/) repo. 
@@ -106,4 +106,4 @@ This allows slightly more expressive testing of the snippets. Since that syntax
 % TEST[s/\n$/\nstartyaml\n  - compare_analyzers: {index: thai_example, first: thai, second: rebuilt_thai}\nendyaml\n/]
 ```
 
-Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.
+Any place you can use json you can use elements like `$body.path.to.thing` which is replaced on the fly with the contents of the thing at `path.to.thing` in the last response.

plugins/discovery-ec2/build.gradle

@@ -46,13 +46,12 @@ esplugin.bundleSpec.from('config/discovery-ec2') {
 }
 
 tasks.register("writeTestJavaPolicy") {
+  boolean inFips = buildParams.inFipsJvm
+  inputs.property("inFipsJvm", inFips)
+  final File javaPolicy = new File(layout.buildDirectory.asFile.get(), "tmp/java.policy")
+  outputs.file(javaPolicy)
   doLast {
-    final File tmp = file("${buildDir}/tmp")
-    if (tmp.exists() == false && tmp.mkdirs() == false) {
-      throw new GradleException("failed to create temporary directory [${tmp}]")
-    }
-    final File javaPolicy = file("${tmp}/java.policy")
-    if (buildParams.inFipsJvm) {
+    if (inFips) {
       javaPolicy.write(
         [
           "grant {",
@@ -95,9 +94,9 @@ tasks.withType(Test).configureEach {
   // this is needed to manipulate com.amazonaws.sdk.ec2MetadataServiceEndpointOverride system property
   // it is better rather disable security manager at all with `systemProperty 'tests.security.manager', 'false'`
   if (buildParams.inFipsJvm){
-    nonInputProperties.systemProperty 'java.security.policy', "=file://${buildDir}/tmp/java.policy"
+    nonInputProperties.systemProperty 'java.security.policy', "=file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   } else {
-    nonInputProperties.systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
+    nonInputProperties.systemProperty 'java.security.policy', "file://${layout.buildDirectory.asFile.get()}/tmp/java.policy"
   }
 }
 

plugins/examples/security-authorization-engine/src/main/java/org/elasticsearch/example/CustomAuthorizationEngine.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.example;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.SubscribableListener;
 import org.elasticsearch.cluster.metadata.IndexAbstraction;
 import org.elasticsearch.cluster.metadata.ProjectMetadata;
 import org.elasticsearch.xpack.core.security.action.user.GetUserPrivilegesResponse;
@@ -86,14 +87,14 @@ public void authorizeClusterAction(RequestInfo requestInfo, AuthorizationInfo au
     }
 
     @Override
-    public void authorizeIndexAction(
+    SubscribableListener<IndexAuthorizationResult> void authorizeIndexAction(
         RequestInfo requestInfo,
         AuthorizationInfo authorizationInfo,
         AsyncSupplier<ResolvedIndices> indicesAsyncSupplier,
-        ProjectMetadata project,
-        ActionListener<IndexAuthorizationResult> listener
+        ProjectMetadata project
     ) {
         if (isSuperuser(requestInfo.getAuthentication().getEffectiveSubject().getUser())) {
+            ActionListener<IndexAuthorizationResult> listener = new SubscribableListener<>();
             indicesAsyncSupplier.getAsync(ActionListener.wrap(resolvedIndices -> {
                 Map<String, IndexAccessControl> indexAccessControlMap = new HashMap<>();
                 for (String name : resolvedIndices.getLocal()) {
@@ -103,8 +104,9 @@ public void authorizeIndexAction(
                     new IndicesAccessControl(true, Collections.unmodifiableMap(indexAccessControlMap));
                 listener.onResponse(new IndexAuthorizationResult(indicesAccessControl));
             }, listener::onFailure));
+            return listener;
         } else {
-            listener.onResponse(new IndexAuthorizationResult(IndicesAccessControl.DENIED));
+            return SubscribableListener.succcess(new IndexAuthorizationResult(IndicesAccessControl.DENIED));
         }
     }
 

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -57,7 +57,6 @@ static TransportVersion def(int id) {
     public static final TransportVersion V_7_3_0 = def(7_03_00_99);
     public static final TransportVersion V_7_3_2 = def(7_03_02_99);
     public static final TransportVersion V_7_4_0 = def(7_04_00_99);
-    public static final TransportVersion V_7_6_0 = def(7_06_00_99);
     public static final TransportVersion V_7_8_0 = def(7_08_00_99);
     public static final TransportVersion V_7_8_1 = def(7_08_01_99);
     public static final TransportVersion V_7_9_0 = def(7_09_00_99);

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineRequest.java

@@ -11,6 +11,7 @@
 
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionRequestValidationException;
+import org.elasticsearch.cluster.metadata.ProjectId;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.bytes.ReleasableBytesReference;
 import org.elasticsearch.common.io.stream.StreamInput;
@@ -130,6 +131,7 @@ record Parsed(Pipeline pipeline, List<IngestDocument> documents, boolean verbose
     static final String SIMULATED_PIPELINE_ID = "_simulate_pipeline";
 
     static Parsed parseWithPipelineId(
+        ProjectId projectId,
         String pipelineId,
         Map<String, Object> config,
         boolean verbose,
@@ -139,7 +141,7 @@ static Parsed parseWithPipelineId(
         if (pipelineId == null) {
             throw new IllegalArgumentException("param [pipeline] is null");
         }
-        Pipeline pipeline = ingestService.getPipeline(pipelineId);
+        Pipeline pipeline = ingestService.getPipeline(projectId, pipelineId);
         if (pipeline == null) {
             throw new IllegalArgumentException("pipeline [" + pipelineId + "] does not exist");
         }

server/src/main/java/org/elasticsearch/action/ingest/SimulatePipelineTransportAction.java

@@ -17,6 +17,7 @@
 import org.elasticsearch.action.support.HandledTransportAction;
 import org.elasticsearch.cluster.node.DiscoveryNode;
 import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.cluster.project.ProjectResolver;
 import org.elasticsearch.common.Randomness;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.util.concurrent.EsExecutors;
@@ -49,6 +50,7 @@ public class SimulatePipelineTransportAction extends HandledTransportAction<Simu
     private final IngestService ingestService;
     private final SimulateExecutionService executionService;
     private final TransportService transportService;
+    private final ProjectResolver projectResolver;
     private volatile TimeValue ingestNodeTransportActionTimeout;
     // ThreadLocal because our unit testing framework does not like sharing Randoms across threads
     private final ThreadLocal<Random> random = ThreadLocal.withInitial(Randomness::get);
@@ -58,7 +60,8 @@ public SimulatePipelineTransportAction(
         ThreadPool threadPool,
         TransportService transportService,
         ActionFilters actionFilters,
-        IngestService ingestService
+        IngestService ingestService,
+        ProjectResolver projectResolver
     ) {
         super(
             SimulatePipelineAction.NAME,
@@ -70,6 +73,7 @@ public SimulatePipelineTransportAction(
         this.ingestService = ingestService;
         this.executionService = new SimulateExecutionService(threadPool);
         this.transportService = transportService;
+        this.projectResolver = projectResolver;
         this.ingestNodeTransportActionTimeout = INGEST_NODE_TRANSPORT_ACTION_TIMEOUT.get(ingestService.getClusterService().getSettings());
         ingestService.getClusterService()
             .getClusterSettings()
@@ -96,9 +100,11 @@ protected void doExecute(Task task, SimulatePipelineRequest request, ActionListe
         }
         try {
             if (discoveryNodes.getLocalNode().isIngestNode()) {
+                final var projectId = projectResolver.getProjectId();
                 final SimulatePipelineRequest.Parsed simulateRequest;
                 if (request.getId() != null) {
                     simulateRequest = SimulatePipelineRequest.parseWithPipelineId(
+                        projectId,
                         request.getId(),
                         source,
                         request.isVerbose(),

server/src/main/java/org/elasticsearch/action/support/SubscribableListener.java

@@ -266,6 +266,13 @@ public final boolean isDone() {
         return isDone(state);
     }
 
+    /**
+     * @return return {@code true} if and only if this listener is done and has been completed successfully
+     */
+    public final boolean isSuccess() {
+        return state instanceof SuccessResult;
+    }
+
     /**
      * @return the result with which this listener completed successfully, or throw the exception with which it failed.
      *

server/src/main/java/org/elasticsearch/cluster/block/ClusterBlocks.java

@@ -438,7 +438,7 @@ private static void writeBlockSet(Set<ClusterBlock> blocks, StreamOutput out) th
     public static ClusterBlocks readFrom(StreamInput in) throws IOException {
         if (in.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)) {
             final Set<ClusterBlock> global = readBlockSet(in);
-            final Map<ProjectId, ProjectBlocks> projectBlocksMap = in.readImmutableMap(ProjectId::new, ProjectBlocks::readFrom);
+            final Map<ProjectId, ProjectBlocks> projectBlocksMap = in.readImmutableMap(ProjectId::readFrom, ProjectBlocks::readFrom);
             if (global.isEmpty()
                 && noProjectOrDefaultProjectOnly(projectBlocksMap)
                 && projectBlocksMap.getOrDefault(Metadata.DEFAULT_PROJECT_ID, ProjectBlocks.EMPTY).indices().isEmpty()) {

server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java

@@ -88,7 +88,7 @@ public class Metadata implements Diffable<Metadata>, ChunkedToXContent {
     public static final String UNKNOWN_CLUSTER_UUID = "_na_";
     // TODO multi-project: verify that usages are really expected to work on the default project only,
     // and that they are not a stop-gap solution to make the tests pass
-    public static final ProjectId DEFAULT_PROJECT_ID = new ProjectId("default");
+    public static final ProjectId DEFAULT_PROJECT_ID = ProjectId.DEFAULT;
 
     public enum XContentContext {
         /* Custom metadata should be returned as part of API call */
@@ -1154,7 +1154,7 @@ public static Metadata readFrom(StreamInput in) throws IOException {
                 builder.put(ReservedStateMetadata.readFrom(in));
             }
 
-            builder.projectMetadata(in.readMap(ProjectId::new, ProjectMetadata::readFrom));
+            builder.projectMetadata(in.readMap(ProjectId::readFrom, ProjectMetadata::readFrom));
         }
         return builder.build();
     }