Project routing should not impact error handling for index resolution (elastic#139177)

Project routing defines the world for which the flat-world index
resolution is applied. Therefore it should not impact the error handling
which is decided by whether the expression is qualified or unqualified.
For example, both `logs` and `logs` + `_alias:*` are successful if logs
is found on any project, while `*:logs` with or without project routing
requires it to be found on all projects.

Author: ywangd

server/src/main/java/org/elasticsearch/search/crossproject/CrossProjectIndexExpressionsRewriter.java

@@ -117,6 +117,8 @@ public static IndexRewriteResult rewriteIndexExpression(
             rewrittenExpression = rewriteUnqualifiedExpression(indexExpression, originProjectAlias, allProjectAliases);
             logger.debug("Rewrote unqualified expression [{}] to [{}]", indexExpression, rewrittenExpression);
         }
+        // Empty rewritten expressions should have been thrown earlier
+        assert false == rewrittenExpression.isEmpty() : "rewritten index expression must not be empty";
         return rewrittenExpression;
     }
 
@@ -221,5 +223,9 @@ public record IndexRewriteResult(@Nullable String localExpression, Set<String> r
         public IndexRewriteResult(String localExpression) {
             this(localExpression, Set.of());
         }
+
+        public boolean isEmpty() {
+            return localExpression == null && remoteExpressions.isEmpty();
+        }
     }
 }

server/src/main/java/org/elasticsearch/search/crossproject/CrossProjectIndexResolutionValidator.java

@@ -97,8 +97,8 @@ public static ElasticsearchException validate(
             String originalExpression = localResolvedIndices.original();
             logger.debug("Checking replaced expression for original expression [{}]", originalExpression);
 
-            // Check if this is a qualified resource (project:index pattern) or has project routing
-            boolean isQualifiedExpression = hasProjectRouting || RemoteClusterAware.isRemoteIndexName(originalExpression);
+            // Check if this is a qualified resource (project:index pattern)
+            boolean isQualifiedExpression = RemoteClusterAware.isRemoteIndexName(originalExpression);
 
             Set<String> remoteExpressions = localResolvedIndices.remoteExpressions();
             ResolvedIndexExpression.LocalExpressions localExpressions = localResolvedIndices.localExpressions();
@@ -129,11 +129,16 @@ public static ElasticsearchException validate(
                     originalExpression,
                     indicesOptions
                 );
-                if (localException == null) {
+                if (localException == null && localExpressions != ResolvedIndexExpression.LocalExpressions.NONE) {
                     // found locally, continue to next expression
                     continue;
                 }
-                boolean isUnauthorized = localException instanceof ElasticsearchSecurityException;
+                ElasticsearchSecurityException unauthorizedException = null;
+                if (localException instanceof ElasticsearchSecurityException securityException) {
+                    unauthorizedException = securityException;
+                }
+                assert localExpressions != ResolvedIndexExpression.LocalExpressions.NONE || false == remoteExpressions.isEmpty()
+                    : "both local expression and remote expressions are empty which should have errored earlier at index rewriting time";
                 boolean foundFlat = false;
                 // checking if flat expression matched remotely
                 for (String remoteExpression : remoteExpressions) {
@@ -150,17 +155,26 @@ public static ElasticsearchException validate(
                         foundFlat = true;
                         break;
                     }
-                    if (false == isUnauthorized && exception instanceof ElasticsearchSecurityException) {
-                        isUnauthorized = true;
+                    if (unauthorizedException == null && exception instanceof ElasticsearchSecurityException securityException) {
+                        unauthorizedException = securityException;
                     }
                 }
                 if (foundFlat) {
                     continue;
                 }
-                if (isUnauthorized) {
+                // Prefer reporting 403 over 404
+                if (unauthorizedException != null) {
+                    return unauthorizedException;
+                }
+                // Prefer reporting 404 on origin over linked projects
+                if (localException != null) {
+                    assert localException instanceof IndexNotFoundException
+                        : "Expected local exception to be IndexNotFoundException, but found: " + localException;
                     return localException;
+                } else {
+                    assert false == remoteExpressions.isEmpty() : "expected remote expressions to be non-empty";
+                    return new IndexNotFoundException(remoteExpressions.iterator().next());
                 }
-                return new IndexNotFoundException(originalExpression);
             }
         }
         // if we didn't throw before it means that we can proceed with the request