Skip to content

Commit 1a07c2f

Browse files
moschemosche
committed
Merge branch 'main' into entitlements/test_rule
2 parents 5eb39d8 + 63258f4 commit 1a07c2f

File tree

3 files changed

+133
-88
lines changed

3 files changed

+133
-88
lines changed

‎docs/reference/query-languages/esql/_snippets/commands/layout/completion.md‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,39 @@ including:
4646
**Requirements**
4747

4848
To use this command, you must deploy your LLM model in Elasticsearch as
49-
an [≈inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
49+
an [inference endpoint](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-inference-put) with the
5050
task type `completion`.
5151

52+
#### Handling timeouts
53+
54+
`COMPLETION` commands may time out when processing large datasets or complex prompts. The default timeout is 10 minutes, but you can increase this limit if necessary.
55+
56+
How you increase the timeout depends on your deployment type:
57+
58+
::::{tab-set}
59+
:::{tab-item} {{ech}}
60+
* You can adjust {{es}} settings in the [Elastic Cloud Console](docs-content://deploy-manage/deploy/elastic-cloud/edit-stack-settings.md)
61+
* You can also adjust the `search.default_search_timeout` cluster setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
62+
:::
63+
64+
:::{tab-item} Self-managed
65+
* You can configure at the cluster level by setting `search.default_search_timeout` in `elasticsearch.yml` or updating via [Cluster Settings API](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-cluster-put-settings)
66+
* You can also adjust the `search:timeout` setting using [Kibana's Advanced settings](kibana://reference/advanced-settings.md#kibana-search-settings)
67+
* Alternatively, you can add timeout parameters to individual queries
68+
:::
69+
70+
:::{tab-item} {{serverless-full}}
71+
* Requires a manual override from Elastic Support because you cannot modify timeout settings directly
72+
:::
73+
::::
74+
75+
If you don't want to increase the timeout limit, try the following:
76+
77+
* Reduce data volume with `LIMIT` or more selective filters before the `COMPLETION` command
78+
* Split complex operations into multiple simpler queries
79+
* Configure your HTTP client's response timeout (Refer to [HTTP client configuration](/reference/elasticsearch/configuration-reference/networking-settings.md#_http_client_configuration))
80+
81+
5282
**Examples**
5383

5484
Use the default column name (results stored in `completion` column):

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/Security.java‎

Lines changed: 4 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,6 @@
1616
import org.elasticsearch.ElasticsearchException;
1717
import org.elasticsearch.ElasticsearchSecurityException;
1818
import org.elasticsearch.ElasticsearchStatusException;
19-
import org.elasticsearch.ExceptionsHelper;
20-
import org.elasticsearch.ResourceAlreadyExistsException;
2119
import org.elasticsearch.TransportVersion;
2220
import org.elasticsearch.action.ActionListener;
2321
import org.elasticsearch.action.ActionResponse;
@@ -30,7 +28,6 @@
3028
import org.elasticsearch.cluster.ClusterState;
3129
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
3230
import org.elasticsearch.cluster.metadata.IndexTemplateMetadata;
33-
import org.elasticsearch.cluster.metadata.ProjectId;
3431
import org.elasticsearch.cluster.node.DiscoveryNode;
3532
import org.elasticsearch.cluster.node.DiscoveryNodes;
3633
import org.elasticsearch.cluster.project.ProjectResolver;
@@ -61,11 +58,9 @@
6158
import org.elasticsearch.common.util.concurrent.ThreadContext;
6259
import org.elasticsearch.common.util.concurrent.ThrottledTaskRunner;
6360
import org.elasticsearch.common.util.set.Sets;
64-
import org.elasticsearch.core.FixForMultiProject;
6561
import org.elasticsearch.core.IOUtils;
6662
import org.elasticsearch.core.Nullable;
6763
import org.elasticsearch.core.Releasable;
68-
import org.elasticsearch.core.TimeValue;
6964
import org.elasticsearch.env.Environment;
7065
import org.elasticsearch.features.FeatureService;
7166
import org.elasticsearch.features.NodeFeature;
@@ -224,7 +219,6 @@
224219
import org.elasticsearch.xpack.core.security.authz.store.ReservedRolesStore;
225220
import org.elasticsearch.xpack.core.security.authz.store.RoleRetrievalResult;
226221
import org.elasticsearch.xpack.core.security.support.Automatons;
227-
import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
228222
import org.elasticsearch.xpack.core.security.user.AnonymousUser;
229223
import org.elasticsearch.xpack.core.ssl.SSLConfigurationSettings;
230224
import org.elasticsearch.xpack.core.ssl.SSLService;
@@ -419,8 +413,6 @@
419413
import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesProviderFactory;
420414
import org.elasticsearch.xpack.security.support.QueryableBuiltInRolesSynchronizer;
421415
import org.elasticsearch.xpack.security.support.ReloadableSecurityComponent;
422-
import org.elasticsearch.xpack.security.support.SecurityIndexManager;
423-
import org.elasticsearch.xpack.security.support.SecurityMigrationExecutor;
424416
import org.elasticsearch.xpack.security.support.SecurityMigrations;
425417
import org.elasticsearch.xpack.security.support.SecuritySystemIndices;
426418
import org.elasticsearch.xpack.security.transport.SecurityHttpSettings;
@@ -449,7 +441,6 @@
449441
import java.util.Objects;
450442
import java.util.Set;
451443
import java.util.concurrent.Executor;
452-
import java.util.concurrent.atomic.AtomicInteger;
453444
import java.util.function.BiConsumer;
454445
import java.util.function.Function;
455446
import java.util.function.Predicate;
@@ -487,8 +478,6 @@ public class Security extends Plugin
487478

488479
public static final String SECURITY_CRYPTO_THREAD_POOL_NAME = XPackField.SECURITY + "-crypto";
489480

490-
private static final int MAX_SECURITY_MIGRATION_RETRY_COUNT = 10;
491-
492481
// TODO: ip filtering does not actually track license usage yet
493482
public static final LicensedFeature.Momentary IP_FILTERING_FEATURE = LicensedFeature.momentaryLenient(
494483
null,
@@ -628,7 +617,6 @@ public class Security extends Plugin
628617
private final SetOnce<GetBuiltinPrivilegesResponseTranslator> getBuiltinPrivilegesResponseTranslator = new SetOnce<>();
629618
private final SetOnce<HasPrivilegesRequestBuilderFactory> hasPrivilegesRequestBuilderFactory = new SetOnce<>();
630619

631-
private final SetOnce<PersistentTasksService> persistentTasksService = new SetOnce<>();
632620
private final SetOnce<FileRolesStore> fileRolesStore = new SetOnce<>();
633621
private final SetOnce<OperatorPrivileges.OperatorPrivilegesService> operatorPrivilegesService = new SetOnce<>();
634622
private final SetOnce<ReservedRoleMappingAction> reservedRoleMappingAction = new SetOnce<>();
@@ -640,13 +628,8 @@ public class Security extends Plugin
640628
private final SetOnce<FileRoleValidator> fileRoleValidator = new SetOnce<>();
641629
private final SetOnce<SecondaryAuthActions> secondaryAuthActions = new SetOnce<>();
642630
private final SetOnce<QueryableBuiltInRolesProviderFactory> queryableRolesProviderFactory = new SetOnce<>();
643-
private final SetOnce<SecurityMigrationExecutor> securityMigrationExecutor = new SetOnce<>();
644-
645-
// Node local retry count for migration jobs that's checked only on the master node to make sure
646-
// submit migration jobs doesn't get out of hand and retries forever if they fail. Reset by a
647-
// restart or master node change.
648-
private final AtomicInteger nodeLocalMigrationRetryCount = new AtomicInteger(0);
649631

632+
private final SetOnce<SecurityMigrations.Manager> migrationManager = new SetOnce<>();
650633
private final SetOnce<List<Closeable>> closableComponents = new SetOnce<>();
651634

652635
public Security(Settings settings) {
@@ -796,23 +779,7 @@ Collection<Object> createComponents(
796779

797780
systemIndices.init(client, featureService, clusterService, projectResolver);
798781

799-
this.securityMigrationExecutor.set(
800-
new SecurityMigrationExecutor(
801-
SecurityMigrationTaskParams.TASK_NAME,
802-
threadPool.executor(ThreadPool.Names.MANAGEMENT),
803-
systemIndices.getMainIndexManager(),
804-
client,
805-
SecurityMigrations.MIGRATIONS_BY_VERSION
806-
)
807-
);
808-
this.persistentTasksService.set(persistentTasksService);
809-
810-
systemIndices.getMainIndexManager().addStateListener((projectId, oldState, newState) -> {
811-
// Only consider applying migrations if it's the master node and the security index exists
812-
if (clusterService.state().nodes().isLocalNodeElectedMaster() && newState.indexExists()) {
813-
applyPendingSecurityMigrations(projectId, newState);
814-
}
815-
});
782+
this.migrationManager.set(new SecurityMigrations.Manager(clusterService, persistentTasksService, systemIndices));
816783

817784
scriptServiceReference.set(scriptService);
818785
// We need to construct the checks here while the secure settings are still available.
@@ -1363,57 +1330,6 @@ private static boolean isInternalExtension(SecurityExtension extension) {
13631330
return canonicalName.startsWith("org.elasticsearch.xpack.") || canonicalName.startsWith("co.elastic.elasticsearch.");
13641331
}
13651332

1366-
@FixForMultiProject
1367-
// TODO : The migration task needs to be project aware
1368-
private void applyPendingSecurityMigrations(ProjectId projectId, SecurityIndexManager.IndexState newState) {
1369-
// If no migrations have been applied and the security index is on the latest version (new index), all migrations can be skipped
1370-
if (newState.migrationsVersion == 0 && newState.createdOnLatestVersion) {
1371-
submitPersistentMigrationTask(SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey(), false);
1372-
return;
1373-
}
1374-
1375-
Map.Entry<Integer, SecurityMigrations.SecurityMigration> nextMigration = SecurityMigrations.MIGRATIONS_BY_VERSION.higherEntry(
1376-
newState.migrationsVersion
1377-
);
1378-
1379-
// Check if next migration that has not been applied is eligible to run on the current cluster
1380-
if (nextMigration == null
1381-
|| systemIndices.getMainIndexManager().getProject(projectId).isEligibleSecurityMigration(nextMigration.getValue()) == false) {
1382-
// Reset retry counter if all eligible migrations have been applied successfully
1383-
nodeLocalMigrationRetryCount.set(0);
1384-
} else if (nodeLocalMigrationRetryCount.get() > MAX_SECURITY_MIGRATION_RETRY_COUNT) {
1385-
logger.warn("Security migration failed [" + nodeLocalMigrationRetryCount.get() + "] times, restart node to retry again.");
1386-
} else if (systemIndices.getMainIndexManager().getProject(projectId).isReadyForSecurityMigration(nextMigration.getValue())) {
1387-
submitPersistentMigrationTask(newState.migrationsVersion);
1388-
}
1389-
}
1390-
1391-
private void submitPersistentMigrationTask(int migrationsVersion) {
1392-
submitPersistentMigrationTask(migrationsVersion, true);
1393-
}
1394-
1395-
private void submitPersistentMigrationTask(int migrationsVersion, boolean securityMigrationNeeded) {
1396-
nodeLocalMigrationRetryCount.incrementAndGet();
1397-
persistentTasksService.get()
1398-
.sendStartRequest(
1399-
SecurityMigrationTaskParams.TASK_NAME,
1400-
SecurityMigrationTaskParams.TASK_NAME,
1401-
new SecurityMigrationTaskParams(migrationsVersion, securityMigrationNeeded),
1402-
TimeValue.THIRTY_SECONDS /* TODO should this be configurable? longer by default? infinite? */,
1403-
ActionListener.wrap((response) -> {
1404-
logger.debug("Security migration task submitted");
1405-
}, (exception) -> {
1406-
// Do nothing if the task is already in progress
1407-
if (ExceptionsHelper.unwrapCause(exception) instanceof ResourceAlreadyExistsException) {
1408-
// Do not count ResourceAlreadyExistsException as failure
1409-
nodeLocalMigrationRetryCount.decrementAndGet();
1410-
} else {
1411-
logger.warn("Submit security migration task failed: " + exception.getCause());
1412-
}
1413-
})
1414-
);
1415-
}
1416-
14171333
private static Executor buildRoleBuildingExecutor(ThreadPool threadPool, Settings settings) {
14181334
final int allocatedProcessors = EsExecutors.allocatedProcessors(settings);
14191335
final ThrottledTaskRunner throttledTaskRunner = new ThrottledTaskRunner("build_roles", allocatedProcessors, threadPool.generic());
@@ -2559,7 +2475,8 @@ public List<PersistentTasksExecutor<?>> getPersistentTasksExecutor(
25592475
SettingsModule settingsModule,
25602476
IndexNameExpressionResolver expressionResolver
25612477
) {
2562-
return this.securityMigrationExecutor.get() != null ? List.of(this.securityMigrationExecutor.get()) : List.of();
2478+
final SecurityMigrations.Manager manager = this.migrationManager.get();
2479+
return manager == null ? List.of() : List.of(manager.getPersistentTasksExecutor(client, threadPool));
25632480
}
25642481

25652482
List<ReservedProjectStateHandler<?>> reservedProjectStateHandlers() {

‎x-pack/plugin/security/src/main/java/org/elasticsearch/xpack/security/support/SecurityMigrations.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,26 +9,36 @@
99

1010
import org.apache.logging.log4j.LogManager;
1111
import org.apache.logging.log4j.Logger;
12+
import org.elasticsearch.ExceptionsHelper;
13+
import org.elasticsearch.ResourceAlreadyExistsException;
1214
import org.elasticsearch.action.ActionListener;
1315
import org.elasticsearch.action.search.SearchRequest;
1416
import org.elasticsearch.action.support.GroupedActionListener;
1517
import org.elasticsearch.action.support.WriteRequest;
1618
import org.elasticsearch.client.internal.Client;
19+
import org.elasticsearch.cluster.metadata.ProjectId;
20+
import org.elasticsearch.cluster.service.ClusterService;
21+
import org.elasticsearch.core.FixForMultiProject;
22+
import org.elasticsearch.core.TimeValue;
1723
import org.elasticsearch.features.NodeFeature;
1824
import org.elasticsearch.index.query.BoolQueryBuilder;
1925
import org.elasticsearch.index.query.QueryBuilders;
2026
import org.elasticsearch.index.reindex.UpdateByQueryAction;
2127
import org.elasticsearch.index.reindex.UpdateByQueryRequest;
28+
import org.elasticsearch.persistent.PersistentTasksExecutor;
29+
import org.elasticsearch.persistent.PersistentTasksService;
2230
import org.elasticsearch.script.Script;
2331
import org.elasticsearch.script.ScriptType;
2432
import org.elasticsearch.search.builder.SearchSourceBuilder;
33+
import org.elasticsearch.threadpool.ThreadPool;
2534
import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingAction;
2635
import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingRequestBuilder;
2736
import org.elasticsearch.xpack.core.security.action.rolemapping.DeleteRoleMappingResponse;
2837
import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsAction;
2938
import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsRequestBuilder;
3039
import org.elasticsearch.xpack.core.security.action.rolemapping.GetRoleMappingsResponse;
3140
import org.elasticsearch.xpack.core.security.authc.support.mapper.ExpressionRoleMapping;
41+
import org.elasticsearch.xpack.core.security.support.SecurityMigrationTaskParams;
3242
import org.elasticsearch.xpack.security.support.SecurityIndexManager.IndexState;
3343

3444
import java.util.Arrays;
@@ -37,6 +47,7 @@
3747
import java.util.Map;
3848
import java.util.Set;
3949
import java.util.TreeMap;
50+
import java.util.concurrent.atomic.AtomicInteger;
4051
import java.util.stream.Collectors;
4152

4253
import static org.elasticsearch.xpack.core.ClientHelper.SECURITY_ORIGIN;
@@ -250,4 +261,91 @@ protected static List<String> getDuplicateRoleMappingNames(ExpressionRoleMapping
250261
.toList();
251262
}
252263
}
264+
265+
public static class Manager {
266+
267+
private static final int MAX_SECURITY_MIGRATION_RETRY_COUNT = 10;
268+
269+
private final PersistentTasksService persistentTasksService;
270+
private final SecuritySystemIndices systemIndices;
271+
272+
// Node local retry count for migration jobs that's checked only on the master node to make sure
273+
// submit migration jobs doesn't get out of hand and retries forever if they fail. Reset by a
274+
// restart or master node change.
275+
private final AtomicInteger nodeLocalMigrationRetryCount;
276+
277+
public Manager(ClusterService clusterService, PersistentTasksService persistentTasksService, SecuritySystemIndices systemIndices) {
278+
this.persistentTasksService = persistentTasksService;
279+
this.systemIndices = systemIndices;
280+
this.nodeLocalMigrationRetryCount = new AtomicInteger(0);
281+
systemIndices.getMainIndexManager().addStateListener((projectId, oldState, newState) -> {
282+
// Only consider applying migrations if it's the master node and the security index exists
283+
if (clusterService.state().nodes().isLocalNodeElectedMaster() && newState.indexExists()) {
284+
applyPendingSecurityMigrations(projectId, newState);
285+
}
286+
});
287+
}
288+
289+
@FixForMultiProject
290+
// TODO : The migration task needs to be project aware
291+
private void applyPendingSecurityMigrations(ProjectId projectId, SecurityIndexManager.IndexState newState) {
292+
// If no migrations have been applied and the security index is on the latest version (new index), all migrations can be skipped
293+
if (newState.migrationsVersion == 0 && newState.createdOnLatestVersion) {
294+
submitPersistentMigrationTask(SecurityMigrations.MIGRATIONS_BY_VERSION.lastKey(), false);
295+
return;
296+
}
297+
298+
Map.Entry<Integer, SecurityMigrations.SecurityMigration> nextMigration = SecurityMigrations.MIGRATIONS_BY_VERSION.higherEntry(
299+
newState.migrationsVersion
300+
);
301+
302+
// Check if next migration that has not been applied is eligible to run on the current cluster
303+
if (nextMigration == null
304+
|| systemIndices.getMainIndexManager()
305+
.getProject(projectId)
306+
.isEligibleSecurityMigration(nextMigration.getValue()) == false) {
307+
// Reset retry counter if all eligible migrations have been applied successfully
308+
nodeLocalMigrationRetryCount.set(0);
309+
} else if (nodeLocalMigrationRetryCount.get() > MAX_SECURITY_MIGRATION_RETRY_COUNT) {
310+
logger.warn("Security migration failed [" + nodeLocalMigrationRetryCount.get() + "] times, restart node to retry again.");
311+
} else if (systemIndices.getMainIndexManager().getProject(projectId).isReadyForSecurityMigration(nextMigration.getValue())) {
312+
submitPersistentMigrationTask(newState.migrationsVersion);
313+
}
314+
}
315+
316+
private void submitPersistentMigrationTask(int migrationsVersion) {
317+
submitPersistentMigrationTask(migrationsVersion, true);
318+
}
319+
320+
private void submitPersistentMigrationTask(int migrationsVersion, boolean securityMigrationNeeded) {
321+
nodeLocalMigrationRetryCount.incrementAndGet();
322+
persistentTasksService.sendStartRequest(
323+
SecurityMigrationTaskParams.TASK_NAME,
324+
SecurityMigrationTaskParams.TASK_NAME,
325+
new SecurityMigrationTaskParams(migrationsVersion, securityMigrationNeeded),
326+
TimeValue.THIRTY_SECONDS /* TODO should this be configurable? longer by default? infinite? */,
327+
ActionListener.wrap((response) -> {
328+
logger.debug("Security migration task submitted");
329+
}, (exception) -> {
330+
// Do nothing if the task is already in progress
331+
if (ExceptionsHelper.unwrapCause(exception) instanceof ResourceAlreadyExistsException) {
332+
// Do not count ResourceAlreadyExistsException as failure
333+
nodeLocalMigrationRetryCount.decrementAndGet();
334+
} else {
335+
logger.warn("Submit security migration task failed: " + exception.getCause());
336+
}
337+
})
338+
);
339+
}
340+
341+
public PersistentTasksExecutor<?> getPersistentTasksExecutor(Client client, ThreadPool threadPool) {
342+
return new SecurityMigrationExecutor(
343+
SecurityMigrationTaskParams.TASK_NAME,
344+
threadPool.executor(ThreadPool.Names.MANAGEMENT),
345+
systemIndices.getMainIndexManager(),
346+
client,
347+
SecurityMigrations.MIGRATIONS_BY_VERSION
348+
);
349+
}
350+
}
253351
}

0 commit comments

Comments
 (0)