From da2566d73120a18265cbce8db7e5646272cdf352 Mon Sep 17 00:00:00 2001 From: David Turner Date: Tue, 1 Oct 2024 13:44:00 +0100 Subject: [PATCH] Misc `ReferenceDocs` improvements (#113667) - Forbid ephemeral `_auto_gen.html` and `page.html#_auto_gen` links. - Remove dangling/unused `BOOTSTRAP_CHECK_G1GC` link. - Separate test suite into individual tests. --- .../setup/bootstrap-checks-xes.asciidoc | 4 ++ .../reference/setup/bootstrap-checks.asciidoc | 12 ++++++ .../elasticsearch/common/ReferenceDocs.java | 10 ++++- .../common/reference-docs-links.json | 33 ++++++++------- .../common/ReferenceDocsTests.java | 41 ++++++++++++++++++- 5 files changed, 80 insertions(+), 20 deletions(-) diff --git a/docs/reference/setup/bootstrap-checks-xes.asciidoc b/docs/reference/setup/bootstrap-checks-xes.asciidoc index bc187e668d0c0..926afe6463f46 100644 --- a/docs/reference/setup/bootstrap-checks-xes.asciidoc +++ b/docs/reference/setup/bootstrap-checks-xes.asciidoc @@ -6,6 +6,7 @@ In addition to the <>, there are checks that are specific to {xpack} features. [discrete] +[[bootstrap-checks-xpack-encrypt-sensitive-data]] === Encrypt sensitive data check //See EncryptSensitiveDAtaBootstrapCheck.java @@ -18,6 +19,7 @@ on each node in the cluster. For more information, see <>. [discrete] +[[bootstrap-checks-xpack-pki-realm]] === PKI realm check //See PkiRealmBootstrapCheckTests.java @@ -31,6 +33,7 @@ To pass this bootstrap check, if a PKI realm is enabled, you must configure TLS and enable client authentication on at least one network communication layer. [discrete] +[[bootstrap-checks-xpack-role-mappings]] === Role mappings check If you authenticate users with realms other than `native` or `file` realms, you @@ -65,6 +68,7 @@ To pass this bootstrap check, you must [discrete] +[[bootstrap-checks-xpack-token-ssl]] === Token SSL check //See TokenSSLBootstrapCheckTests.java diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 64977ae4e4611..7793e3e3c3359 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -60,6 +60,7 @@ in the <>. We strongly encourage you to do this if you are in this specific situation. This system property can be used to force execution of the bootstrap checks independent of the node configuration. +[[bootstrap-checks-heap-size]] === Heap size check By default, {es} automatically sizes JVM heap based on a node's @@ -71,6 +72,7 @@ size on startup. If the initial heap size is not equal to the maximum heap size, some JVM heap may not be locked after a resize. To avoid these issues, start the JVM with an initial heap size equal to the maximum heap size. +[[bootstrap-checks-file-descriptor]] === File descriptor check File descriptors are a Unix construct for tracking open "files". In Unix @@ -83,6 +85,7 @@ bootstrap check is enforced on OS X and Linux. To pass the file descriptor check, you might have to configure <>. +[[bootstrap-checks-memory-lock]] === Memory lock check When the JVM does a major garbage collection it touches every page of @@ -116,6 +119,7 @@ least 4096 threads. This can be done via `/etc/security/limits.conf` using the `nproc` setting (note that you might have to increase the limits for the `root` user too). +[[bootstrap-checks-max-file-size]] === Max file size check The segment files that are the components of individual shards and the translog @@ -144,6 +148,7 @@ address space. This can be done via adding ` - as unlimited` to `/etc/security/limits.conf`. This may require you to increase the limits for the `root` user too. +[[bootstrap-checks-max-map-count]] === Maximum map count check Continuing from the previous <>, to @@ -158,6 +163,7 @@ Alternatively, the maximum map count check is only needed if you are using indices. If you <> the use of `mmap` then this bootstrap check will not be enforced. +[[bootstrap-checks-client-jvm]] === Client JVM check There are two different JVMs provided by OpenJDK-derived JVMs: the @@ -171,6 +177,7 @@ JVM check, you must start Elasticsearch with the server VM. On modern systems and operating systems, the server VM is the default. +[[bootstrap-checks-serial-collector]] === Use serial collector check There are various garbage collectors for the OpenJDK-derived JVMs @@ -187,6 +194,7 @@ configuration that ships with Elasticsearch configures Elasticsearch to use the G1GC garbage collector with JDK14 and later versions. For earlier JDK versions, the configuration defaults to the CMS collector. +[[bootstrap-checks-syscall-filter]] === System call filter check Elasticsearch installs system call filters of various flavors depending on the operating system (e.g., seccomp on Linux). These system call @@ -198,6 +206,7 @@ installed. To pass the system call filter check you must fix any configuration errors on your system that prevented system call filters from installing (check your logs). +[[bootstrap-checks-onerror]] === OnError and OnOutOfMemoryError checks The JVM options `OnError` and `OnOutOfMemoryError` enable executing @@ -214,6 +223,7 @@ use the JVM flag `ExitOnOutOfMemoryError`. While this does not have the full capabilities of `OnError` nor `OnOutOfMemoryError`, arbitrary forking will not be supported with seccomp enabled. +[[bootstrap-checks-early-access]] === Early-access check The OpenJDK project provides early-access snapshots of upcoming releases. These @@ -221,12 +231,14 @@ releases are not suitable for production. The early-access check detects these early-access snapshots. To pass this check, you must start Elasticsearch on a release build of the JVM. +[[bootstrap-checks-all-permission]] === All permission check The all permission check ensures that the security policy used during bootstrap does not grant the `java.security.AllPermission` to Elasticsearch. Running with the all permission granted is equivalent to disabling the security manager. +[[bootstrap-checks-discovery-configuration]] === Discovery configuration check By default, when Elasticsearch first starts up it will try and discover other diff --git a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java index c0d2b904396e8..97dc9244141e0 100644 --- a/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java +++ b/server/src/main/java/org/elasticsearch/common/ReferenceDocs.java @@ -61,7 +61,6 @@ public enum ReferenceDocs { BOOTSTRAP_CHECK_SYSTEM_CALL_FILTER, BOOTSTRAP_CHECK_ONERROR_AND_ONOUTOFMEMORYERROR, BOOTSTRAP_CHECK_EARLY_ACCESS, - BOOTSTRAP_CHECK_G1GC, BOOTSTRAP_CHECK_ALL_PERMISSION, BOOTSTRAP_CHECK_DISCOVERY_CONFIGURATION, BOOTSTRAP_CHECKS, @@ -120,6 +119,15 @@ static Map readLinksBySymbol(InputStream inputStream) throws Exc if (iterator.hasNext()) { throw new IllegalStateException("found unexpected extra value: " + iterator.next()); } + + // We must only link to anchors with fixed IDs (defined by [[fragment-name]] in the docs) because auto-generated fragment IDs + // depend on the heading text and are too easy to break inadvertently. Auto-generated fragment IDs begin with an underscore. + for (final var entry : result.entrySet()) { + if (entry.getValue().startsWith("_") || entry.getValue().contains("#_")) { + throw new IllegalStateException("found auto-generated fragment ID at " + entry.getKey()); + } + } + return result; } } diff --git a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json index 14c377682b081..38bbbecdcce35 100644 --- a/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json +++ b/server/src/main/resources/org/elasticsearch/common/reference-docs-links.json @@ -9,27 +9,26 @@ "ARCHIVE_INDICES": "archive-indices.html", "HTTP_TRACER": "modules-network.html#http-rest-request-tracer", "LOGGING": "logging.html", - "BOOTSTRAP_CHECK_HEAP_SIZE": "_heap_size_check.html", - "BOOTSTRAP_CHECK_FILE_DESCRIPTOR": "_file_descriptor_check.html", - "BOOTSTRAP_CHECK_MEMORY_LOCK": "_memory_lock_check.html", + "BOOTSTRAP_CHECK_HEAP_SIZE": "bootstrap-checks-heap-size.html", + "BOOTSTRAP_CHECK_FILE_DESCRIPTOR": "bootstrap-checks-file-descriptor.html", + "BOOTSTRAP_CHECK_MEMORY_LOCK": "bootstrap-checks-memory-lock.html", "BOOTSTRAP_CHECK_MAX_NUMBER_THREADS": "max-number-threads-check.html", - "BOOTSTRAP_CHECK_MAX_FILE_SIZE": "_max_file_size_check.html", + "BOOTSTRAP_CHECK_MAX_FILE_SIZE": "bootstrap-checks-max-file-size.html", "BOOTSTRAP_CHECK_MAX_SIZE_VIRTUAL_MEMORY": "max-size-virtual-memory-check.html", - "BOOTSTRAP_CHECK_MAXIMUM_MAP_COUNT": "_maximum_map_count_check.html", - "BOOTSTRAP_CHECK_CLIENT_JVM": "_client_jvm_check.html", - "BOOTSTRAP_CHECK_USE_SERIAL_COLLECTOR": "_use_serial_collector_check.html", - "BOOTSTRAP_CHECK_SYSTEM_CALL_FILTER": "_system_call_filter_check.html", - "BOOTSTRAP_CHECK_ONERROR_AND_ONOUTOFMEMORYERROR": "_onerror_and_onoutofmemoryerror_checks.html", - "BOOTSTRAP_CHECK_EARLY_ACCESS": "_early_access_check.html", - "BOOTSTRAP_CHECK_G1GC": "_g1gc_check.html", - "BOOTSTRAP_CHECK_ALL_PERMISSION": "_all_permission_check.html", - "BOOTSTRAP_CHECK_DISCOVERY_CONFIGURATION": "_discovery_configuration_check.html", + "BOOTSTRAP_CHECK_MAXIMUM_MAP_COUNT": "bootstrap-checks-max-map-count.html", + "BOOTSTRAP_CHECK_CLIENT_JVM": "bootstrap-checks-client-jvm.html", + "BOOTSTRAP_CHECK_USE_SERIAL_COLLECTOR": "bootstrap-checks-serial-collector.html", + "BOOTSTRAP_CHECK_SYSTEM_CALL_FILTER": "bootstrap-checks-syscall-filter.html", + "BOOTSTRAP_CHECK_ONERROR_AND_ONOUTOFMEMORYERROR": "bootstrap-checks-onerror.html", + "BOOTSTRAP_CHECK_EARLY_ACCESS": "bootstrap-checks-early-access.html", + "BOOTSTRAP_CHECK_ALL_PERMISSION": "bootstrap-checks-all-permission.html", + "BOOTSTRAP_CHECK_DISCOVERY_CONFIGURATION": "bootstrap-checks-discovery-configuration.html", "BOOTSTRAP_CHECKS": "bootstrap-checks.html", - "BOOTSTRAP_CHECK_ENCRYPT_SENSITIVE_DATA": "bootstrap-checks-xpack.html#_encrypt_sensitive_data_check", - "BOOTSTRAP_CHECK_PKI_REALM": "bootstrap-checks-xpack.html#_pki_realm_check", - "BOOTSTRAP_CHECK_ROLE_MAPPINGS": "bootstrap-checks-xpack.html#_role_mappings_check", + "BOOTSTRAP_CHECK_ENCRYPT_SENSITIVE_DATA": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-encrypt-sensitive-data", + "BOOTSTRAP_CHECK_PKI_REALM": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-pki-realm", + "BOOTSTRAP_CHECK_ROLE_MAPPINGS": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-role-mappings", "BOOTSTRAP_CHECK_TLS": "bootstrap-checks-xpack.html#bootstrap-checks-tls", - "BOOTSTRAP_CHECK_TOKEN_SSL": "bootstrap-checks-xpack.html#_token_ssl_check", + "BOOTSTRAP_CHECK_TOKEN_SSL": "bootstrap-checks-xpack.html#bootstrap-checks-xpack-token-ssl", "BOOTSTRAP_CHECK_SECURITY_MINIMAL_SETUP": "security-minimal-setup.html", "CONTACT_SUPPORT": "troubleshooting.html#troubleshooting-contact-support", "UNASSIGNED_SHARDS": "red-yellow-cluster-status.html", diff --git a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java index b383ce7b869ea..a5efd578df36c 100644 --- a/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java +++ b/server/src/test/java/org/elasticsearch/common/ReferenceDocsTests.java @@ -39,8 +39,7 @@ public void testVersionComponent() { assertEquals("master", getVersionComponent("ABCDEF", true)); } - public void testResourceValidation() throws Exception { - + public void testReadsValidLinkDefinitions() throws Exception { try (var builder = XContentFactory.jsonBuilder()) { builder.startObject(); for (ReferenceDocs link : ReferenceDocs.values()) { @@ -54,11 +53,15 @@ public void testResourceValidation() throws Exception { assertEquals("TEST", map.get(link.name())); } } + } + public void testRejectsInvalidJSON() throws Exception { try (var stream = new ByteArrayInputStream("{\"invalid\":".getBytes(StandardCharsets.UTF_8))) { expectThrows(XContentParseException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); } + } + public void testRejectsBadStructure() throws Exception { try (var builder = XContentFactory.jsonBuilder()) { builder.startObject(); for (ReferenceDocs link : ReferenceDocs.values()) { @@ -70,7 +73,9 @@ public void testResourceValidation() throws Exception { expectThrows(IllegalArgumentException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); } } + } + public void testRejectsExtraSymbol() throws Exception { try (var builder = XContentFactory.jsonBuilder()) { builder.startObject(); for (ReferenceDocs link : ReferenceDocs.values()) { @@ -82,7 +87,9 @@ public void testResourceValidation() throws Exception { expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); } } + } + public void testRejectsMissingSymbol() throws Exception { try (var builder = XContentFactory.jsonBuilder()) { builder.startObject(); var skipped = randomFrom(ReferenceDocs.values()); @@ -97,7 +104,9 @@ public void testResourceValidation() throws Exception { expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); } } + } + public void testRejectsIncorrectOrder() throws Exception { try (var builder = XContentFactory.jsonBuilder()) { var shuffled = Arrays.copyOf(ReferenceDocs.values(), ReferenceDocs.values().length); var i = between(0, ReferenceDocs.values().length - 1); @@ -117,4 +126,32 @@ public void testResourceValidation() throws Exception { } } } + + public void testRejectsAutoGeneratedFragment() throws Exception { + try (var builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + for (ReferenceDocs link : ReferenceDocs.values()) { + builder.field(link.name(), "test.html#_auto_generated_fragment"); + } + builder.endObject(); + + try (var stream = BytesReference.bytes(builder).streamInput()) { + expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); + } + } + } + + public void testRejectsAutoGeneratedPageName() throws Exception { + try (var builder = XContentFactory.jsonBuilder()) { + builder.startObject(); + for (ReferenceDocs link : ReferenceDocs.values()) { + builder.field(link.name(), "_auto_generated_page.html"); + } + builder.endObject(); + + try (var stream = BytesReference.bytes(builder).streamInput()) { + expectThrows(IllegalStateException.class, () -> ReferenceDocs.readLinksBySymbol(stream)); + } + } + } }