diff --git a/docs/changelog/118785.yaml b/docs/changelog/118785.yaml new file mode 100644 index 0000000000000..5d294b9b52867 --- /dev/null +++ b/docs/changelog/118785.yaml @@ -0,0 +1,6 @@ +pr: 118785 +summary: "[Draft] Allow searchable snapshot indices in version N-2 to be recovered\ + \ on a V9 cluster" +area: Allocation +type: enhancement +issues: [] diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java index c42e879f84892..d7239e33ffd34 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/AbstractLuceneIndexCompatibilityTestCase.java @@ -15,6 +15,8 @@ import com.carrotsearch.randomizedtesting.annotations.TestCaseOrdering; import org.elasticsearch.client.Request; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.core.Strings; import org.elasticsearch.test.cluster.ElasticsearchCluster; import org.elasticsearch.test.cluster.local.LocalClusterConfigProvider; import org.elasticsearch.test.cluster.local.distribution.DistributionType; @@ -28,12 +30,15 @@ import java.util.Comparator; import java.util.Locale; +import java.util.stream.IntStream; import java.util.stream.Stream; import static org.elasticsearch.test.cluster.util.Version.CURRENT; import static org.elasticsearch.test.cluster.util.Version.fromString; import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; +import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; /** @@ -113,6 +118,12 @@ protected String suffix(String name) { return name + '-' + getTestName().split(" ")[0].toLowerCase(Locale.ROOT); } + protected Settings repositorySettings() { + return Settings.builder() + .put("location", REPOSITORY_PATH.getRoot().toPath().resolve(suffix("location")).toFile().getPath()) + .build(); + } + protected static Version clusterVersion() throws Exception { var response = assertOK(client().performRequest(new Request("GET", "/"))); var responseBody = createFromResponse(response); @@ -121,12 +132,24 @@ protected static Version clusterVersion() throws Exception { return version; } - protected static Version indexLuceneVersion(String indexName) throws Exception { + protected static Version indexVersion(String indexName) throws Exception { var response = assertOK(client().performRequest(new Request("GET", "/" + indexName + "/_settings"))); int id = Integer.parseInt(createFromResponse(response).evaluate(indexName + ".settings.index.version.created")); return new Version((byte) ((id / 1000000) % 100), (byte) ((id / 10000) % 100), (byte) ((id / 100) % 100)); } + protected static void indexDocs(String indexName, int numDocs) throws Exception { + var request = new Request("POST", "/_bulk"); + var docs = new StringBuilder(); + IntStream.range(0, numDocs).forEach(n -> docs.append(Strings.format(""" + {"index":{"_id":"%s","_index":"%s"}} + {"test":"test"} + """, n, indexName))); + request.setJsonEntity(docs.toString()); + var response = assertOK(client().performRequest(request)); + assertThat(entityAsMap(response).get("errors"), allOf(notNullValue(), is(false))); + } + /** * Execute the test suite with the parameters provided by the {@link #parameters()} in version order. */ diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java index d6dd949b843d6..79106f3b3712b 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/LuceneCompatibilityIT.java @@ -10,20 +10,18 @@ package org.elasticsearch.lucene; import org.elasticsearch.client.Request; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.cluster.metadata.IndexMetadata; import org.elasticsearch.common.Strings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexSettings; import org.elasticsearch.repositories.fs.FsRepository; +import org.elasticsearch.rest.RestStatus; import org.elasticsearch.test.cluster.util.Version; -import java.util.stream.IntStream; - -import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; public class LuceneCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { @@ -42,13 +40,7 @@ public void testRestoreIndex() throws Exception { final int numDocs = 1234; logger.debug("--> registering repository [{}]", repository); - registerRepository( - client(), - repository, - FsRepository.TYPE, - true, - Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build() - ); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); if (VERSION_MINUS_2.equals(clusterVersion())) { logger.debug("--> creating index [{}]", index); @@ -63,17 +55,7 @@ public void testRestoreIndex() throws Exception { ); logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); - final var bulks = new StringBuilder(); - IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format(""" - {"index":{"_id":"%s","_index":"%s"}} - {"test":"test"} - """, n, index))); - - var bulkRequest = new Request("POST", "/_bulk"); - bulkRequest.setJsonEntity(bulks.toString()); - var bulkResponse = client().performRequest(bulkRequest); - assertOK(bulkResponse); - assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false))); + indexDocs(index, numDocs); logger.debug("--> creating snapshot [{}]", snapshot); createSnapshot(client(), repository, snapshot, true); @@ -83,7 +65,7 @@ public void testRestoreIndex() throws Exception { if (VERSION_MINUS_1.equals(clusterVersion())) { ensureGreen(index); - assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2)); + assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); assertDocCount(client(), index, numDocs); logger.debug("--> deleting index [{}]", index); @@ -106,9 +88,19 @@ public void testRestoreIndex() throws Exception { "rename_replacement": "%s", "include_aliases": false }""", index, restoredIndex)); - var responseBody = createFromResponse(client().performRequest(request)); - assertThat(responseBody.evaluate("snapshot.shards.total"), equalTo((int) responseBody.evaluate("snapshot.shards.failed"))); - assertThat(responseBody.evaluate("snapshot.shards.successful"), equalTo(0)); + var responseException = expectThrows(ResponseException.class, () -> client().performRequest(request)); + assertEquals(RestStatus.INTERNAL_SERVER_ERROR.getStatus(), responseException.getResponse().getStatusLine().getStatusCode()); + assertThat( + responseException.getMessage(), + allOf( + containsString("cannot restore index"), + containsString("because it cannot be upgraded"), + containsString("has current compatibility version [" + VERSION_MINUS_2 + '-' + VERSION_MINUS_1.getMajor() + ".0.0]"), + containsString("but the minimum compatible version is [" + VERSION_MINUS_1.getMajor() + ".0.0]."), + containsString("It should be re-indexed in Elasticsearch " + VERSION_MINUS_1.getMajor() + ".x"), + containsString("before upgrading to " + VERSION_CURRENT) + ) + ); } } } diff --git a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java index 4f348b7fb122f..1ece449c1f0f0 100644 --- a/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java +++ b/qa/lucene-index-compatibility/src/javaRestTest/java/org/elasticsearch/lucene/SearchableSnapshotCompatibilityIT.java @@ -17,13 +17,8 @@ import org.elasticsearch.repositories.fs.FsRepository; import org.elasticsearch.test.cluster.util.Version; -import java.util.stream.IntStream; - import static org.elasticsearch.test.rest.ObjectPath.createFromResponse; -import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.notNullValue; public class SearchableSnapshotCompatibilityIT extends AbstractLuceneIndexCompatibilityTestCase { @@ -46,13 +41,7 @@ public void testSearchableSnapshot() throws Exception { final int numDocs = 1234; logger.debug("--> registering repository [{}]", repository); - registerRepository( - client(), - repository, - FsRepository.TYPE, - true, - Settings.builder().put("location", REPOSITORY_PATH.getRoot().getPath()).build() - ); + registerRepository(client(), repository, FsRepository.TYPE, true, repositorySettings()); if (VERSION_MINUS_2.equals(clusterVersion())) { logger.debug("--> creating index [{}]", index); @@ -67,17 +56,7 @@ public void testSearchableSnapshot() throws Exception { ); logger.debug("--> indexing [{}] docs in [{}]", numDocs, index); - final var bulks = new StringBuilder(); - IntStream.range(0, numDocs).forEach(n -> bulks.append(Strings.format(""" - {"index":{"_id":"%s","_index":"%s"}} - {"test":"test"} - """, n, index))); - - var bulkRequest = new Request("POST", "/_bulk"); - bulkRequest.setJsonEntity(bulks.toString()); - var bulkResponse = client().performRequest(bulkRequest); - assertOK(bulkResponse); - assertThat(entityAsMap(bulkResponse).get("errors"), allOf(notNullValue(), is(false))); + indexDocs(index, numDocs); logger.debug("--> creating snapshot [{}]", snapshot); createSnapshot(client(), repository, snapshot, true); @@ -87,7 +66,7 @@ public void testSearchableSnapshot() throws Exception { if (VERSION_MINUS_1.equals(clusterVersion())) { ensureGreen(index); - assertThat(indexLuceneVersion(index), equalTo(VERSION_MINUS_2)); + assertThat(indexVersion(index), equalTo(VERSION_MINUS_2)); assertDocCount(client(), index, numDocs); logger.debug("--> deleting index [{}]", index); diff --git a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java index 74a8dc7851c89..e3eaca7467708 100644 --- a/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java +++ b/server/src/main/java/org/elasticsearch/cluster/coordination/NodeJoinExecutor.java @@ -49,6 +49,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +import static org.elasticsearch.cluster.metadata.IndexMetadataVerifier.isReadOnlySupportedVersion; import static org.elasticsearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; public class NodeJoinExecutor implements ClusterStateTaskExecutor { @@ -179,8 +180,12 @@ public ClusterState execute(BatchExecutionContext batchExecutionContex Set newNodeEffectiveFeatures = enforceNodeFeatureBarrier(node, effectiveClusterFeatures, features); // we do this validation quite late to prevent race conditions between nodes joining and importing dangling indices // we have to reject nodes that don't support all indices we have in this cluster - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), initialState.getMetadata()); - + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + initialState.getMetadata() + ); nodesBuilder.add(node); compatibilityVersionsMap.put(node.getId(), compatibilityVersions); // store the actual node features here, not including assumed features, as this is persisted in cluster state @@ -396,7 +401,12 @@ private Set calculateEffectiveClusterFeatures(DiscoveryNodes nodes, Map< * @see IndexVersions#MINIMUM_COMPATIBLE * @throws IllegalStateException if any index is incompatible with the given version */ - public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, IndexVersion maxSupportedVersion, Metadata metadata) { + public static void ensureIndexCompatibility( + IndexVersion minSupportedVersion, + IndexVersion minReadOnlySupportedVersion, + IndexVersion maxSupportedVersion, + Metadata metadata + ) { // we ensure that all indices in the cluster we join are compatible with us no matter if they are // closed or not we can't read mappings of these indices so we need to reject the join... for (IndexMetadata idxMetadata : metadata) { @@ -411,14 +421,17 @@ public static void ensureIndexCompatibility(IndexVersion minSupportedVersion, In ); } if (idxMetadata.getCompatibilityVersion().before(minSupportedVersion)) { - throw new IllegalStateException( - "index " - + idxMetadata.getIndex() - + " version not supported: " - + idxMetadata.getCompatibilityVersion().toReleaseVersion() - + " minimum compatible index version is: " - + minSupportedVersion.toReleaseVersion() - ); + boolean isReadOnlySupported = isReadOnlySupportedVersion(idxMetadata, minSupportedVersion, minReadOnlySupportedVersion); + if (isReadOnlySupported == false) { + throw new IllegalStateException( + "index " + + idxMetadata.getIndex() + + " version not supported: " + + idxMetadata.getCompatibilityVersion().toReleaseVersion() + + " minimum compatible index version is: " + + minSupportedVersion.toReleaseVersion() + ); + } } } } @@ -542,7 +555,12 @@ public static Collection> addBuiltInJoin final Collection> validators = new ArrayList<>(); validators.add((node, state) -> { ensureNodesCompatibility(node.getVersion(), state.getNodes()); - ensureIndexCompatibility(node.getMinIndexVersion(), node.getMaxIndexVersion(), state.getMetadata()); + ensureIndexCompatibility( + node.getMinIndexVersion(), + node.getMinReadOnlyIndexVersion(), + node.getMaxIndexVersion(), + state.getMetadata() + ); }); validators.addAll(onJoinValidators); return Collections.unmodifiableCollection(validators); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java index 0b8095c24519f..4526a7aaa9712 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifier.java @@ -88,8 +88,12 @@ public IndexMetadataVerifier( * If the index does not need upgrade it returns the index metadata unchanged, otherwise it returns a modified index metadata. If index * cannot be updated the method throws an exception. */ - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion); + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + checkSupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion); // First convert any shared_cache searchable snapshot indices to only use _tier_preference: data_frozen IndexMetadata newMetadata = convertSharedCacheTierPreference(indexMetadata); @@ -105,26 +109,81 @@ public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersi } /** - * Check that the index version is compatible. Elasticsearch does not support indices created before the - * previous major version. + * Check that the index version is compatible. Elasticsearch supports reading and writing indices created in the current version ("N") + + as well as the previous major version ("N-1"). Elasticsearch only supports reading indices created down to the penultimate version + + ("N-2") and does not support reading nor writing any version below that. */ - private static void checkSupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { - boolean isSupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); - if (isSupportedVersion == false) { - throw new IllegalStateException( - "The index " - + indexMetadata.getIndex() - + " has current compatibility version [" - + indexMetadata.getCompatibilityVersion().toReleaseVersion() - + "] but the minimum compatible version is [" - + minimumIndexCompatibilityVersion.toReleaseVersion() - + "]. It should be re-indexed in Elasticsearch " - + (Version.CURRENT.major - 1) - + ".x before upgrading to " - + Build.current().version() - + "." - ); + private static void checkSupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { + if (isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion)) { + return; + } + if (isReadOnlySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion, minimumReadOnlyIndexCompatibilityVersion)) { + return; + } + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " has current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] but the minimum compatible version is [" + + minimumIndexCompatibilityVersion.toReleaseVersion() + + "]. It should be re-indexed in Elasticsearch " + + (Version.CURRENT.major - 1) + + ".x before upgrading to " + + Build.current().version() + + "." + ); + } + + private static boolean isFullySupportedVersion(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + return indexMetadata.getCompatibilityVersion().onOrAfter(minimumIndexCompatibilityVersion); + } + + /** + * Returns {@code true} if the index version is compatible in read-only mode. As of today, only searchable snapshots and archive indices + * in version N-2 with a write block are read-only compatible. This method throws an {@link IllegalStateException} if the index is + * either a searchable snapshot or an archive index with a read-only compatible version but is missing the write block. + * + * @param indexMetadata the index metadata + * @param minimumIndexCompatibilityVersion the min. index compatible version for reading and writing indices (used in assertion) + * @param minReadOnlyIndexCompatibilityVersion the min. index compatible version for only reading indices + * + * @return {@code true} if the index version is compatible in read-only mode, {@code false} otherwise. + * @throws IllegalStateException if the index is read-only compatible but has no write block in place. + */ + public static boolean isReadOnlySupportedVersion( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minReadOnlyIndexCompatibilityVersion + ) { + boolean isReadOnlySupportedVersion = indexMetadata.getCompatibilityVersion().onOrAfter(minReadOnlyIndexCompatibilityVersion); + assert isFullySupportedVersion(indexMetadata, minimumIndexCompatibilityVersion) == false; + + if (isReadOnlySupportedVersion + && (indexMetadata.isSearchableSnapshot() || indexMetadata.getCreationVersion().isLegacyIndexVersion())) { + boolean isReadOnly = IndexMetadata.INDEX_BLOCKS_WRITE_SETTING.get(indexMetadata.getSettings()); + if (isReadOnly == false) { + throw new IllegalStateException( + "The index " + + indexMetadata.getIndex() + + " created in version [" + + indexMetadata.getCreationVersion() + + "] with current compatibility version [" + + indexMetadata.getCompatibilityVersion().toReleaseVersion() + + "] must be marked as read-only using the setting [" + + IndexMetadata.SETTING_BLOCKS_WRITE + + "] set to [true] before upgrading to " + + Build.current().version() + + "." + ); + } + return true; } + return false; } /** diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java index 0c33878b01229..95d1c37ec41ae 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/MetadataIndexStateService.java @@ -1120,6 +1120,7 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre final Metadata.Builder metadata = Metadata.builder(currentState.metadata()); final ClusterBlocks.Builder blocks = ClusterBlocks.builder(currentState.blocks()); final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion(); + final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion(); for (IndexMetadata indexMetadata : indicesToOpen) { final Index index = indexMetadata.getIndex(); @@ -1137,7 +1138,11 @@ private ClusterState openIndices(final Index[] indices, final ClusterState curre // The index might be closed because we couldn't import it due to an old incompatible // version, so we need to verify its compatibility. - newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(newIndexMetadata, minIndexCompatibilityVersion); + newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + newIndexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); try { indicesService.verifyIndexMetadata(newIndexMetadata, newIndexMetadata); } catch (Exception e) { diff --git a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java index bf2387453145d..6038a83130db5 100644 --- a/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java +++ b/server/src/main/java/org/elasticsearch/gateway/GatewayMetaState.java @@ -295,7 +295,11 @@ static Metadata upgradeMetadata(Metadata metadata, IndexMetadataVerifier indexMe boolean changed = false; final Metadata.Builder upgradedMetadata = Metadata.builder(metadata); for (IndexMetadata indexMetadata : metadata) { - IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, IndexVersions.MINIMUM_COMPATIBLE); + IndexMetadata newMetadata = indexMetadataVerifier.verifyIndexMetadata( + indexMetadata, + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE + ); changed |= indexMetadata != newMetadata; upgradedMetadata.put(newMetadata, false); } diff --git a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java index a15fef653dabe..9359f6e377ef4 100644 --- a/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java +++ b/server/src/main/java/org/elasticsearch/gateway/LocalAllocateDangledIndices.java @@ -125,6 +125,7 @@ public ClusterState execute(ClusterState currentState) { currentState.routingTable() ); IndexVersion minIndexCompatibilityVersion = currentState.nodes().getMinSupportedIndexVersion(); + IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.nodes().getMinReadOnlySupportedIndexVersion(); IndexVersion maxIndexCompatibilityVersion = currentState.nodes().getMaxDataNodeCompatibleIndexVersion(); boolean importNeeded = false; StringBuilder sb = new StringBuilder(); @@ -176,7 +177,11 @@ public ClusterState execute(ClusterState currentState) { try { // The dangled index might be from an older version, we need to make sure it's compatible // with the current version. - newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(indexMetadata, minIndexCompatibilityVersion); + newIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + indexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); newIndexMetadata = IndexMetadata.builder(newIndexMetadata) .settings( Settings.builder() diff --git a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java index ddb1e3d384fbe..5aec8e0e3253c 100644 --- a/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java +++ b/server/src/main/java/org/elasticsearch/snapshots/RestoreService.java @@ -1348,6 +1348,7 @@ public ClusterState execute(ClusterState currentState) { final Map shards = new HashMap<>(); final IndexVersion minIndexCompatibilityVersion = currentState.getNodes().getMinSupportedIndexVersion(); + final IndexVersion minReadOnlyIndexCompatibilityVersion = currentState.getNodes().getMinReadOnlySupportedIndexVersion(); final String localNodeId = clusterService.state().nodes().getLocalNodeId(); for (Map.Entry indexEntry : indicesToRestore.entrySet()) { final IndexId index = indexEntry.getValue(); @@ -1360,12 +1361,16 @@ public ClusterState execute(ClusterState currentState) { request.indexSettings(), request.ignoreIndexSettings() ); - if (snapshotIndexMetadata.getCompatibilityVersion().before(minIndexCompatibilityVersion)) { + if (snapshotIndexMetadata.getCompatibilityVersion().isLegacyIndexVersion()) { // adapt index metadata so that it can be understood by current version snapshotIndexMetadata = convertLegacyIndex(snapshotIndexMetadata, currentState, indicesService); } try { - snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata(snapshotIndexMetadata, minIndexCompatibilityVersion); + snapshotIndexMetadata = indexMetadataVerifier.verifyIndexMetadata( + snapshotIndexMetadata, + minIndexCompatibilityVersion, + minReadOnlyIndexCompatibilityVersion + ); } catch (Exception ex) { throw new SnapshotRestoreException(snapshot, "cannot restore index [" + index + "] because it cannot be upgraded", ex); } diff --git a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java index 492a142492e18..a8c81b1db5514 100644 --- a/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/coordination/NodeJoinExecutorTests.java @@ -39,6 +39,7 @@ import org.elasticsearch.features.NodeFeature; import org.elasticsearch.index.IndexVersion; import org.elasticsearch.index.IndexVersions; +import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ClusterServiceUtils; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.MockLog; @@ -58,6 +59,7 @@ import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.assertDesiredNodesStatusIsCorrect; import static org.elasticsearch.cluster.metadata.DesiredNodesTestCase.randomDesiredNode; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.elasticsearch.test.VersionUtils.maxCompatibleVersion; import static org.elasticsearch.test.VersionUtils.randomCompatibleVersion; import static org.elasticsearch.test.VersionUtils.randomVersion; @@ -89,12 +91,18 @@ public void testPreventJoinClusterWithNewerIndices() { .build(); metaBuilder.put(indexMetadata, false); Metadata metadata = metaBuilder.build(); - NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata); + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ); expectThrows( IllegalStateException.class, () -> NodeJoinExecutor.ensureIndexCompatibility( IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, IndexVersionUtils.getPreviousVersion(IndexVersion.current()), metadata ) @@ -113,10 +121,136 @@ public void testPreventJoinClusterWithUnsupportedIndices() { Metadata metadata = metaBuilder.build(); expectThrows( IllegalStateException.class, - () -> NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata) + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ) ); } + public void testJoinClusterWithReadOnlyCompatibleIndices() { + { + var indexMetadata = IndexMetadata.builder("searchable-snapshot") + .settings( + Settings.builder() + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ); + } + { + var indexMetadata = IndexMetadata.builder("searchable-snapshot-no-write-block") + .settings( + Settings.builder() + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("archive") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099))) + .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ); + } + { + var indexMetadata = IndexMetadata.builder("archive-no-write-block") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099))) + .put(IndexMetadata.SETTING_VERSION_COMPATIBILITY, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("legacy") + .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.fromId(randomFrom(5000099, 6000099)))) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + { + var indexMetadata = IndexMetadata.builder("read-only-compatible-but-unsupported") + .settings( + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + ) + .numberOfShards(1) + .numberOfReplicas(1) + .build(); + + expectThrows( + IllegalStateException.class, + () -> NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + Metadata.builder().put(indexMetadata, false).build() + ) + ); + } + } + public void testPreventJoinClusterWithUnsupportedNodeVersions() { DiscoveryNodes.Builder builder = DiscoveryNodes.builder(); final Version version = randomCompatibleVersion(random(), Version.CURRENT); @@ -467,7 +601,12 @@ public void testSuccess() { .build(); metaBuilder.put(indexMetadata, false); Metadata metadata = metaBuilder.build(); - NodeJoinExecutor.ensureIndexCompatibility(IndexVersions.MINIMUM_COMPATIBLE, IndexVersion.current(), metadata); + NodeJoinExecutor.ensureIndexCompatibility( + IndexVersions.MINIMUM_COMPATIBLE, + IndexVersions.MINIMUM_READONLY_COMPATIBLE, + IndexVersion.current(), + metadata + ); } public static Settings.Builder randomCompatibleVersionSettings() { diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java index 3b122864aa472..032a174a7c6c0 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/IndexMetadataVerifierTests.java @@ -18,11 +18,13 @@ import org.elasticsearch.index.mapper.MapperMetrics; import org.elasticsearch.index.mapper.MapperRegistry; import org.elasticsearch.plugins.MapperPlugin; +import org.elasticsearch.snapshots.SearchableSnapshotsSettings; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.index.IndexVersionUtils; import java.util.Collections; +import static org.elasticsearch.index.IndexModule.INDEX_STORE_TYPE_SETTING; import static org.hamcrest.Matchers.equalTo; public class IndexMetadataVerifierTests extends ESTestCase { @@ -97,7 +99,7 @@ public void testCustomSimilarity() { .put("index.similarity.my_similarity.after_effect", "l") .build() ); - service.verifyIndexMetadata(src, IndexVersions.MINIMUM_COMPATIBLE); + service.verifyIndexMetadata(src, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); } public void testIncompatibleVersion() { @@ -110,7 +112,7 @@ public void testIncompatibleVersion() { ); String message = expectThrows( IllegalStateException.class, - () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE) + () -> service.verifyIndexMetadata(metadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) ).getMessage(); assertThat( message, @@ -132,7 +134,78 @@ public void testIncompatibleVersion() { indexCreated = IndexVersionUtils.randomVersionBetween(random(), minCompat, IndexVersion.current()); IndexMetadata goodMeta = newIndexMeta("foo", Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated).build()); - service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE); + service.verifyIndexMetadata(goodMeta, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); + } + + public void testReadOnlyVersionCompatibility() { + var service = getIndexMetadataVerifier(); + var indexCreated = IndexVersions.MINIMUM_READONLY_COMPATIBLE; + { + var idxMetadata = newIndexMeta( + "not-searchable-snapshot", + Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .build() + ); + String message = expectThrows( + IllegalStateException.class, + () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ).getMessage(); + assertThat( + message, + equalTo( + "The index [not-searchable-snapshot/" + + idxMetadata.getIndexUUID() + + "] has current compatibility version [" + + indexCreated.toReleaseVersion() + + "] " + + "but the minimum compatible version is [" + + IndexVersions.MINIMUM_COMPATIBLE.toReleaseVersion() + + "]. It should be re-indexed in Elasticsearch " + + (Version.CURRENT.major - 1) + + ".x before upgrading to " + + Build.current().version() + + "." + ) + ); + } + { + var idxMetadata = newIndexMeta( + "not-read-only", + Settings.builder() + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .build() + ); + String message = expectThrows( + IllegalStateException.class, + () -> service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE) + ).getMessage(); + assertThat( + message, + equalTo( + "The index [not-read-only/" + + idxMetadata.getIndexUUID() + + "] with current compatibility version [" + + indexCreated.toReleaseVersion() + + "] must be marked as read-only using the setting [index.blocks.write] set to [true] before upgrading to " + + Build.current().version() + + "." + ) + ); + } + { + var idxMetadata = newIndexMeta( + "good", + Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + .put(IndexMetadata.SETTING_VERSION_CREATED, indexCreated) + .put(INDEX_STORE_TYPE_SETTING.getKey(), SearchableSnapshotsSettings.SEARCHABLE_SNAPSHOT_STORE_TYPE) + .build() + ); + service.verifyIndexMetadata(idxMetadata, IndexVersions.MINIMUM_COMPATIBLE, IndexVersions.MINIMUM_READONLY_COMPATIBLE); + } } private IndexMetadataVerifier getIndexMetadataVerifier() { diff --git a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java index a161794e35b91..fd0718e5280fe 100644 --- a/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java +++ b/server/src/test/java/org/elasticsearch/gateway/GatewayMetaStateTests.java @@ -264,7 +264,11 @@ private static class MockIndexMetadataVerifier extends IndexMetadataVerifier { } @Override - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { return upgrade ? IndexMetadata.builder(indexMetadata).build() : indexMetadata; } } diff --git a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java index 82154848ea039..39c327ddee228 100644 --- a/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/elasticsearch/indices/cluster/ClusterStateChanges.java @@ -252,7 +252,11 @@ public Transport.Connection getConnection(DiscoveryNode node) { ) { // metadata upgrader should do nothing @Override - public IndexMetadata verifyIndexMetadata(IndexMetadata indexMetadata, IndexVersion minimumIndexCompatibilityVersion) { + public IndexMetadata verifyIndexMetadata( + IndexMetadata indexMetadata, + IndexVersion minimumIndexCompatibilityVersion, + IndexVersion minimumReadOnlyIndexCompatibilityVersion + ) { return indexMetadata; } };