Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,7 @@ private static boolean getOptimizedMergeEnabledDefault() {

final int skipIndexIntervalSize;
final int minDocsPerOrdinalForRangeEncoding;
private final boolean enableOptimizedMerge;
final boolean enableOptimizedMerge;

/** Default constructor. */
public ES819TSDBDocValuesFormat() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import java.util.Map;
import java.util.function.IntSupplier;

import static org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormatTests.TestES819TSDBDocValuesFormatVersion0;
import static org.hamcrest.Matchers.equalTo;

public class TsdbDocValueBwcTests extends ESTestCase {
Expand All @@ -62,6 +63,13 @@ public void testMixedIndex() throws Exception {
testMixedIndex(oldCodec, newCodec);
}

// TODO update Current to Version1 once version is incremented
public void testMixedIndexDocValueVersion0ToCurrent() throws Exception {
var oldCodec = TestUtil.alwaysDocValuesFormat(new TestES819TSDBDocValuesFormatVersion0());
var newCodec = TestUtil.alwaysDocValuesFormat(new ES819TSDBDocValuesFormat());
testMixedIndex(oldCodec, newCodec, this::assertVersion819, this::assertVersion819);
}

public void testMixedIndex816To900Lucene101() throws Exception {
var oldCodec = new Elasticsearch816Codec() {

Expand All @@ -84,8 +92,37 @@ public DocValuesFormat getDocValuesFormatForField(String field) {
testMixedIndex(oldCodec, newCodec);
}

void assertFieldInfoDocValuesFormat(DirectoryReader reader, String expectedSuffix, String expectedFormat) throws IOException,
NoSuchFieldException, IllegalAccessException {
// Assert per field format field info attributes:
// (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC.
// Otherwise, doc values fields may disappear)
for (var leaf : reader.leaves()) {
for (var fieldInfo : leaf.reader().getFieldInfos()) {
assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", expectedSuffix));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", expectedFormat));
}
}
}

void assertVersion87(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException {
assert87DocValuesFormatVersion(reader);
assertFieldInfoDocValuesFormat(reader, "0", "ES87TSDB");
}

void assertVersion819(DirectoryReader reader) throws IOException, NoSuchFieldException, ClassNotFoundException, IllegalAccessException {
assert819DocValuesFormatVersion(reader);
assertFieldInfoDocValuesFormat(reader, "0", "ES819TSDB");
}

void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFieldException, IllegalAccessException,
ClassNotFoundException {
testMixedIndex(oldCodec, newCodec, this::assertVersion87, this::assertVersion819);
}

void testMixedIndex(Codec oldCodec, Codec newCodec, VersionAssert assertOldVersion, VersionAssert assertNewVersion) throws IOException,
NoSuchFieldException, IllegalAccessException, ClassNotFoundException {
String timestampField = "@timestamp";
String hostnameField = "host.name";
long baseTimestamp = 1704067200000L;
Expand Down Expand Up @@ -136,17 +173,7 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi
}
// Check documents before force merge:
try (var reader = DirectoryReader.open(dir)) {
assertOldDocValuesFormatVersion(reader);
// Assert per field format field info attributes:
// (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC.
// Otherwise, doc values fields may disappear)
for (var leaf : reader.leaves()) {
for (var fieldInfo : leaf.reader().getFieldInfos()) {
assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0"));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES87TSDB"));
}
}
assertOldVersion.run(reader);

var hostNameDV = MultiDocValues.getSortedValues(reader, hostnameField);
assertNotNull(hostNameDV);
Expand Down Expand Up @@ -205,17 +232,9 @@ void testMixedIndex(Codec oldCodec, Codec newCodec) throws IOException, NoSuchFi
try (var reader = DirectoryReader.open(iw)) {
assertEquals(1, reader.leaves().size());
assertEquals(numDocs, reader.maxDoc());
assertNewDocValuesFormatVersion(reader);
var leaf = reader.leaves().get(0).reader();
// Assert per field format field info attributes:
// (XPerFieldDocValuesFormat must produce the same attributes as PerFieldDocValuesFormat for BWC.
// Otherwise, doc values fields may disappear)
for (var fieldInfo : leaf.getFieldInfos()) {
assertThat(fieldInfo.attributes(), Matchers.aMapWithSize(2));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.suffix", "0"));
assertThat(fieldInfo.attributes(), Matchers.hasEntry("PerFieldDocValuesFormat.format", "ES819TSDB"));
}
assertNewVersion.run(reader);

var leaf = reader.leaves().get(0).reader();
var hostNameDV = leaf.getSortedDocValues(hostnameField);
assertNotNull(hostNameDV);
var timestampDV = DocValues.unwrapSingleton(leaf.getSortedNumericDocValues(timestampField));
Expand Down Expand Up @@ -385,7 +404,7 @@ private IndexWriterConfig getTimeSeriesIndexWriterConfig(String hostnameField, S

// A hacky way to figure out whether doc values format is written in what version. Need to use reflection, because
// PerFieldDocValuesFormat hides the doc values formats it wraps.
private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException {
private void assert87DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException {
if (System.getSecurityManager() != null) {
// With jvm version 24 entitlements are used and security manager is nog longer used.
// Making this assertion work with security manager requires granting the entire test codebase privileges to use
Expand All @@ -406,7 +425,7 @@ private void assertOldDocValuesFormatVersion(DirectoryReader reader) throws NoSu
}
}

private void assertNewDocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException,
private void assert819DocValuesFormatVersion(DirectoryReader reader) throws NoSuchFieldException, IllegalAccessException, IOException,
ClassNotFoundException {

for (var leafReaderContext : reader.leaves()) {
Expand Down Expand Up @@ -451,4 +470,7 @@ private static Field getFormatsFieldFromPerFieldFieldsReader(Class<?> c) throws
return field;
}

interface VersionAssert {
void run(DirectoryReader reader) throws IOException, NoSuchFieldException, IllegalAccessException, ClassNotFoundException;
}
}
Loading