IGNITE-26502 Catalog. Improve serialisation compatibility tests to ensure that every object descriptor has a test case (#6834)

Author: lowka

modules/catalog/src/test/java/org/apache/ignite/internal/catalog/storage/CatalogSerializationChecker.java

@@ -29,21 +29,29 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Consumer;
 import org.apache.ignite.internal.catalog.storage.serialization.CatalogEntrySerializerProvider;
 import org.apache.ignite.internal.catalog.storage.serialization.CatalogObjectSerializer;
+import org.apache.ignite.internal.catalog.storage.serialization.CatalogSerializer;
 import org.apache.ignite.internal.catalog.storage.serialization.MarshallableEntry;
 import org.apache.ignite.internal.catalog.storage.serialization.MarshallableEntryType;
 import org.apache.ignite.internal.catalog.storage.serialization.UpdateLogMarshaller;
 import org.apache.ignite.internal.catalog.storage.serialization.UpdateLogMarshallerImpl;
 import org.apache.ignite.internal.logger.IgniteLogger;
 import org.assertj.core.api.BDDAssertions;
+import org.jetbrains.annotations.Nullable;
 
 final class CatalogSerializationChecker {
 
     private static final String UPDATE_TIMESTAMP_FIELD_REGEX = ".*updateTimestamp";
 
+    private static final String V1_SERDE_NOTE = "MANUAL CALL TO SUPPORT V1 SERIALIZATION";
+
     private final Map<Integer, Integer> expectedEntryVersions = new HashMap<>();
 
     private boolean writeSnapshot;
@@ -52,24 +60,32 @@ final class CatalogSerializationChecker {
 
     private final String directory;
 
-    private final int entryVersion;
+    private final int defaultEntryVersion;
 
     private final boolean expectExactProtocolVersion;
 
     private final int protocolVersion;
 
+    private final Set<SerializerClass> collectedSerializers = new HashSet<>();
+
+    private final Consumer<SerializerClass> recordSerializer;
+
+    private boolean addSerializerManually;
+
     CatalogSerializationChecker(
             IgniteLogger log,
             String directory,
-            int entryVersion,
+            int defaultEntryVersion,
             boolean expectExactProtocolVersion,
-            int protocolVersion
+            int protocolVersion,
+            Consumer<SerializerClass> recordSerializer
     ) {
         this.log = log;
         this.directory = directory;
-        this.entryVersion = entryVersion;
+        this.defaultEntryVersion = defaultEntryVersion;
         this.expectExactProtocolVersion = expectExactProtocolVersion;
         this.protocolVersion = protocolVersion;
+        this.recordSerializer = recordSerializer;
     }
 
     void writeSnapshot(boolean value) {
@@ -80,6 +96,10 @@ void addExpectedVersion(int typeId, int entryVersion) {
         expectedEntryVersions.put(typeId, entryVersion);
     }
 
+    void addClassesManually(boolean value) {
+        addSerializerManually = value;
+    }
+
     void reset() {
         expectedEntryVersions.clear();
         writeSnapshot = false;
@@ -96,7 +116,7 @@ void compareSnapshotEntry(SnapshotEntry expectedEntry, String fileName, int vers
         var assertion = BDDAssertions.assertThat(expectedEntry.snapshot())
                 .usingRecursiveComparison();
 
-        if (entryVersion == 1) {
+        if (defaultEntryVersion == 1) {
             // Ignoring update timestamp for version 1.
             assertion = assertion.ignoringFieldsMatchingRegexes(UPDATE_TIMESTAMP_FIELD_REGEX);
         }
@@ -112,20 +132,42 @@ void compareEntries(List<UpdateEntry> entries, String fileName, int version) {
             UpdateEntry expectedEntry = entries.get(i);
             UpdateEntry actualEntry = actual.get(i);
 
-            var assertion = BDDAssertions.assertThat(actualEntry).as("entry#" + i)
-                    .usingRecursiveComparison();
+            var assertion = BDDAssertions.assertThat(actualEntry)
+                    .as("entry#" + i).usingRecursiveComparison();
 
-            if (entryVersion == 1) {
+            if (defaultEntryVersion == 1) {
                 // Ignoring update timestamp for version 1.
                 assertion = assertion.ignoringFieldsMatchingRegexes(UPDATE_TIMESTAMP_FIELD_REGEX);
             }
 
+            if (addSerializerManually) {
+                // Record entry version to support v1 serialization.
+                // This is not needed by v2 serialization.
+                recordSerializer(expectedEntry.typeId(), version, V1_SERDE_NOTE);
+            }
+
             assertion.isEqualTo(expectedEntry);
         }
     }
 
+    private void recordSerializer(int typeId, int version, @Nullable String note) {
+        SerializerClass sc = new SerializerClass(typeId, version);
+        if (!collectedSerializers.add(sc)) {
+            return;
+        }
+        recordSerializer.accept(sc);
+
+        MarshallableEntryType entryType = getMarshallableEntryType(typeId);
+        if (note != null) {
+            log.info("{} uses version: {}, NOTE: {}", entryType, version, note);
+        } else {
+            log.info("{} uses version: {}", entryType, version);
+        }
+    }
+
     @SuppressWarnings({"unchecked", "rawtypes"})
     private <T extends UpdateEntry> List<T> checkEntries(List<? extends T> entries, String fileName, int version) {
+        // The version number is ignored, checkEntry uses the concrete serializer version. 
         VersionedUpdate update = new VersionedUpdate(1, 100L, (List<UpdateEntry>) entries);
         VersionedUpdate deserializedUpdate = checkEntry(VersionedUpdate.class, fileName, version, update);
 
@@ -151,7 +193,8 @@ private <T extends UpdateLogEvent> T checkEntry(Class<T> entryClass, String entr
 
         log.info("Read fileName: {}, class: {}, entryVersion: {}", fileName, entryClass.getSimpleName(), entryVersion);
 
-        UpdateLogMarshaller marshaller = new UpdateLogMarshallerImpl(provider, protocolVersion);
+        SerializerVersionCollectingProvider versionCollectingProvider = new SerializerVersionCollectingProvider(provider);
+        UpdateLogMarshaller marshaller = new UpdateLogMarshallerImpl(versionCollectingProvider, protocolVersion);
 
         if (writeSnapshot) {
             writeEntry(entry, Path.of("src", "test", "resources", directory, fileName), marshaller);
@@ -183,11 +226,36 @@ private void writeEntry(UpdateLogEvent entry, Path resourcePath, UpdateLogMarsha
         }
     }
 
-    private static class VersionCheckingProvider implements CatalogEntrySerializerProvider {
+    private final class SerializerVersionCollectingProvider implements CatalogEntrySerializerProvider {
+
+        private final CatalogEntrySerializerProvider delegate;
+
+        private SerializerVersionCollectingProvider(CatalogEntrySerializerProvider delegate) {
+            this.delegate = delegate;
+        }
+
+        @Override
+        public <T extends MarshallableEntry> CatalogObjectSerializer<T> get(int version, int typeId) {
+            recordSerializer(typeId, version, null);
+
+            return delegate.get(version, typeId);
+        }
+
+        @Override
+        public int latestSerializerVersion(int typeId) {
+            int version = delegate.latestSerializerVersion(typeId);
+
+            recordSerializer(typeId, version, null);
+
+            return version;
+        }
+    }
+
+    private static final class VersionCheckingProvider implements CatalogEntrySerializerProvider {
 
         private final CatalogEntrySerializerProvider provider;
 
-        private final int protocolVersion;
+        private final int expectedProtocolVersion;
 
         private final Map<Integer, Integer> entryVersions = new HashMap<>();
 
@@ -197,7 +265,7 @@ private VersionCheckingProvider(
                 Map<Integer, Integer> entryVersions
         ) {
             this.provider = provider;
-            this.protocolVersion = expectedProtocolVersion;
+            this.expectedProtocolVersion = expectedProtocolVersion;
             this.entryVersions.putAll(entryVersions);
         }
 
@@ -219,7 +287,7 @@ public int latestSerializerVersion(int typeId) {
         }
 
         private void checkVersion(int typeId, int entryVersion) {
-            int expectedEntryVersion = entryVersions.getOrDefault(typeId, protocolVersion);
+            int expectedEntryVersion = entryVersions.getOrDefault(typeId, expectedProtocolVersion);
             if (entryVersion != expectedEntryVersion) {
                 MarshallableEntryType type = null;
 
@@ -238,4 +306,70 @@ private void checkVersion(int typeId, int entryVersion) {
             }
         }
     }
+
+    static Set<SerializerClass> findEntrySerializers() {
+        Set<SerializerClass> classes = new HashSet<>();
+
+        for (var entryType : MarshallableEntryType.values()) {
+            for (Class<?> declaredClass : entryType.container().getDeclaredClasses()) {
+                if (CatalogObjectSerializer.class.isAssignableFrom(declaredClass)) {
+                    CatalogSerializer catalogSerializer = declaredClass.getAnnotation(CatalogSerializer.class);
+
+                    classes.add(new SerializerClass(entryType.id(), catalogSerializer.version()));
+                }
+            }
+        }
+
+        return classes;
+    }
+
+    protected static final class SerializerClass implements Comparable<SerializerClass> {
+        final int entryTypeId;
+        final int serializerVersion;
+
+        SerializerClass(int entryTypeId, int serializerVersion) {
+            this.entryTypeId = entryTypeId;
+            this.serializerVersion = serializerVersion;
+        }
+
+        @Override
+        public String toString() {
+            MarshallableEntryType entryType = getMarshallableEntryType(entryTypeId);
+            return "SerializerClass{" + entryType + "#" + serializerVersion + "}";
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (o == null || getClass() != o.getClass()) {
+                return false;
+            }
+            SerializerClass that = (SerializerClass) o;
+            return serializerVersion == that.serializerVersion && entryTypeId == that.entryTypeId;
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(entryTypeId, serializerVersion);
+        }
+
+        @Override
+        public int compareTo(CatalogSerializationChecker.SerializerClass o) {
+            int c1 = Integer.compare(entryTypeId, o.entryTypeId);
+            if (c1 != 0) {
+                return c1;
+            } else {
+                return Integer.compare(serializerVersion, o.serializerVersion);
+            }
+        }
+    }
+
+    private static MarshallableEntryType getMarshallableEntryType(int typeId) {
+        for (MarshallableEntryType t : MarshallableEntryType.values()) {
+            if (t.id() == typeId) {
+                return t;
+            }
+        }
+
+        throw new IllegalArgumentException("Unexpected type: " + typeId);
+    }
 }