Merge branch 'main' into ignite-26722

Author: tkalkirill

gradle/libs.versions.toml

@@ -87,12 +87,12 @@ tree-sitter = "0.25.3"
 tree-sitter-json = "0.24.8"
 tree-sitter-sql = "gh-pages-a"
 tree-sitter-hocon = "master-a"
-otel = "1.54.1"
+otel = "1.55.0"
 spring-boot = "3.5.6"
 spring-data = "3.5.4"
 testcontainers = "1.21.3"
 gradleToolingApi = "8.6"
-protobuf = "4.32.1"
+protobuf = "4.33.0"
 cytodynamics = "0.2.0"
 japicmp = "0.24.2"
 

modules/cluster-management/build.gradle

@@ -51,12 +51,12 @@ dependencies {
     testImplementation project(':ignite-configuration')
     testImplementation project(':ignite-network')
     testImplementation project(':ignite-system-disaster-recovery')
-    testImplementation project(':ignite-raft')
     testImplementation testFixtures(project(':ignite-core'))
     testImplementation testFixtures(project(':ignite-configuration'))
     testImplementation testFixtures(project(':ignite-network'))
     testImplementation testFixtures(project(':ignite-vault'))
     testImplementation testFixtures(project(':ignite-failure-handler'))
+    testImplementation testFixtures(project(':ignite-raft'))
 
     testFixturesImplementation project(':ignite-core')
     testFixturesImplementation project(':ignite-failure-handler')

modules/cluster-management/src/test/java/org/apache/ignite/internal/cluster/management/raft/commands/CmgCommandsCompatibilityTest.java

@@ -20,44 +20,44 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 
-import java.util.Base64;
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.UUID;
 import org.apache.ignite.internal.cluster.management.ClusterState;
 import org.apache.ignite.internal.cluster.management.ClusterTag;
 import org.apache.ignite.internal.cluster.management.network.messages.CmgMessagesFactory;
 import org.apache.ignite.internal.cluster.management.network.messages.CmgMessagesSerializationRegistryInitializer;
-import org.apache.ignite.internal.network.MessageSerializationRegistryImpl;
-import org.apache.ignite.internal.network.serialization.MessageSerializationRegistry;
+import org.apache.ignite.internal.network.serialization.MessageSerializationRegistryInitializer;
+import org.apache.ignite.internal.raft.BaseCommandsCompatibilityTest;
 import org.apache.ignite.internal.raft.Command;
-import org.apache.ignite.internal.raft.Marshaller;
-import org.apache.ignite.internal.raft.util.ThreadLocalOptimizedMarshaller;
-import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /**
  * Compatibility testing for serialization/deserialization of CMG raft commands. It is verified that deserialization of commands that were
  * created on earlier versions of the product will be error-free.
- *
- * <p>For MAC users with aarch64 architecture, you will need to add {@code || "aarch64".equals(arch)} to the
- * {@code GridUnsafe#unaligned()} for the tests to pass. For more details, see
- * <a href="https://lists.apache.org/thread/67coyvm8mo7106mkndt24yqwtbvb7590">discussion</a>.</p>
- *
- * <p>To serialize commands, use {@link #serializeAll()} and insert the result into the appropriate tests.</p>
  */
-public class CmgCommandsCompatibilityTest extends BaseIgniteAbstractTest {
-    private final MessageSerializationRegistry registry = new MessageSerializationRegistryImpl();
-
-    private final Marshaller marshaller = new ThreadLocalOptimizedMarshaller(registry);
-
+public class CmgCommandsCompatibilityTest extends BaseCommandsCompatibilityTest {
     private final CmgMessagesFactory factory = new CmgMessagesFactory();
 
-    @BeforeEach
-    void setUp() {
-        new CmgMessagesSerializationRegistryInitializer().registerFactories(registry);
+    @Override
+    protected Collection<MessageSerializationRegistryInitializer> initializers() {
+        return List.of(new CmgMessagesSerializationRegistryInitializer());
+    }
+
+    @Override
+    protected Collection<Command> commandsToSerialize() {
+        return List.of(
+                createChangeMetaStorageInfoCommand(),
+                createInitCmgStateCommand(),
+                createJoinReadyCommand(),
+                createJoinRequestCommand(),
+                createNodesLeaveCommand(),
+                createReadLogicalTopologyCommand(),
+                createReadMetaStorageInfoCommand(),
+                createReadStateCommand(),
+                createReadValidatedNodesCommand()
+        );
     }
 
     @Test
@@ -138,10 +138,6 @@ void testReadValidatedNodesCommand() {
         assertInstanceOf(ReadValidatedNodesCommand.class, command);
     }
 
-    private static UUID uuid() {
-        return new UUID(42, 69);
-    }
-
     private ClusterNodeMessage createClusterNodeMessage() {
         return factory.clusterNodeMessage()
                 .id(uuid())
@@ -165,33 +161,6 @@ private ClusterState createClusterState() {
                 .build();
     }
 
-    private <T extends Command> T deserializeCommand(byte[] bytes) {
-        return marshaller.unmarshall(bytes);
-    }
-
-    private <T extends Command> T decodeCommand(String base64) {
-        return deserializeCommand(Base64.getDecoder().decode(base64));
-    }
-
-    @SuppressWarnings("unused")
-    private void serializeAll() {
-        List<Command> commands = List.of(
-                createChangeMetaStorageInfoCommand(),
-                createInitCmgStateCommand(),
-                createJoinReadyCommand(),
-                createJoinRequestCommand(),
-                createNodesLeaveCommand(),
-                createReadLogicalTopologyCommand(),
-                createReadMetaStorageInfoCommand(),
-                createReadStateCommand(),
-                createReadValidatedNodesCommand()
-        );
-
-        for (Command c : commands) {
-            log.info(">>>>> Serialized command: [c={}, base64='{}']", c.getClass().getSimpleName(), encodeCommand(c));
-        }
-    }
-
     private ReadValidatedNodesCommand createReadValidatedNodesCommand() {
         return factory.readValidatedNodesCommand().build();
     }
@@ -241,12 +210,4 @@ private ChangeMetaStorageInfoCommand createChangeMetaStorageInfoCommand() {
                 .metastorageRepairingConfigIndex(42L)
                 .build();
     }
-
-    private byte[] serializeCommand(Command c) {
-        return marshaller.marshall(c);
-    }
-
-    private String encodeCommand(Command c) {
-        return Base64.getEncoder().encodeToString(serializeCommand(c));
-    }
 }

modules/metastorage/build.gradle

@@ -44,14 +44,14 @@ dependencies {
 
     annotationProcessor project(':ignite-network-annotation-processor')
 
-    testImplementation project(':ignite-network')
     testImplementation testFixtures(project(':ignite-core'))
     testImplementation testFixtures(project(':ignite-vault'))
     testImplementation testFixtures(project(':ignite-configuration'))
     testImplementation testFixtures(project(':ignite-replicator'))
     testImplementation testFixtures(project(':ignite-failure-handler'))
     testImplementation testFixtures(project(':ignite-metrics'))
     testImplementation testFixtures(project(':ignite-configuration-system'))
+    testImplementation testFixtures(project(':ignite-raft'))
 
     integrationTestImplementation project(':ignite-cluster-management')
     integrationTestImplementation project(':ignite-network')

modules/metastorage/src/test/java/org/apache/ignite/internal/metastorage/command/MetastorageCommandsCompatibilityTest.java

@@ -24,9 +24,8 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import java.nio.ByteBuffer;
-import java.util.Base64;
+import java.util.Collection;
 import java.util.List;
-import java.util.UUID;
 import org.apache.ignite.internal.hlc.HybridTimestamp;
 import org.apache.ignite.internal.lang.ByteArray;
 import org.apache.ignite.internal.metastorage.CommandId;
@@ -36,36 +35,46 @@
 import org.apache.ignite.internal.metastorage.dsl.MetaStorageMessagesSerializationRegistryInitializer;
 import org.apache.ignite.internal.metastorage.dsl.Operations;
 import org.apache.ignite.internal.metastorage.dsl.Statements;
-import org.apache.ignite.internal.network.MessageSerializationRegistryImpl;
-import org.apache.ignite.internal.network.serialization.MessageSerializationRegistry;
+import org.apache.ignite.internal.network.serialization.MessageSerializationRegistryInitializer;
+import org.apache.ignite.internal.raft.BaseCommandsCompatibilityTest;
 import org.apache.ignite.internal.raft.Command;
-import org.apache.ignite.internal.raft.Marshaller;
-import org.apache.ignite.internal.raft.util.ThreadLocalOptimizedMarshaller;
-import org.apache.ignite.internal.testframework.BaseIgniteAbstractTest;
-import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 /**
  * Compatibility testing for serialization/deserialization of metastorage raft commands. It is verified that deserialization of commands
  * that were created on earlier versions of the product will be error-free.
- *
- * <p>For MAC users with aarch64 architecture, you will need to add {@code || "aarch64".equals(arch)} to the
- * {@code GridUnsafe#unaligned()} for the tests to pass. For more details, see
- * <a href="https://lists.apache.org/thread/67coyvm8mo7106mkndt24yqwtbvb7590">discussion</a>.</p>
- *
- * <p>To serialize commands, use {@link #serializeAll()} and insert the result into the appropriate tests.</p>
  */
-public class MetastorageCommandsCompatibilityTest extends BaseIgniteAbstractTest {
-    private final MessageSerializationRegistry registry = new MessageSerializationRegistryImpl();
-
-    private final Marshaller marshaller = new ThreadLocalOptimizedMarshaller(registry);
-
+public class MetastorageCommandsCompatibilityTest extends BaseCommandsCompatibilityTest {
     private final MetaStorageCommandsFactory factory = new MetaStorageCommandsFactory();
 
-    @BeforeEach
-    void setUp() {
-        new MetaStorageCommandsSerializationRegistryInitializer().registerFactories(registry);
-        new MetaStorageMessagesSerializationRegistryInitializer().registerFactories(registry);
+    @Override
+    protected Collection<MessageSerializationRegistryInitializer> initializers() {
+        return List.of(
+                new MetaStorageCommandsSerializationRegistryInitializer(),
+                new MetaStorageMessagesSerializationRegistryInitializer()
+        );
+    }
+
+    @Override
+    protected Collection<Command> commandsToSerialize() {
+        return List.of(
+                createCompactionCommand(),
+                createEvictIdempotentCommandsCacheCommand(),
+                createGetAllCommand(),
+                createGetChecksumCommand(),
+                createGetCommand(),
+                createGetCurrentRevisionsCommand(),
+                createGetPrefixCommand(),
+                createGetRangeCommand(),
+                createInvokeCommand(),
+                createMultiInvokeCommand(),
+                createPutAllCommand(),
+                createPutCommand(),
+                createRemoveAllCommand(),
+                createRemoveByPrefixCommand(),
+                createRemoveCommand(),
+                createSyncTimeCommand()
+        );
     }
 
     @Test
@@ -255,18 +264,6 @@ void testSyncTimeCommand() {
         assertEquals(42, command.initiatorTerm());
     }
 
-    private static HybridTimestamp initiatorTime() {
-        return HybridTimestamp.hybridTimestamp(70);
-    }
-
-    private static HybridTimestamp safeTime() {
-        return HybridTimestamp.hybridTimestamp(69);
-    }
-
-    private static UUID uuid() {
-        return new UUID(42, 69);
-    }
-
     private static CommandId commandId() {
         return CommandId.fromString(uuid() + "_cnt_" + 70);
     }
@@ -283,40 +280,6 @@ private static ByteArray keyAsByteArray(String key) {
         return ByteArray.fromString(key);
     }
 
-    private <T extends Command> T deserializeCommand(byte[] bytes) {
-        return marshaller.unmarshall(bytes);
-    }
-
-    private <T extends Command> T decodeCommand(String base64) {
-        return deserializeCommand(Base64.getDecoder().decode(base64));
-    }
-
-    @SuppressWarnings("unused")
-    private void serializeAll() {
-        List<Command> commands = List.of(
-                createCompactionCommand(),
-                createEvictIdempotentCommandsCacheCommand(),
-                createGetAllCommand(),
-                createGetChecksumCommand(),
-                createGetCommand(),
-                createGetCurrentRevisionsCommand(),
-                createGetPrefixCommand(),
-                createGetRangeCommand(),
-                createInvokeCommand(),
-                createMultiInvokeCommand(),
-                createPutAllCommand(),
-                createPutCommand(),
-                createRemoveAllCommand(),
-                createRemoveByPrefixCommand(),
-                createRemoveCommand(),
-                createSyncTimeCommand()
-        );
-
-        for (Command c : commands) {
-            log.info(">>>>> Serialized command: [c={}, base64='{}']", c.getClass().getSimpleName(), encodeCommand(c));
-        }
-    }
-
     private SyncTimeCommand createSyncTimeCommand() {
         return factory.syncTimeCommand()
                 .safeTime(safeTime())
@@ -472,12 +435,4 @@ private CompactionCommand createCompactionCommand() {
                 .compactionRevision(42)
                 .build();
     }
-
-    private byte[] serializeCommand(Command c) {
-        return marshaller.marshall(c);
-    }
-
-    private String encodeCommand(Command c) {
-        return Base64.getEncoder().encodeToString(serializeCommand(c));
-    }
 }

modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/checkpoint/Checkpoint.java

@@ -30,6 +30,9 @@ class Checkpoint {
     /** Number of dirty pages. */
     final int dirtyPagesSize;
 
+    /** Number of written dirty pages. */
+    int writtenPages;
+
     /**
      * Constructor.
      *
@@ -49,7 +52,17 @@ class Checkpoint {
     /**
      * Returns {@code true} if this checkpoint contains at least one dirty page.
      */
-    public boolean hasDelta() {
+    boolean hasDelta() {
         return dirtyPagesSize != 0;
     }
+
+    void finishCheckpoint() {
+        if (hasDelta()) {
+            writtenPages = progress.writtenPages();
+
+            progress.pagesToWrite(null);
+
+            progress.clearCounters();
+        }
+    }
 }

modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/checkpoint/CheckpointWorkflow.java

@@ -318,11 +318,7 @@ public void markCheckpointEnd(Checkpoint chp) throws IgniteInternalCheckedExcept
             }
         }
 
-        if (chp.hasDelta()) {
-            chp.progress.pagesToWrite(null);
-
-            chp.progress.clearCounters();
-        }
+        chp.finishCheckpoint();
 
         for (CheckpointListener listener : collectCheckpointListeners(dataRegions)) {
             listener.afterCheckpointEnd(chp.progress);

modules/page-memory/src/main/java/org/apache/ignite/internal/pagememory/persistence/checkpoint/Checkpointer.java

@@ -428,7 +428,7 @@ void doCheckpoint() throws IgniteInternalCheckedException {
 
             if (chp.hasDelta()) {
                 if (log.isInfoEnabled()) {
-                    int totalWrittenPages = chp.progress.writtenPages();
+                    int totalWrittenPages = chp.writtenPages;
                     long totalWriteBytes = (long) pageSize * totalWrittenPages;
                     long totalDurationInNanos = tracker.checkpointDuration(NANOSECONDS);
 

modules/partition-replicator/build.gradle

@@ -57,6 +57,7 @@ dependencies {
     testImplementation testFixtures(project(':ignite-table'))
     testImplementation testFixtures(project(':ignite-transactions'))
     testImplementation testFixtures(project(':ignite-placement-driver-api'))
+    testImplementation testFixtures(project(':ignite-raft'))
 
     integrationTestImplementation testFixtures(project(':ignite-cluster-management'))
     integrationTestImplementation testFixtures(project(':ignite-core'))