chore: reproduce MultiLeavesTxQueue fails on multi-shard

Signed-off-by: kostas <[email protected]>

Author: kostasrim

.github/workflows/ci.yml

@@ -129,10 +129,10 @@ jobs:
       - name: Build
         run: |
           cd ${GITHUB_WORKSPACE}/build
-          ninja search_family_test
+          #ninja search_family_test
           df -h
           echo "-----------------------------"
-          ninja src/all
+          ninja multi_test
 
       - name: PostFail
         if: failure()
@@ -145,60 +145,62 @@ jobs:
           cd ${GITHUB_WORKSPACE}/build
           echo Run ctest -V -L DFLY
 
-          GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \
-          FLAGS_fiber_safety_margin=4096 FLAGS_list_experimental_v2=true timeout 20m ctest -V -L DFLY -E allocation_tracker_test
-
-          # Run allocation tracker test separately without alsologtostderr because it generates a TON of logs.
-          FLAGS_fiber_safety_margin=4096 timeout 5m ./allocation_tracker_test
-
-          timeout 5m ./dragonfly_test
-          timeout 5m ./json_family_test --jsonpathv2=false
-          timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr
-
-
-      - name: C++ Unit Tests - Epoll
-        run: |
-          cd ${GITHUB_WORKSPACE}/build
-
-          # Create a rule that automatically prints stacktrace upon segfault
-          cat > ./init.gdb <<EOF
-          catch signal SIGSEGV
-          command
-          bt
-          end
-          EOF
-
-          gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll
-          FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 timeout 20m ctest -V -L DFLY -E allocation_tracker_test
-
-          FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true timeout 5m ./allocation_tracker_test
-
-      - name: C++ Unit Tests - IoUring with cluster mode
-        run: |
-          FLAGS_fiber_safety_margin=4096 FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY
-
-      - name: C++ Unit Tests - IoUring with cluster mode and FLAGS_lock_on_hashtags
-        run: |
-          FLAGS_fiber_safety_margin=4096 FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY
-
+          #GLOG_alsologtostderr=1 GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1,op_manager=1,op_manager_test=1 \
+          #FLAGS_fiber_safety_margin=4096 FLAGS_list_experimental_v2=true timeout 20m ctest -V -L DFLY -E allocation_tracker_test
+
+          ## Run allocation tracker test separately without alsologtostderr because it generates a TON of logs.
+          #FLAGS_fiber_safety_margin=4096 timeout 5m ./allocation_tracker_test
+
+          #timeout 5m ./dragonfly_test
+          #timeout 5m ./json_family_test --jsonpathv2=false
+          #timeout 5m ./tiered_storage_test --vmodule=db_slice=2 --logtostderr
+          FLAGS_cluster_mode=emulated ./multi_test
+          FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true ./multi_Test
+
+
+          #      - name: C++ Unit Tests - Epoll
+          #        run: |
+          #          cd ${GITHUB_WORKSPACE}/build
+          #
+          #          # Create a rule that automatically prints stacktrace upon segfault
+          #          cat > ./init.gdb <<EOF
+          #          catch signal SIGSEGV
+          #          command
+          #          bt
+          #          end
+          #          EOF
+          #
+          #          gdb -ix ./init.gdb --batch -ex r --args ./dragonfly_test --force_epoll
+          #          FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true GLOG_vmodule=rdb_load=1,rdb_save=1,snapshot=1 timeout 20m ctest -V -L DFLY -E allocation_tracker_test
+          #
+          #          FLAGS_fiber_safety_margin=4096 FLAGS_force_epoll=true timeout 5m ./allocation_tracker_test
+          #
+          #      - name: C++ Unit Tests - IoUring with cluster mode
+          #        run: |
+          #          FLAGS_fiber_safety_margin=4096 FLAGS_cluster_mode=emulated timeout 20m ctest -V -L DFLY
+          #
+          #      - name: C++ Unit Tests - IoUring with cluster mode and FLAGS_lock_on_hashtags
+          #        run: |
+          #          FLAGS_fiber_safety_margin=4096 FLAGS_cluster_mode=emulated FLAGS_lock_on_hashtags=true timeout 20m ctest -V -L DFLY
+          #
       - name: Upload unit logs on failure
         if: failure()
         uses: actions/upload-artifact@v4
         with:
           name: unit_logs
           path: /tmp/*INFO*
 
-      - name: Run regression tests
-        if: matrix.container == 'ubuntu-dev:20'
-        uses: ./.github/actions/regression-tests
-        with:
-          dfly-executable: dragonfly
-          run-only-on-ubuntu-latest: true
-          build-folder-name: build
-          # Non-release build will not run tests marked as slow or opt_only
-          # "not empty" string is needed for release build because pytest command can not get empty string for filter
-          filter: ${{ matrix.build-type == 'Release' && 'not empty' || '(not slow) and (not opt_only)' }}
-
+            #      - name: Run regression tests
+            #        if: matrix.container == 'ubuntu-dev:20'
+            #        uses: ./.github/actions/regression-tests
+            #        with:
+            #          dfly-executable: dragonfly
+            #          run-only-on-ubuntu-latest: true
+            #          build-folder-name: build
+            #          # Non-release build will not run tests marked as slow or opt_only
+            #          # "not empty" string is needed for release build because pytest command can not get empty string for filter
+            #          filter: ${{ matrix.build-type == 'Release' && 'not empty' || '(not slow) and (not opt_only)' }}
+            #
       - name: Upload regression logs on failure
         if: failure()
         uses: actions/upload-artifact@v4

src/server/cluster_support.cc

@@ -71,6 +71,7 @@ void InitializeCluster() {
 
   if (cluster_mode != ClusterMode::kNoCluster) {
     cluster_shard_by_slot = absl::GetFlag(FLAGS_experimental_cluster_shard_by_slot);
+    LOG(INFO) << "FLAG IS " << cluster_shard_by_slot;
   }
 }
 

src/server/multi_test.cc

@@ -21,6 +21,7 @@ ABSL_DECLARE_FLAG(bool, lua_auto_async);
 ABSL_DECLARE_FLAG(bool, lua_allow_undeclared_auto_correct);
 ABSL_DECLARE_FLAG(std::string, default_lua_flags);
 ABSL_DECLARE_FLAG(std::vector<std::string>, lua_force_atomicity_shas);
+ABSL_DECLARE_FLAG(bool, experimental_cluster_shard_by_slot);
 
 namespace dfly {
 
@@ -890,14 +891,16 @@ TEST_F(MultiTest, MultiLeavesTxQueue) {
   // tx (mget) after it that runs and tests for atomicity.
   absl::FlagSaver fs;
   absl::SetFlag(&FLAGS_multi_exec_squash, false);
+  absl::SetFlag(&FLAGS_experimental_cluster_shard_by_slot, true);
+  InitializeCluster();
 
   for (unsigned i = 0; i < 20; ++i) {
     string key = StrCat("x", i);
     LOG(INFO) << key << ": shard " << Shard(key, shard_set->size());
   }
 
   Run({"mget", "x5", "x8", "x9", "x13", "x16", "x17"});
-  ASSERT_EQ(1, GetDebugInfo().shards_count);
+  ASSERT_EQ(3, GetDebugInfo().shards_count);
 
   auto fb1 = pp_->at(1)->LaunchFiber(Launch::post, [&] {
     // Runs multi on shard0 1000 times.