diff --git a/src/rsz/src/ConcreteSwapArithModules.cc b/src/rsz/src/ConcreteSwapArithModules.cc
index 34b7ef74f8a..463b7f41e49 100644
--- a/src/rsz/src/ConcreteSwapArithModules.cc
+++ b/src/rsz/src/ConcreteSwapArithModules.cc
@@ -235,12 +235,14 @@ bool ConcreteSwapArithModules::hasArithOperatorProperty(
   return false;
 }
 
-bool ConcreteSwapArithModules::doSwapInstances(
-    const std::set<dbModInst*>& insts,
-    const std::string& target)
+bool ConcreteSwapArithModules::doSwapInstances(std::set<dbModInst*>& insts,
+                                               const std::string& target)
 {
   int swapped_count = 0;
 
+  // Create a new inst set since old insts are destroyed
+  std::set<dbModInst*> swappedInsts;
+
   for (dbModInst* inst : insts) {
     dbModule* old_master = inst->getMaster();
     if (!old_master) {
@@ -271,11 +273,18 @@ bool ConcreteSwapArithModules::doSwapInstances(
                     inst->getName(),
                     old_name,
                     new_name);
-      inst->swapMaster(new_master);
-      swapped_count++;
+      dbModInst* new_inst = inst->swapMaster(new_master);
+
+      if (new_inst) {
+        swapped_count++;
+        swappedInsts.insert(new_inst);
+      }
     }
   }
 
+  insts.clear();
+  insts.insert(swappedInsts.begin(), swappedInsts.end());
+
   logger_->info(RSZ,
                 160,
                 "{} arithmetic instances have swapped to improve '{}' target",
diff --git a/src/rsz/src/ConcreteSwapArithModules.hh b/src/rsz/src/ConcreteSwapArithModules.hh
index bdf9bfe1b32..4d4de0d6eef 100644
--- a/src/rsz/src/ConcreteSwapArithModules.hh
+++ b/src/rsz/src/ConcreteSwapArithModules.hh
@@ -26,7 +26,7 @@ class ConcreteSwapArithModules : public SwapArithModules
                              const std::string& target,
                              float slack_threshold,
                              std::set<dbModInst*>& insts) override;
-  bool doSwapInstances(const std::set<dbModInst*>& insts,
+  bool doSwapInstances(std::set<dbModInst*>& insts,
                        const std::string& target) override;
 
  protected:
diff --git a/src/rsz/src/SwapArithModules.hh b/src/rsz/src/SwapArithModules.hh
index 89f3fa0977d..0f8c59c3ba7 100644
--- a/src/rsz/src/SwapArithModules.hh
+++ b/src/rsz/src/SwapArithModules.hh
@@ -60,7 +60,7 @@ class SwapArithModules : public sta::dbStaState
                                      float slack_threshold,
                                      std::set<dbModInst*>& insts)
       = 0;
-  virtual bool doSwapInstances(const std::set<dbModInst*>& insts,
+  virtual bool doSwapInstances(std::set<dbModInst*>& insts,
                                const std::string& target)
       = 0;