Ensure modified flag is returned correctly for RemovePermutesAroundElementwiseOps. (pytorch#16047)

Summary: Pull Request resolved: pytorch#16047

Differential Revision: D88109865

Author: eigen-k

backends/cadence/aot/remove_ops.py

@@ -566,13 +566,18 @@ def call(self, graph_module: torch.fx.GraphModule) -> PassResult:
                     for node in subgraph.nodes:
                         processed_nodes.add(node)
 
+        changed = False
         for subgraph in subgraphs_found:
             self.permute_subgraph(subgraph)
+            changed = True
 
-        graph_module.graph.eliminate_dead_code()
-        graph_module.recompile()
 
-        return super().call(graph_module)
+        if changed:
+            graph_module.graph.eliminate_dead_code()
+            graph_module.recompile()
+            return super().call(graph_module)
+
+        return PassResult(graph_module, False)
 
     def visit(
         self,