Revert change to attention spec handling in sdxl test

Author: monorimet

models/turbine_models/tests/sdxl_test.py

@@ -99,11 +99,6 @@ def setUp(self):
                 else True
             ),
         }
-        attn_spec = {
-            "text_encoder": None,
-            "unet": arguments["attn_spec"],
-            "vae": arguments["attn_spec"],
-        }
         self.pipe = SharkSDPipeline(
             arguments["hf_model_name"],
             arguments["height"],
@@ -114,7 +109,7 @@ def setUp(self):
             arguments["device"],
             arguments["iree_target_triple"],
             ireec_flags=None,
-            attn_spec=attn_spec,
+            attn_spec=arguments["attn_spec"],
             decomp_attn=decomp_attn,
             pipeline_dir="test_vmfbs",
             external_weights_dir="test_weights",