remove pile-val backup from tests

Signed-off-by: Brian Dellabetta <[email protected]>

Author: brian-dellabetta

tests/e2e/vLLM/configs/w4a16_grouped_quant_asym_awq.yaml

@@ -2,7 +2,7 @@ cadence: "nightly"
 test_type: "regression"
 model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
 recipe: tests/e2e/vLLM/recipes/WNA16/recipe_w4a16_group_quant_asym_awq.yaml
-dataset_id: "mit-han-lab/pile-val-backup"
-dataset_split: validation
+dataset_id: HuggingFaceH4/ultrachat_200k
+dataset_split: train_sft
 scheme: W4A16_weight_asym_awq
 save_dir: TinyLlama-1.1B-Chat-v1.0-w4a16-asym-awq

tests/e2e/vLLM/configs/w4a16_grouped_quant_sym_awq.yaml

@@ -2,7 +2,7 @@ cadence: "nightly"
 test_type: "regression"
 model: TinyLlama/TinyLlama-1.1B-Chat-v1.0
 recipe: tests/e2e/vLLM/recipes/WNA16/recipe_w4a16_awq_sym.yaml
-dataset_id: "mit-han-lab/pile-val-backup"
-dataset_split: validation
+dataset_id: HuggingFaceH4/ultrachat_200k
+dataset_split: train_sft
 scheme: W4A16_weight_sym_awq
 save_dir: TinyLlama-1.1B-Chat-v1.0-w4a16-sym-awq

tests/testing_utils.py

@@ -285,18 +285,6 @@ def process(sample):
                 "images": sample["image"],
             }
 
-    elif ds_name == "pile-val-backup":
-
-        def preprocess(example):
-            return {
-                "input_ids": processor.encode(example["text"].strip())[:max_seq_length]
-            }
-
-        ds = ds.map(preprocess, remove_columns=ds.column_names)
-        # Note: potentially swap filtering to pad for AWQ
-        ds = ds.filter(lambda example: len(example["input_ids"]) >= max_seq_length)
-        return ds
-
     else:
         raise NotImplementedError(f"Cannot preprocess dataset {ds.info.dataset_name}")