Adding test artifacts

Signed-off-by: Thara Palanivel <[email protected]>

Author: tharapalanivel

tests/__init__.py

@@ -0,0 +1,13 @@
+# Copyright The FMS Model Optimizer Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

tests/artifacts/configs/dummy_job_config.json

@@ -0,0 +1,12 @@
+{
+    "accelerate_launch_args": {
+        "main_process_port": 1234
+    },
+    "model_name_or_path": "Maykeye/TinyLLama-v0",
+    "training_data_path": "data_train",
+    "quant_method": "gptq",
+    "bits": 4,
+    "group_size": 128,
+    "output_dir": "models/Maykeye/TinyLLama-v0-GPTQ",
+    "log_level":"DEBUG"
+  }

tests/artifacts/testdata/__init__.py

@@ -0,0 +1,28 @@
+# Copyright The FMS Model Optimizer Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Helpful datasets for configuring individual unit tests.
+"""
+# Standard
+import os
+
+### Constants used for data
+MODEL_NAME = "Maykeye/TinyLLama-v0"
+DATA_DIR = os.path.join(os.path.dirname(__file__))
+
+WIKITEXT_TOKENIZED_DATA_JSON = os.path.join(
+    DATA_DIR, "wiki_maykeye_tinyllama_v0_numsamp2_seqlen2048"
+)
+EMPTY_DATA = os.path.join(DATA_DIR, "empty_data.json")
+MALFORMATTED_DATA = os.path.join(DATA_DIR, "malformatted_data.json")

tests/artifacts/testdata/wiki_maykeye_tinyllama_v0_numsamp2_seqlen2048/data-00000-of-00001.arrow

@@ -0,0 +1,22 @@
+{
+  "citation": "",
+  "description": "",
+  "features": {
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    },
+    "attention_mask": {
+      "feature": {
+        "dtype": "int8",
+        "_type": "Value"
+      },
+      "_type": "Sequence"
+    }
+  },
+  "homepage": "",
+  "license": ""
+}

tests/artifacts/testdata/wiki_maykeye_tinyllama_v0_numsamp2_seqlen2048/dataset_info.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "27301518aea394cd",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

tests/artifacts/testdata/wiki_maykeye_tinyllama_v0_numsamp2_seqlen2048/state.json

@@ -0,0 +1,13 @@
+# Copyright The FMS Model Optimizer Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.