Merge branch 'main' into multi_turn_chat

Author: dushyantbehl

tests/artifacts/testdata/__init__.py

@@ -19,37 +19,47 @@
 
 ### Constants used for data
 DATA_DIR = os.path.join(os.path.dirname(__file__))
+JSON_DATA_DIR = os.path.join(os.path.dirname(__file__), "json")
+JSONL_DATA_DIR = os.path.join(os.path.dirname(__file__), "jsonl")
+ARROW_DATA_DIR = os.path.join(os.path.dirname(__file__), "arrow")
 PARQUET_DATA_DIR = os.path.join(os.path.dirname(__file__), "parquet")
-TWITTER_COMPLAINTS_DATA_JSON = os.path.join(DATA_DIR, "twitter_complaints_small.json")
-TWITTER_COMPLAINTS_DATA_JSONL = os.path.join(DATA_DIR, "twitter_complaints_small.jsonl")
-TWITTER_COMPLAINTS_DATA_ARROW = os.path.join(DATA_DIR, "twitter_complaints_small.arrow")
+
+TWITTER_COMPLAINTS_DATA_JSON = os.path.join(
+    JSON_DATA_DIR, "twitter_complaints_small.json"
+)
+TWITTER_COMPLAINTS_DATA_JSONL = os.path.join(
+    JSONL_DATA_DIR, "twitter_complaints_small.jsonl"
+)
+TWITTER_COMPLAINTS_DATA_ARROW = os.path.join(
+    ARROW_DATA_DIR, "twitter_complaints_small.arrow"
+)
 TWITTER_COMPLAINTS_DATA_PARQUET = os.path.join(
     PARQUET_DATA_DIR, "twitter_complaints_small.parquet"
 )
 TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSON = os.path.join(
-    DATA_DIR, "twitter_complaints_input_output.json"
+    JSON_DATA_DIR, "twitter_complaints_input_output.json"
 )
 TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_JSONL = os.path.join(
-    DATA_DIR, "twitter_complaints_input_output.jsonl"
+    JSONL_DATA_DIR, "twitter_complaints_input_output.jsonl"
 )
 TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_ARROW = os.path.join(
-    DATA_DIR, "twitter_complaints_input_output.arrow"
+    ARROW_DATA_DIR, "twitter_complaints_input_output.arrow"
 )
 TWITTER_COMPLAINTS_DATA_INPUT_OUTPUT_PARQUET = os.path.join(
     PARQUET_DATA_DIR, "twitter_complaints_input_output.parquet"
 )
 TWITTER_COMPLAINTS_TOKENIZED_JSON = os.path.join(
-    DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json"
+    JSON_DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.json"
 )
 TWITTER_COMPLAINTS_TOKENIZED_JSONL = os.path.join(
-    DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.jsonl"
+    JSONL_DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.jsonl"
 )
 TWITTER_COMPLAINTS_TOKENIZED_ARROW = os.path.join(
-    DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.arrow"
+    ARROW_DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.arrow"
 )
 TWITTER_COMPLAINTS_TOKENIZED_PARQUET = os.path.join(
     PARQUET_DATA_DIR, "twitter_complaints_tokenized_with_maykeye_tinyllama_v0.parquet"
 )
-EMPTY_DATA = os.path.join(DATA_DIR, "empty_data.json")
-MALFORMATTED_DATA = os.path.join(DATA_DIR, "malformatted_data.json")
+EMPTY_DATA = os.path.join(JSON_DATA_DIR, "empty_data.json")
+MALFORMATTED_DATA = os.path.join(JSON_DATA_DIR, "malformatted_data.json")
 MODEL_NAME = "Maykeye/TinyLLama-v0"