Merge branch 'feature/llm-complete-updates' of github.com:zenml-io/zenml-projects into feature/llm-complete-updates

Author: AlexejPenner

.gitignore

@@ -162,6 +162,7 @@ llm-lora-finetuning/configs/shopify.yaml
 finetuned-matryoshka/
 finetuned-all-MiniLM-L6-v2/
 finetuned-snowflake-arctic-embed-m/
+finetuned-snowflake-arctic-embed-m-v1.5/
 
 # ollama ignores
 nohup.out

llm-complete-guide/constants.py

@@ -57,16 +57,17 @@
 
 # embeddings finetuning constants
 EMBEDDINGS_MODEL_NAME_ZENML = "finetuned-zenml-docs-embeddings"
-DATASET_NAME_DEFAULT = "zenml/rag_qa_embedding_questions_0_60_0"
+# DATASET_NAME_DEFAULT = "zenml/rag_qa_embedding_questions_0_60_0"
+DATASET_NAME_DEFAULT = "zenml/rag_qa_embedding_questions"
 DATASET_NAME_DISTILABEL = f"{DATASET_NAME_DEFAULT}_distilabel"
 DATASET_NAME_ARGILLA = DATASET_NAME_DEFAULT.replace("zenml/", "")
 OPENAI_MODEL_GEN = "gpt-4o"
 OPENAI_MODEL_GEN_KWARGS_EMBEDDINGS = {
     "temperature": 0.7,
     "max_new_tokens": 512,
 }
-EMBEDDINGS_MODEL_ID_BASELINE = "Snowflake/snowflake-arctic-embed-m"
-EMBEDDINGS_MODEL_ID_FINE_TUNED = "finetuned-snowflake-arctic-embed-m"
+EMBEDDINGS_MODEL_ID_BASELINE = "Snowflake/snowflake-arctic-embed-m-v1.5"
+EMBEDDINGS_MODEL_ID_FINE_TUNED = "finetuned-snowflake-arctic-embed-m-v1.5"
 EMBEDDINGS_MODEL_MATRYOSHKA_DIMS: list[int] = [
     384,
     256,

llm-complete-guide/requirements-argilla.txt

@@ -1,9 +1,10 @@
-zenml[server]>=0.63.0
-sentence-transformers>=3
-transformers
+zenml[server]>=0.63.0,<0.68.0
+sentence-transformers>=3,<=3.0.1
+transformers<=4.44.0
 litellm
 ollama
-polars
-datasets
-argilla
-distilabel
+polars<=1.4.1
+datasets<=2.20.0
+argilla<=2.0.0
+distilabel<=1.3.1
+accelerate<=0.33.0