update scripts and data ingestion documentation

Author: Andrew Desousa

scripts/.env.sample

@@ -1,21 +1,12 @@
 # resource switch 
-FLAG_EMBEDDING_MODEL = "AOAI" # "AOAI" or "COHERE"
-FLAG_COHERE = "ENGLISH" # "MULTILINGUAL" or "ENGLISH" options for Cohere embedding models
-FLAG_AOAI = "V3" # "V2" or "V3" options for AOAI embedding models
+FLAG_EMBEDDING_MODEL = "AOAI"
 
 # update vector dimension based on model chosen
-VECTOR_DIMENSION = 1536 # change it to desired, e.g., 1536 for AOAI ada 002, 1024 for COHERE
+VECTOR_DIMENSION = 1536
 
 # AOAI resource
 AZURE_OPENAI_API_VERSION = '2023-05-15'
 AZURE_OPENAI_ENDPOINT = ""
 AZURE_OPENAI_API_KEY = ""
-
-# Cohere multilingual resource
-COHERE_MULTILINGUAL_ENDPOINT = ""
-COHERE_MULTILINGUAL_API_KEY = ""
-
-# Cohere English resource
-COHERE_ENGLISH_ENDPOINT = ""
-COHERE_ENGLISH_API_KEY = ""
+EMBEDDING_MODEL_ENDPOINT = ""
 

scripts/data_utils.py

@@ -740,8 +740,6 @@ def get_embedding(text, embedding_model_endpoint=None, embedding_model_key=None,
     endpoint = embedding_model_endpoint if embedding_model_endpoint else os.environ.get("EMBEDDING_MODEL_ENDPOINT")
 
     FLAG_EMBEDDING_MODEL = os.getenv("FLAG_EMBEDDING_MODEL", "AOAI")
-    FLAG_COHERE = os.getenv("FLAG_COHERE", "ENGLISH")
-    FLAG_AOAI = os.getenv("FLAG_AOAI", "V3")
 
     if azure_credential is None and (endpoint is None):
         raise Exception("EMBEDDING_MODEL_ENDPOINT and EMBEDDING_MODEL_KEY are required for embedding")

scripts/push_to_acs.py

@@ -1,9 +1,8 @@
 # Data Preparation
 
 # Prepare Data Locally
-Follow the instructions in this section to prepare your data locally. This is easiest for small data sets. For much larger sets, see the [instructions to use AML below](#use-aml-to-prepare-data).
 ## Setup
-- Install the necessary packages listed in requirements.txt, e.g. `pip install --user -r requirements.txt`
+- Install the necessary packages listed in requirements.txt, e.g. `pip install --user -r requirements-dev.txt`
 
 ## Configure
 - Create a .env file similar to the .env.example file. Fill in the values for the environment variables.