feat: adding new defaults to openai Model and embeddings (#973)

change default models for OpenAI
- LLM:  gpt-3.5-turbo
- embeddings: text-embedding-ada-002

Also fixes the MacOS runners and CI

Author: jjmachan

.github/workflows/ci.yaml

@@ -52,7 +52,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     if: ${{ (github.event_name == 'pull_request' && needs.diff.outputs.ragas == 'true') || github.event_name == 'push' }}
     name: python${{ matrix.python-version }}_unit_tests (${{ matrix.os }})
@@ -64,10 +64,10 @@ jobs:
           fetch-depth: 0 # fetch all tags and branches
 
       - name: Setup python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
+          architecture: ${{ matrix.os == 'macos-latest' && 'arm64' || 'x64' }}
 
       - name: Get pip cache dir
         id: cache-dir

docs/howtos/customisations/gcp-vertexai.ipynb

@@ -100,10 +100,10 @@
    "source": [
     "from ragas.metrics import (\n",
     "    context_precision,\n",
-    "    answer_relevancy,  \n",
+    "    answer_relevancy,\n",
     "    faithfulness,\n",
     "    context_recall,\n",
-    "    answer_similarity, \n",
+    "    answer_similarity,\n",
     "    answer_correctness,\n",
     ")\n",
     "from ragas.metrics.critique import harmfulness\n",
@@ -150,13 +150,12 @@
     "\n",
     "# create Langchain LLM and Embeddings\n",
     "vertextai_llm = ChatVertexAI(\n",
-    "        credentials=creds,\n",
-    "        model_name=config['chat_model_id'],\n",
-    "    )\n",
-    "vertextai_embeddings = VertexAIEmbeddings(\n",
     "    credentials=creds,\n",
-    "    model_name=config[\"embedding_model_id\"]\n",
-    "    )"
+    "    model_name=config[\"chat_model_id\"],\n",
+    ")\n",
+    "vertextai_embeddings = VertexAIEmbeddings(\n",
+    "    credentials=creds, model_name=config[\"embedding_model_id\"]\n",
+    ")"
    ]
   },
   {

docs/howtos/integrations/langfuse.ipynb

@@ -162,6 +162,7 @@
     "from ragas.run_config import RunConfig\n",
     "from ragas.metrics.base import MetricWithLLM, MetricWithEmbeddings\n",
     "\n",
+    "\n",
     "# util function to init Ragas Metrics\n",
     "def init_ragas_metrics(metrics, llm, embedding):\n",
     "    for metric in metrics:\n",
@@ -183,7 +184,7 @@
     "from langchain_openai.chat_models import ChatOpenAI\n",
     "from langchain_openai.embeddings import OpenAIEmbeddings\n",
     "\n",
-    "# wrappers \n",
+    "# wrappers\n",
     "from ragas.llms import LangchainLLMWrapper\n",
     "from ragas.embeddings import LangchainEmbeddingsWrapper\n",
     "\n",
@@ -368,14 +369,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# the logic of the dummy application is \n",
+    "# the logic of the dummy application is\n",
     "# given a question fetch the correspoinding contexts and answers from a dict\n",
     "\n",
     "import hashlib\n",
     "\n",
+    "\n",
     "def hash_string(input_string):\n",
     "    return hashlib.sha256(input_string.encode()).hexdigest()\n",
     "\n",
+    "\n",
     "q_to_c = {}  # map between question and context\n",
     "q_to_a = {}  # map between question and answer\n",
     "for row in amnesty_qa:\n",
@@ -394,6 +397,7 @@
     "# if your running this in a notebook - please run this cell\n",
     "# to manage asyncio event loops\n",
     "import nest_asyncio\n",
+    "\n",
     "nest_asyncio.apply()"
    ]
   },
@@ -407,14 +411,17 @@
     "from langfuse.decorators import observe, langfuse_context\n",
     "from asyncio import run\n",
     "\n",
+    "\n",
     "@observe()\n",
     "def retriver(question: str):\n",
     "    return q_to_c[question]\n",
     "\n",
+    "\n",
     "@observe()\n",
     "def generator(question):\n",
     "    return q_to_a[question]\n",
     "\n",
+    "\n",
     "@observe()\n",
     "def rag_pipeline(question):\n",
     "    q_hash = hash_string(question)\n",
@@ -424,10 +431,7 @@
     "    # score the runs\n",
     "    score = run(score_with_ragas(question, contexts, answer=generated_answer))\n",
     "    for s in score:\n",
-    "        langfuse_context.score_current_trace(\n",
-    "            name=s,\n",
-    "            value=score[s]\n",
-    "        )\n",
+    "        langfuse_context.score_current_trace(name=s, value=score[s])\n",
     "    return generated_answer"
    ]
   },

src/ragas/embeddings/base.py

@@ -153,8 +153,10 @@ def predict(self, texts: List[List[str]]) -> List[List[float]]:
         return predictions.tolist()
 
 
-def embedding_factory(run_config: t.Optional[RunConfig] = None) -> BaseRagasEmbeddings:
-    openai_embeddings = OpenAIEmbeddings()
+def embedding_factory(
+    model: str = "text-embedding-ada-002", run_config: t.Optional[RunConfig] = None
+) -> BaseRagasEmbeddings:
+    openai_embeddings = OpenAIEmbeddings(model=model)
     if run_config is not None:
         openai_embeddings.request_timeout = run_config.timeout
     else:

src/ragas/llms/base.py

@@ -61,7 +61,8 @@ def generate_text(
         temperature: float = 1e-8,
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = None,
-    ) -> LLMResult: ...
+    ) -> LLMResult:
+        ...
 
     @abstractmethod
     async def agenerate_text(
@@ -71,7 +72,8 @@ async def agenerate_text(
         temperature: float = 1e-8,
         stop: t.Optional[t.List[str]] = None,
         callbacks: Callbacks = None,
-    ) -> LLMResult: ...
+    ) -> LLMResult:
+        ...
 
     async def generate(
         self,
@@ -202,7 +204,7 @@ def set_run_config(self, run_config: RunConfig):
 
 
 def llm_factory(
-    model: str = "gpt-3.5-turbo-16k", run_config: t.Optional[RunConfig] = None
+    model: str = "gpt-3.5-turbo", run_config: t.Optional[RunConfig] = None
 ) -> BaseRagasLLM:
     timeout = None
     if run_config is not None:

src/ragas/testset/generator.py

@@ -104,7 +104,6 @@ def from_langchain(
             docstore=docstore,
         )
 
-
     @classmethod
     @deprecated("0.1.4", removal="0.2.0", alternative="from_langchain")
     def with_openai(

tests/benchmarks/benchmark_eval.py

@@ -34,7 +34,7 @@
 ]
 
 # os.environ["PYTHONASYNCIODEBUG"] = "1"
-IGNORE_THREADS = False
+IGNORE_THREADS = True
 IGNORE_ASYNCIO = False
 
 if __name__ == "__main__":