clean up makefile and scripts

Author: tylerhutcherson

.github/workflows/test.yml

@@ -83,12 +83,12 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          poetry run test-verbose
+          make test-all
 
       - name: Run tests
         if: matrix.connection != 'plain' || matrix.redis-stack-version != 'latest'
         run: |
-          SKIP_VECTORIZERS=True SKIP_RERANKERS=True poetry run test-verbose
+          make test
 
       - name: Run notebooks
         if: matrix.connection == 'plain' && matrix.redis-stack-version == 'latest'
@@ -106,7 +106,7 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
         run: |
-          cd docs/ && poetry run pytest --nbval-lax ./user_guide -vv 
+          make test-notebooks 
 
   docs:
     runs-on: ubuntu-latest

Makefile

@@ -1,4 +1,4 @@
-.PHONY: install format lint test test-all clean redis-start redis-stop check-types docs-build docs-serve check
+.PHONY: install format lint test test-all test-notebooks clean redis-start redis-stop check-types docs-build docs-serve check
 
 install:
 	poetry install --all-extras
@@ -19,10 +19,13 @@ check-types:
 lint: format check-types
 
 test:
-	SKIP_RERANKERS=true SKIP_VECTORIZERS=true poetry run test-verbose
+	poetry run test-verbose 
 
 test-all:
-	poetry run test-verbose
+	poetry run test-verbose --run-api-tests
+
+test-notebooks:
+	poetry run test-notebooks
 
 check: lint test
 

scripts.py

@@ -48,7 +48,7 @@ def test_verbose():
 
 
 def test_notebooks():
-    subprocess.run(["cd", "docs/", "&&", "poetry run treon", "-v"], check=True)
+    subprocess.run(["cd", "docs/", "&&", "poetry run pytest --nbval-lax ./user_guide", "-vv"], check=True)
 
 
 def build_docs():

tests/conftest.py

@@ -58,11 +58,6 @@ async def async_client(redis_url):
         redis_url
     ) as client:
         yield client
-        # try:
-        #     await client.aclose()
-        # except RuntimeError as e:
-        #     if "Event loop is closed" not in str(e):
-        #         raise
 
 
 @pytest.fixture
@@ -72,51 +67,6 @@ def client(redis_url):
     """
     conn = RedisConnectionFactory.get_redis_connection(redis_url)
     yield conn
-    # conn.close()
-
-
-# @pytest.fixture
-# def openai_key():
-#     return os.getenv("OPENAI_API_KEY")
-
-
-# @pytest.fixture
-# def openai_version():
-#     return os.getenv("OPENAI_API_VERSION")
-
-
-# @pytest.fixture
-# def azure_endpoint():
-#     return os.getenv("AZURE_OPENAI_ENDPOINT")
-
-
-# @pytest.fixture
-# def cohere_key():
-#     return os.getenv("COHERE_API_KEY")
-
-
-# @pytest.fixture
-# def mistral_key():
-#     return os.getenv("MISTRAL_API_KEY")
-
-
-# @pytest.fixture
-# def gcp_location():
-#     return os.getenv("GCP_LOCATION")
-
-
-# @pytest.fixture
-# def gcp_project_id():
-#     return os.getenv("GCP_PROJECT_ID")
-
-
-# @pytest.fixture
-# def aws_credentials():
-#     return {
-#         "aws_access_key_id": os.getenv("AWS_ACCESS_KEY_ID"),
-#         "aws_secret_access_key": os.getenv("AWS_SECRET_ACCESS_KEY"),
-#         "aws_region": os.getenv("AWS_REGION", "us-east-1"),
-#     }
 
 
 @pytest.fixture
@@ -201,6 +151,8 @@ def pytest_collection_modifyitems(
 ) -> None:
     if config.getoption("--run-api-tests"):
         return
+
+    # Otherwise skip all tests requiring an API key
     skip_api = pytest.mark.skip(
         reason="Skipping test because API keys are not provided. Use --run-api-tests to run these tests."
     )

tests/integration/test_rerankers.py

@@ -9,24 +9,14 @@
 )
 
 
-@pytest.fixture
-def skip_reranker() -> bool:
-    # os.getenv returns a string
-    v = os.getenv("SKIP_RERANKERS", "False").lower() == "true"
-    return v
-
-
 # Fixture for the reranker instance
 @pytest.fixture(
     params=[
         CohereReranker,
         VoyageAIReranker,
     ]
 )
-def reranker(request, skip_reranker):
-    if skip_reranker:
-        pytest.skip("Skipping reranker instantiation...")
-
+def reranker(request):
     if request.param == CohereReranker:
         return request.param()
     elif request.param == VoyageAIReranker:
@@ -43,7 +33,7 @@ def hfCrossEncoderRerankerWithCustomModel():
     return HFCrossEncoderReranker("cross-encoder/stsb-distilroberta-base")
 
 
-# Test for basic ranking functionality
+@pytest.mark.requires_api_keys
 def test_rank_documents(reranker):
     docs = ["document one", "document two", "document three"]
     query = "search query"
@@ -55,7 +45,7 @@ def test_rank_documents(reranker):
     assert all(isinstance(score, float) for score in scores)  # Scores should be floats
 
 
-# Test for asynchronous ranking functionality
+@pytest.mark.requires_api_keys
 @pytest.mark.asyncio
 async def test_async_rank_documents(reranker):
     docs = ["document one", "document two", "document three"]
@@ -68,7 +58,7 @@ async def test_async_rank_documents(reranker):
     assert all(isinstance(score, float) for score in scores)  # Scores should be floats
 
 
-# Test handling of bad input
+@pytest.mark.requires_api_keys
 def test_bad_input(reranker):
     with pytest.raises(Exception):
         reranker.rank("", [])  # Empty query or documents

tests/integration/test_vectorizers.py

@@ -15,12 +15,6 @@
 )
 
 
-@pytest.fixture
-def skip_vectorizer() -> bool:
-    v = os.getenv("SKIP_VECTORIZERS", "False").lower() == "true"
-    return v
-
-
 @pytest.fixture(
     params=[
         HFTextVectorizer,
@@ -34,10 +28,7 @@ def skip_vectorizer() -> bool:
         VoyageAITextVectorizer,
     ]
 )
-def vectorizer(request, skip_vectorizer):
-    if skip_vectorizer:
-        pytest.skip("Skipping vectorizer instantiation...")
-
+def vectorizer(request):
     if request.param == HFTextVectorizer:
         return request.param()
     elif request.param == OpenAITextVectorizer:
@@ -70,10 +61,7 @@ def embed_many(texts):
 
 
 @pytest.fixture
-def bedrock_vectorizer(skip_vectorizer):
-    if skip_vectorizer:
-        pytest.skip("Skipping Bedrock vectorizer tests...")
-
+def bedrock_vectorizer():
     return BedrockTextVectorizer(
         model=os.getenv("BEDROCK_MODEL_ID", "amazon.titan-embed-text-v2:0")
     )
@@ -108,6 +96,7 @@ def embed_many_with_args(self, texts, param=True):
     return MyEmbedder
 
 
+@pytest.mark.requires_api_keys
 def test_vectorizer_embed(vectorizer):
     text = "This is a test sentence."
     if isinstance(vectorizer, CohereTextVectorizer):
@@ -121,6 +110,7 @@ def test_vectorizer_embed(vectorizer):
     assert len(embedding) == vectorizer.dims
 
 
+@pytest.mark.requires_api_keys
 def test_vectorizer_embed_many(vectorizer):
     texts = ["This is the first test sentence.", "This is the second test sentence."]
     if isinstance(vectorizer, CohereTextVectorizer):
@@ -137,6 +127,7 @@ def test_vectorizer_embed_many(vectorizer):
     )
 
 
+@pytest.mark.requires_api_keys
 def test_vectorizer_bad_input(vectorizer):
     with pytest.raises(TypeError):
         vectorizer.embed(1)
@@ -148,6 +139,7 @@ def test_vectorizer_bad_input(vectorizer):
         vectorizer.embed_many(42)
 
 
+@pytest.mark.requires_api_keys
 def test_bedrock_bad_credentials():
     with pytest.raises(ValueError):
         BedrockTextVectorizer(
@@ -158,6 +150,7 @@ def test_bedrock_bad_credentials():
         )
 
 
+@pytest.mark.requires_api_keys
 def test_bedrock_invalid_model(bedrock_vectorizer):
     with pytest.raises(ValueError):
         bedrock = BedrockTextVectorizer(model="invalid-model")
@@ -250,64 +243,48 @@ def bad_return_type(text: str) -> str:
         )
 
 
-@pytest.mark.parametrize(
-    "vector_class",
-    [
-        AzureOpenAITextVectorizer,
-        BedrockTextVectorizer,
-        CohereTextVectorizer,
-        CustomTextVectorizer,
-        HFTextVectorizer,
-        MistralAITextVectorizer,
-        OpenAITextVectorizer,
-        VertexAITextVectorizer,
-        VoyageAITextVectorizer,
-    ],
-)
-def test_dtypes(vector_class, skip_vectorizer):
-    if skip_vectorizer:
-        pytest.skip("Skipping vectorizer instantiation...")
-
-    # test dtype defaults to float32
-    if issubclass(vector_class, CustomTextVectorizer):
-        vectorizer = vector_class(embed=lambda x, input_type=None: [1.0, 2.0, 3.0])
-    elif issubclass(vector_class, AzureOpenAITextVectorizer):
-        vectorizer = vector_class(
-            model=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002")
-        )
-    else:
-        vectorizer = vector_class()
-
-    assert vectorizer.dtype == "float32"
-
-    # test initializing dtype in constructor
-    for dtype in ["float16", "float32", "float64", "bfloat16"]:
-        if issubclass(vector_class, CustomTextVectorizer):
-            vectorizer = vector_class(embed=lambda x: [1.0, 2.0, 3.0], dtype=dtype)
-        elif issubclass(vector_class, AzureOpenAITextVectorizer):
-            vectorizer = vector_class(
-                model=os.getenv(
-                    "AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002"
-                ),
-                dtype=dtype,
-            )
-        else:
-            vectorizer = vector_class(dtype=dtype)
-
-        assert vectorizer.dtype == dtype
-
-    # test validation of dtype on init
-    if issubclass(vector_class, CustomTextVectorizer):
-        pytest.skip("skipping custom text vectorizer")
+# @pytest.mark.requires_api_keys
+# def test_dtypes(vectorizer):
+#     # # test dtype defaults to float32
+#     # if issubclass(vectorizer, CustomTextVectorizer):
+#     #     vectorizer = vectorizer(embed=lambda x, input_type=None: [1.0, 2.0, 3.0])
+#     # elif issubclass(vectorizer, AzureOpenAITextVectorizer):
+#     #     vectorizer = vectorizer(
+#     #         model=os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002")
+#     #     )
+#     # else:
+#     #     vectorizer = vector_class()
 
-    with pytest.raises(ValueError):
-        vectorizer = vector_class(dtype="float25")
+#     assert vectorizer.dtype == "float32"
 
-    with pytest.raises(ValueError):
-        vectorizer = vector_class(dtype=7)
+#     # test initializing dtype in constructor
+#     for dtype in ["float16", "float32", "float64", "bfloat16"]:
+#         if issubclass(vectorizer, CustomTextVectorizer):
+#             vectorizer = vectorizer(embed=lambda x: [1.0, 2.0, 3.0], dtype=dtype)
+#         elif issubclass(vectorizer, AzureOpenAITextVectorizer):
+#             vectorizer = vectorizer(
+#                 model=os.getenv(
+#                     "AZURE_OPENAI_DEPLOYMENT_NAME", "text-embedding-ada-002"
+#                 ),
+#                 dtype=dtype,
+#             )
+#         else:
+#             vectorizer = vectorizer(dtype=dtype)
 
-    with pytest.raises(ValueError):
-        vectorizer = vector_class(dtype=None)
+#         assert vectorizer.dtype == dtype
+
+#     # test validation of dtype on init
+#     if issubclass(vectorizer, CustomTextVectorizer):
+#         pytest.skip("skipping custom text vectorizer")
+
+#     with pytest.raises(ValueError):
+#         vectorizer = vectorizer(dtype="float25")
+
+#     with pytest.raises(ValueError):
+#         vectorizer = vectorizer(dtype=7)
+
+#     with pytest.raises(ValueError):
+#         vectorizer = vectorizer(dtype=None)
 
 
 @pytest.fixture(
@@ -319,10 +296,7 @@ def test_dtypes(vector_class, skip_vectorizer):
         VoyageAITextVectorizer,
     ]
 )
-def avectorizer(request, skip_vectorizer):
-    if skip_vectorizer:
-        pytest.skip("Skipping vectorizer instantiation...")
-
+def avectorizer(request):
     if request.param == CustomTextVectorizer:
 
         def embed_func(text):
@@ -341,6 +315,7 @@ async def aembed_many_func(texts):
         return request.param()
 
 
+@pytest.mark.requires_api_keys
 @pytest.mark.asyncio
 async def test_vectorizer_aembed(avectorizer):
     text = "This is a test sentence."
@@ -350,6 +325,7 @@ async def test_vectorizer_aembed(avectorizer):
     assert len(embedding) == avectorizer.dims
 
 
+@pytest.mark.requires_api_keys
 @pytest.mark.asyncio
 async def test_vectorizer_aembed_many(avectorizer):
     texts = ["This is the first test sentence.", "This is the second test sentence."]
@@ -362,6 +338,7 @@ async def test_vectorizer_aembed_many(avectorizer):
     )
 
 
+@pytest.mark.requires_api_keys
 @pytest.mark.asyncio
 async def test_avectorizer_bad_input(avectorizer):
     with pytest.raises(TypeError):