feat/support python 3.13 (#331)

rbiseck3 · web-flow · commit 3a5d8243ae46 · 2025-01-14T13:38:26.000-05:00
* add 3.13 to supported python versions

* fix tidy

* isolate unstructured unit tests

* fix file path for unit tests

* explicitly download nltk data
diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml
@@ -17,7 +17,7 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11" ]
+        python-version: [ "3.9","3.10","3.11", "3.12", "3.13"]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -67,7 +67,7 @@ jobs:
     environment: ci
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11" ]
+        python-version: [ "3.9","3.10","3.11", "3.12", "3.13" ]
     runs-on: ubuntu-latest
     needs: [ lint ]
     steps:
@@ -86,7 +86,7 @@ jobs:
     environment: ci
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11" ]
+        python-version: [ "3.9","3.10","3.11", "3.12", "3.13" ]
     runs-on: ubuntu-latest
     needs: [ lint ]
     steps:
@@ -105,7 +105,27 @@ jobs:
   test_ingest_unit:
     strategy:
       matrix:
-        python-version: [ "3.9","3.10" ]
+        python-version: [ "3.9","3.10","3.11", "3.12", "3.13" ]
+    runs-on: ubuntu-latest
+    needs: [ lint ]
+    steps:
+      # actions/checkout MUST come before auth
+      - uses: 'actions/checkout@v4'
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Test Ingest (unit)
+        run: |
+          make install-base
+          make install-test
+          make unit-test
+
+  test_ingest_unit_unstructured:
+    strategy:
+      matrix:
+        # TODO include 3.13 when unstructured supports it
+        python-version: [ "3.9","3.10","3.11", "3.12"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data
@@ -122,4 +142,5 @@ jobs:
           make install-base
           make install-test
           pip install unstructured
-          make unit-test
+          python -m nltk.downloader -d $NLTK_DATA punkt_tab averaged_perceptron_tagger_eng
+          make unit-test-unstructured
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## 0.3.13-dev3
+## 0.3.13-dev4
 
 ### Fixes
 
@@ -9,6 +9,7 @@
 ### Enhancements
 
 * **Async support for all IO-bounded embedders**
+* **Expand support to Python 3.13**
 
 ## 0.3.12
 
diff --git a/Makefile b/Makefile
@@ -119,7 +119,11 @@ check-version:
 ###########
 .PHONY: unit-test
 unit-test:
-	PYTHONPATH=. pytest -sv --cov unstructured_ingest/ test/unit
+	PYTHONPATH=. pytest -sv --cov unstructured_ingest/ test/unit --ignore test/unit/unstructured
+
+.PHONY: unit-test-unstructured
+unit-test-unstructured:
+	PYTHONPATH=. pytest -sv --cov unstructured_ingest/ test/unit/unstructured
 
 .PHONY: integration-test
 integration-test:
diff --git a/setup.py b/setup.py
@@ -173,7 +173,7 @@ def load_requirements(file: Union[str, Path]) -> List[str]:
     long_description_content_type="text/markdown",
     keywords="NLP PDF HTML CV XML parsing preprocessing",
     url="https://github.com/Unstructured-IO/unstructured-ingest",
-    python_requires=">=3.9.0,<3.13",
+    python_requires=">=3.9.0,<3.14",
     classifiers=[
         "Development Status :: 4 - Beta",
         "Intended Audience :: Developers",
diff --git a/test/unit/embed/test_mixedbreadai.py b/test/unit/embed/test_mixedbreadai.py
@@ -31,7 +31,7 @@ def mock_embeddings(
         )
     )
 
-    raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]
+    raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]
     elements = encoder.embed_documents(
         elements=raw_elements,
     )
diff --git a/test/unit/embed/test_octoai.py b/test/unit/embed/test_octoai.py
@@ -17,7 +17,7 @@ def test_embed_documents_does_not_break_element_to_dict(mocker):
     mocker.patch.object(OctoAiEmbeddingConfig, "get_client", return_value=mock_client)
 
     encoder = OctoAIEmbeddingEncoder(config=OctoAiEmbeddingConfig(api_key="api_key"))
-    raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]
+    raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]
 
     elements = encoder.embed_documents(
         elements=raw_elements,
diff --git a/test/unit/embed/test_openai.py b/test/unit/embed/test_openai.py
@@ -10,7 +10,7 @@ def test_embed_documents_does_not_break_element_to_dict(mocker):
     mocker.patch.object(OpenAIEmbeddingConfig, "get_client", return_value=mock_client)
 
     encoder = OpenAIEmbeddingEncoder(config=OpenAIEmbeddingConfig(api_key="api_key"))
-    raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]
+    raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]
 
     elements = encoder.embed_documents(
         elements=raw_elements,
diff --git a/test/unit/embed/test_vertexai.py b/test/unit/embed/test_vertexai.py
@@ -15,7 +15,7 @@ def test_embed_documents_does_not_break_element_to_dict(mocker):
     # Mock create_client to return our mock_client
     mocker.patch.object(VertexAIEmbeddingConfig, "get_client", return_value=mock_client)
     encoder = VertexAIEmbeddingEncoder(config=VertexAIEmbeddingConfig(api_key={"api_key": "value"}))
-    raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]
+    raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]
 
     elements = encoder.embed_documents(
         elements=raw_elements,
diff --git a/test/unit/embed/test_voyageai.py b/test/unit/embed/test_voyageai.py
@@ -14,7 +14,7 @@ def test_embed_documents_does_not_break_element_to_dict(mocker):
     encoder = VoyageAIEmbeddingEncoder(
         config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-law-2")
     )
-    raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]
+    raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]
 
     elements = encoder.embed_documents(
         elements=raw_elements,
diff --git a/test/unit/pipeline/reformat/test_chunking.py b/test/unit/pipeline/reformat/test_chunking.py
diff --git a/test/unit/unstructured/test_chunking_utils.py b/test/unit/unstructured/test_chunking_utils.py
@@ -10,7 +10,7 @@
 )
 
 test_file_path = Path(__file__).resolve()
-project_root = test_file_path.parents[2]
+project_root = test_file_path.parents[3]
 docs_path = project_root / "example-docs"
 
 
diff --git a/test/unit/unstructured/test_interfaces.py b/test/unit/unstructured/test_interfaces.py
@@ -18,7 +18,7 @@
     ReadConfig,
 )
 
-DIRECTORY = pathlib.Path(__file__).parents[2].resolve()
+DIRECTORY = pathlib.Path(__file__).parents[3].resolve()
 EXAMPLE_DOCS_DIRECTORY = DIRECTORY / "example-docs"
 TEST_DOWNLOAD_DIR = "/tmp"
 TEST_OUTPUT_DIR = "/tmp"
diff --git a/unstructured_ingest/__version__.py b/unstructured_ingest/__version__.py
@@ -1 +1 @@
-__version__ = "0.3.13-dev3"  # pragma: no cover
+__version__ = "0.3.13-dev4"  # pragma: no cover
diff --git a/unstructured_ingest/connector/notion/helpers.py b/unstructured_ingest/connector/notion/helpers.py
@@ -471,7 +471,7 @@ def build_columned_list(client: Client, column_parent: Block) -> HtmlTag:
         ):
             columns_content.append(
                 Div(
-                    [Style(f"width:{100/num_columns}%; float: left")],
+                    [Style(f"width:{100 / num_columns}%; float: left")],
                     [content.block.get_html() for content in column_content_chunk],
                 ),
             )
diff --git a/unstructured_ingest/v2/examples/chroma.py b/unstructured_ingest/v2/examples/chroma.py
@@ -45,7 +45,7 @@
             access_config=ChromaAccessConfig(settings=None, headers=None),
             host="localhost",
             port=8047,
-            collection_name=f"test-collection-{random.randint(1000,9999)}",
+            collection_name=f"test-collection-{random.randint(1000, 9999)}",
             tenant="default_tenant",
             database="default_database",
         ),
diff --git a/unstructured_ingest/v2/examples/mongodb.py b/unstructured_ingest/v2/examples/mongodb.py
@@ -45,7 +45,7 @@
             access_config=MongoDBAccessConfig(uri=None),
             host="localhost",
             port=27017,
-            collection=f"test-collection-{random.randint(1000,9999)}",
+            collection=f"test-collection-{random.randint(1000, 9999)}",
             database="testDatabase",
         ),
         stager_config=MongoDBUploadStagerConfig(),
diff --git a/unstructured_ingest/v2/processes/connectors/notion/helpers.py b/unstructured_ingest/v2/processes/connectors/notion/helpers.py
@@ -133,7 +133,7 @@ def build_html(
         if isinstance(current_block["block"].block, notion_blocks.Column):
             html.append(
                 Div(
-                    [Style(f"width:{100/current_block['peers_count']}%; float: left")],
+                    [Style(f"width:{100 / current_block['peers_count']}%; float: left")],
                     children_html,
                 )
             )

Original file line number	Diff line number	Diff line change
`@@ -31,7 +31,7 @@ def mock_embeddings(`
`31`	`31`	`)`
`32`	`32`	`)`
`33`	`33`
`34`		`- raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]`
	`34`	`+ raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]`
`35`	`35`	`elements = encoder.embed_documents(`
`36`	`36`	`elements=raw_elements,`
`37`	`37`	`)`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@ def test_embed_documents_does_not_break_element_to_dict(mocker):`
`14`	`14`	`encoder = VoyageAIEmbeddingEncoder(`
`15`	`15`	`config=VoyageAIEmbeddingConfig(api_key="api_key", model_name="voyage-law-2")`
`16`	`16`	`)`
`17`		`- raw_elements = [{"text": f"This is sentence {i+1}"} for i in range(2)]`
	`17`	`+ raw_elements = [{"text": f"This is sentence {i + 1}"} for i in range(2)]`
`18`	`18`
`19`	`19`	`elements = encoder.embed_documents(`
`20`	`20`	`elements=raw_elements,`