chore: drop support for python 3.9 (#572)

Dropped support for python `3.9` in lights of upcoming EOL in October (https://devguide.python.org/versions/).
Upgraded dependencies.

---------

Co-authored-by: Filip Knefel <[email protected]>

Author: ds-filipknefel

.github/workflows/unit_tests.yml

@@ -17,7 +17,7 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11", "3.12" ]
+        python-version: [ "3.10","3.11", "3.12" ]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -79,7 +79,7 @@ jobs:
     environment: ci
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11", "3.12" ]
+        python-version: [ "3.10","3.11", "3.12" ]
     runs-on: ubuntu-latest
     needs: [ lint ]
     steps:
@@ -100,7 +100,7 @@ jobs:
     environment: ci
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11", "3.12" ]
+        python-version: [ "3.10","3.11", "3.12" ]
     runs-on: ubuntu-latest
     needs: [ lint ]
     steps:
@@ -119,7 +119,7 @@ jobs:
   test_ingest_unit:
     strategy:
       matrix:
-        python-version: [ "3.9","3.10","3.11", "3.12" ]
+        python-version: [ "3.10","3.11", "3.12" ]
     runs-on: ubuntu-latest
     needs: [ lint ]
     steps:
@@ -144,7 +144,7 @@ jobs:
     strategy:
       matrix:
         # TODO include 3.13 when unstructured supports it
-        python-version: [ "3.9","3.10","3.11", "3.12"]
+        python-version: [ "3.10","3.11", "3.12"]
     runs-on: ubuntu-latest
     env:
       NLTK_DATA: ${{ github.workspace }}/nltk_data

CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.2.0
+* **Drop Python 3.9 support**
+
 ## 1.1.3
 
 * **Fix: Remove unnecessary deletion operation in ES connector**

examples/sql.py

@@ -31,7 +31,6 @@
 SQLITE_DB_PATH = "test-sql-db.sqlite"
 
 if __name__ == "__main__":
-
     configs = {
         "context": ProcessorConfig(work_dir=str(work_dir.resolve()), verbose=True),
         "indexer_config": LocalIndexerConfig(input_path=str(docs_path.resolve()) + "/multisimple/"),
@@ -56,8 +55,7 @@
 
     query = None
     script_path = (
-        Path(__file__).parent.parent
-        / Path("test_e2e/env_setup/sql/sqlite-schema.sql")
+        Path(__file__).parent.parent / Path("test_e2e/env_setup/sql/sqlite-schema.sql")
     ).resolve()
     with open(script_path) as f:
         query = f.read()

pyproject.toml

@@ -1,7 +1,7 @@
 [project]
 name = "unstructured_ingest"
 description = "Local ETL data pipeline to get data RAG ready"
-requires-python = ">=3.9, <3.13"
+requires-python = ">=3.10, <3.13"
 authors = [{name = "Unstructured Technologies", email = "[email protected]"}]
 classifiers = [
     "Development Status :: 4 - Beta",
@@ -11,7 +11,6 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Operating System :: OS Independent",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
@@ -147,21 +146,13 @@ test = [
 ]
 # Add constraints needed for CI
 ci = [
-    # consistency with local-inference-pin
-    "protobuf<4.24",
     "grpcio>=1.65.5",
     # TODO: Pinned in transformers package, remove when that gets updated
     "tokenizers>=0.19,<0.20",
-    # TODO: Constaint due to boto, with python before 3.10 not requiring openssl 1.1.1, remove when that gets
-    # updated or we drop support for 3.9
-    "urllib3<1.27",
     # TODO: Constriant due to aiobotocore, remove when that gets updates:
     "botocore<1.34.132",
     # TODO: Constriant due to both 8.5.0 and 8.4.0 being installed during pip-compile
     "importlib-metadata>=8.5.0",
-    # TODO: Constraint due to boto, with python before 3.10 not requiring openssl 1.1.1, remove when that gets
-    # updated or we drop support for 3.9
-    "urllib3<1.27",
     "unstructured-client>= 0.25.8",
     "fsspec==2024.5.0",
     # python 3.12 support

scripts/pip-compile.sh

@@ -2,8 +2,8 @@
 
 set -e
 
-# python version must match lowest supported (3.9)
-python_version=${UV_PYTHON_VERSION:-"3.9"}
+# python version must match lowest supported (3.10)
+python_version=${UV_PYTHON_VERSION:-"3.10"}
 
 # if major and minor python version (x.y) is not equal to current python_version, error out
 if [[ $(python --version | cut -d ' ' -f 2 | cut -d '.' -f 1-2) != $(echo "$python_version" | cut -d '.' -f 1-2) ]]; then

unstructured_ingest/__version__.py

@@ -1 +1 @@
-__version__ = "1.1.3"  # pragma: no cover
+__version__ = "1.2.0"  # pragma: no cover

unstructured_ingest/otel.py

@@ -32,21 +32,23 @@ def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
             self.log_out(self.formatter(span))
         return SpanExportResult.SUCCESS
 
+
 def get_log_out() -> Callable:
     level_names_mapping = {
-        'CRITICAL': logging.CRITICAL,
-        'FATAL': logging.FATAL,
-        'ERROR': logging.ERROR,
-        'WARN': logging.WARNING,
-        'WARNING': logging.WARNING,
-        'INFO': logging.INFO,
-        'DEBUG': logging.DEBUG,
-        'NOTSET': logging.NOTSET,
+        "CRITICAL": logging.CRITICAL,
+        "FATAL": logging.FATAL,
+        "ERROR": logging.ERROR,
+        "WARN": logging.WARNING,
+        "WARNING": logging.WARNING,
+        "INFO": logging.INFO,
+        "DEBUG": logging.DEBUG,
+        "NOTSET": logging.NOTSET,
     }
     log_level = os.getenv("OTEL_LOG_LEVEL", "DEBUG").upper()
     log_level_int = level_names_mapping.get(log_level, logging.DEBUG)
     return lambda message: logger.log(log_level_int, message)
 
+
 @dataclass
 class OtelHandler:
     otel_endpoint: Optional[str] = None

unstructured_ingest/processes/connectors/elasticsearch/elasticsearch.py

@@ -335,7 +335,7 @@ class ElasticsearchUploadStager(UploadStager):
 
     def conform_dict(self, element_dict: dict, file_data: FileData) -> dict:
         data = element_dict.copy()
-        # when _op_type is not specified, it defaults to "index": 
+        # when _op_type is not specified, it defaults to "index":
         # Overwrites if exists, creates if not.
         resp = {
             "_index": self.upload_stager_config.index_name,

unstructured_ingest/processes/connectors/fsspec/fsspec.py

@@ -385,7 +385,7 @@ def fs(self) -> "AbstractFileSystem":
 
     def __post_init__(self):
         super().__post_init__()
-        # TODO once python3.9 no longer supported and kw_only is allowed in dataclasses, remove:
+        # TODO: Consider using `kw_only` instead
         if not self.upload_config:
             raise TypeError(
                 f"{self.__class__.__name__}.__init__() "

unstructured_ingest/utils/filesystem.py

@@ -11,17 +11,17 @@
 def mkdir_concurrent_safe(path: Path) -> None:
     """
     Create directory safely in concurrent environments, handling race conditions.
-    
+
     This addresses the issue where Path.mkdir(parents=True, exist_ok=True) can still
-    raise FileExistsError when multiple processes attempt to create overlapping 
+    raise FileExistsError when multiple processes attempt to create overlapping
     directory structures simultaneously. In this codebase, this occurs when multiple
     files are being downloaded in parallel and archive extraction is happening in parallel.
-    
+
     Related: https://github.com/python/cpython/pull/112966/files
     Python core team used the same approach to fix zipfile race conditions.
     """
     try:
         path.mkdir(parents=True, exist_ok=True)
     except FileExistsError:
         if not (path.exists() and path.is_dir()):
-            raise
+            raise