python migration from 3.9 to 3.12 (#178)

* python migration from 3.9 to 3.12

* github workflow updated

* pre-commit updated

* README updated based on UV changes

* build system updated in each and every pyproject.toml fiel

* add pyproject.toml

* build related changes reverted back

* build system commented on azure_open_ai

* reverted build change

* RC Version updated

* Version updated

Author: jaseemjaskp

.github/workflows/pypi-publish.yml

@@ -3,12 +3,12 @@ name: Publish Python Package
 on:
   release:
     types:
-        - published
+      - published
 
 jobs:
   pypi-publish:
     name: upload release to PyPI
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     permissions:
       contents: read
       id-token: write
@@ -19,13 +19,16 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-            python-version: '3.9.6'
+          python-version: "3.12.9"
 
-      - name: Setup PDM
-        uses: pdm-project/setup-pdm@v4
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
         with:
-          python-version: 3.9.6
-          version: 2.12.3
+          # Install a specific version of uv.
+          version: "0.6.11"
 
-      - name: Publish package distributions to PyPI
-        run: pdm publish
+      - name: Build package
+        run: uv build
+
+      - name: Publish to PyPI
+        run: uv publish

.pre-commit-config.yaml

@@ -1,15 +1,10 @@
----
-# See https://pre-commit.com for more information
-# See https://pre-commit.com/hooks.html for more hooks
-# - Added pkgs feature flag auto generated code to flake8 exclude list
-# Force all unspecified python hooks to run python 3.10
 default_language_version:
-  python: python3.9
+  python: python3.12
 default_stages:
-  - commit
+  - pre-commit
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         exclude_types:
@@ -34,125 +29,58 @@ repos:
       - id: destroyed-symlinks
       - id: forbid-new-submodules
       - id: mixed-line-ending
-      - id: no-commit-to-branch
-  - repo: https://github.com/adrienverge/yamllint
-    rev: v1.35.1
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.3.4
     hooks:
-      - id: yamllint
-        args: ["-d", "relaxed"]
-        language: system
-  - repo: https://github.com/rhysd/actionlint
-    rev: v1.6.26
-    hooks:
-      - id: actionlint-docker
-        args: [-ignore, 'label ".+" is unknown']
-  - repo: https://github.com/psf/black
-    rev: 24.2.0
-    hooks:
-      - id: black
-        args: [--config=pyproject.toml, -l 88]
-        language: system
-        exclude: |
-          (?x)^(
-              pkgs/unstract-flags/src/unstract/flags/evaluation_.*\.py|
-          )$
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
-    hooks:
-      - id: flake8
-        args: [--max-line-length=88]
-        exclude: |
-          (?x)^(
-              .*migrations/.*\.py|
-              unstract-core/tests/.*|
-              pkgs/unstract-flags/src/unstract/flags/evaluation_.*\.py|
-          )$
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        files: "\\.(py)$"
-        args:
-          [
-            "--profile",
-            "black",
-            "--filter-files",
-            --settings-path=pyproject.toml,
-          ]
+      - id: ruff
+        args: [--fix]
+      - id: ruff-format
+
   - repo: https://github.com/hadialqattan/pycln
     rev: v2.4.0
     hooks:
       - id: pycln
+        entry: uv run pycln
         args: [--config=pyproject.toml]
+
   - repo: https://github.com/pycqa/docformatter
     rev: v1.7.5
     hooks:
       - id: docformatter
-  # - repo: https://github.com/MarcoGorelli/absolufy-imports
-  #   rev: v0.3.1
-  #   hooks:
-  #     - id: absolufy-imports
+        language: python
+
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
+    rev: v3.17.0
     hooks:
       - id: pyupgrade
         entry: pyupgrade --py39-plus --keep-runtime-typing
         types:
           - python
-  - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.18.2
+
+  - repo: https://github.com/astral-sh/uv-pre-commit
+    rev: 0.6.11
     hooks:
-      - id: gitleaks
-  - repo: https://github.com/asottile/yesqa
-    rev: v1.5.0
+      - id: uv-lock
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
     hooks:
-      - id: yesqa
-  # TODO: Uncomment after typing the SDK
-  # - repo: https://github.com/pre-commit/mirrors-mypy
-  #   rev: v1.8.0
-  #   hooks:
-  #     - id: mypy
-  #       language: system
-  #       entry: mypy .
-  #       pass_filenames: false
-  #       # IMPORTANT!
-  #       # Keep args same as tool.mypy section in pyproject.toml
-  #       args:
-  #         [
-  #           --allow-subclassing-any,
-  #           --allow-untyped-decorators,
-  #           --check-untyped-defs,
-  #           --exclude, ".*migrations/.*.py",
-  #           --exclude, "backend/prompt/.*",
-  #           --exclude, "document_display_service/.*",
-  #           --exclude, "pkgs/unstract-connectors/tests/.*",
-  #           --exclude, "pkgs/unstract-core/.*",
-  #           --exclude, "pkgs/unstract-flags/src/unstract/flags/.*",
-  #           --exclude, "sdks/.*",
-  #           --exclude, "unstract-document-service/.*",
-  #           --exclude, "__pypackages__/.*",
-  #           --follow-imports, "silent",
-  #           --ignore-missing-imports,
-  #           --implicit-reexport,
-  #           --pretty,
-  #           --python-version=3.9,
-  #           --show-column-numbers,
-  #           --show-error-codes,
-  #           --strict,
-  #           --warn-redundant-casts,
-  #           --warn-return-any,
-  #           --warn-unreachable,
-  #           --warn-unused-configs,
-  #           --warn-unused-ignores,
-  #         ]
+      - id: mypy
+        language: system
+        entry: uv run mypy .
+        pass_filenames: false
+        additional_dependencies: []
+
   - repo: https://github.com/igorshubovych/markdownlint-cli
-    rev: v0.39.0
+    rev: v0.42.0
     hooks:
       - id: markdownlint
         args: [--disable, MD013]
       - id: markdownlint-fix
         args: [--disable, MD013]
-  - repo: https://github.com/pdm-project/pdm
-    rev: 2.12.3
+
+  - repo: https://github.com/gitleaks/gitleaks
+    rev: v8.18.2
     hooks:
-      - id: pdm-lock-check
+      - id: gitleaks

README.md

@@ -61,25 +61,27 @@ Index Version **0.9.28** as on January 14th, 2024
 
 Ensure that you have all the required dependencies and pre-commit hooks installed
 ```shell
-pdm install
+uv sync
 pre-commit install
 ```
 
 Once the changes have been made, it can be tested with [Unstract](https://github.com/Zipstack/unstract) through the following means.
 
-#### With PDM
-Specify the SDK as a dependency to a project using a tool like `pdm` by adding the following to your `pyproject.toml`
+#### With UV
+Specify the SDK as a dependency to a project using a tool like `uv` by adding the following to your `pyproject.toml`
 
 ```toml
-[tool.pdm.dev-dependencies]
-local_copies = [
-    "-e unstract-adapters @ file:///${UNSTRACT_ADAPTERS_PATH}",
-    "-e unstract-sdk @ file:///${UNSTRACT_SDK_PATH}",
+dependencies = [
+    "unstract-sdk"
 ]
-```
-Or by running the below command
-```shell
-pdm add -e /path/to/unstract-sdk --dev
+
+[dependency-groups]
+dev = [
+"unstract-sdk"
+]
+
+[tool.uv.sources]
+unstract-sdk = { path = "${UNSTRACT_SDK_PATH", editable = true  }
 ```
 
 #### With pip
@@ -90,23 +92,24 @@ pdm add -e /path/to/unstract-sdk --dev
 NOTE: Building locally might require the below section to be replaced in the `unstract-sdk`'s build system configuration
 ```
 [build-system]
-requires = ["setuptools", "wheel"]
-build-backend = "setuptools.build_meta"
+requires = ["hatchling"]
+build-backend = "hatchling.build"
 ```
 - Another option is to provide a git URL in `requirements.txt`, this can come in handy while building tool
 docker images. Don't forget to run `apt install git` within the `Dockerfile` for this
 ```shell
-unstract-sdk @ git+https://github.com/Zipstack/unstract-sdk@feature-branch
+[tool.uv.sources]
+unstract-sdk = { git = "git+https://github.com/Zipstack/unstract-sdk@feature-branch" }
 ```
 
 - Or try installing a [local PyPI server](https://pypi.org/project/pypiserver/) and upload / download your package from this server
 
 #### Additonal dependencies for tool
-Tools may need to be backed up by a file storage. unstract.sdk.file_storage contains the required interfaces for the 
-same. fssepc is being used underneath to implement these interfaces. Hence, one can choose to use a file_system 
-supported by fsspec for this. However, the required dependencies need to be added in the tool dependency manager. 
-Eg. If the tool is using Minio as the underlying file storage, then s3fs can be added to support it. 
-Similarly, for Google Cloud Storage, gcsfs is to be added. 
+Tools may need to be backed up by a file storage. unstract.sdk.file_storage contains the required interfaces for the
+same. fssepc is being used underneath to implement these interfaces. Hence, one can choose to use a file_system
+supported by fsspec for this. However, the required dependencies need to be added in the tool dependency manager.
+Eg. If the tool is using Minio as the underlying file storage, then s3fs can be added to support it.
+Similarly, for Google Cloud Storage, gcsfs is to be added.
 The following versions are tested in the SDK using unit test cases for the above package.
     gcsfs==2024.10.0
     s3fs==2024.10.0