Migrate dev dependencies to pyproject.toml, remove dev-requirements.txt and update related files and workflows.

Dump flake8 and use ruff that covers format and linting.

Update readme

Signed-off-by: Filinto Duran <[email protected]>

Author: filintod

.flake8

@@ -28,7 +28,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install -r dev-requirements.txt
+        pip install '.[dev]'
     - name: Lint with flake8
       run: |
         flake8 . --count --show-source --statistics --exit-zero

.github/workflows/pr-validation.yml

@@ -162,33 +162,48 @@ The following is more information about how to develop this project. Note that d
 ### Generating protobufs
 
 ```sh
-pip3 install -r dev-requirements.txt
+# install dev dependencies for generating protobufs and running tests
+pip3 install '.[dev]'
+
 make gen-proto
 ```
 
 This will download the `orchestrator_service.proto` from the `microsoft/durabletask-protobuf` repo and compile it using `grpcio-tools`. The version of the source proto file that was downloaded can be found in the file `durabletask/internal/PROTO_SOURCE_COMMIT_HASH`.
 
 ### Running unit tests
 
-Unit tests can be run using the following command from the project root. Unit tests _don't_ require a sidecar process to be running.
+Unit tests can be run using the following command from the project root. 
+Unit tests _don't_ require a sidecar process to be running.
+
+To run on a specific python version (eg: 3.11), run the following command from the project root:
 
 ```sh
-make test-unit
+tox -e py311
 ```
 
 ### Running E2E tests
 
-The E2E (end-to-end) tests require a sidecar process to be running. You can use the Dapr sidecar for this or run a Durable Task test sidecar using the following command:
+The E2E (end-to-end) tests require a sidecar process to be running. 
+
+For non-multi app activities test you can use the Durable Task test sidecar using the following command:
 
 ```sh
 go install github.com/dapr/durabletask-go@main
 durabletask-go --port 4001
 ```
 
-To run the E2E tests, run the following command from the project root:
+Certain aspects like multi-app activities require the full dapr runtime to be running.
+
+```shell
+dapr init || true
+
+dapr run --app-id test-app --dapr-grpc-port  4001 --components-path ./examples/components/
+```
+
+To run the E2E tests on a specific python version (eg: 3.11), run the following command from the project root:
 
 ```sh
-make test-e2e
+tox -e py311-e2e
 ```
 
 ## Contributing

README.md

@@ -137,12 +137,14 @@ def schedule_new_orchestration(
     ) -> str:
         name = orchestrator if isinstance(orchestrator, str) else task.get_name(orchestrator)
 
+        input_pb = (
+            wrappers_pb2.StringValue(value=shared.to_json(input)) if input is not None else None
+        )
+
         req = pb.CreateInstanceRequest(
             name=name,
             instanceId=instance_id if instance_id else uuid.uuid4().hex,
-            input=wrappers_pb2.StringValue(value=shared.to_json(input))
-            if input is not None
-            else None,
+            input=input_pb,
             scheduledStartTimestamp=helpers.new_timestamp(start_at) if start_at else None,
             version=wrappers_pb2.StringValue(value=""),
             orchestrationIdReusePolicy=reuse_id_policy,

dev-requirements.txt

@@ -0,0 +1,16 @@
+apiVersion: dapr.io/v1alpha1
+kind: Component
+metadata:
+  name: statestore
+spec:
+  type: state.redis
+  version: v1
+  metadata:
+  - name: redisHost
+    value: localhost:6379
+  - name: redisPassword
+    value: ""
+  - name: actorStateStore
+    value: "true"
+  - name: keyPrefix
+    value: "workflow"

durabletask/client.py

@@ -49,6 +49,19 @@ markers = [
     "e2e: mark a test as an end-to-end test that requires a running sidecar"
 ]
 
+[project.optional-dependencies]
+dev = [
+    "pytest",
+    "pytest-asyncio>=0.23",
+    "flake8==7.3.0",
+    "tox>=4.0.0",
+    "pytest-cov",
+    "ruff",
+
+    # grpc gen
+    "grpcio-tools==1.75.1",
+]
+
 [tool.ruff]
 target-version = "py39" # TODO: update to py310 when we drop support for py39
 line-length = 100
@@ -59,7 +72,6 @@ select = [
     "E",  # pycodestyle errors
     "W",  # pycodestyle warnings
     "F",  # pyflakes
-    "I",  # isort
     "C",  # flake8-comprehensions
     "B",  # flake8-bugbear
     "UP",  # pyupgrade

examples/components/statestore.yaml

@@ -1,6 +1 @@
-autopep8
-grpcio>=1.60.0 # 1.60.0 is the version introducing protobuf 1.25.X support, newer versions are backwards compatible
-protobuf
-pytest
-pytest-cov
-asyncio
+# requirements in pyproject.toml

pyproject.toml

@@ -6,6 +6,8 @@ envlist =
     flake8,
     ruff,
     mypy,
+# TODO: switch runner to uv (tox-uv plugin)
+runner = virtualenv
 
 [testenv]
 # you can run tox with the e2e pytest marker using tox factors:
@@ -16,7 +18,7 @@ envlist =
 #   DAPR_GRPC_ENDPOINT=localhost:12345 tox -e py310-e2e -- -s
 setenv =
     PYTHONDONTWRITEBYTECODE=1
-deps = -rdev-requirements.txt
+deps = .[dev]
 commands =
     !e2e: pytest {posargs} -q -k "not e2e" --cov=durabletask --cov-branch --cov-report=term-missing --cov-report=xml
     e2e: pytest {posargs} -q -k e2e
@@ -25,79 +27,9 @@ commands_pre =
 allowlist_externals = pip3
 pass_env = DAPR_GRPC_ENDPOINT,DAPR_HTTP_ENDPOINT,DAPR_RUNTIME_HOST,DAPR_GRPC_PORT,DAPR_HTTP_PORT
 
-[testenv:flake8]
-basepython = python3
-usedevelop = False
-deps = 
-    flake8==7.3.0
-    pip
-commands =
-    flake8 .
-
 [testenv:ruff]
 basepython = python3
 usedevelop = False
-deps = ruff
 commands =
     ruff check --select I --fix
     ruff format
-
-[testenv:examples]
-passenv = HOME
-basepython = python3
-changedir = ./examples/
-deps =
-    mechanical-markdown
-commands =
-    ./validate.sh conversation
-    ./validate.sh crypto
-    ./validate.sh metadata
-    ./validate.sh error_handling
-    ./validate.sh pubsub-simple
-    ./validate.sh pubsub-streaming
-    ./validate.sh pubsub-streaming-async
-    ./validate.sh state_store
-    ./validate.sh state_store_query
-    ./validate.sh secret_store
-    ./validate.sh invoke-simple
-    ./validate.sh invoke-custom-data
-    ./validate.sh demo_actor
-    ./validate.sh invoke-binding
-    ./validate.sh grpc_proxying
-    ./validate.sh w3c-tracing
-    ./validate.sh distributed_lock
-    ./validate.sh configuration
-    ./validate.sh demo_workflow
-    ./validate.sh workflow
-    ./validate.sh jobs
-    ./validate.sh ../
-commands_pre =
-    pip3 install -e {toxinidir}/
-allowlist_externals=*
-
-[testenv:example-component]
-; This environment is used to validate a specific example component.
-; Usage: tox -e example-component -- component_name
-; Example: tox -e example-component -- conversation
-passenv = HOME
-basepython = python3
-changedir = ./examples/
-deps =
-    mechanical-markdown
-commands =
-    ./validate.sh {posargs}
-
-commands_pre =
-    pip3 install -e {toxinidir}/
-allowlist_externals=*
-
-[testenv:type]
-basepython = python3
-usedevelop = False
-deps = -rdev-requirements.txt
-commands =
-    mypy --config-file mypy.ini
-commands_pre =
-    pip3 install -e {toxinidir}/
-allowlist_externals=*
-