Creation of pipeline to publish dts python package to pypi

.github/workflows/pr-validation.yml

@@ -1,51 +1,58 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
-
-name: Build Validation
-
-on:
-  push:
-    branches: [ "main" ]
-  pull_request:
-    branches: [ "main" ]
-  merge_group:
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        pip install -r requirements.txt
-    - name: Lint with flake8
-      run: |
-        flake8 . --count --show-source --statistics --exit-zero
-    - name: Pytest unit tests
-      run: |
-        pytest -m "not e2e" --verbose
-
-    # Sidecar for running e2e tests requires Go SDK
-    - name: Install Go SDK
-      uses: actions/setup-go@v5
-      with:
-        go-version: 'stable'
-
-    # Install and run the durabletask-go sidecar for running e2e tests
-    - name: Pytest e2e tests
-      run: |
-        go install github.com/microsoft/durabletask-go@main
-        durabletask-go --port 4001 &
-        pytest -m "e2e" --verbose
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+
+name: Build Validation
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+  merge_group:
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install durabletask dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        pip install -r requirements.txt
+    - name: Install durabletask-azuremanaged dependencies
+      working-directory: examples/dts
+      run: |
+        python -m pip install --upgrade pip
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        flake8 . --count --show-source --statistics --exit-zero
+    - name: Pytest unit tests
+      working-directory: tests/durabletask
+      run: |
+        pytest -m "not e2e and not dts" --verbose
+
+    # Sidecar for running e2e tests requires Go SDK
+    - name: Install Go SDK
+      uses: actions/setup-go@v5
+      with:
+        go-version: 'stable'
+
+    # Install and run the durabletask-go sidecar for running e2e tests
+    - name: Pytest e2e tests
+      working-directory: tests/durabletask
+      run: |
+        go install github.com/microsoft/durabletask-go@main
+        durabletask-go --port 4001 &
+        pytest -m "e2e and not dts" --verbose

.github/workflows/publish-dts-sdk.yml

@@ -0,0 +1,111 @@
+name: Publish Durable Task Scheduler to PyPI
+
+on:
+  push:
+    tags:
+      - 'azuremanaged-v*'  # Only run for tags starting with "azuremanaged-v"
+
+# on:
+#   push:
+#     branches: [ "main" ]
+#   pull_request:
+#     branches: [ "main" ]
+
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python 3.12
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.12
+    - name: Install dependencies
+      working-directory: durabletask-azuremanaged
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools wheel tox
+        pip install flake8
+    - name: Run flake8 Linter
+      working-directory: durabletask-azuremanaged
+      run: flake8 .
+
+
+  run-docker-tests:
+    needs: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Pull Docker image
+        run: docker pull mcr.microsoft.com/dts/dts-emulator:v0.0.4
+
+      - name: Run Docker container
+        run: |
+          docker run --name dtsemulator -d -p 8080:8080 mcr.microsoft.com/dts/dts-emulator:v0.0.4
+
+      - name: Wait for container to be ready
+        run: sleep 10  # Adjust if your service needs more time to start
+
+      - name: Set environment variables
+        run: |
+          echo "TASKHUB=default" >> $GITHUB_ENV
+          echo "ENDPOINT=http://localhost:8080" >> $GITHUB_ENV
+
+      - name: Install durabletask dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          pip install -r requirements.txt
+
+      - name: Install durabletask-azuremanaged dependencies
+        working-directory: examples/dts
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Run the tests
+        working-directory: tests/durabletask-azuremanaged
+        run: |
+          pytest -m "dts" --verbose
+
+  publish:
+    if: startsWith(github.ref, 'refs/tags/azuremanaged-v')  # Only run if a matching tag is pushed
+    needs: run-docker-tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Extract version from tag
+        run: echo "VERSION=${GITHUB_REF#refs/tags/azuremanaged-v}" >> $GITHUB_ENV  # Extract version from the tag
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"  # Adjust Python version as needed
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build twine
+
+      - name: Build package from directory durabletask-azuremanaged
+        working-directory: durabletask-azuremanaged
+        run: |
+          python -m build
+
+      - name: Check package
+        working-directory: durabletask-azuremanaged
+        run: |
+          twine check dist/* 
+
+      - name: Publish package to PyPI
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN_AZUREMANAGED  }}  # Store your PyPI API token in GitHub Secrets
+        working-directory: durabletask-azuremanaged
+        run: | 
+          twine upload dist/* 

durabletask-azuremanaged/pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durabletask.azuremanaged"
-version = "0.1b1"
+version = "0.1.2"
 description = "Extensions for the Durable Task Python SDK for integrating with the Durable Task Scheduler in Azure"
 keywords = [
     "durable",

examples/dts/README.md

@@ -4,8 +4,13 @@ This directory contains examples of how to author durable orchestrations using t
 
 ## Prerequisites
 
-All the examples assume that you have a Durable Task Scheduler taskhub created.
+There are 2 separate ways to run an example:
+1. Using the emulator.
+2. Using a real scheduler and taskhub.
 
+All the examples by defualt assume that you have a Durable Task Scheduler taskhub created.
+
+## Running with a scheduler and taskhub resource
 The simplest way to create a taskhub is by using the az cli commands:
 
 1. Create a scheduler:
@@ -46,6 +51,29 @@ The simplest way to create a taskhub is by using the az cli commands:
 
 1. Grant yourself the `Durable Task Data Contributor` role over your scheduler
 
+## Running with the emulator
+The emulator is a simulation of a scheduler and taskhub. It is the 'backend' of the durabletask-azuremanaged system packaged up into an easy to use docker container. For these steps, it is assumed that you are using port 8080.
+
+In order to use the emulator for the examples, perform the following steps:
+1. Install docker if it is not already installed.
+
+2. Pull down the docker image for the emulator:
+ `docker pull mcr.microsoft.com/dts/dts-emulator:v0.0.4`
+
+3. Run the emulator and wait a few seconds for the container to be ready:
+`docker run --name dtsemulator -d -p 8080:8080 mcr.microsoft.com/dts/dts-emulator:v0.0.4`
+
+4. Set the environment variables that are referenced and used in the examples:
+    1. If you are using windows powershell:
+    `$env:TASKHUB="default"`
+    `$env:ENDPOINT="http://localhost:8080"`
+    2. If you are using bash:
+    `export TASKHUB=default`
+    `export ENDPOINT=http://localhost:8080`
+
+5. Finally, edit the examples to change the `token_credential` input of both the `DurableTaskSchedulerWorker` and `DurableTaskSchedulerClient` to a value of `None` 
+
+
 ## Running the examples
 
 Now, you can simply execute any of the examples in this directory using `python3`:

examples/dts/requirements.txt

@@ -0,0 +1,8 @@
+autopep8
+grpcio>=1.60.0 # 1.60.0 is the version introducing protobuf 1.25.X support, newer versions are backwards compatible
+protobuf
+pytest
+pytest-cov
+azure-identity
+durabletask-azuremanaged
+durabletask

pyproject.toml

@@ -9,7 +9,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "durabletask"
-version = "0.2b1"
+version = "0.2.0"
 description = "A Durable Task Client SDK for Python"
 keywords = [
     "durable",

tests/durabletask-azuremanaged/test_dts_activity_sequence.py

@@ -0,0 +1,69 @@
+"""End-to-end sample that demonstrates how to configure an orchestrator
+that calls an activity function in a sequence and prints the outputs."""
+import os
+
+from durabletask import client, task
+from durabletask.azuremanaged.client import DurableTaskSchedulerClient
+from durabletask.azuremanaged.worker import DurableTaskSchedulerWorker
+
+import pytest
+
+
+pytestmark = pytest.mark.dts
+
+def hello(ctx: task.ActivityContext, name: str) -> str:
+    """Activity function that returns a greeting"""
+    return f'Hello {name}!'
+
+
+def sequence(ctx: task.OrchestrationContext, _):
+    """Orchestrator function that calls the 'hello' activity function in a sequence"""
+    # call "hello" activity function in a sequence
+    result1 = yield ctx.call_activity(hello, input='Tokyo')
+    result2 = yield ctx.call_activity(hello, input='Seattle')
+    result3 = yield ctx.call_activity(hello, input='London')
+
+    # return an array of results
+    return [result1, result2, result3]
+
+
+# Read the environment variable
+taskhub_name = os.getenv("TASKHUB")
+
+# Check if the variable exists
+if taskhub_name:
+    print(f"The value of TASKHUB is: {taskhub_name}")
+else:
+    print("TASKHUB is not set. Please set the TASKHUB environment variable to the name of the taskhub you wish to use")
+    print("If you are using windows powershell, run the following: $env:TASKHUB=\"<taskhubname>\"")
+    print("If you are using bash, run the following: export TASKHUB=\"<taskhubname>\"")
+    exit()
+
+# Read the environment variable
+endpoint = os.getenv("ENDPOINT")
+
+# Check if the variable exists
+if endpoint:
+    print(f"The value of ENDPOINT is: {endpoint}")
+else:
+    print("ENDPOINT is not set. Please set the ENDPOINT environment variable to the endpoint of the scheduler")
+    print("If you are using windows powershell, run the following: $env:ENDPOINT=\"<schedulerEndpoint>\"")
+    print("If you are using bash, run the following: export ENDPOINT=\"<schedulerEndpoint>\"")
+    exit()
+
+# configure and start the worker
+with DurableTaskSchedulerWorker(host_address=endpoint, secure_channel=True,
+                                taskhub=taskhub_name, token_credential=None) as w:
+    w.add_orchestrator(sequence)
+    w.add_activity(hello)
+    w.start()
+
+    # Construct the client and run the orchestrations
+    c = DurableTaskSchedulerClient(host_address=endpoint, secure_channel=True,
+                                   taskhub=taskhub_name, token_credential=None)
+    instance_id = c.schedule_new_orchestration(sequence)
+    state = c.wait_for_orchestration_completion(instance_id, timeout=60)
+    if state and state.runtime_status == client.OrchestrationStatus.COMPLETED:
+        print(f'Orchestration completed! Result: {state.serialized_output}')
+    elif state:
+        print(f'Orchestration failed: {state.failure_details}')