Feat/iac

.gitignore

@@ -162,3 +162,6 @@ cython_debug/
 #.idea/
 
 openapitools.json
+
+hatchet-cloud/
+hatchet/

.gitmodules

@@ -1,4 +1,8 @@
 [submodule "hatchet"]
 	path = hatchet
-	url = git@github.com:hatchet-dev/hatchet.git
+	url = https://github.com/hatchet-dev/hatchet.git
 	branch = main
+[submodule "hatchet-cloud"]
+	path = hatchet-cloud
+	url = https://github.com/hatchet-dev/hatchet-cloud.git
+	branch = feat/iac

branch-sync.sh

@@ -1,28 +1,79 @@
 #!/bin/bash
 
+# Default values
+mode="oss"
+repo_url="https://github.com/hatchet-dev/hatchet.git"
+submodule_name="hatchet"
+
+# Parse command line options
+while getopts ":n:" opt; do
+  case $opt in
+    n)
+      mode=$OPTARG
+      ;;
+    \?)
+      echo "Invalid option: -$OPTARG" >&2
+      exit 1
+      ;;
+    :)
+      echo "Option -$OPTARG requires an argument." >&2
+      exit 1
+      ;;
+  esac
+done
+
+# Set the repository URL and submodule name based on the mode
+if [ "$mode" = "cloud" ]; then
+  repo_url="https://github.com/hatchet-dev/hatchet-cloud.git"
+  submodule_name="hatchet-cloud"
+else
+  repo_url="https://github.com/hatchet-dev/hatchet.git"
+  submodule_name="hatchet"
+fi
+
+echo "Mode: $mode"
+echo "Repository URL: $repo_url"
+echo "Submodule name: $submodule_name"
+
 # 1. Get the current branch name
 current_branch=$(echo $GITHUB_HEAD_REF | sed 's/refs\/heads\///')
 
 if [ -z "$current_branch" ]; then
     current_branch=$(git rev-parse --abbrev-ref HEAD)
 fi
 
-# 2. Check a different repo and determine if a branch with the same name exists
-git ls-remote --heads https://github.com/hatchet-dev/hatchet.git $current_branch | grep -q refs/heads/$current_branch
+echo "Current branch: $current_branch"
+
+# 2. Check the repo and determine if a branch with the same name exists
+git ls-remote --heads $repo_url $current_branch | grep -q refs/heads/$current_branch
 branch_exists=$?
 
 # 3. If it does, update the .gitmodules to set `branch = {the branch name}`
 if [ $branch_exists -eq 0 ]; then
-    git config -f .gitmodules submodule.hatchet.branch $current_branch
+    git config -f .gitmodules submodule.$submodule_name.branch $current_branch
+    git config -f .gitmodules submodule.$submodule_name.path $submodule_name
+    git config -f .gitmodules submodule.$submodule_name.url $repo_url
     git add .gitmodules
     echo "Updated .gitmodules with branch $current_branch"
 else
-    echo "Branch $current_branch does not exist in the remote repository. Pulling main branch instead."
-    git config -f .gitmodules submodule.hatchet.branch main
+    echo "Branch $current_branch does not exist in the remote repository. Using main branch instead."
+    git config -f .gitmodules submodule.$submodule_name.branch main
+    git config -f .gitmodules submodule.$submodule_name.path $submodule_name
+    git config -f .gitmodules submodule.$submodule_name.url $repo_url
     git add .gitmodules
     echo "Updated .gitmodules with branch main"
 fi
 
-# 4. Initialize and update the submodule
+# 4. Remove existing submodule if it exists
+git submodule deinit -f -- $submodule_name
+rm -rf .git/modules/$submodule_name
+git rm -f $submodule_name
+
+# 5. Re-add and initialize the submodule
+git submodule add -b $(git config -f .gitmodules submodule.$submodule_name.branch) $repo_url $submodule_name
 git submodule init
+
+# 6. Update the submodule
 git submodule update --remote --merge
+
+echo "Hatchet submodule ($mode mode) updated successfully"

examples/blocked_check/worker.py

@@ -0,0 +1,52 @@
+import asyncio
+import time
+
+from dotenv import load_dotenv
+
+from hatchet_sdk import Context, Hatchet
+
+load_dotenv()
+
+hatchet = Hatchet(debug=True)
+
+
+async def check_blocking_workers(context: Context):
+    start_time = time.time()
+    context.log("Starting blocking worker check")
+
+    await asyncio.sleep(30)
+
+    end_time = time.time()
+    elapsed_time = end_time - start_time
+
+    if 25 <= elapsed_time <= 35:
+        context.log(f"Check completed in {elapsed_time:.2f} seconds")
+    else:
+        raise ValueError(
+            f"Blockage detected: Task took {elapsed_time:.2f} seconds, expected 25-35"
+            " seconds"
+        )
+
+
+@hatchet.workflow(on_crons=["*/5 * * * *"])
+class CheckBlockingWorkersWorkflow:
+    @hatchet.step(timeout="1m")
+    async def iter_1(self, context: Context):
+        await check_blocking_workers(context)
+
+    @hatchet.step(parents=["iter_1"], timeout="1m")
+    async def iter_2(self, context):
+        await check_blocking_workers(context)
+
+    @hatchet.step(parents=["iter_2"], timeout="1m")
+    async def iter_3(self, context):
+        await check_blocking_workers(context)
+
+def main():
+    workflow = CheckBlockingWorkersWorkflow()
+    worker = hatchet.worker("test-worker", max_runs=1)
+    worker.register_workflow(workflow)
+    worker.start()
+
+if __name__ == "__main__":
+    main()

examples/managed/event.py

@@ -0,0 +1,41 @@
+from typing import List
+
+from dotenv import load_dotenv
+
+from hatchet_sdk import new_client
+from hatchet_sdk.clients.events import BulkPushEventWithMetadata
+
+load_dotenv()
+
+client = new_client()
+
+# client.event.push("user:create", {"test": "test"})
+client.event.push(
+    "user:create", {"test": "test"}, options={"additional_metadata": {"hello": "moon"}}
+)
+
+events: List[BulkPushEventWithMetadata] = [
+    {
+        "key": "event1",
+        "payload": {"message": "This is event 1"},
+        "additional_metadata": {"source": "test", "user_id": "user123"},
+    },
+    {
+        "key": "event2",
+        "payload": {"message": "This is event 2"},
+        "additional_metadata": {"source": "test", "user_id": "user456"},
+    },
+    {
+        "key": "event3",
+        "payload": {"message": "This is event 3"},
+        "additional_metadata": {"source": "test", "user_id": "user789"},
+    },
+]
+
+
+result = client.event.bulk_push(
+    events,
+    options={"namespace": "bulk-test"},
+)
+
+print(result)

examples/managed/worker.py

@@ -0,0 +1,72 @@
+import time
+
+from dotenv import load_dotenv
+
+from hatchet_sdk import Context, Hatchet
+from hatchet_sdk.compute.configs import Compute
+
+load_dotenv()
+
+hatchet = Hatchet()
+
+# Default compute
+
+default_compute = Compute(cpu_kind="shared", cpus=1, memory_mb=1024, regions=["ewr"])
+
+blocked_compute = Compute(
+    pool="blocked-pool",
+    cpu_kind="shared",
+    cpus=1,
+    memory_mb=1024,
+    regions=["ewr"],
+)
+
+gpu_compute = Compute(cpu_kind="gpu", cpus=2, memory_mb=1024, regions=["ewr"])
+
+
+@hatchet.workflow(on_events=["user:create"])
+class ManagedWorkflow:
+    @hatchet.step(timeout="11s", retries=3, compute=default_compute)
+    def step1(self, context: Context):
+        print("executed step1")
+        time.sleep(10)
+        # raise Exception("test")
+        return {
+            "step1": "step1",
+        }
+
+    @hatchet.step(timeout="11s", retries=3, compute=gpu_compute)
+    def step2(self, context: Context):
+        print("executed step2")
+        time.sleep(10)
+        # raise Exception("test")
+        return {
+            "step2": "step2",
+        }
+
+    @hatchet.step(timeout="11s", retries=3, compute=blocked_compute)
+    def step3(self, context: Context):
+        print("executed step3")
+
+        return {
+            "step3": "step3",
+        }
+
+    @hatchet.step(timeout="11s", retries=3, compute=default_compute)
+    def step4(self, context: Context):
+        print("executed step4")
+        time.sleep(10)
+        return {
+            "step4": "step4",
+        }
+
+
+def main():
+    workflow = ManagedWorkflow()
+    worker = hatchet.worker("test-worker", max_runs=1)
+    worker.register_workflow(workflow)
+    worker.start()
+
+
+if __name__ == "__main__":
+    main()

examples/simple/worker.py

@@ -20,6 +20,14 @@ def step1(self, context: Context):
             "step1": "step1",
         }
 
+    @hatchet.step(timeout="11s", retries=3)
+    def step2(self, context: Context):
+        print("executed step2")
+        time.sleep(10)
+        return {
+            "step2": "step2",
+        }
+
 
 def main():
     workflow = MyWorkflow()

generate.sh

@@ -6,6 +6,15 @@ set -eux
 
 ROOT_DIR=$(pwd)
 
+# Check if hatchet-cloud submodule exists and has been pulled
+if [ -d "hatchet-cloud" ] && [ "$(ls -A hatchet-cloud)" ]; then
+    echo "hatchet-cloud submodule detected"
+    CLOUD_MODE=true
+else
+    echo "hatchet-cloud submodule not detected or empty"
+    CLOUD_MODE=false
+fi
+
 # deps
 version=7.3.0
 
@@ -58,9 +67,20 @@ cp $tmp_dir/hatchet_sdk/clients/rest/api/__init__.py $dst_dir/api/__init__.py
 # remove tmp folder
 rm -rf $tmp_dir
 
-poetry run python -m grpc_tools.protoc --proto_path=hatchet/api-contracts/dispatcher --python_out=./hatchet_sdk/contracts --pyi_out=./hatchet_sdk/contracts --grpc_python_out=./hatchet_sdk/contracts dispatcher.proto
-poetry run python -m grpc_tools.protoc --proto_path=hatchet/api-contracts/events --python_out=./hatchet_sdk/contracts --pyi_out=./hatchet_sdk/contracts --grpc_python_out=./hatchet_sdk/contracts events.proto
-poetry run python -m grpc_tools.protoc --proto_path=hatchet/api-contracts/workflows --python_out=./hatchet_sdk/contracts --pyi_out=./hatchet_sdk/contracts --grpc_python_out=./hatchet_sdk/contracts workflows.proto
+# Generate protobuf files for both hatchet and hatchet-cloud (if exists)
+generate_proto() {
+    local submodule=$1
+    local proto_file=$2
+    poetry run python -m grpc_tools.protoc --proto_path=$submodule/api-contracts/$proto_file \
+        --python_out=./hatchet_sdk/contracts --pyi_out=./hatchet_sdk/contracts \
+        --grpc_python_out=./hatchet_sdk/contracts $proto_file.proto
+}
+
+proto_files=("dispatcher" "events" "workflows")
+
+for proto in "${proto_files[@]}"; do
+    generate_proto "hatchet" $proto
+done
 
 # Fix relative imports in _grpc.py files
 if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -71,6 +91,52 @@ else
     find ./hatchet_sdk/contracts -type f -name '*_grpc.py' -print0 | xargs -0 sed -i 's/^import \([^ ]*\)_pb2/from . import \1_pb2/'
 fi
 
+if [ "$CLOUD_MODE" = true ]; then
+    echo "Generating cloud-specific OpenAPI"
+
+    # Generate cloud-specific OpenAPI
+    cloud_dst_dir=./hatchet_sdk/clients/cloud_rest
+    cloud_tmp_dir=./cloud_tmp
+
+    mkdir -p $cloud_dst_dir
+
+    # generate into cloud tmp folder
+    openapi-generator-cli generate -i ./hatchet-cloud/api-contracts/openapi/openapi.yaml -g python -o ./cloud_tmp --skip-validate-spec \
+        --library asyncio \
+        --global-property=apiTests=false \
+        --global-property=apiDocs=true \
+        --global-property=modelTests=false \
+        --global-property=modelDocs=true \
+        --package-name hatchet_sdk.clients.cloud_rest
+
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/api_client.py $cloud_dst_dir/api_client.py
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/configuration.py $cloud_dst_dir/configuration.py
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/api_response.py $cloud_dst_dir/api_response.py
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/exceptions.py $cloud_dst_dir/exceptions.py
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/__init__.py $cloud_dst_dir/__init__.py
+    mv $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/rest.py $cloud_dst_dir/rest.py
+
+    openapi-generator-cli generate -i ./hatchet-cloud/api-contracts/openapi/openapi.yaml -g python -o . --skip-validate-spec \
+        --library asyncio \
+        --global-property=apis,models \
+        --global-property=apiTests=false \
+        --global-property=apiDocs=false \
+        --global-property=modelTests=false \
+        --global-property=modelDocs=false \
+        --package-name hatchet_sdk.clients.cloud_rest
+
+    # copy the __init__ files from cloud tmp to the destination since they are not generated for some reason
+    cp $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/models/__init__.py $cloud_dst_dir/models/__init__.py
+    cp $cloud_tmp_dir/hatchet_sdk/clients/cloud_rest/api/__init__.py $cloud_dst_dir/api/__init__.py
+
+    # remove cloud tmp folder
+    rm -rf $cloud_tmp_dir
+
+    echo "Generation completed for both OSS and Cloud versions"
+else
+    echo "Generation completed for OSS version only"
+fi
+
 # ensure that pre-commit is applied without errors
 pre-commit run --all-files || pre-commit run --all-files