feat: in BYOC mode, ray helm chart should also submit the job

starpit · starpit · commit 333e8d9f8ed4 · 2023-03-03T19:40:23.000-05:00
diff --git a/.github/workflows/kind.yml b/.github/workflows/kind.yml
@@ -47,5 +47,4 @@ jobs:
       env:
         EXECUTABLE_PATH: github-actions-production
         DEBUG_KUBERNETES: true
-        TEST_LOG_AGGREGATOR: true
       run: ./tests/kind/run.sh ${{ matrix.profile }}
diff --git a/package-lock.json b/package-lock.json
diff --git a/plugins/plugin-codeflare/package.json b/plugins/plugin-codeflare/package.json
@@ -30,13 +30,13 @@
     "@types/split2": "^3.2.1"
   },
   "dependencies": {
-    "@guidebooks/store": "^5.3.4",
+    "@guidebooks/store": "^5.4.8",
     "@logdna/tail-file": "^3.0.1",
     "@patternfly/react-charts": "^6.94.18",
     "@patternfly/react-core": "^4.276.6",
     "asciinema-player": "^3.1.0",
     "chokidar": "^3.5.3",
-    "madwizard": "^6.2.5",
+    "madwizard": "^6.2.8",
     "needle": "^3.2.0",
     "open": "^8.4.2",
     "pretty-bytes": "^6.1.0",
diff --git a/tests/kind/profiles/non-gpu1/keep-it-simple b/tests/kind/profiles/non-gpu1/keep-it-simple
@@ -2,7 +2,7 @@
   "name": "keep-it-simple",
   "creationTime": 1664149787016,
   "lastModifiedTime": 1676828268772,
-  "lastUsedTime": 1676836671254,
+  "lastUsedTime": 1677870297449,
   "choices": {
     "madwizard/apriori/use-gpu": "don't use gpus",
     "madwizard/apriori/arch": "x64",
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"500m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
@@ -22,4 +22,4 @@
     "ml/ray/cluster/choose/kubernetes": "codeflare-test-ray-cluster",
     "ml/ray/cluster/kubernetes/choose-pod-scheduler": "Keep It Simple"
   }
-}
+}
diff --git a/tests/kind/profiles/non-gpu1/mcad-coscheduler b/tests/kind/profiles/non-gpu1/mcad-coscheduler
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"200m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
@@ -24,4 +24,4 @@
     "kubernetes/mcad/choose/job-priority": "Default Priority",
     "kubernetes/mcad/choose/scheduler": "MCAD with the Advanced Coscheduler"
   }
-}
+}
diff --git a/tests/kind/profiles/non-gpu1/mcad-default b/tests/kind/profiles/non-gpu1/mcad-default
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"200m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/profiles/non-gpu1/mcad-preinstalled b/tests/kind/profiles/non-gpu1/mcad-preinstalled
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"200m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/profiles/non-gpu1/ray-autoscaler b/tests/kind/profiles/non-gpu1/ray-autoscaler
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"200m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"0\",\"Maximum Workers\":\"0\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"2.5Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/profiles/non-gpu2/keep-it-simple b/tests/kind/profiles/non-gpu2/keep-it-simple
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"500m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/profiles/non-gpu3/keep-it-simple b/tests/kind/profiles/non-gpu3/keep-it-simple
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"500m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/profiles/non-gpu4/keep-it-simple b/tests/kind/profiles/non-gpu4/keep-it-simple
@@ -11,7 +11,7 @@
     "madwizard/apriori/in-terminal": "HTML",
     "ml/codeflare": "Submit a new Run",
     "ml/codeflare/run": "Bring Your Own Code",
-    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
+    "ml/codeflare/training/byoc/form": "{\"Path to source\":\"$PWD/tests/kind/inputs/qiskit\",\"Base image\":\"rayproject/ray:2.1.0\"}",
     "kubernetes/choose/secret/image-pull": "No secret needed, since my image is public",
     "s3/choose/bucket/maybe": "My data is not stored in S3",
     "ml/ray/start/resources": "{\"Number of CPUs\":\"500m\",\"Number of GPUs\":\"0\",\"Minimum Workers\":\"1\",\"Maximum Workers\":\"1\",\"Worker Memory\":\"500Mi\",\"Head Memory\":\"3Gi\",\"Ephemeral Storage\":\"5Gi\"}",
diff --git a/tests/kind/run.sh b/tests/kind/run.sh
@@ -10,6 +10,7 @@
 
 set -e
 set -o pipefail
+if [ -z "$TERM" ]; then export TERM=xterm-256color; fi
 
 SCRIPTDIR=$(cd $(dirname "$0") && pwd)
 . "$SCRIPTDIR"/values.sh
@@ -87,7 +88,7 @@ function run {
     fi
 
     echo "[Test] Running with variant=$variant profile=$profile yes=$yes"
-    GUIDEBOOK_NAME="main-job-run" "$ROOT"/bin/codeflare -p $profile $yes $guidebook
+    GUIDEBOOK_NAME="main-job-run" "$ROOT"/bin/codeflare -p $profile $yes $guidebook | tee $OUTPUT
 }
 
 #
@@ -159,6 +160,9 @@ function onexit {
         (pkill -P $EVENTS_PID || exit 0)
     fi
 
+    # just in case...
+    (kubectl delete job ray-cleaner-codeflare-test-ray-cluster || exit 0)
+
     if [ -z "$NO_KIND" ]; then
         # don't kill ourselves if we're running in a container
         sleep 10
@@ -200,21 +204,20 @@ function test {
     echo "[Test] Using JOB_ID=$JOB_ID"
 
     # 1. launch codeflare guidebook run
-    run "$1" | tee $OUTPUT &
-    local RUN_PID=$!
-    echo "[Test] Run submitted pid=$RUN_PID"
+    echo "[Test] Submitting run"
+    run "$1"
 
     # wait to attach until the job has been submitted
-    while true; do
-        grep -q 'Run it' "$OUTPUT" && break
-        echo "[Test] Waiting to attach log streamer..."
-        sleep 1
-    done
+    # while true; do
+    #    grep -q 'Run it' "$OUTPUT" && break
+    #    echo "[Test] Waiting to attach log streamer..."
+    #    sleep 1
+    #done
 
     # echo "[Test] Preparing to attach log streamer jobid=$JOB_ID"
     # attach "$1" "$JOB_ID"
 
-    wait $RUN_PID
+    # wait $RUN_PID
     echo "[Test] Run has finished"
     # the job should be done now
 
diff --git a/tests/self-test/self-test.yaml b/tests/self-test/self-test.yaml
@@ -9,8 +9,14 @@ metadata:
   name: codeflare-self-test-role
 rules:
 - apiGroups: [""]
-  resources: ["pods", "pods/exec", "services", "events", "secrets", "serviceaccounts"]
+  resources: ["pods", "pods/exec", "services", "events", "serviceaccounts", "configmaps"]
   verbs: ["create", "delete", "get", "watch", "list"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["create", "get", "list", "delete", "update"]
+- apiGroups: ["batch"]
+  resources: ["jobs"]
+  verbs: ["create", "get", "list", "delete"]
 - apiGroups: ["rbac.authorization.k8s.io"]
   resources: ["roles", "rolebindings"]
   verbs: ["create", "delete", "get", "list"]
