diff --git a/.github/workflows/armada.yml b/.github/workflows/armada.yml new file mode 100644 index 0000000000000..f45d3d50559c9 --- /dev/null +++ b/.github/workflows/armada.yml @@ -0,0 +1,51 @@ +name: Aramda + +on: + pull_request: + +jobs: + armada: + name: Armada integration + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + path: spark-armada + - uses: actions/checkout@v4 + with: + repository: armadaproject/armada-operator + path: armada-operator + - run: | + cd spark-armada + ./build/sbt package -Parmada -Pkubernetes + ./bin/docker-image-tool.sh -t testing build + docker image save -o ../spark_testing.tar spark:testing + cd .. + + cd armada-operator + make kind-all + ./bin/tooling/kind load image-archive ../spark_testing.tar --name armada + + # sleep a bit, or we see: create queue request failed: rpc error: code = DeadlineExceeded + sleep 60 + + ./bin/app/armadactl create queue test + + # sleep a bit, or we see: rpc error: code = PermissionDenied desc = could not find queue "test" + sleep 60 + + ./bin/app/armadactl submit ../spark-armada/examples/spark-driver-job.yaml + ./bin/app/armadactl submit ../spark-armada/examples/spark-executor-job.yaml + + # wait for the jobs to start + sleep 60 + + # inspect jobs + kubectl get pods + for pod in $(kubectl get pods | grep armada | cut -d " " -f 1) + do + echo "$pod" + kubectl logs pod/$pod + echo + done + diff --git a/examples/spark-driver-job.yaml b/examples/spark-driver-job.yaml new file mode 100644 index 0000000000000..66a8a3b9d1c1e --- /dev/null +++ b/examples/spark-driver-job.yaml @@ -0,0 +1,31 @@ + queue: test + jobSetId: job-set-1 + jobs: + - namespace: default + priority: 0 + podSpec: + terminationGracePeriodSeconds: 0 + restartPolicy: Never + containers: + - name: spark-driver + image: spark:testing + env: + - name: SPARK_DRIVER_BIND_ADDRESS + value: "0.0.0.0:1234" + command: + - /opt/entrypoint.sh + args: + - driver + - --verbose + - --class + - org.apache.spark.examples.LocalPi + - --master + - armada://192.168.1.167:50051 + - submit + resources: + limits: + memory: 1Gi + cpu: 1 + requests: + memory: 1Gi + cpu: 1 diff --git a/examples/spark-executor-job.yaml b/examples/spark-executor-job.yaml new file mode 100644 index 0000000000000..f243c3e54fb57 --- /dev/null +++ b/examples/spark-executor-job.yaml @@ -0,0 +1,39 @@ + queue: test + jobSetId: job-set-1 + jobs: + - namespace: default + priority: 0 + podSpec: + terminationGracePeriodSeconds: 0 + restartPolicy: Never + containers: + - name: spark-executor + image: spark:testing + env: + - name: SPARK_EXECUTOR_MEMORY + value: "512m" + - name: SPARK_DRIVER_URL + value: "spark://localhost:1337" + - name: SPARK_EXECUTOR_ID + value: "1" + - name: SPARK_EXECUTOR_CORES + value: "1" + - name: SPARK_APPLICATION_ID + value: "test_spark_app_id" + - name: SPARK_EXECUTOR_POD_IP + value: "localhost" + - name: SPARK_RESOURCE_PROFILE_ID + value: "1" + - name: SPARK_EXECUTOR_POD_NAME + value: "test-pod-name" + command: + - /opt/entrypoint.sh + args: + - executor + resources: + limits: + memory: 1Gi + cpu: 1 + requests: + memory: 1Gi + cpu: 1