Merge pull request #295 from stackhpc/fix/eessi-tests

Disable EESSI tests in CI and make them debuggable

Author: sjpb

.github/workflows/stackhpc.yml

@@ -90,11 +90,11 @@ jobs:
           . environments/.stackhpc/activate
           ansible-playbook -vv ansible/adhoc/hpctests.yml
 
-      - name: Run EESSI tests
-        run: |
-          . venv/bin/activate
-          . environments/.stackhpc/activate
-          ansible-playbook -vv ansible/ci/check_eessi.yml
+      # - name: Run EESSI tests
+      #   run: |
+      #     . venv/bin/activate
+      #     . environments/.stackhpc/activate
+      #     ansible-playbook -vv ansible/ci/check_eessi.yml
 
       - name: Confirm Open Ondemand is up (via SOCKS proxy)
         run: |

ansible/ci/check_eessi.yml

@@ -17,18 +17,34 @@
         repo: "https://github.com/eessi/eessi-demo.git"
         dest: "{{ eessi_test_rootdir }}/eessi-demo"
 
-    - name: Run test job
-      ansible.builtin.shell:
-        cmd: |
+    - name: Create batch script
+      copy:
+        dest: "{{ eessi_test_rootdir }}/eessi-demo/TensorFlow/tensorflow.sh"
+        content: |
+          #!/usr/bin/env bash
+          #SBATCH --output=%x.out
+          #SBATCH --error=%x.out
           source /cvmfs/pilot.eessi-hpc.org/latest/init/bash
           srun ./run.sh
+
+    - name: Run test job
+      ansible.builtin.shell:
+        cmd: sbatch --wait tensorflow.sh
         chdir: "{{ eessi_test_rootdir }}/eessi-demo/TensorFlow"
-        executable: /bin/bash
       register: job_output
 
+    - name: Retrieve job output
+      slurp:
+        src: "{{ eessi_test_rootdir }}/eessi-demo/TensorFlow/tensorflow.sh.out"
+      register: _tensorflow_out
+      no_log: true # as its base64 encoded so useless
+
+    - name: Show job output
+      debug:
+        msg: "{{ _tensorflow_out.content | b64decode }}"
+
     - name: Fail if job output contains error
       fail:
         # Note: Job prints live progress bar to terminal, so use regex filter to remove this from stdout
         msg: "Test job using EESSI modules failed. Job output was: {{ job_output.stdout | regex_replace('\b', '') }}"        
-      when: '"Epoch 5/5" not in job_output.stdout'
-      
+      when: '"Epoch 5/5" not in _tensorflow_out.content | b64decode'

environments/.stackhpc/ansible.cfg

@@ -11,5 +11,6 @@ roles_path = ../../ansible/roles
 filter_plugins = ../../ansible/filter_plugins
 
 [ssh_connection]
-ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
+#ssh_args = -o ControlMaster=auto -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
+ssh_args = -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null
 pipelining = True