added ci check for persistence

Author: wtripp180901

.github/workflows/stackhpc.yml

@@ -170,6 +170,11 @@ jobs:
         env:
           TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
 
+      - name: Get timestamp in milliseconds before rebuild
+        id: timestamp
+        run: |
+          echo "TIME_BEFORE_REBUILD=$(date +%s%N | cut -b1-13)" >> $GITHUB_OUTPUT
+
       # - name: Build environment-specific compute image
       #   id: packer_build
       #   run: |
@@ -209,6 +214,14 @@ jobs:
           . environments/.stackhpc/activate
           ansible-playbook -vv ansible/ci/check_grafana.yml
 
+      - name: Check Loki Slurm logs persisted through rebuild
+        run: |
+          . venv/bin/activate
+          . environments/.stackhpc/activate
+          ansible-playbook -vv ansible/ci/check_loki.yml -e "end_timestamp=${{ steps.timestamp.outputs.TIME_BEFORE_REBUILD }} testuser_password=$TESTUSER_PASSWORD"
+        env:
+          TESTUSER_PASSWORD: ${{ secrets.TEST_USER_PASSWORD }}
+
       - name: Delete infrastructure
         run: |
           . venv/bin/activate

ansible/ci/check_loki.yml

@@ -0,0 +1,42 @@
+- hosts: control
+  gather_facts: no
+  become: no
+  tasks:
+    - name: Get Loki datasource uid
+      ansible.builtin.uri:
+        url: http://localhost:{{ grafana_port }}/api/datasources/name/Loki
+        url_username: 'testuser'
+        url_password: "{{ testuser_password }}"
+        follow_redirects: all
+      register: uid_response
+
+    - name: Get Loki logs from before rebuild
+      vars:
+        testuser_password: mypassword
+      ansible.builtin.uri:
+        url: http://localhost:{{ grafana_port }}/api/ds/query
+        follow_redirects: all
+        url_username: testuser
+        url_password: "{{ testuser_password }}"
+        method: POST	
+        body_format: json
+        headers:
+          Accept: application/json
+          Content-Type: application/json
+        # Queries from 20 mins before timestamp to timestamp
+        body: |
+          {
+            "queries":[
+              {
+                "expr":"{unit=\"slurmd.service\"} |= ``",
+                "datasource":{"uid":"{{ uid_response.json.uid }}"},
+                "format":"time_series"
+              }],
+            "from":"{{ end_timestamp | int - 1200000 }}",
+            "to":"{{ end_timestamp }}"
+          }
+      register: log_query_content
+
+    - name: Check that logs exist
+      ansible.builtin.assert:
+        that: log_query_content.json.results.A.frames[0].data['values'][2] | length > 0