Add more monitoring

Signed-off-by: Simon Davies <[email protected]>

Author: simongdavies

.github/workflows/dep_rust.yml

@@ -69,6 +69,26 @@ jobs:
         run: |
           echo "Starting continuous /dev/kvm monitoring with inotify..."
           
+          # Set up audit rules for /dev/kvm monitoring
+          echo "Setting up audit rules for /dev/kvm monitoring..."
+          
+          # Add audit rules to monitor /dev/kvm access and changes
+          sudo auditctl -w /dev/kvm -p rwxa -k hypervisor_kvm || echo "Failed to add /dev/kvm watch rule"
+          
+          # Add audit rules for device file operations (using syscall monitoring)
+          sudo auditctl -a always,exit -F arch=b64 -S chmod,fchmod,fchmodat -F path=/dev/kvm -k device_changes || echo "Failed to add chmod audit rule"
+          sudo auditctl -a always,exit -F arch=b64 -S chown,fchown,lchown,fchownat -F path=/dev/kvm -k device_changes || echo "Failed to add chown audit rule"
+          
+          # Add audit rules for permission-related syscalls that might affect device access
+          sudo auditctl -a always,exit -F arch=b64 -S setuid,setgid,setreuid,setregid,setresuid,setresgid -k permission_syscalls || echo "Failed to add permission syscalls audit rule"
+          
+          # Add audit rules for file operations on the /dev directory that might affect kvm
+          sudo auditctl -w /dev/ -p wa -k device_changes || echo "Failed to add /dev/ directory watch rule"
+          
+          # Verify audit rules were added
+          echo "Current audit rules:"
+          sudo auditctl -l | grep -E "(kvm|device_changes|permission_syscalls|hypervisor_kvm)" || echo "No relevant audit rules found"
+          
           # Create monitoring script using inotify
           cat > /tmp/monitor_kvm_device.sh << 'EOF'
           #!/bin/bash
@@ -149,6 +169,23 @@ jobs:
           
           echo "$(date): Starting inotify monitoring on /dev/" >> "$LOGFILE"
           
+          # Test inotify functionality first
+          echo "$(date): Testing inotify functionality..." >> "$LOGFILE"
+          timeout 2 inotifywait -e create,delete /tmp/ &
+          INOTIFY_TEST_PID=$!
+          touch /tmp/inotify_test_$$
+          rm /tmp/inotify_test_$$ 2>/dev/null
+          wait $INOTIFY_TEST_PID 2>/dev/null
+          if [ $? -eq 0 ]; then
+            echo "$(date): inotify test successful" >> "$LOGFILE"
+          else
+            echo "$(date): WARNING: inotify test failed - events may not be detected" >> "$LOGFILE"
+          fi
+          
+          # Check current /dev filesystem type and mount options
+          echo "$(date): /dev filesystem info:" >> "$LOGFILE"
+          mount | grep "/dev " >> "$LOGFILE" 2>/dev/null || echo "Could not get /dev mount info" >> "$LOGFILE"
+          
           # Monitor the /dev directory for KVM device changes
           # We monitor /dev because /dev/kvm might not exist initially
           inotifywait -m -e create,delete,modify,attrib,moved_to,moved_from /dev/ 2>/dev/null | while read path action file; do
@@ -163,14 +200,32 @@ jobs:
           # Also monitor /dev/kvm directly if it exists
           if [ -e /dev/kvm ]; then
             echo "$(date): Also monitoring /dev/kvm directly" >> "$LOGFILE"
-            inotifywait -m -e modify,attrib,access /dev/kvm 2>/dev/null | while read path action file; do
+            # Use more comprehensive event monitoring for device files
+            inotifywait -m -e modify,attrib,access,open,close_write,close_nowrite /dev/kvm 2>/dev/null | while read path action file; do
               echo "$(date): Direct inotify event on /dev/kvm: $action" >> "$LOGFILE"
               handle_change "direct_kvm_$action"
             done &
             KVM_MONITOR_PID=$!
             echo "$(date): Started direct /dev/kvm monitor with PID: $KVM_MONITOR_PID" >> "$LOGFILE"
           fi
           
+          # Add periodic validation alongside inotify (in case inotify misses changes)
+          while true; do
+            sleep 15  # Check every 15 seconds
+            if [ -e /dev/kvm ]; then
+              current_check=$(ls -la /dev/kvm 2>/dev/null)
+              if [ -f "$BASELINE_FILE" ]; then
+                baseline_check=$(cat "$BASELINE_FILE" 2>/dev/null)
+                if [ "$current_check" != "$baseline_check" ] && [ "$baseline_check" != "NO_DEVICE" ]; then
+                  echo "$(date): PERIODIC CHECK: Change detected that inotify may have missed!" >> "$LOGFILE"
+                  handle_change "periodic_validation"
+                fi
+              fi
+            fi
+          done &
+          PERIODIC_CHECK_PID=$!
+          echo "$(date): Started periodic validation with PID: $PERIODIC_CHECK_PID" >> "$LOGFILE"
+          
           # Periodic check to ensure monitor processes are still running
           while true; do
             sleep 30
@@ -199,9 +254,17 @@ jobs:
               kill $MAIN_PID 2>/dev/null || echo "Main process already terminated"
             fi
             
-            # Kill any remaining inotifywait processes
+            if [ -f /tmp/audit_kvm_monitor.pid ]; then
+              AUDIT_PID=$(cat /tmp/audit_kvm_monitor.pid)
+              echo "Terminating audit monitor process: $AUDIT_PID"
+              kill $AUDIT_PID 2>/dev/null || echo "Audit process already terminated"
+            fi
+            
+            # Kill any remaining monitoring processes
             pkill -f "inotifywait.*kvm" 2>/dev/null || echo "No inotifywait processes to clean up"
             pkill -f "monitor_kvm_device" 2>/dev/null || echo "No monitor script processes to clean up"
+            pkill -f "audit_kvm_monitor" 2>/dev/null || echo "No audit monitor processes to clean up"
+            pkill -f "ausearch.*hypervisor_kvm" 2>/dev/null || echo "No ausearch processes to clean up"
             
             echo "Monitor cleanup completed"
           }
@@ -213,7 +276,10 @@ jobs:
           sleep 3
           
           echo "Monitor status:"
-          ps aux | grep "monitor_kvm_device\|inotifywait" | grep -v grep || echo "Monitor process not found"
+          ps aux | grep "monitor_kvm_device\|inotifywait\|audit_kvm_monitor\|ausearch" | grep -v grep || echo "Monitor processes not found"
+          echo ""
+          echo "Monitoring log files:"
+          ls -la /tmp/kvm_device_changes.log /tmp/kvm_audit_changes.log 2>/dev/null || echo "Log files not yet created"
 
       - name: clippy
         run: |
@@ -345,6 +411,19 @@ jobs:
             echo ""
             echo "Recent audit events related to changes:"
             grep -A 10 "Recent audit events:" /tmp/kvm_device_changes.log | head -20 || echo "No audit events captured"
+            echo ""
+            echo "=== Auditd KVM Events ==="
+            if [ -f /tmp/kvm_audit_changes.log ]; then
+              echo "🔍 KVM-related audit events detected:"
+              cat /tmp/kvm_audit_changes.log
+              echo ""
+              echo "📊 Audit event statistics:"
+              echo "Total KVM audit events: $(grep -c "AUDIT EVENT" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+              echo "Permission syscalls detected: $(grep -c "Permission/ownership syscall" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+              echo "KVM-related syscalls: $(grep -c "KVM-related syscall detected" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+            else
+              echo "📝 No auditd events captured"
+            fi
           else
             echo "✅ No device permission/ownership changes detected during job execution"
           fi
@@ -365,6 +444,14 @@ jobs:
             sleep 2
           fi
           
+          # Stop the audit monitoring process if still running
+          if [ -f /tmp/audit_kvm_monitor.pid ]; then
+            AUDIT_PID=$(cat /tmp/audit_kvm_monitor.pid)
+            echo "Stopping audit monitor process (PID: $AUDIT_PID)..."
+            kill $AUDIT_PID 2>/dev/null || echo "Audit monitor process already stopped"
+            sleep 2
+          fi
+          
           echo "KVM device listing:"
           sudo ls -al /dev/kvm 2>/dev/null || echo "❌ /dev/kvm device not found or not accessible"
           echo ""
@@ -374,13 +461,57 @@ jobs:
           
           echo "=== Complete Device Change Summary ==="
           if [ -f /tmp/kvm_device_changes.log ]; then
-            echo "📊 Full monitoring log:"
+            echo "📊 Full inotify monitoring log:"
             cat /tmp/kvm_device_changes.log
             echo ""
-            echo "🔢 Change statistics:"
+            echo "🔢 inotify Change statistics:"
             echo "Device creation events: $(grep -c "DEVICE CREATED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
             echo "Device removal events: $(grep -c "DEVICE REMOVED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
             echo "Permission/ownership changes: $(grep -c "CHANGE DETECTED" /tmp/kvm_device_changes.log 2>/dev/null || echo "0")"
           else
-            echo "📝 No monitoring log available"
+            echo "📝 No inotify monitoring log available"
+          fi
+          echo ""
+          
+          if [ -f /tmp/kvm_audit_changes.log ]; then
+            echo "📊 Full auditd monitoring log:"
+            cat /tmp/kvm_audit_changes.log
+            echo ""
+            echo "🔢 Auditd event statistics:"
+            echo "Total KVM audit events: $(grep -c "AUDIT EVENT" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+            echo "Permission syscalls detected: $(grep -c "Permission/ownership syscall" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+            echo "KVM-related syscalls: $(grep -c "KVM-related syscall detected" /tmp/kvm_audit_changes.log 2>/dev/null || echo "0")"
+          else
+            echo "📝 No auditd monitoring log available"
           fi
+      - name: Setup auditd rules for KVM monitoring
+        if: runner.os == 'Linux'
+        run: |
+          echo "Setting up auditd rules for /dev/kvm monitoring..."
+          
+          # Ensure auditd is running
+          sudo systemctl start auditd || echo "Failed to start auditd, trying to continue..."
+          
+          # Add audit rules for KVM device monitoring
+          echo "Adding audit rules..."
+          
+          # Monitor file operations on /dev/kvm
+          sudo auditctl -w /dev/kvm -p rwxa -k hypervisor_kvm || echo "Failed to add /dev/kvm watch rule"
+          
+          # Monitor device creation/deletion in /dev/
+          sudo auditctl -w /dev/ -p wa -k device_changes || echo "Failed to add /dev/ watch rule"
+          
+          # Monitor permission/ownership change syscalls
+          sudo auditctl -a always,exit -F arch=b64 -S chmod,fchmod,chown,fchown,lchown,fchownat,fchmodat -k permission_syscalls || echo "Failed to add permission syscalls rule"
+          
+          # Monitor udev-related processes
+          sudo auditctl -a always,exit -F comm=udevd -k device_changes || echo "Failed to add udevd monitoring rule"
+          sudo auditctl -a always,exit -F comm=systemd-udevd -k device_changes || echo "Failed to add systemd-udevd monitoring rule"
+          
+          # Show current audit rules
+          echo "Current audit rules:"
+          sudo auditctl -l || echo "Failed to list audit rules"
+          
+          # Check audit status
+          echo "Audit system status:"
+          sudo auditctl -s || echo "Failed to get audit status"