tool: [WIN-NPM] minor updates to windows script, and new linux script (#1685)

* minor fixes to win-debug.ps1 and same script for linux

* delete previous results when rerunning pod_exec.ps1

* fix filename that we copy from

Author: huntergregory

debug/windows/npm/README.md

@@ -3,5 +3,11 @@
 This script will collect Windows NPM logs and the HNS and VFP state of the cluster and write them to a new local folder.
 
 ## How to collect logs
+In a PowerShell terminal, navigate to the `azure-container-networking/debug/windows/npm folder`. Make sure your kubectl is configured to point to the cluster you want to collect logs from (`az aks get-credentials -g <resource-group> -n <cluster-name>`)
+### Windows
+Run `.\win-debug.ps1`. The script will create a new folder called logs_DATE containing the results.
 
-In a PowerShell terminal, navigate to the `azure-container-networking/debug/windows/npm folder`. Make sure your kubectl is configured to point to the cluster you want to collect logs from (`az aks get-credentials -g <resource-group> -n <cluster-name>`) and run `.\win-debug.ps1`. The script will create a new folder called logs_DATE containing the results.
+### Linux
+Run `.\win-debug.sh`. The script will create a new folder called logs_DATE containing the results.
+
+Note: You may not be able to unzip logs.zip in Linux since it was compressed in Windows.

debug/windows/npm/pod_exec.ps1

@@ -1,5 +1,8 @@
 param([string]$podIps)
-$filepath = "logs"
+$filepath = "npm-exec-logs"
+
+Write-Output "attempting to delete previous results if they exist"
+Remove-Item -path $filepath -recurse
 
 $podIp = @()
 foreach ($r in ($podIps -split " ")) {
@@ -21,6 +24,6 @@ foreach ($r in ($podIps -split " ")) {
     }
  }
 
- Compress-Archive -Path 'logs' -DestinationPath 'logs.zip' -Force
+ Compress-Archive -Path "$filepath" -DestinationPath "$filepath.zip" -Force
 
 exit

debug/windows/npm/win-debug.ps1

@@ -1,5 +1,10 @@
 $filepath = "logs_$((Get-Date).ToString('MM-dd-yyyy'))"
-kubectl get pod -A -o wide >> ( New-Item -Path ./$filepath/allpods.out -Force )
+Write-Output "gathering logs and writing to $filepath/"
+
+kubectl get pod -A -o wide --show-labels > ( New-Item -Path ./$filepath/allpods.out -Force )
+kubectl get netpol -A -o yaml > ( New-Item -Path ./$filepath/all-netpol-yamls.out -Force )
+kubectl describe netpol -A >  ( New-Item -Path ./$filepath/all-netpol-descriptions.out -Force )
+
 $npmpod = kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | Select-String "npm-win"
 $rows = @()
 foreach ($row in (-split $npmpod)) {
@@ -10,9 +15,10 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) {
     $npm = $rows[$i]
     $node = $rows[$i + 1]
 
-    Write-Output "Gathering logs for node $node"
+    Write-Output "Gathering logs. npm pod: $npm. node: $node"
+    kubectl logs -n kube-system $npm > $filepath/logs_$npm.out
 
-    $ip = kubectl get pod -n kube-system -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node"
+    $ip = kubectl get pod -A -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | Select-String "$node"
     $ip = (-split $ip)
     [string] $ips = ""
     for ($j = 0; $j -lt $ip.Length; $j += 2) {
@@ -21,10 +27,17 @@ for ($i = 0; $i -lt $rows.Length; $i += 2) {
             else{
                 $ips += $ip[$j]
             }
-    }  
-    
-    kubectl logs -n kube-system $npm >> $filepath/logs_$npm.out
+    }
+    echo "node $node has IPs: $ips"
+
+    Write-Output "copying ps1 file into $npm"
     kubectl cp ./pod_exec.ps1 kube-system/"$npm":execw.ps1
+
+    Write-Output echo "executing ps1 file on $npm"
     kubectl exec -it -n kube-system $npm -- powershell.exe -Command  .\execw.ps1 "'$ips'"
-    kubectl cp kube-system/"$npm":logs.zip ./$filepath/logs_$node.zip
+
+    Write-Output "copying logs.zip from $npm"
+    kubectl cp kube-system/"$npm":npm-exec-logs.zip ./$filepath/npm-exec-logs_$node.zip
 }
+
+Write-Output "finished capturing all logs. written to $filepath/"

debug/windows/npm/win-debug.sh

@@ -0,0 +1,54 @@
+# NOTE: you may not be able to unzip logs.zip in Linux since it was compressed in Windows
+set -e
+dateString=`date -I` # like 2022-09-24
+filepath=logs_$dateString
+mkdir $filepath
+
+echo "gathering logs and writing to $filepath/"
+
+kubectl get pod -A -o wide --show-labels > $filepath/allpods.out
+kubectl get netpol -A -o yaml > $filepath/all-netpol-yamls.out
+kubectl describe netpol -A > $filepath/all-netpol-descriptions.out
+
+npmPods=()
+nodes=()
+for npmPodOrNode in `kubectl get pod -n kube-system -owide --output=custom-columns='Name:.metadata.name,Node:spec.nodeName' | grep "npm-win"`; do
+    # for loop will go over each item (npm pod, then its node, then the next npm pod, then its node, ...)
+    set +e
+    echo $npmPodOrNode | grep -q azure-npm-win-
+    if [ $? -eq 0 ]; then
+        npmPods+=($npmPodOrNode)
+    else
+        nodes+=($npmPodOrNode)
+    fi
+done
+set -e
+
+echo "npm pods: ${npmPods[@]}"
+echo "nodes of npm pods: ${nodes[@]}"
+
+for i in $(seq 1 ${#npmPods[*]}); do
+    j=$((i-1))
+    npmPod=${npmPods[$j]}
+    node=${nodes[$j]}
+
+    echo "gathering logs. npm pod: $npmPod. node: $node"
+    kubectl logs -n kube-system $npmPod > $filepath/logs_$npmPod.out
+
+    ips=()
+    for ip in `kubectl get pod -A -owide --output=custom-columns='IP:.status.podIP,Node:spec.nodeName' | grep $node | grep -oP "\d+\.\d+\.\d+\.\d+"`; do 
+        ips+=($ip)
+    done
+    echo "node $node has IPs: ${ips[@]}"
+
+    echo "copying ps1 file into $npmPod"
+    kubectl cp ./pod_exec.ps1 kube-system/"$npmPod":execw.ps1
+
+    echo "executing ps1 file on $npmPod"
+    kubectl exec -it -n kube-system $npmPod -- powershell.exe -Command  .\\execw.ps1 "'${ips[@]}'"
+
+    echo "copying logs.zip from $npmPod. NOTE: this will be a windows-based compressed archive (probably need windows to expand it)"
+    kubectl cp kube-system/"$npmPod":npm-exec-logs.zip $filepath/npm-exec-logs_$node.zip
+done
+
+echo "finished gathering all logs. written to $filepath/"