Skip to content

Commit fd5759e

Browse files
zhiying-linzhiying-lin
authored
chore: collect logs after running e2e tests (#133)
Signed-off-by: Zhiying Lin <[email protected]>
1 parent aeb3c6c commit fd5759e

File tree

3 files changed

+189
-1
lines changed

3 files changed

+189
-1
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -147,4 +147,19 @@ jobs:
147147
PROPERTY_PROVIDER: 'azure'
148148
RESOURCE_SNAPSHOT_CREATION_MINIMUM_INTERVAL: ${{ matrix.resource-snapshot-creation-minimum-interval }}
149149
RESOURCE_CHANGES_COLLECTION_DURATION: ${{ matrix.resource-changes-collection-duration }}
150-
150+
151+
- name: Collect logs
152+
if: always()
153+
run: |
154+
make collect-e2e-logs
155+
env:
156+
KUBECONFIG: '/home/runner/.kube/config'
157+
LOG_DIR: 'logs-${{ matrix.customized-settings }}'
158+
159+
- name: Upload logs
160+
if: always()
161+
uses: actions/upload-artifact@v4
162+
with:
163+
name: e2e-logs-${{ matrix.customized-settings }}
164+
path: test/e2e/logs-${{ matrix.customized-settings }}/
165+
retention-days: 3

‎Makefile‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -222,6 +222,10 @@ e2e-tests-custom: setup-clusters
222222
setup-clusters:
223223
cd ./test/e2e && chmod +x ./setup.sh && ./setup.sh $(MEMBER_CLUSTER_COUNT)
224224

225+
.PHONY: collect-e2e-logs
226+
collect-e2e-logs: ## Collect logs from hub and member agent pods after e2e tests
227+
cd ./test/e2e && chmod +x ./collect-logs.sh && ./collect-logs.sh $(MEMBER_CLUSTER_COUNT)
228+
225229
## reviewable
226230
.PHONY: reviewable
227231
reviewable: fmt vet lint staticcheck

‎test/e2e/collect-logs.sh‎

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#!/usr/bin/env bash
2+
3+
set -o errexit
4+
set -o nounset
5+
set -o pipefail
6+
7+
# Configuration
8+
KUBECONFIG="${KUBECONFIG:-$HOME/.kube/config}"
9+
MEMBER_CLUSTER_COUNT="${1:-3}"
10+
NAMESPACE="fleet-system"
11+
LOG_DIR="${LOG_DIR:-logs}"
12+
TIMESTAMP=$(date '+%Y%m%d-%H%M%S')
13+
LOG_DIR="${LOG_DIR}/${TIMESTAMP}"
14+
15+
# Cluster names
16+
HUB_CLUSTER="hub"
17+
declare -a MEMBER_CLUSTERS=()
18+
19+
for (( i=1;i<=MEMBER_CLUSTER_COUNT;i++ ))
20+
do
21+
MEMBER_CLUSTERS+=("cluster-$i")
22+
done
23+
24+
# Colors for output
25+
RED='\033[0;31m'
26+
GREEN='\033[0;32m'
27+
YELLOW='\033[1;33m'
28+
NC='\033[0m' # No Color
29+
30+
# Create log directory
31+
mkdir -p "${LOG_DIR}"
32+
33+
echo -e "${GREEN}Starting log collection at ${TIMESTAMP}${NC}"
34+
echo "Logs will be saved to: ${LOG_DIR}"
35+
echo ""
36+
37+
# Function to collect logs from a pod
38+
collect_pod_logs() {
39+
local pod_name=$1
40+
local cluster_name=$2
41+
local log_file_prefix=$3
42+
43+
echo -e "${YELLOW}Collecting logs from pod ${pod_name} in cluster ${cluster_name}${NC}"
44+
45+
# Get all containers in the pod
46+
containers=$(kubectl get pod "${pod_name}" -n "${NAMESPACE}" -o jsonpath='{.spec.containers[*].name}' 2>/dev/null || echo "")
47+
48+
if [ -z "$containers" ]; then
49+
echo -e "${RED}No containers found in pod ${pod_name}${NC}"
50+
return
51+
fi
52+
53+
# Collect logs for each container
54+
for container in $containers; do
55+
log_file="${log_file_prefix}-${container}.log"
56+
echo " - Container ${container} -> ${log_file}"
57+
58+
# Get current logs
59+
kubectl logs "${pod_name}" -n "${NAMESPACE}" -c "${container}" > "${log_file}" 2>&1 || \
60+
echo "Failed to get logs for container ${container}" > "${log_file}"
61+
62+
# Try to get previous logs if pod was restarted
63+
previous_log_file="${log_file_prefix}-${container}-previous.log"
64+
if kubectl logs "${pod_name}" -n "${NAMESPACE}" -c "${container}" --previous > "${previous_log_file}" 2>&1; then
65+
echo " - Previous logs for ${container} -> ${previous_log_file}"
66+
else
67+
rm -f "${previous_log_file}"
68+
fi
69+
done
70+
}
71+
72+
# Collect hub cluster logs
73+
echo -e "${GREEN}=== Collecting Hub Cluster Logs ===${NC}"
74+
kind export kubeconfig --name "${HUB_CLUSTER}" 2>/dev/null || {
75+
echo -e "${RED}Failed to export kubeconfig for hub cluster${NC}"
76+
exit 1
77+
}
78+
79+
# Create hub logs directory
80+
HUB_LOG_DIR="${LOG_DIR}/hub"
81+
mkdir -p "${HUB_LOG_DIR}"
82+
83+
# Get all hub-agent pods
84+
hub_pods=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=hub-agent -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "")
85+
86+
if [ -z "$hub_pods" ]; then
87+
echo -e "${RED}No hub-agent pods found${NC}"
88+
else
89+
for pod in $hub_pods; do
90+
collect_pod_logs "${pod}" "${HUB_CLUSTER}" "${HUB_LOG_DIR}/${pod}"
91+
done
92+
fi
93+
94+
# Collect member cluster logs
95+
for cluster in "${MEMBER_CLUSTERS[@]}"; do
96+
echo -e "${GREEN}=== Collecting Member Cluster Logs: ${cluster} ===${NC}"
97+
98+
# Export kubeconfig for the member cluster
99+
if ! kind export kubeconfig --name "${cluster}" 2>/dev/null; then
100+
echo -e "${RED}Failed to export kubeconfig for cluster ${cluster}, skipping...${NC}"
101+
continue
102+
fi
103+
104+
# Create member logs directory
105+
MEMBER_LOG_DIR="${LOG_DIR}/${cluster}"
106+
mkdir -p "${MEMBER_LOG_DIR}"
107+
108+
# Get all member-agent pods
109+
member_pods=$(kubectl get pods -n "${NAMESPACE}" -l app.kubernetes.io/name=member-agent -o jsonpath='{.items[*].metadata.name}' 2>/dev/null || echo "")
110+
111+
if [ -z "$member_pods" ]; then
112+
echo -e "${RED}No member-agent pods found in cluster ${cluster}${NC}"
113+
else
114+
for pod in $member_pods; do
115+
collect_pod_logs "${pod}" "${cluster}" "${MEMBER_LOG_DIR}/${pod}"
116+
done
117+
fi
118+
119+
echo ""
120+
done
121+
122+
# Collect additional debugging information
123+
echo -e "${GREEN}=== Collecting Additional Debug Information ===${NC}"
124+
125+
# Hub cluster debug info
126+
kind export kubeconfig --name "${HUB_CLUSTER}" 2>/dev/null
127+
{
128+
echo "=== Hub Cluster Pod Status ==="
129+
kubectl get pods -n "${NAMESPACE}" -o wide
130+
echo ""
131+
echo "=== Hub Cluster Events ==="
132+
kubectl get events -n "${NAMESPACE}" --sort-by='.lastTimestamp'
133+
} > "${LOG_DIR}/hub-debug-info.txt" 2>&1
134+
135+
# Member clusters debug info
136+
for cluster in "${MEMBER_CLUSTERS[@]}"; do
137+
if kind export kubeconfig --name "${cluster}" 2>/dev/null; then
138+
{
139+
echo "=== ${cluster} Pod Status ==="
140+
kubectl get pods -n "${NAMESPACE}" -o wide
141+
echo ""
142+
echo "=== ${cluster} Events ==="
143+
kubectl get events -n "${NAMESPACE}" --sort-by='.lastTimestamp'
144+
} > "${LOG_DIR}/${cluster}-debug-info.txt" 2>&1
145+
fi
146+
done
147+
148+
# Create a summary file
149+
echo -e "${GREEN}=== Creating Summary ===${NC}"
150+
{
151+
echo "Log Collection Summary"
152+
echo "====================="
153+
echo "Timestamp: ${TIMESTAMP}"
154+
echo "Hub Cluster: ${HUB_CLUSTER}"
155+
echo "Member Clusters: ${MEMBER_CLUSTERS[*]}"
156+
echo ""
157+
echo "Directory Structure:"
158+
find "${LOG_DIR}" -type f -name "*.log" -o -name "*.txt" | sort
159+
} > "${LOG_DIR}/summary.txt"
160+
161+
echo ""
162+
echo -e "${GREEN}Log collection completed!${NC}"
163+
echo "All logs saved to: ${LOG_DIR}"
164+
echo ""
165+
echo "To view the summary:"
166+
echo " cat ${LOG_DIR}/summary.txt"
167+
echo ""
168+
echo "To search across all logs:"
169+
echo " grep -r 'ERROR' ${LOG_DIR}"

0 commit comments

Comments
 (0)