Skip to content

Commit 82d7ebb

Browse files
authored
fix: prevent panic when JSON_REPORT missing, add retry logic (#13)
- Add bounds check for strings.LastIndex to prevent slice bounds panic when JSON_REPORT is not found in pod logs - Add retry mechanism for getL2Disc with 3 attempts and 15s wait between retries to handle transient failures during L2 discovery
1 parent 4a90126 commit 82d7ebb

File tree

1 file changed

+18
-2
lines changed

1 file changed

+18
-2
lines changed

l2lib.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ const (
5757
L2ContainerMemReq = "100M"
5858
// ptpIfIndent defines the indentation level for PtpIf detailed output
5959
ptpIfIndent = 6
60+
// L2DiscoveryRetries is the number of retries for getL2Disc
61+
L2DiscoveryRetries = 3
6062
)
6163

6264
type L2DaemonsetMode int64
@@ -270,9 +272,20 @@ func (config *L2DiscoveryConfig) DiscoverL2Connectivity(ptpInterfacesOnly, allIF
270272
if err != nil {
271273
return fmt.Errorf("could not get l2 discovery pods, err=%s", err)
272274
}
273-
err = config.getL2Disc(ptpInterfacesOnly)
275+
// Retry getL2Disc up to L2DiscoveryRetries times
276+
for attempt := 1; attempt <= L2DiscoveryRetries; attempt++ {
277+
err = config.getL2Disc(ptpInterfacesOnly)
278+
if err == nil {
279+
break
280+
}
281+
logrus.Warnf("attempt %d/%d: error getting l2 discovery data, err=%s", attempt, L2DiscoveryRetries, err)
282+
if attempt < L2DiscoveryRetries {
283+
logrus.Infof("waiting %v before retry...", L2DiscoveryDuration)
284+
time.Sleep(L2DiscoveryDuration)
285+
}
286+
}
274287
if err != nil {
275-
logrus.Errorf("error getting l2 discovery data, err=%s", err)
288+
logrus.Errorf("failed to get l2 discovery data after %d attempts, err=%s", L2DiscoveryRetries, err)
276289
}
277290
// Delete L2 discovery daemonset
278291
if config.L2DsMode == Managed {
@@ -319,6 +332,9 @@ func (config *L2DiscoveryConfig) getL2Disc(ptpInterfacesOnly bool) error {
319332
for _, k := range keys {
320333
podLogs, _ := pods.GetLog(config.L2DiscoveryPods[k], config.L2DiscoveryPods[k].Spec.Containers[0].Name)
321334
indexReport := strings.LastIndex(podLogs, "JSON_REPORT")
335+
if indexReport == -1 {
336+
return fmt.Errorf("no JSON_REPORT found in pod logs for pod %s on node %s", config.L2DiscoveryPods[k].Name, config.L2DiscoveryPods[k].Spec.NodeName)
337+
}
322338
report := strings.Split(strings.Split(podLogs[indexReport:], `\n`)[0], "JSON_REPORT")[1]
323339
var discDataPerNode map[string]map[string]*exports.Neighbors
324340
if err := json.Unmarshal([]byte(report), &discDataPerNode); err != nil {

0 commit comments

Comments
 (0)