util: add a blocklist cooldown period of 5 mins

This commit adds a blocklist cool period of 5 mins
making sure cephcsi will wait for atleast 5 mins
before auto unfencing a node.

Signed-off-by: Rakshith R <[email protected]>

Author: Rakshith-R

internal/csi-addons/networkfence/fencing.go

@@ -37,7 +37,11 @@ import (
 
 const (
 	ISO8601TimeLayout = "2006-01-02T15:04:05.000000-0700"
-	invalidCommandStr = "invalid command"
+	// BlockListCoolDownPeriod defines the time duration
+	// after which a blocklist entry can be removed.
+	// TODO: Make this configurable.
+	blockListCoolDownPeriod = 5 * time.Minute
+	invalidCommandStr       = "invalid command"
 	// we can always use mds rank 0, since all the clients have a session with rank-0.
 	mdsRank = 0
 )
@@ -592,7 +596,17 @@ func containsMatchingBlockListEntry(
 				entry.Until, err)
 		}
 
-		if until.Sub(time.Now()) <= util.AutoBlocklistTime {
+		timeLeftUntilExpire := time.Until(until)
+		// Check if the blocklist entry is eligible for auto-unfencing if
+		// 1. the time left until expiry is less than or equal to AutoBlocklistTime,
+		// 2. the blocklist was done at least blockListCoolDownPeriod seconds (5 Minutes) ago.
+		if timeLeftUntilExpire <= util.AutoBlocklistTime {
+			if timeLeftUntilExpire > (util.AutoBlocklistTime - blockListCoolDownPeriod) {
+				// still in cool-down period
+				return false, fmt.Errorf("blocklist entry %q is still in cool-down period for %s",
+					entry.Addr, (timeLeftUntilExpire - (util.AutoBlocklistTime - blockListCoolDownPeriod)))
+			}
+
 			return true, nil
 		}
 	}

internal/csi-addons/networkfence/fencing_test.go

@@ -274,23 +274,153 @@ func Test_containsMatchingBlockListEntry(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			name: "matching entry found",
+			name: "matching entry found blocked outside cool down period 1",
 			args: args{
 				blocklist: &[]osdAdmin.Blocklist{
 					{
 						Addr:  "192.0.1.0:0/32",
-						Until: time.Now().Format(ISO8601TimeLayout),
+						Until: time.Now().Add(1 * time.Hour).Format(ISO8601TimeLayout),
 					},
 					{
 						Addr:  "192.0.2.0:0/32",
-						Until: time.Now().Format(ISO8601TimeLayout),
+						Until: time.Now().Add(1 * time.Hour).Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "192.0.1.0",
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "matching entry found blocked outside cool down period 2",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr:  "192.0.1.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime - blockListCoolDownPeriod).Format(ISO8601TimeLayout),
+					},
+					{
+						Addr:  "192.0.2.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime - blockListCoolDownPeriod).Format(ISO8601TimeLayout),
 					},
 				},
 				addr: "192.0.1.0",
 			},
 			want:    true,
 			wantErr: false,
 		},
+		{
+			name: "matching entry found blocked in cool down period 1",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr:  "192.0.1.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime).Format(ISO8601TimeLayout),
+					},
+					{
+						Addr:  "192.0.2.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime).Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "192.0.1.0",
+			},
+			want:    false,
+			wantErr: true,
+		},
+		{
+			name: "matching entry found blocked in cool down period 2",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr: "192.0.1.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime - 2*time.Minute).
+							Format(ISO8601TimeLayout),
+					},
+					{
+						Addr: "192.0.2.0:0/32",
+						Until: time.Now().Add(util.AutoBlocklistTime - 2*time.Minute).
+							Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "192.0.1.0",
+			},
+			want:    false,
+			wantErr: true,
+		},
+		{
+			name: "matching IPv6 entry found blocked outside cool down period 1",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr:  "2001:db8::1:0/128",
+						Until: time.Now().Add(1 * time.Hour).Format(ISO8601TimeLayout),
+					},
+					{
+						Addr:  "2001:db8::2:0/128",
+						Until: time.Now().Add(1 * time.Hour).Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "2001:db8::1",
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "matching IPv6 entry found blocked outside cool down period 2",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr:  "2001:db8::1:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime - blockListCoolDownPeriod).Format(ISO8601TimeLayout),
+					},
+					{
+						Addr:  "2001:db8::2:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime - blockListCoolDownPeriod).Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "2001:db8::1",
+			},
+			want:    true,
+			wantErr: false,
+		},
+		{
+			name: "matching IPv6 entry found blocked in cool down period 1",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr:  "2001:db8::1:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime).Format(ISO8601TimeLayout),
+					},
+					{
+						Addr:  "2001:db8::2:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime).Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "2001:db8::1",
+			},
+			want:    false,
+			wantErr: true,
+		},
+		{
+			name: "matching IPv6 entry found blocked in cool down period 2",
+			args: args{
+				blocklist: &[]osdAdmin.Blocklist{
+					{
+						Addr: "2001:db8::1:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime - 2*time.Minute).
+							Format(ISO8601TimeLayout),
+					},
+					{
+						Addr: "2001:db8::2:0/128",
+						Until: time.Now().Add(util.AutoBlocklistTime - 2*time.Minute).
+							Format(ISO8601TimeLayout),
+					},
+				},
+				addr: "2001:db8::1",
+			},
+			want:    false,
+			wantErr: true,
+		},
 		{
 			name: "address does not match",
 			args: args{