NETOBSERV-2126: cleanup unused code wrt Duplicate (#1261)

Author: jotak

apis/flowmetrics/v1alpha1/flowmetric_types.go

@@ -78,8 +78,7 @@ type FlowMetricSpec struct {
 	// +optional
 	ValueField string `json:"valueField,omitempty"`
 
-	// `filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must
-	// be used to eliminate duplicates: `Duplicate != "true"` and `FlowDirection = "0"`.
+	// `filters` is a list of fields and values used to restrict which flows are taken into account.
 	// Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/observability/network_observability/json-flows-format-reference.html.
 	// +optional
 	Filters []MetricFilter `json:"filters"`

bundle/manifests/flows.netobserv.io_flowmetrics.yaml

@@ -167,8 +167,7 @@ spec:
                 type: string
               filters:
                 description: |-
-                  `filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must
-                  be used to eliminate duplicates: `Duplicate != "true"` and `FlowDirection = "0"`.
+                  `filters` is a list of fields and values used to restrict which flows are taken into account.
                   Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/observability/network_observability/json-flows-format-reference.html.
                 items:
                   properties:

config/crd/bases/flows.netobserv.io_flowmetrics.yaml

@@ -157,8 +157,7 @@ spec:
                 type: string
               filters:
                 description: |-
-                  `filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must
-                  be used to eliminate duplicates: `Duplicate != "true"` and `FlowDirection = "0"`.
+                  `filters` is a list of fields and values used to restrict which flows are taken into account.
                   Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/observability/network_observability/json-flows-format-reference.html.
                 items:
                   properties:

controllers/consoleplugin/config/config.go

@@ -108,11 +108,6 @@ type FieldConfig struct {
 	Filter      string `yaml:"filter,omitempty" json:"filter,omitempty"`
 }
 
-type Deduper struct {
-	Mark  bool `yaml:"mark" json:"mark"`
-	Merge bool `yaml:"merge" json:"merge"`
-}
-
 type FrontendConfig struct {
 	RecordTypes     []api.ConnTrackOutputRecordTypeEnum `yaml:"recordTypes" json:"recordTypes"`
 	PortNaming      flowslatest.ConsolePluginPortConfig `yaml:"portNaming,omitempty" json:"portNaming,omitempty"`
@@ -123,7 +118,6 @@ type FrontendConfig struct {
 	AlertNamespaces []string                            `yaml:"alertNamespaces,omitempty" json:"alertNamespaces,omitempty"`
 	Sampling        int                                 `yaml:"sampling" json:"sampling"`
 	Features        []string                            `yaml:"features" json:"features"`
-	Deduper         Deduper                             `yaml:"deduper" json:"deduper"`
 	Fields          []FieldConfig                       `yaml:"fields" json:"fields"`
 }
 

controllers/consoleplugin/config/static-frontend-config.yaml

@@ -1336,9 +1336,6 @@ fields:
   - name: IcmpCode
     type: number
     description: ICMP code
-  - name: Duplicate
-    type: boolean
-    description: Indicates if this flow was also captured from another interface on the same host
   - name: FlowDirection
     type: number
     description: |

controllers/consoleplugin/consoleplugin_objects.go

@@ -24,7 +24,6 @@ import (
 	flowslatest "github.com/netobserv/network-observability-operator/apis/flowcollector/v1beta2"
 	cfg "github.com/netobserv/network-observability-operator/controllers/consoleplugin/config"
 	"github.com/netobserv/network-observability-operator/controllers/constants"
-	"github.com/netobserv/network-observability-operator/controllers/ebpf"
 	"github.com/netobserv/network-observability-operator/controllers/reconcilers"
 	"github.com/netobserv/network-observability-operator/pkg/helper"
 	"github.com/netobserv/network-observability-operator/pkg/helper/loki"
@@ -432,15 +431,6 @@ func (b *builder) getPromConfig(ctx context.Context) cfg.PrometheusConfig {
 }
 
 func (b *builder) setFrontendConfig(fconf *cfg.FrontendConfig) error {
-	var err error
-	dedupJustMark, err := strconv.ParseBool(ebpf.DedupeJustMarkDefault)
-	if err != nil {
-		return err
-	}
-	dedupMerge, err := strconv.ParseBool(ebpf.DedupeMergeDefault)
-	if err != nil {
-		return err
-	}
 	if helper.IsPktDropEnabled(&b.desired.Agent.EBPF) {
 		fconf.Features = append(fconf.Features, "pktDrop")
 	}
@@ -469,30 +459,11 @@ func (b *builder) setFrontendConfig(fconf *cfg.FrontendConfig) error {
 		fconf.Features = append(fconf.Features, "ipsec")
 	}
 
-	if b.desired.Agent.EBPF.Advanced != nil {
-		if v, ok := b.desired.Agent.EBPF.Advanced.Env[ebpf.EnvDedupeJustMark]; ok {
-			dedupJustMark, err = strconv.ParseBool(v)
-			if err != nil {
-				return err
-			}
-		}
-
-		if v, ok := b.desired.Agent.EBPF.Advanced.Env[ebpf.EnvDedupeMerge]; ok {
-			dedupMerge, err = strconv.ParseBool(v)
-			if err != nil {
-				return err
-			}
-		}
-	}
 	fconf.RecordTypes = helper.GetRecordTypes(&b.desired.Processor)
 	fconf.PortNaming = b.desired.ConsolePlugin.PortNaming
 	fconf.QuickFilters = b.desired.ConsolePlugin.QuickFilters
 	fconf.AlertNamespaces = []string{b.info.Namespace}
 	fconf.Sampling = helper.GetSampling(b.desired)
-	fconf.Deduper = cfg.Deduper{
-		Mark:  dedupJustMark,
-		Merge: dedupMerge,
-	}
 	if helper.IsMultiClusterEnabled(&b.desired.Processor) {
 		fconf.Features = append(fconf.Features, "multiCluster")
 	}

controllers/consoleplugin/consoleplugin_test.go

@@ -338,59 +338,6 @@ func TestConfigMapContent(t *testing.T) {
 	assert.NotEmpty(config.Frontend.Filters)
 	assert.NotEmpty(config.Frontend.Scopes)
 	assert.Equal(config.Frontend.Sampling, 1)
-	assert.Equal(config.Frontend.Deduper.Mark, false)
-	assert.Equal(config.Frontend.Deduper.Merge, true)
-}
-
-func TestConfigMapError(t *testing.T) {
-	assert := assert.New(t)
-
-	agentSpec := flowslatest.FlowCollectorAgent{
-		Type: "eBPF",
-		EBPF: flowslatest.FlowCollectorEBPF{
-			Sampling: ptr.To(int32(1)),
-			Advanced: &flowslatest.AdvancedAgentConfig{
-				Env: map[string]string{
-					"DEDUPER_JUST_MARK": "invalid",
-				},
-			},
-		},
-	}
-	lokiSpec := flowslatest.FlowCollectorLoki{}
-	loki := helper.NewLokiConfig(&lokiSpec, "any")
-
-	// spec with invalid flag
-	spec := flowslatest.FlowCollectorSpec{
-		Agent:         agentSpec,
-		ConsolePlugin: getPluginConfig(),
-		Loki:          lokiSpec,
-	}
-	builder := getBuilder(&spec, &loki)
-	cm, _, err := builder.configMap(context.Background())
-	assert.Nil(cm)
-	assert.NotNil(err)
-
-	// update to valid flags
-	agentSpec.EBPF.Advanced.Env = map[string]string{
-		"DEDUPER_JUST_MARK": "false",
-		"DEDUPER_MERGE":     "true",
-	}
-	spec = flowslatest.FlowCollectorSpec{
-		Agent:         agentSpec,
-		ConsolePlugin: getPluginConfig(),
-		Loki:          lokiSpec,
-	}
-	builder = getBuilder(&spec, &loki)
-	cm, _, err = builder.configMap(context.Background())
-	assert.NotNil(cm)
-	assert.Nil(err)
-
-	// parse output config and check expected values
-	var config config.PluginConfig
-	err = yaml.Unmarshal([]byte(cm.Data["config.yaml"]), &config)
-	assert.Nil(err)
-	assert.Equal(config.Frontend.Deduper.Mark, false)
-	assert.Equal(config.Frontend.Deduper.Merge, true)
 }
 
 func TestServiceUpdateCheck(t *testing.T) {

controllers/ebpf/agent_controller.go

@@ -52,8 +52,6 @@ const (
 	envKafkaSASLIDPath            = "KAFKA_SASL_CLIENT_ID_PATH"
 	envKafkaSASLSecretPath        = "KAFKA_SASL_CLIENT_SECRET_PATH"
 	envLogLevel                   = "LOG_LEVEL"
-	envDedupe                     = "DEDUPER"
-	dedupeDefault                 = "firstCome"
 	envGoMemLimit                 = "GOMEMLIMIT"
 	envEnablePktDrop              = "ENABLE_PKT_DROPS"
 	envEnableDNSTracking          = "ENABLE_DNS_TRACKING"
@@ -94,11 +92,7 @@ const (
 )
 
 const (
-	EnvDedupeJustMark      = "DEDUPER_JUST_MARK"
-	EnvDedupeMerge         = "DEDUPER_MERGE"
 	envDNSTrackingPort     = "DNS_TRACKING_PORT"
-	DedupeJustMarkDefault  = "false"
-	DedupeMergeDefault     = "true"
 	defaultDNSTrackingPort = "53"
 	bpfmanMapsVolumeName   = "bpfman-maps"
 	bpfManBpfFSPath        = "/run/netobserv/maps"
@@ -752,9 +746,6 @@ func (c *AgentController) setEnvConfig(coll *flowslatest.FlowCollector) []corev1
 		})
 	}
 
-	dedup := dedupeDefault
-	dedupJustMark := DedupeJustMarkDefault
-	dedupMerge := DedupeMergeDefault
 	dnsTrackingPort := defaultDNSTrackingPort
 	networkEventsGroupID := defaultNetworkEventsGroupID
 	// we need to sort env map to keep idempotency,
@@ -763,12 +754,6 @@ func (c *AgentController) setEnvConfig(coll *flowslatest.FlowCollector) []corev1
 	for _, pair := range helper.KeySorted(advancedConfig.Env) {
 		k, v := pair[0], pair[1]
 		switch k {
-		case envDedupe:
-			dedup = v
-		case EnvDedupeJustMark:
-			dedupJustMark = v
-		case EnvDedupeMerge:
-			dedupMerge = v
 		case envDNSTrackingPort:
 			dnsTrackingPort = v
 		case envNetworkEventsGroupID:
@@ -778,8 +763,6 @@ func (c *AgentController) setEnvConfig(coll *flowslatest.FlowCollector) []corev1
 		}
 	}
 
-	config = append(config, corev1.EnvVar{Name: envDedupe, Value: dedup})
-	config = append(config, corev1.EnvVar{Name: EnvDedupeJustMark, Value: dedupJustMark})
 	config = append(config, corev1.EnvVar{Name: envDNSTrackingPort, Value: dnsTrackingPort})
 	config = append(config, corev1.EnvVar{Name: envNetworkEventsGroupID, Value: networkEventsGroupID})
 	config = append(config, corev1.EnvVar{
@@ -792,7 +775,6 @@ func (c *AgentController) setEnvConfig(coll *flowslatest.FlowCollector) []corev1
 		},
 	},
 	)
-	config = append(config, corev1.EnvVar{Name: EnvDedupeMerge, Value: dedupMerge})
 
 	return config
 }

controllers/flowcollector_controller_certificates_test.go

@@ -145,13 +145,7 @@ func flowCollectorCertificatesSpecs() {
 			DeploymentModel: flowslatest.DeploymentModelKafka,
 			Agent: flowslatest.FlowCollectorAgent{
 				Type: "eBPF",
-				EBPF: flowslatest.FlowCollectorEBPF{
-					Advanced: &flowslatest.AdvancedAgentConfig{
-						Env: map[string]string{
-							"DEDUPER_JUST_MARK": "true",
-						},
-					},
-				},
+				EBPF: flowslatest.FlowCollectorEBPF{},
 			},
 			Loki: flowslatest.FlowCollectorLoki{
 				Enable: ptr.To(true),

docs/FlowMetric.md

@@ -146,8 +146,7 @@ When set to `Egress`, it is equivalent to adding the regular expression filter o
         <td><b><a href="#flowmetricspecfiltersindex">filters</a></b></td>
         <td>[]object</td>
         <td>
-          `filters` is a list of fields and values used to restrict which flows are taken into account. Oftentimes, these filters must
-be used to eliminate duplicates: `Duplicate != "true"` and `FlowDirection = "0"`.
+          `filters` is a list of fields and values used to restrict which flows are taken into account.
 Refer to the documentation for the list of available fields: https://docs.openshift.com/container-platform/latest/observability/network_observability/json-flows-format-reference.html.<br/>
         </td>
         <td>false</td>