diff --git a/Makefile b/Makefile index 468f375da..4d9a166a6 100644 --- a/Makefile +++ b/Makefile @@ -41,7 +41,7 @@ ifneq ($(CLEAN_BUILD),) LDFLAGS ?= -X 'main.buildVersion=${VERSION}-${BUILD_SHA}' -X 'main.buildDate=${BUILD_DATE}' endif -GOLANGCI_LINT_VERSION = v2.2.1 +GOLANGCI_LINT_VERSION = v2.6.0 KIND_VERSION = v0.22.0 FLP_BIN_FILE=flowlogs-pipeline @@ -119,7 +119,7 @@ vendors: ## Check go vendors .PHONY: lint lint: prereqs ## Lint the code - ./bin/golangci-lint-${GOLANGCI_LINT_VERSION} run ./... --timeout=3m + ./bin/golangci-lint-${GOLANGCI_LINT_VERSION} run ./... --timeout=5m .PHONY: compile compile: ## Compile main flowlogs-pipeline and config generator diff --git a/README.md b/README.md index e1a417d62..aa7e04a44 100644 --- a/README.md +++ b/README.md @@ -357,32 +357,44 @@ parameters: ### Transform Filter -The filter transform module allows setting rules to remove complete entries from -the output, or just remove specific keys and values from entries. +The filter transform module allows setting rules to remove complete flow logs from the output, or just remove specific keys and values from logs. For example, suppose we have a flow log with the following syntax: ```json -{"Bytes":20800,"DstAddr":"10.130.2.2","DstPort":36936,"Packets":400,"Proto":6,"SequenceNum":1919,"SrcAddr":"10.130.2.13","SrcHostIP":"10.0.197.206","SrcPort":3100,"TCPFlags":0,"TimeFlowStart":0,"TimeReceived":1637501832} +{ + "Bytes":20800, + "DstAddr":"10.130.2.2", + "DstPort":36936, + "Packets":400, + "Proto":6, + "SequenceNum":1919, + "SrcAddr":"10.130.2.13", + "SrcHostIP":"10.0.197.206", + "SrcPort":3100, + "TCPFlags":0, + "TimeFlowStart":0, + "TimeReceived":1637501832 +} ``` -The below configuration will remove (filter) the entry from the output +The below configuration will skip that log, removing it from the output. ```yaml -pipeline: - transform: - - type: filter +parameters: + - name: filter1 + transform: + type: filter filter: rules: - type: remove_entry_if_exists removeEntry: - input: SrcPort + input: TCPFlags ``` -Using `remove_entry_if_doesnt_exist` in the rule reverses the logic and will not remove the above example entry -Using `remove_field` in the rule `type` instead, results in outputting the entry after -removal of only the `SrcPort` key and value -Using `remove_entry_if_equal` will remove the entry if the specified field exists and is equal to the specified value. -Using `remove_entry_if_not_equal` will remove the entry if the specified field exists and is not equal to the specified value. +- `type: remove_entry_if_doesnt_exist` reverses the logic and will not remove the above example entry. +- `type: remove_field` keeps the entry but changes its content, removing the `TCPFlags` key and value. +- `type: remove_entry_if_equal` removes the entry if the specified field exists and is equal to the specified value. +- `type: remove_entry_if_not_equal` removes the entry if the specified field exists and is not equal to the specified value. #### Transform Filter: query language diff --git a/docs/api.md b/docs/api.md index 188126b4a..d14052828 100644 --- a/docs/api.md +++ b/docs/api.md @@ -159,13 +159,13 @@ Following is the supported API format for filter transformations: filter: rules: list of filter rules, each includes: type: (enum) one of the following: - remove_field: removes the field from the entry - remove_entry_if_exists: removes the entry if the field exists - remove_entry_if_doesnt_exist: removes the entry if the field does not exist - remove_entry_if_equal: removes the entry if the field value equals specified value - remove_entry_if_not_equal: removes the entry if the field value does not equal specified value - remove_entry_all_satisfied: removes the entry if all of the defined rules are satisfied - keep_entry_query: keeps the entry if it matches the query + remove_field: removes the field from the flow log + remove_entry_if_exists: removes the entry (ie. the flow log) if the field exists + remove_entry_if_doesnt_exist: removes the entry (ie. the flow log) if the field does not exist + remove_entry_if_equal: removes the entry (ie. the flow log) if the field value equals the specified value + remove_entry_if_not_equal: removes the entry (ie. the flow log) if the field value does not equal the specified value + remove_entry_all_satisfied: removes the entry (ie. the flow log) if all of the defined rules are satisfied + keep_entry_query: keeps the entry (ie. the flow log) if it matches the query add_field: adds (input) field to the entry; overrides previous value if present (key=input, value=value) add_field_if_doesnt_exist: adds a field to the entry if the field does not exist add_field_if: add output field set to assignee if input field satisfies criteria from parameters field @@ -183,10 +183,10 @@ Following is the supported API format for filter transformations: castInt: set true to cast the value field as an int (numeric values are float64 otherwise) removeEntryAllSatisfied: configuration for remove_entry_all_satisfied rule type: (enum) one of the following: - remove_entry_if_exists: removes the entry if the field exists - remove_entry_if_doesnt_exist: removes the entry if the field does not exist - remove_entry_if_equal: removes the entry if the field value equals specified value - remove_entry_if_not_equal: removes the entry if the field value does not equal specified value + remove_entry_if_exists: removes the entry (ie. the flow log) if the field exists + remove_entry_if_doesnt_exist: removes the entry (ie. the flow log) if the field does not exist + remove_entry_if_equal: removes the entry (ie. the flow log) if the field value equals the specified value + remove_entry_if_not_equal: removes the entry (ie. the flow log) if the field value does not equal the specified value removeEntry: configuration for remove_entry_* rules input: entry input field value: specified value of input field: @@ -224,10 +224,10 @@ Following is the supported API format for filter transformations: value: sampling interval: 1 flow on is kept rules: rules to be satisfied for this sampling configuration type: (enum) one of the following: - remove_entry_if_exists: removes the entry if the field exists - remove_entry_if_doesnt_exist: removes the entry if the field does not exist - remove_entry_if_equal: removes the entry if the field value equals specified value - remove_entry_if_not_equal: removes the entry if the field value does not equal specified value + remove_entry_if_exists: removes the entry (ie. the flow log) if the field exists + remove_entry_if_doesnt_exist: removes the entry (ie. the flow log) if the field does not exist + remove_entry_if_equal: removes the entry (ie. the flow log) if the field value equals the specified value + remove_entry_if_not_equal: removes the entry (ie. the flow log) if the field value does not equal the specified value removeEntry: configuration for remove_entry_* rules input: entry input field value: specified value of input field: diff --git a/pkg/api/transform_filter.go b/pkg/api/transform_filter.go index a933438e0..fc8e90cbb 100644 --- a/pkg/api/transform_filter.go +++ b/pkg/api/transform_filter.go @@ -32,13 +32,13 @@ type TransformFilterEnum string const ( // For doc generation, enum definitions must match format `Constant Type = "value" // doc` - RemoveField TransformFilterEnum = "remove_field" // removes the field from the entry - RemoveEntryIfExists TransformFilterEnum = "remove_entry_if_exists" // removes the entry if the field exists - RemoveEntryIfDoesntExist TransformFilterEnum = "remove_entry_if_doesnt_exist" // removes the entry if the field does not exist - RemoveEntryIfEqual TransformFilterEnum = "remove_entry_if_equal" // removes the entry if the field value equals specified value - RemoveEntryIfNotEqual TransformFilterEnum = "remove_entry_if_not_equal" // removes the entry if the field value does not equal specified value - RemoveEntryAllSatisfied TransformFilterEnum = "remove_entry_all_satisfied" // removes the entry if all of the defined rules are satisfied - KeepEntryQuery TransformFilterEnum = "keep_entry_query" // keeps the entry if it matches the query + RemoveField TransformFilterEnum = "remove_field" // removes the field from the flow log + RemoveEntryIfExists TransformFilterEnum = "remove_entry_if_exists" // removes the entry (ie. the flow log) if the field exists + RemoveEntryIfDoesntExist TransformFilterEnum = "remove_entry_if_doesnt_exist" // removes the entry (ie. the flow log) if the field does not exist + RemoveEntryIfEqual TransformFilterEnum = "remove_entry_if_equal" // removes the entry (ie. the flow log) if the field value equals the specified value + RemoveEntryIfNotEqual TransformFilterEnum = "remove_entry_if_not_equal" // removes the entry (ie. the flow log) if the field value does not equal the specified value + RemoveEntryAllSatisfied TransformFilterEnum = "remove_entry_all_satisfied" // removes the entry (ie. the flow log) if all of the defined rules are satisfied + KeepEntryQuery TransformFilterEnum = "keep_entry_query" // keeps the entry (ie. the flow log) if it matches the query AddField TransformFilterEnum = "add_field" // adds (input) field to the entry; overrides previous value if present (key=input, value=value) AddFieldIfDoesntExist TransformFilterEnum = "add_field_if_doesnt_exist" // adds a field to the entry if the field does not exist AddFieldIf TransformFilterEnum = "add_field_if" // add output field set to assignee if input field satisfies criteria from parameters field @@ -51,10 +51,10 @@ const ( type TransformFilterRemoveEntryEnum string const ( - RemoveEntryIfExistsD TransformFilterRemoveEntryEnum = "remove_entry_if_exists" // removes the entry if the field exists - RemoveEntryIfDoesntExistD TransformFilterRemoveEntryEnum = "remove_entry_if_doesnt_exist" // removes the entry if the field does not exist - RemoveEntryIfEqualD TransformFilterRemoveEntryEnum = "remove_entry_if_equal" // removes the entry if the field value equals specified value - RemoveEntryIfNotEqualD TransformFilterRemoveEntryEnum = "remove_entry_if_not_equal" // removes the entry if the field value does not equal specified value + RemoveEntryIfExistsD TransformFilterRemoveEntryEnum = "remove_entry_if_exists" // removes the entry (ie. the flow log) if the field exists + RemoveEntryIfDoesntExistD TransformFilterRemoveEntryEnum = "remove_entry_if_doesnt_exist" // removes the entry (ie. the flow log) if the field does not exist + RemoveEntryIfEqualD TransformFilterRemoveEntryEnum = "remove_entry_if_equal" // removes the entry (ie. the flow log) if the field value equals the specified value + RemoveEntryIfNotEqualD TransformFilterRemoveEntryEnum = "remove_entry_if_not_equal" // removes the entry (ie. the flow log) if the field value does not equal the specified value ) type TransformFilterRule struct {