Add documentation and make target

Author: jotak

Makefile

@@ -197,5 +197,11 @@ else
 	DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${IMAGE} docker://${IMAGE};
 endif
 
+.PHONY: goyacc
+goyacc: ## Regenerate filters query langage
+	@echo "### Regenerate filters query langage"
+	GOFLAGS="" go install golang.org/x/tools/cmd/[email protected]
+	goyacc -o pkg/dsl/expr.y.go pkg/dsl/expr.y
+
 include .mk/development.mk
 include .mk/shortcuts.mk

README.md

@@ -386,6 +386,16 @@ removal of only the `SrcPort` key and value
 Using `remove_entry_if_equal` will remove the entry if the specified field exists and is equal to the specified value.
 Using `remove_entry_if_not_equal` will remove the entry if the specified field exists and is not equal to the specified value.
 
+#### Transform Filter: query language
+
+Alternatively, a query language allows to filter flows, keeping entries rather than removing them.
+
+```
+(srcnamespace="netobserv" OR (srcnamespace="ingress" AND dstnamespace="netobserv")) AND srckind!="service"
+```
+
+[See here](./docs/filtering.md) for more information about this language.
+
 ### Transform Network
 
 `transform network` provides specific functionality that is useful for transformation of network flow-logs:

docs/filtering.md

@@ -0,0 +1,66 @@
+# FLP filtering language
+
+Flowlogs-pipeline uses a simple query language to filter network flows:
+
+```
+(srcnamespace="netobserv" OR (srcnamespace="ingress" AND dstnamespace="netobserv")) AND srckind!="service"
+```
+
+The syntax includes:
+
+- Logical boolean operators (case insensitive)
+  - `and`
+  - `or`
+- String comparison operators
+  - equals `=`
+  - not equals `!=`
+  - matches regexp `=~`
+  - not matches regexp `!~`
+- Unary operations
+  - field is present: `with(field)`
+  - field is absent: `without(field)`
+- Parenthesis-based priority
+
+## API integration
+
+The language is currently integrated in the "keep_entry" transform/filtering API. Example:
+
+```yaml
+    transform:
+      type: filter
+      filter:
+        rules:
+        - type: keep_entry_query
+          keepEntryQuery: (namespace="A" and with(workload)) or service=~"abc.+"
+          keepEntrySampling: 10 # Optionally, a sampling ratio can be associated with the filter
+```
+
+## Integration with the NetObserv operator
+
+In the [NetObserv operator](https://github.com/netobserv/network-observability-operator), the filtering query language is used in `FlowCollector` `spec.processor.filters`. Example:
+
+```yaml
+spec:
+  processor:
+    filters:
+      - query: |
+          (SrcK8S_Namespace="netobserv" OR (SrcK8S_Namespace="openshift-ingress" AND DstK8S_Namespace="netobserv"))
+        outputTarget: Loki  # The filter can target a specific output (such as Loki logs or exported data), or all outputs.
+        sampling: 10        # Optionally, a sampling ratio can be associated with the filter
+```
+
+See also the [list of field names](https://github.com/netobserv/network-observability-operator/blob/main/docs/flows-format.adoc) that are available for queries, and the [API documentation](https://github.com/netobserv/network-observability-operator/blob/main/docs/FlowCollector.md#flowcollectorspecprocessorfiltersindex-1).
+
+## Internals
+
+This language is designed using [Yacc](https://en.wikipedia.org/wiki/Yacc) / goyacc.
+
+The [definition file](../pkg/dsl/expr.y) describes the syntax based on a list of tokens. It is derived to a [go source file](../pkg/dsl/expr.y.go) using [goyacc](https://pkg.go.dev/golang.org/x/tools/cmd/goyacc), which defines constants for the tokens, among other things. The [lexer](../pkg/dsl/lexer.go) file defines structures and helpers that can be used from `expr.y`, the logic used to interpret the language in a structured way, and is also where actual characters/strings are mapped to syntax tokens. Finally, [eval.go](../pkg/dsl/eval.go) runs the desired query on actual data.
+
+When adding features to the language, you'll likely have to change `expr.y` and `lexer.go`.
+
+To regenerate `expr.y.go`, run:
+
+```bash
+make goyacc
+```

pkg/dsl/expr.y

@@ -10,7 +10,7 @@ package dsl
 %type <expr> root
 %type <expr> expr
 
-%token <value> VAR STRING NUMBER AND OR EQ NEQ REG NREG OPEN_PARENTHESIS CLOSE_PARENTHESIS WITH WITHOUT
+%token <value> NF_FIELD STRING NUMBER AND OR EQ NEQ REG NREG OPEN_PARENTHESIS CLOSE_PARENTHESIS WITH WITHOUT
 %left AND
 %left OR
 %%
@@ -25,10 +25,10 @@ expr:
 	OPEN_PARENTHESIS expr CLOSE_PARENTHESIS { $$ = ParenthesisExpr{inner: $2} }
 	| expr AND expr { $$ = LogicalExpr{left: $1, operator: operatorAnd, right: $3} }
 	| expr OR expr { $$ = LogicalExpr{left: $1, operator: operatorOr, right: $3} }
-	| WITH OPEN_PARENTHESIS VAR CLOSE_PARENTHESIS { $$ = WithExpr{key: $3} }
-	| WITHOUT OPEN_PARENTHESIS VAR CLOSE_PARENTHESIS { $$ = WithoutExpr{key: $3} }
-	| VAR EQ STRING { $$ = EqExpr{key: $1, value: $3} }
-	| VAR NEQ STRING { $$ = NEqExpr{key: $1, value: $3} }
-	| VAR REG STRING { $$ = RegExpr{key: $1, value: $3} }
-	| VAR NREG STRING { $$ = NRegExpr{key: $1, value: $3} }
+	| WITH OPEN_PARENTHESIS NF_FIELD CLOSE_PARENTHESIS { $$ = WithExpr{key: $3} }
+	| WITHOUT OPEN_PARENTHESIS NF_FIELD CLOSE_PARENTHESIS { $$ = WithoutExpr{key: $3} }
+	| NF_FIELD EQ STRING { $$ = EqExpr{key: $1, value: $3} }
+	| NF_FIELD NEQ STRING { $$ = NEqExpr{key: $1, value: $3} }
+	| NF_FIELD REG STRING { $$ = RegExpr{key: $1, value: $3} }
+	| NF_FIELD NREG STRING { $$ = NRegExpr{key: $1, value: $3} }
 %%