Skip to content

Commit 6974f21

Browse files
eranraeranra
committed
ocp-correctness
1 parent 1869eb7 commit 6974f21

File tree

11 files changed

+451
-24
lines changed

11 files changed

+451
-24
lines changed

‎Makefile‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ FL2M_BIN_FILE=flowlogs2metrics
2020
CG_BIN_FILE=confgenerator
2121
NETFLOW_GENERATOR=nflow-generator
2222
CMD_DIR=./cmd/
23+
FL2M_CONF_FILE ?= contrib/kubernetes/flowlogs2metrics.conf.yaml
2324

2425
.DEFAULT_GOAL := help
2526

@@ -58,7 +59,7 @@ lint: $(GOLANGCI_LINT) ## Lint the code
5859
.PHONY: build_code
5960
build_code: validate_go lint
6061
@go mod vendor
61-
go build "${CMD_DIR}${FL2M_BIN_FILE}"
62+
VERSION=$$(date); go build -ldflags "-X 'main.Version=$$VERSION'" "${CMD_DIR}${FL2M_BIN_FILE}"
6263

6364
.PHONY: build
6465
build: build_code docs ## Build flowlogs2metrics executable and update the docs
@@ -113,7 +114,7 @@ push-image: build-image ## Push latest image
113114
.PHONY: deploy
114115
deploy: ## Deploy the image
115116
sed 's|%DOCKER_IMG%|$(DOCKER_IMG)|g;s|%DOCKER_TAG%|$(DOCKER_TAG)|g' contrib/kubernetes/deployment.yaml > /tmp/deployment.yaml
116-
kubectl create configmap flowlogs2metrics-configuration --from-file=flowlogs2metrics.conf.yaml=contrib/kubernetes/flowlogs2metrics.conf.yaml
117+
kubectl create configmap flowlogs2metrics-configuration --from-file=flowlogs2metrics.conf.yaml=$(FL2M_CONF_FILE)
117118
kubectl apply -f /tmp/deployment.yaml
118119
kubectl rollout status "deploy/flowlogs2metrics" --timeout=600s
119120

@@ -177,7 +178,7 @@ delete-kind-cluster: $(KIND) ## Delete cluster
177178
generate-configuration: $(KIND) ## Generate metrics configuration
178179
go build "${CMD_DIR}${CG_BIN_FILE}"
179180
./${CG_BIN_FILE} --log-level debug --srcFolder network_definitions \
180-
--destConfFile contrib/kubernetes/flowlogs2metrics.conf.yaml \
181+
--destConfFile $(FL2M_CONF_FILE) \
181182
--destDocFile docs/metrics.md \
182183
--destGrafanaJsonnetFolder contrib/dashboards/jsonnet/
183184

‎cmd/flowlogs2metrics/main.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ import (
3333
)
3434

3535
var (
36+
Version string
3637
cfgFile string
3738
logLevel string
3839
envPrefix = "FLOWLOGS2METRICS"
@@ -89,8 +90,8 @@ func initLogger() {
8990
}
9091

9192
func dumpConfig() {
92-
configAsJSON, _ := json.MarshalIndent(config.Opt, "", "\t")
93-
log.Infof("configuration:\n%s\n", configAsJSON)
93+
configAsJSON, _ := json.MarshalIndent(config.Opt, "", " ")
94+
fmt.Printf("Using configuration:\n%s\n", configAsJSON)
9495
}
9596

9697
func bindFlags(cmd *cobra.Command, v *viper.Viper) {
@@ -155,7 +156,7 @@ func run() {
155156
)
156157

157158
// Starting log message
158-
log.Infof("starting %s", filepath.Base(os.Args[0]))
159+
fmt.Printf("%s starting - version [%s]\n\n", filepath.Base(os.Args[0]), Version)
159160
// Dump the configuration
160161
dumpConfig()
161162
// creating a new pipeline

‎contrib/dashboards/dashboard_details.json‎

Lines changed: 181 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,91 @@
136136
"spaceLength": 10,
137137
"stack": false,
138138
"steppedLine": false,
139+
"targets": [
140+
{
141+
"expr": "topk(10,rate(fl2m_bandwidth_per_source_destination_subnet[1m]))",
142+
"format": "time_series",
143+
"intervalFactor": 2,
144+
"legendFormat": "",
145+
"refId": "A"
146+
}
147+
],
148+
"thresholds": [ ],
149+
"timeFrom": null,
150+
"timeShift": null,
151+
"title": "Bandwidth per src and destination subnet",
152+
"tooltip": {
153+
"shared": true,
154+
"sort": 0,
155+
"value_type": "individual"
156+
},
157+
"type": "graph",
158+
"xaxis": {
159+
"buckets": null,
160+
"mode": "time",
161+
"name": null,
162+
"show": true,
163+
"values": [ ]
164+
},
165+
"yaxes": [
166+
{
167+
"format": "short",
168+
"label": null,
169+
"logBase": 1,
170+
"max": null,
171+
"min": null,
172+
"show": true
173+
},
174+
{
175+
"format": "short",
176+
"label": null,
177+
"logBase": 1,
178+
"max": null,
179+
"min": null,
180+
"show": true
181+
}
182+
]
183+
},
184+
{
185+
"aliasColors": { },
186+
"bars": false,
187+
"dashLength": 10,
188+
"dashes": false,
189+
"datasource": "prometheus",
190+
"fill": 1,
191+
"fillGradient": 0,
192+
"gridPos": {
193+
"h": 20,
194+
"w": 25,
195+
"x": 0,
196+
"y": 0
197+
},
198+
"id": 4,
199+
"legend": {
200+
"alignAsTable": false,
201+
"avg": false,
202+
"current": false,
203+
"max": false,
204+
"min": false,
205+
"rightSide": false,
206+
"show": true,
207+
"sideWidth": null,
208+
"total": false,
209+
"values": false
210+
},
211+
"lines": true,
212+
"linewidth": 1,
213+
"links": [ ],
214+
"nullPointMode": "null",
215+
"percentage": false,
216+
"pointradius": 5,
217+
"points": false,
218+
"renderer": "flot",
219+
"repeat": null,
220+
"seriesOverrides": [ ],
221+
"spaceLength": 10,
222+
"stack": false,
223+
"steppedLine": false,
139224
"targets": [
140225
{
141226
"expr": "topk(10,rate(fl2m_bandwidth_per_source_subnet[1m]))",
@@ -195,7 +280,7 @@
195280
"x": 0,
196281
"y": 0
197282
},
198-
"id": 4,
283+
"id": 5,
199284
"legend": {
200285
"alignAsTable": false,
201286
"avg": false,
@@ -280,7 +365,7 @@
280365
"x": 0,
281366
"y": 0
282367
},
283-
"id": 5,
368+
"id": 6,
284369
"legend": {
285370
"alignAsTable": false,
286371
"avg": false,
@@ -365,7 +450,7 @@
365450
"x": 0,
366451
"y": 0
367452
},
368-
"id": 6,
453+
"id": 7,
369454
"legend": {
370455
"alignAsTable": false,
371456
"avg": false,
@@ -450,7 +535,7 @@
450535
"x": 0,
451536
"y": 0
452537
},
453-
"id": 7,
538+
"id": 8,
454539
"legend": {
455540
"alignAsTable": false,
456541
"avg": false,
@@ -535,7 +620,7 @@
535620
"x": 0,
536621
"y": 0
537622
},
538-
"id": 8,
623+
"id": 9,
539624
"legend": {
540625
"alignAsTable": false,
541626
"avg": false,
@@ -620,7 +705,92 @@
620705
"x": 0,
621706
"y": 0
622707
},
623-
"id": 9,
708+
"id": 10,
709+
"legend": {
710+
"alignAsTable": false,
711+
"avg": false,
712+
"current": false,
713+
"max": false,
714+
"min": false,
715+
"rightSide": false,
716+
"show": true,
717+
"sideWidth": null,
718+
"total": false,
719+
"values": false
720+
},
721+
"lines": true,
722+
"linewidth": 1,
723+
"links": [ ],
724+
"nullPointMode": "null",
725+
"percentage": false,
726+
"pointradius": 5,
727+
"points": false,
728+
"renderer": "flot",
729+
"repeat": null,
730+
"seriesOverrides": [ ],
731+
"spaceLength": 10,
732+
"stack": false,
733+
"steppedLine": false,
734+
"targets": [
735+
{
736+
"expr": "topk(10,rate(fl2m_count_per_source_destination_subnet[1m]))",
737+
"format": "time_series",
738+
"intervalFactor": 2,
739+
"legendFormat": "",
740+
"refId": "A"
741+
}
742+
],
743+
"thresholds": [ ],
744+
"timeFrom": null,
745+
"timeShift": null,
746+
"title": "Connections rate of src / destination subnet occurences",
747+
"tooltip": {
748+
"shared": true,
749+
"sort": 0,
750+
"value_type": "individual"
751+
},
752+
"type": "graph",
753+
"xaxis": {
754+
"buckets": null,
755+
"mode": "time",
756+
"name": null,
757+
"show": true,
758+
"values": [ ]
759+
},
760+
"yaxes": [
761+
{
762+
"format": "short",
763+
"label": null,
764+
"logBase": 1,
765+
"max": null,
766+
"min": null,
767+
"show": true
768+
},
769+
{
770+
"format": "short",
771+
"label": null,
772+
"logBase": 1,
773+
"max": null,
774+
"min": null,
775+
"show": true
776+
}
777+
]
778+
},
779+
{
780+
"aliasColors": { },
781+
"bars": false,
782+
"dashLength": 10,
783+
"dashes": false,
784+
"datasource": "prometheus",
785+
"fill": 1,
786+
"fillGradient": 0,
787+
"gridPos": {
788+
"h": 20,
789+
"w": 25,
790+
"x": 0,
791+
"y": 0
792+
},
793+
"id": 11,
624794
"legend": {
625795
"alignAsTable": false,
626796
"avg": false,
@@ -705,7 +875,7 @@
705875
"x": 0,
706876
"y": 0
707877
},
708-
"id": 10,
878+
"id": 12,
709879
"legend": {
710880
"alignAsTable": false,
711881
"avg": false,
@@ -790,7 +960,7 @@
790960
"x": 0,
791961
"y": 0
792962
},
793-
"id": 11,
963+
"id": 13,
794964
"legend": {
795965
"alignAsTable": false,
796966
"avg": false,
@@ -875,7 +1045,7 @@
8751045
"x": 0,
8761046
"y": 0
8771047
},
878-
"id": 12,
1048+
"id": 14,
8791049
"legend": {
8801050
"alignAsTable": false,
8811051
"avg": false,
@@ -960,7 +1130,7 @@
9601130
"x": 0,
9611131
"y": 0
9621132
},
963-
"id": 13,
1133+
"id": 15,
9641134
"legend": {
9651135
"alignAsTable": false,
9661136
"avg": false,
@@ -1045,7 +1215,7 @@
10451215
"x": 0,
10461216
"y": 0
10471217
},
1048-
"id": 14,
1218+
"id": 16,
10491219
"legend": {
10501220
"alignAsTable": false,
10511221
"avg": false,

‎contrib/docker/Dockerfile‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,14 @@ COPY cmd/ cmd/
1313
COPY pkg/ pkg/
1414

1515
# Build
16-
RUN CGO_ENABLED=0 GOOS=linux GO111MODULE=on GOARCH=amd64 go build ./cmd/flowlogs2metrics
16+
RUN VERSION=$(date); CGO_ENABLED=0 GOOS=linux GO111MODULE=on GOARCH=amd64 go build -ldflags "-X 'main.Version=$VERSION'" ./cmd/flowlogs2metrics
1717

1818
# final stage
1919
FROM ubuntu
2020
RUN apt-get update -y
2121
RUN apt-get install -y netbase
2222
RUN apt-get install -y curl
23+
RUN apt-get install -y net-tools
2324
COPY --from=builder /app/flowlogs2metrics /app/
2425

2526
# expose ports

‎contrib/kubernetes/flowlogs2metrics.conf.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# This file was generated automatically by flowlogs2metrics confgenerator
2-
log-level: debug
2+
log-level: error
33
pipeline:
44
decode:
55
type: json
@@ -173,7 +173,7 @@ pipeline:
173173
- srcK8S_Namespace
174174
- srcK8S_Type
175175
operation: sum
176-
recordkey: Bytes
176+
recordkey: bytes
177177
- name: dest_connection_location_count
178178
by:
179179
- dstLocation_CountryName

‎docs/development.md‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,22 @@ go tool pprof -web /tmp/flowlogs2metrics-cpu-profile.out
1616
```
1717
> Note: installed graphviz is a pre-req.
1818
19+
## OCP Flow logs correctness
20+
21+
use the script `hack/deploy-and-monitor-k8s-network-workload.sh` to deploy on OCP synthetic demo workloads
22+
and `flowlogs2metrics` components to observe and check correctness for the raw flow-logs generated by OCP.
23+
24+
The script deploys three workloads:
25+
(1) egress from the cluster
26+
(2) ingress to the cluster
27+
(3) pod-to-pod traffic
28+
29+
Each workload is deployed into separate namespace to ease monitoring.
30+
The script also deploys `flowlogs2metrics` on OCP with specific configuration that doesn't include any
31+
aggregation and/or export to prometheus, just simple dump of the logs into stdout after enrichment with
32+
k8s information (e.g. pod-name, namespace etc.). This allows usage of simple tools, such as `grep` to
33+
focus on specific flow-logs generated by OCP.
34+
1935
## Edit diagrams
2036
- To edit drawio diagrams (with .png extension) use [https://app.diagrams.net/](https://app.diagrams.net/)
2137

0 commit comments

Comments
 (0)