Skip to content

Commit a3323b8

Browse files
gizasgizas
authored
Adding more realistic cpu and memory stats for k8s (#126)
* adding cpu and memory stats more realistic * adding object for usage and cardinality in values * changing percentages and keep mul only in generation
1 parent 5dccdd4 commit a3323b8

File tree

8 files changed

+81
-31
lines changed

8 files changed

+81
-31
lines changed

‎assets/templates/kubernetes.container/schema-b/configs.yml‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,28 @@ fields:
3333
range:
3434
min: 0
3535
max: 10000
36-
- name: Percentage
36+
- name: PercentageMemory
3737
range:
38-
min: 0
39-
max: 5000
40-
fuzziness: 0.005
38+
min: 0.0
39+
max: 1.0
40+
- name: PercentageCPU
41+
range:
42+
min: 0.0
43+
max: 1.0
44+
- name: usage.*
45+
object_keys:
46+
- nanoseconds
47+
- nanocores
48+
- name: usage.nanoseconds
49+
range:
50+
min: 100000
51+
max: 9000000000
52+
cardinality: 10000
53+
- name: usage.nanocores
54+
range:
55+
min: 100000
56+
max: 9000000
57+
cardinality: 10000
4158
- name: container.name
4259
enum: ["web", "default-http-backend", "dnsmasq", "csi-driver", "web", "web", "web", "prometheus", "konnectivity-agent", "sidecar", "kubedns", "metrics-server-nanny", "web", "web", "fluentbit", "autoscaler", "gke-metrics-agent", "elastic-agent", "web", "kube-state-metrics", "metrics-server", "fluentbit", "elastic-agent", "web", "prometheus-to-sd-exporter"]
4360
- name: timedate

‎assets/templates/kubernetes.container/schema-b/fields.yml‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,19 @@
2626
type: long
2727
- name: Ip
2828
type: ip
29-
- name: Percentage
30-
type: double
3129
- name: rangeofid
3230
type: integer
31+
- name: PercentageMemory
32+
type: double
33+
- name: PercentageCPU
34+
type: double
35+
- name: usage
36+
type: group
37+
fields:
38+
- name: nanoseconds
39+
type: integer
40+
- name: nanocores
41+
type: integer
3342
- name: agent.snapshot
3443
type: boolean
3544
- name: container.name

‎assets/templates/kubernetes.container/schema-b/gotext.tpl‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@
1111
{{- $rangeofid := generate "rangeofid" -}}
1212
{{- $nodeid := div $rangeofid 110 -}}
1313
{{- $faults := generate "faults" -}}
14-
{{- $pct := generate "Percentage" -}}
14+
{{- $pctmem := generate "PercentageMemory" }}
15+
{{- $pctcpu := generate "PercentageCPU" }}
16+
{{- $usage_nanoseconds := generate "usage.nanoseconds" | mul 1000-}}
17+
{{- $usage_nanocores := generate "usage.nanocores" | mul 1000 -}}
1518
{{- $name := generate "container.name" -}}
16-
{ "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "container":{ "memory":{ "usage": {{divf $pct 1000000}} }, "name":"{{ $name }}", "runtime":"containerd", "cpu":{ "usage": {{divf $pct 1000000}} }, "id":"container-{{ $rangeofid }}" }, "kubernetes": { "container":{ "start_time":"{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "memory":{ "rss":{ "bytes": {{generate "Bytes"}} }, "majorpagefaults": {{ $faults }}, "usage":{ "node":{ "pct": {{divf $pct 1000000}} }, "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{divf $pct 1000000}} } }, "available":{ "bytes": {{generate "Bytes"}} }, "workingset":{ "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{divf $pct 1000000}} } }, "pagefaults": "{{ $faults }}" }, "rootfs":{ "inodes":{ "used": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } }, "name":"{{ $name }}", "cpu":{ "usage":{ "core":{ "ns": 41129679 }, "node":{ "pct": {{divf $pct 1000000}} }, "nanocores":0, "limit":{ "pct": {{divf $pct 1000000}} } } }, "logs":{ "inodes":{ "count": {{ generate "kubernetes.container.rootfs.inodes.used" }}, "used":5, "free": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } } }, "node":{ "uid": "host-{{ $nodeid }}" , "hostname":"host-{{ $nodeid }}", "name":"host-{{ $nodeid }}", "labels":{ "cloud_google_com/machine-family":"e2", "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", "kubernetes_io/hostname":"host-{{ $nodeid }}", "cloud_google_com/gke-os-distribution":"cos", "topology_kubernetes_io/zone":"europe-west1-d", "topology_gke_io/zone":"europe-west1-d", "topology_kubernetes_io/region":"europe-west1", "kubernetes_io/arch":"amd64", "cloud_google_com/gke-cpu-scaling-level":"4", "env":"kubernetes-scale", "failure-domain_beta_kubernetes_io/region":"europe-west1", "cloud_google_com/gke-max-pods-per-node":"110", "cloud_google_com/gke-container-runtime":"containerd", "beta_kubernetes_io/instance-type":"e2-standard-4", "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", "node_kubernetes_io/instance-type":"e2-standard-4", "beta_kubernetes_io/os":"linux", "cloud_google_com/gke-boot-disk":"pd-balanced", "kubernetes_io/os":"linux", "cloud_google_com/private-node":"false", "cloud_google_com/gke-logging-variant":"DEFAULT", "beta_kubernetes_io/arch":"amd64" } }, "pod":{ "uid": "demo-pod-{{ $rangeofid }}", "ip":"{{generate "Ip"}}", "name":"demo-pod-{{ $rangeofid }}", "namespace":"demo-{{ $rangeofid }}", "namespace_uid":"demo-{{ $rangeofid }}", "replicaset":{ "name":"demo-deployment-{{ $rangeofid }}" }, "namespace_labels":{ "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" }, "labels":{ "app":"demo", "pod-template-hash":"{{ $rangeofid }}", "app-2":"demo-2", "app-1":"demo-1" }, "deployment":{ "name":"demo-deployment-{{ $rangeofid }}" } } }, "cloud": { "provider": "gcp", "availability_zone": "europe-west1-d", "instance":{ "name": "{{ $agentName }}" , "id": "{{ $agentId }}" }, "machine":{ "type":"e2-standard-4" }, "service":{ "name":"GCE" }, "project":{ "id":"elastic-obs-integrations-dev" }, "account":{ "id":"elastic-obs-integrations-dev" } }, "orchestrator":{ "cluster":{ "name":"kubernetes-scale", "url":"https://{{ generate "Ip" }}" } }, "service":{ "address": "https://{{ $agentName }}:10250/stats/summary", "type":"kubernetes" }, "data_stream":{ "namespace":"default", "type":"metrics", "dataset":"kubernetes.container" }, "ecs": { "version": "8.2.0" }, "agent": { "id": "{{ $agentId}}", "name": "{{ $agentName }}" , "type": "metricbeat", "version": "{{ $agentVersion }}", "ephemeral_id": "{{ $agentEphemeralid }}" }, "elastic_agent": { "id": "{{ $agentId }}" , "version": "{{ $agentVersion }}", "snapshot": {{ generate "agent.snapshot" }} }, "metricset":{ "period": "{{ $period }}" , "name":"pod" }, "event":{ "duration": "{{generate "event.duration"}}", "agent_id_status": "verified", "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "module":"kubernetes", "dataset":"kubernetes.container" }, "host":{ "hostname":"host-{{ $nodeid }}", "os":{ "kernel":"5.10.161+", "codename":"focal", "name":"Ubuntu", "type":"linux", "family":"debian", "version":"20.04.5 LTS (Focal Fossa)", "platform":"ubuntu" }, "containerized":false, "name": "host-{{ $nodeid }}", "id": "host-{{ $nodeid }}", "architecture":"x86_64" } }
19+
{ "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "container":{ "memory":{ "usage": {{$pctmem}} }, "name":"{{ $name }}", "runtime":"containerd", "cpu":{ "usage": {{$pctcpu}} }, "id":"container-{{ $rangeofid }}" }, "kubernetes": { "container":{ "start_time":"{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "memory":{ "rss":{ "bytes": {{generate "Bytes"}} }, "majorpagefaults": {{ $faults }}, "usage":{ "node":{ "pct": {{$pctmem}} }, "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{$pctmem}} } }, "available":{ "bytes": {{generate "Bytes"}} }, "workingset":{ "bytes": {{generate "Bytes"}}, "limit":{ "pct": {{$pctmem}} } }, "pagefaults": "{{ $faults }}" }, "rootfs":{ "inodes":{ "used": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } }, "name":"{{ $name }}", "cpu":{ "usage":{ "core":{ "ns": {{$usage_nanoseconds}} }, "node":{ "pct": {{$pctcpu}} }, "nanocores":{{$usage_nanocores}}, "limit":{ "pct": {{$pctcpu}} } } }, "logs":{ "inodes":{ "count": {{ generate "kubernetes.container.rootfs.inodes.used" }}, "used":5, "free": {{ generate "kubernetes.container.rootfs.inodes.used" }} }, "available":{ "bytes": {{generate "Bytes"}} }, "used":{ "bytes": {{generate "Bytes"}} }, "capacity":{ "bytes": {{generate "Bytes"}} } } }, "node":{ "uid": "host-{{ $nodeid }}" , "hostname":"host-{{ $nodeid }}", "name":"host-{{ $nodeid }}", "labels":{ "cloud_google_com/machine-family":"e2", "cloud_google_com/gke-nodepool":"kubernetes-scale-nl", "kubernetes_io/hostname":"host-{{ $nodeid }}", "cloud_google_com/gke-os-distribution":"cos", "topology_kubernetes_io/zone":"europe-west1-d", "topology_gke_io/zone":"europe-west1-d", "topology_kubernetes_io/region":"europe-west1", "kubernetes_io/arch":"amd64", "cloud_google_com/gke-cpu-scaling-level":"4", "env":"kubernetes-scale", "failure-domain_beta_kubernetes_io/region":"europe-west1", "cloud_google_com/gke-max-pods-per-node":"110", "cloud_google_com/gke-container-runtime":"containerd", "beta_kubernetes_io/instance-type":"e2-standard-4", "failure-domain_beta_kubernetes_io/zone":"europe-west1-d", "node_kubernetes_io/instance-type":"e2-standard-4", "beta_kubernetes_io/os":"linux", "cloud_google_com/gke-boot-disk":"pd-balanced", "kubernetes_io/os":"linux", "cloud_google_com/private-node":"false", "cloud_google_com/gke-logging-variant":"DEFAULT", "beta_kubernetes_io/arch":"amd64" } }, "pod":{ "uid": "demo-pod-{{ $rangeofid }}", "ip":"{{generate "Ip"}}", "name":"demo-pod-{{ $rangeofid }}", "namespace":"demo-{{ $rangeofid }}", "namespace_uid":"demo-{{ $rangeofid }}", "replicaset":{ "name":"demo-deployment-{{ $rangeofid }}" }, "namespace_labels":{ "kubernetes_io/metadata_name":"demo-{{ $rangeofid }}" }, "labels":{ "app":"demo", "pod-template-hash":"{{ $rangeofid }}", "app-2":"demo-2", "app-1":"demo-1" }, "deployment":{ "name":"demo-deployment-{{ $rangeofid }}" } } }, "cloud": { "provider": "gcp", "availability_zone": "europe-west1-d", "instance":{ "name": "{{ $agentName }}" , "id": "{{ $agentId }}" }, "machine":{ "type":"e2-standard-4" }, "service":{ "name":"GCE" }, "project":{ "id":"elastic-obs-integrations-dev" }, "account":{ "id":"elastic-obs-integrations-dev" } }, "orchestrator":{ "cluster":{ "name":"kubernetes-scale", "url":"https://{{ generate "Ip" }}" } }, "service":{ "address": "https://{{ $agentName }}:10250/stats/summary", "type":"kubernetes" }, "data_stream":{ "namespace":"default", "type":"metrics", "dataset":"kubernetes.container" }, "ecs": { "version": "8.2.0" }, "agent": { "id": "{{ $agentId}}", "name": "{{ $agentName }}" , "type": "metricbeat", "version": "{{ $agentVersion }}", "ephemeral_id": "{{ $agentEphemeralid }}" }, "elastic_agent": { "id": "{{ $agentId }}" , "version": "{{ $agentVersion }}", "snapshot": {{ generate "agent.snapshot" }} }, "metricset":{ "period": "{{ $period }}" , "name":"pod" }, "event":{ "duration": "{{generate "event.duration"}}", "agent_id_status": "verified", "ingested": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}", "module":"kubernetes", "dataset":"kubernetes.container" }, "host":{ "hostname":"host-{{ $nodeid }}", "os":{ "kernel":"5.10.161+", "codename":"focal", "name":"Ubuntu", "type":"linux", "family":"debian", "version":"20.04.5 LTS (Focal Fossa)", "platform":"ubuntu" }, "containerized":false, "name": "host-{{ $nodeid }}", "id": "host-{{ $nodeid }}", "architecture":"x86_64" } }

‎assets/templates/kubernetes.container/schema-b/gotext_multiline.tpl‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -9,19 +9,22 @@
99
{{- $picktimedate := generate "timedate" }}
1010
{{- $timehour := generate "timehour" }}
1111
{{- $faults := generate "faults" }}
12-
{{- $pct := generate "Percentage" }}
12+
{{- $pctmem := generate "PercentageMemory" }}
13+
{{- $pctcpu := generate "PercentageCPU" }}
14+
{{- $usage_nanoseconds := generate "usage.nanoseconds" | mul 1000 -}}
15+
{{- $usage_nanocores := generate "usage.nanocores" | mul 1000 -}}
1316
{{- $rangeofid := generate "rangeofid" -}}
1417
{{- $nodeid := div $rangeofid 110 -}}
1518
{{- $name := generate "container.name" }}
1619
{ "@timestamp": "{{$picktimedate}}T{{$timehour}}:{{ $resttime._1 }}:{{ $resttime._2 }}:{{ $resttime._3}}",
1720
"container":{
1821
"memory":{
19-
"usage": {{divf $pct 1000000}}
22+
"usage": {{$pctmem}}
2023
},
2124
"name":"{{ $name }}",
2225
"runtime":"containerd",
2326
"cpu":{
24-
"usage": {{divf $pct 1000000}}
27+
"usage": {{$pctcpu}}
2528
},
2629
"id":"container-{{ $rangeofid }}"
2730
},
@@ -35,11 +38,11 @@
3538
"majorpagefaults": {{ $faults }},
3639
"usage":{
3740
"node":{
38-
"pct": {{divf $pct 1000000}}
41+
"pct": {{$pctmem}}
3942
},
4043
"bytes": {{generate "Bytes"}},
4144
"limit":{
42-
"pct": {{divf $pct 1000000}}
45+
"pct": {{$pctmem}}
4346
}
4447
},
4548
"available":{
@@ -48,7 +51,7 @@
4851
"workingset":{
4952
"bytes": {{generate "Bytes"}},
5053
"limit":{
51-
"pct": {{divf $pct 1000000}}
54+
"pct": {{$pctmem}}
5255
}
5356
},
5457
"pagefaults": "{{ $faults }}"
@@ -71,14 +74,14 @@
7174
"cpu":{
7275
"usage":{
7376
"core":{
74-
"ns": 41129679
77+
"ns": {{$usage_nanoseconds}}
7578
},
7679
"node":{
77-
"pct": {{divf $pct 1000000}}
80+
"pct": {{$pctcpu}}
7881
},
79-
"nanocores":0,
82+
"nanocores":{{$usage_nanocores}},
8083
"limit":{
81-
"pct": {{divf $pct 1000000}}
84+
"pct": {{$pctcpu}}
8285
}
8386
}
8487
},

‎assets/templates/kubernetes.pod/schema-b/configs.yml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,21 @@ fields:
3333
range:
3434
min: 9000
3535
max: 10000
36-
- name: Percentage
36+
- name: PercentageMemory
3737
range:
38-
min: 0
39-
max: 5000
38+
min: 0.0
39+
max: 1.0
40+
fuzziness: 0.005
41+
- name: PercentageCPU
42+
range:
43+
min: 0.0
44+
max: 1.0
4045
fuzziness: 0.005
46+
- name: Nanocores
47+
range:
48+
min: 100000
49+
max: 9000000
50+
cardinality: 10000
4151
- name: timedate
4252
# Add more dates below in case you want your data to be spread. Those are fixed because this is required in the nightly tests of Rally
4353
enum: ["2023-05-15", "2023-05-16"]

‎assets/templates/kubernetes.pod/schema-b/fields.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,11 @@
3030
type: integer
3131
- name: Ip
3232
type: ip
33-
- name: Percentage
33+
- name: PercentageMemory
3434
type: double
35+
- name: PercentageCPU
36+
type: double
37+
- name: Nanocores
38+
type: integer
3539
- name: agent.snapshot
3640
type: boolean

0 commit comments

Comments
 (0)