Skip to content

Commit 2c64e07

Browse files
endoramaendorama
authored
aws.billing template schema-b (#43)
1 parent 3b681a1 commit 2c64e07

File tree

4 files changed

+254
-1
lines changed

4 files changed

+254
-1
lines changed

‎assets/templates/aws.billing/schema-b/configs.yml‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
- name: cloud.region
2+
enum: ["us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-south-1", "ap-northeast-3", "ap-northeast-2", "ap-southeast-1", "ap-southeast-2", "ap-northeast-1", "ca-central-1", "eu-central-1", "eu-west-1", "eu-west-2", "eu-west-3", "eu-north-1", "sa-east-1", "af-south-1", "ap-east-1", "ap-south-2", "ap-southeast-3", "eu-south-2", "eu-central-2", "me-south-1", "me-central-1"]
3+
cardinality:
4+
numerator: 1
5+
denominator: 25
6+
- name: cloud.account.id
7+
value: "123456789"
8+
- name: cloud.account.name
9+
value: sample-account
10+
- name: aws.billing.currency
11+
value: "USD"
12+
- name: aws.billing.ServiceName
13+
# NOTE: When empty the data refers to estimated charged for the entire account. We cannot reproduce the content (as it's a sum of previous data) but we want to provide the case.
14+
enum: ["", "AWSCloudTrail", "AWSCodeArtifact", "AWSConfig", "AWSCostExplorer", "AWSDataTransfer", "AWSELB", "AWSLambda", "AWSMarketplace", "AWSQueueService", "AWSSecretsManager", "AWSServiceCatalog", "AWSSystemsManager", "AWSXRay", "AmazonApiGateway", "AmazonCloudWatch", "AmazonCognito", "AmazonDynamoDB", "AmazonEC2", "AmazonECR", "AmazonEKS", "AmazonKinesis", "AmazonKinesisFirehose", "AmazonRDS", "AmazonRedshift", "AmazonRoute53", "AmazonS3", "AmazonSNS", "AmazonVPC", "awskms"]
15+
- name: agent.id
16+
value: "12f376ef-5186-4e8b-a175-70f1140a8f30"
17+
- name: agent.ephemeral_id
18+
value: "5fd278ce-2a12-4a09-a125-0c5b39aa69e3"
19+
- name: agent.name
20+
value: "host.local"
21+
- name: metricset.period
22+
value: 86400
23+
- name: aws.billing.group_definition.key
24+
# NOTE: repeated values are needed to produce 10% cases with "" value
25+
enum: ["", "AZ", "INSTANCE_TYPE", "SERVICE", "LINKED_ACCOUNT", "AZ", "INSTANCE_TYPE", "SERVICE", "LINKED_ACCOUNT"]
26+
27+
- name: event.duration
28+
range:
29+
min: 1
30+
max: 1000
31+
- name: aws.billing.EstimatedCharges
32+
cardinality:
33+
numerator: 1
34+
denominator: 25
35+
fuzziness:
36+
numerator: 1
37+
denominator: 5
38+
- name: aws.billing.AmortizedCost.amount
39+
cardinality:
40+
numerator: 1
41+
denominator: 25
42+
fuzziness:
43+
numerator: 1
44+
denominator: 5
45+
- name: aws.billing.BlendedCost.amount
46+
cardinality:
47+
numerator: 1
48+
denominator: 25
49+
fuzziness:
50+
numerator: 1
51+
denominator: 5
52+
- name: aws.billing.NormalizedUsageAmount.amount
53+
cardinality:
54+
numerator: 1
55+
denominator: 25
56+
fuzziness:
57+
numerator: 1
58+
denominator: 5
59+
- name: aws.billing.UnblendedCost.amount
60+
cardinality:
61+
numerator: 1
62+
denominator: 25
63+
fuzziness:
64+
numerator: 1
65+
denominator: 5
66+
- name: aws.billing.UsageQuantity.amount
67+
cardinality:
68+
numerator: 1
69+
denominator: 25
70+
fuzziness:
71+
numerator: 1
72+
denominator: 5
73+
74+
- name: aws.billing.group_definition.type
75+
value: "DIMENSION"
76+
- name: aws.billing.group_by.INSTANCE_TYPE
77+
enum: ["NoInstanceType", "a1.large", "c5.2xlarge", "c5.xlarge", "c6i.2xlarge", "db.r6g.2xlarge", "db.t2.micro", "dc2.large", "m5.large", "t1.micro", "t2.medium", "t2.micro", "t2.small", "t2.xlarge", "t3.2xlarge", "t3.medium", "t3.xlarge","t3.xlarge"]
78+
- name: aws.billing.group_by.SERVICE
79+
enum: ["Amazon Simple Storage Service", "Amazon Elastic Compute Cloud - Compute", "EC2 - Other", "Amazon Kinesis", "Amazon Relational Database Service", "Amazon Elastic Load Balancing", "AmazonCloudWatch", "AWS CloudTrail", "AWS Config", "AWS Key Management Service", "AWS Lambda", "AWS Secrets Manager", "AWS Service Catalog", "Amazon API Gateway", "Amazon DynamoDB", "Amazon EC2 Container Registry (ECR)", "Amazon Elastic Container Service for Kubernetes", "Amazon Kinesis Firehose", "Amazon Redshift", "Amazon Simple Notification Service", "Amazon Simple Queue Service", "Amazon Virtual Private Cloud"]
80+

‎assets/templates/aws.billing/schema-b/fields.yml‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
- name: timestamp
2+
type: date
3+
- name: cloud.region
4+
type: keyword
5+
- name: cloud.account.id
6+
type: keyword
7+
- name: cloud.account.name
8+
type: keyword
9+
- name: event.duration
10+
type: long
11+
- name: metricset.period
12+
type: long
13+
- name: aws.billing.currency
14+
type: keyword
15+
- name: aws.billing.EstimatedCharges
16+
type: float
17+
# positive
18+
- name: aws.billing.ServiceName
19+
type: keyword
20+
- name: aws.billing.AmortizedCost.amount
21+
type: float
22+
# positive
23+
- name: aws.billing.BlendedCost.amount
24+
type: float
25+
# positive
26+
- name: aws.billing.NormalizedUsageAmount.amount
27+
type: integer
28+
# positive
29+
- name: aws.billing.UnblendedCost.amount
30+
type: float
31+
# positive
32+
- name: aws.billing.UsageQuantity.amount
33+
type: integer
34+
# positive
35+
- name: agent.id
36+
type: keyword
37+
- name: agent.name
38+
type: keyword
39+
- name: agent.ephemeral_id
40+
type: keyword
41+
example: 12f376ef-5186-4e8b-a175-70f1140a8f30
42+
- name: aws.billing.group_definition.key
43+
type: keyword
44+
- name: aws.billing.start_date
45+
type: date
46+
- name: aws.billing.group_definition.type
47+
type: keyword
48+
- name: aws.billing.group_by.INSTANCE_TYPE
49+
type: keyword
50+
- name: aws.billing.group_by.SERVICE
51+
type: keyword

‎assets/templates/aws.billing/schema-b/gotext.tpl‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
{{- $currency := generate "aws.billing.currency" }}
2+
{{- $groupBy := generate "aws.billing.group_definition.key" }}
3+
{{- $period := generate "metricset.period" }}
4+
{{- $cloudId := generate "cloud.account.id" }}
5+
{{- $cloudRegion := generate "cloud.region" }}
6+
{
7+
"@timestamp": "{{generate "timestamp"}}",
8+
"cloud": {
9+
"provider": "aws",
10+
"region": "{{$cloudRegion}}",
11+
"account": {
12+
"id": "{{$cloudId}}",
13+
"name": "{{generate "cloud.account.name"}}"
14+
}
15+
},
16+
"event": {
17+
"dataset": "aws.billing",
18+
"module": "aws",
19+
"duration": {{generate "event.duration"}}
20+
},
21+
"metricset": {
22+
"name": "billing",
23+
"period": {{$period}}
24+
},
25+
"ecs": {
26+
"version": "8.2.0"
27+
},
28+
"aws": {
29+
"billing": {
30+
{{- if eq $groupBy "" }}
31+
"Currency": "{{$currency}}",
32+
"EstimatedCharges": {{generate "aws.billing.EstimatedCharges"}},
33+
"ServiceName": "{{generate "aws.billing.ServiceName"}}"
34+
{{- else }}
35+
{{- $sd := generate "aws.billing.start_date" }}
36+
"start_date": "{{ $sd }}",
37+
"end_date": "{{ $sd | date_modify (print "+" $period "s") }}",
38+
"AmortizedCost": {
39+
"amount": {{printf "%.2f" (generate "aws.billing.AmortizedCost.amount")}},
40+
"unit": "{{$currency}}"
41+
},
42+
"BlendedCost": {
43+
"amount": {{printf "%.2f" (generate "aws.billing.BlendedCost.amount")}},
44+
"unit": "{{$currency}}"
45+
},
46+
"NormalizedUsageAmount": {
47+
"amount": {{generate "aws.billing.NormalizedUsageAmount.amount"}},
48+
"unit": "N/A"
49+
},
50+
"UnblendedCost": {
51+
"amount": {{printf "%.2f" (generate "aws.billing.UnblendedCost.amount")}},
52+
"unit": "{{$currency}}"
53+
},
54+
"UsageQuantity": {
55+
"amount": {{generate "aws.billing.UsageQuantity.amount"}},
56+
"unit": "N/A"
57+
},
58+
"group_definition": {
59+
"key": "{{$groupBy}}",
60+
"type": "{{generate "aws.billing.group_definition.type"}}"
61+
},
62+
"group_by": {
63+
{{- if eq $groupBy "AZ"}}
64+
"AZ": "{{awsAZFromRegion $cloudRegion}}"
65+
{{- else if eq $groupBy "INSTANCE_TYPE"}}
66+
"INSTANCE_TYPE": "{{generate "aws.billing.group_by.INSTANCE_TYPE"}}"
67+
{{- else if eq $groupBy "SERVICE"}}
68+
"SERVICE": "{{generate "aws.billing.group_by.SERVICE"}}"
69+
{{- else if eq $groupBy "LINKED_ACCOUNT"}}
70+
"LINKED_ACCOUNT": "{{$cloudId}}"
71+
{{- end}}
72+
}
73+
{{- end}}
74+
}
75+
},
76+
"service": {
77+
"type": "aws"
78+
},
79+
"agent": {
80+
"id": "{{generate "agent.id"}}",
81+
"name": "{{generate "agent.name"}}",
82+
"type": "metricbeat",
83+
"version": "8.0.0",
84+
"ephemeral_id": "{{generate "agent.ephemeral_id"}}"
85+
}
86+
}

‎pkg/genlib/generator_with_text_template.go‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,12 @@ package genlib
77
import (
88
"bytes"
99
"errors"
10-
"github.com/Masterminds/sprig/v3"
1110
"io"
11+
"math/rand"
1212
"text/template"
1313
"time"
14+
15+
"github.com/Masterminds/sprig/v3"
1416
)
1517

1618
var generateOnFieldNotInFieldsYaml = errors.New("generate called on a field not present in fields yaml definition")
@@ -23,6 +25,31 @@ type GeneratorWithTextTemplate struct {
2325
totEvents uint64
2426
}
2527

28+
// awsAZs list all possible AZs for a specific AWS region
29+
// NOTE: this list is not comprehensive
30+
// missing regions: af-south-1, ap-south-2, ap-southeast-3, ap-southeast-4, eu-central-2, eu-south-1, eu-south-2, me-central-1
31+
var awsAZs map[string][]string = map[string][]string{
32+
"ap-east-1": {"ap-east-1a", "ap-east-1b", "ap-east-1c"},
33+
"ap-northeast-1": {"ap-northeast-1a", "ap-northeast-1c", "ap-northeast-1d"},
34+
"ap-northeast-2": {"ap-northeast-2a", "ap-northeast-2b", "ap-northeast-2c", "ap-northeast-2d"},
35+
"ap-northeast-3": {"ap-northeast-3a", "ap-northeast-3b", "ap-northeast-3c"},
36+
"ap-south-1": {"ap-south-1a", "ap-south-1b", "ap-south-1c"},
37+
"ap-southeast-1": {"ap-southeast-1a", "ap-southeast-1b", "ap-southeast-1c"},
38+
"ap-southeast-2": {"ap-southeast-2a", "ap-southeast-2b", "ap-southeast-2c"},
39+
"ca-central-1": {"ca-central-1a", "ca-central-1b", "ca-central-1d"},
40+
"eu-central-1": {"eu-central-1a", "eu-central-1b", "eu-central-1c"},
41+
"eu-north-1": {"eu-north-1a", "eu-north-1b", "eu-north-1c"},
42+
"eu-west-1": {"eu-west-1a", "eu-west-1b", "eu-west-1c"},
43+
"eu-west-2": {"eu-west-2a", "eu-west-2b", "eu-west-2c"},
44+
"eu-west-3": {"eu-west-3a", "eu-west-3b", "eu-west-3c"},
45+
"me-south-1": {"me-south-1a", "me-south-1b", "me-south-1c"},
46+
"sa-east-1": {"sa-east-1a", "sa-east-1b", "sa-east-1c"},
47+
"us-east-1": {"us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", "us-east-1e", "us-east-1f"},
48+
"us-east-2": {"us-east-2a", "us-east-2b", "us-east-2c"},
49+
"us-west-1": {"us-west-1a", "us-west-1b"},
50+
"us-west-2": {"us-west-2a", "us-west-2b", "us-west-2c", "us-west-2d"},
51+
}
52+
2653
func calculateTotEventsWithTextTemplate(totSize uint64, fieldMap map[string]any, errChan chan error, tpl []byte) (uint64, error) {
2754
if totSize == 0 {
2855
return 0, nil
@@ -113,6 +140,15 @@ func NewGeneratorWithTextTemplate(tpl []byte, cfg Config, fields Fields, totSize
113140
templateFns["timeDuration"] = func(duration int64) time.Duration {
114141
return time.Duration(duration)
115142
}
143+
144+
templateFns["awsAZFromRegion"] = func(region string) string {
145+
azs, ok := awsAZs[region]
146+
if !ok {
147+
return "NoAZ"
148+
}
149+
150+
return azs[rand.Intn(len(azs))]
151+
}
116152

117153
templateFns["generate"] = func(field string) any {
118154
bindF, ok := fieldMap[field].(EmitF)

0 commit comments

Comments
 (0)