Skip to content

Commit 977e185

Browse files
csatib02csatib02
authored
Merge pull request #2091 from kube-logging/feat/add-new-axosyslog-s3-fields
feat: add new axosyslog s3 fields
2 parents 7f64da9 + 514ea74 commit 977e185

File tree

9 files changed

+100
-14
lines changed

9 files changed

+100
-14
lines changed

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_syslogngclusteroutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2207,6 +2207,8 @@ spec:
22072207
type: boolean
22082208
compresslevel:
22092209
type: integer
2210+
content_type:
2211+
type: string
22102212
disk_buffer:
22112213
properties:
22122214
compaction:
@@ -2233,6 +2235,8 @@ spec:
22332235
type: object
22342236
flush_grace_period:
22352237
type: integer
2238+
kms_key:
2239+
type: string
22362240
log-fifo-size:
22372241
type: integer
22382242
max_object_size:
@@ -2249,6 +2253,8 @@ spec:
22492253
type: string
22502254
retries:
22512255
type: integer
2256+
role:
2257+
type: string
22522258
secret_key:
22532259
properties:
22542260
mountFrom:
@@ -2286,6 +2292,8 @@ spec:
22862292
x-kubernetes-map-type: atomic
22872293
type: object
22882294
type: object
2295+
server_side_encryption:
2296+
type: string
22892297
storage_class:
22902298
type: string
22912299
template:

‎charts/logging-operator/charts/logging-operator-crds/templates/logging.banzaicloud.io_syslogngoutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2205,6 +2205,8 @@ spec:
22052205
type: boolean
22062206
compresslevel:
22072207
type: integer
2208+
content_type:
2209+
type: string
22082210
disk_buffer:
22092211
properties:
22102212
compaction:
@@ -2231,6 +2233,8 @@ spec:
22312233
type: object
22322234
flush_grace_period:
22332235
type: integer
2236+
kms_key:
2237+
type: string
22342238
log-fifo-size:
22352239
type: integer
22362240
max_object_size:
@@ -2247,6 +2251,8 @@ spec:
22472251
type: string
22482252
retries:
22492253
type: integer
2254+
role:
2255+
type: string
22502256
secret_key:
22512257
properties:
22522258
mountFrom:
@@ -2284,6 +2290,8 @@ spec:
22842290
x-kubernetes-map-type: atomic
22852291
type: object
22862292
type: object
2293+
server_side_encryption:
2294+
type: string
22872295
storage_class:
22882296
type: string
22892297
template:

‎charts/logging-operator/crds/logging.banzaicloud.io_syslogngclusteroutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,6 +2204,8 @@ spec:
22042204
type: boolean
22052205
compresslevel:
22062206
type: integer
2207+
content_type:
2208+
type: string
22072209
disk_buffer:
22082210
properties:
22092211
compaction:
@@ -2230,6 +2232,8 @@ spec:
22302232
type: object
22312233
flush_grace_period:
22322234
type: integer
2235+
kms_key:
2236+
type: string
22332237
log-fifo-size:
22342238
type: integer
22352239
max_object_size:
@@ -2246,6 +2250,8 @@ spec:
22462250
type: string
22472251
retries:
22482252
type: integer
2253+
role:
2254+
type: string
22492255
secret_key:
22502256
properties:
22512257
mountFrom:
@@ -2283,6 +2289,8 @@ spec:
22832289
x-kubernetes-map-type: atomic
22842290
type: object
22852291
type: object
2292+
server_side_encryption:
2293+
type: string
22862294
storage_class:
22872295
type: string
22882296
template:

‎charts/logging-operator/crds/logging.banzaicloud.io_syslogngoutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2202,6 +2202,8 @@ spec:
22022202
type: boolean
22032203
compresslevel:
22042204
type: integer
2205+
content_type:
2206+
type: string
22052207
disk_buffer:
22062208
properties:
22072209
compaction:
@@ -2228,6 +2230,8 @@ spec:
22282230
type: object
22292231
flush_grace_period:
22302232
type: integer
2233+
kms_key:
2234+
type: string
22312235
log-fifo-size:
22322236
type: integer
22332237
max_object_size:
@@ -2244,6 +2248,8 @@ spec:
22442248
type: string
22452249
retries:
22462250
type: integer
2251+
role:
2252+
type: string
22472253
secret_key:
22482254
properties:
22492255
mountFrom:
@@ -2281,6 +2287,8 @@ spec:
22812287
x-kubernetes-map-type: atomic
22822288
type: object
22832289
type: object
2290+
server_side_encryption:
2291+
type: string
22842292
storage_class:
22852293
type: string
22862294
template:

‎config/crd/bases/logging.banzaicloud.io_syslogngclusteroutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2204,6 +2204,8 @@ spec:
22042204
type: boolean
22052205
compresslevel:
22062206
type: integer
2207+
content_type:
2208+
type: string
22072209
disk_buffer:
22082210
properties:
22092211
compaction:
@@ -2230,6 +2232,8 @@ spec:
22302232
type: object
22312233
flush_grace_period:
22322234
type: integer
2235+
kms_key:
2236+
type: string
22332237
log-fifo-size:
22342238
type: integer
22352239
max_object_size:
@@ -2246,6 +2250,8 @@ spec:
22462250
type: string
22472251
retries:
22482252
type: integer
2253+
role:
2254+
type: string
22492255
secret_key:
22502256
properties:
22512257
mountFrom:
@@ -2283,6 +2289,8 @@ spec:
22832289
x-kubernetes-map-type: atomic
22842290
type: object
22852291
type: object
2292+
server_side_encryption:
2293+
type: string
22862294
storage_class:
22872295
type: string
22882296
template:

‎config/crd/bases/logging.banzaicloud.io_syslogngoutputs.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2202,6 +2202,8 @@ spec:
22022202
type: boolean
22032203
compresslevel:
22042204
type: integer
2205+
content_type:
2206+
type: string
22052207
disk_buffer:
22062208
properties:
22072209
compaction:
@@ -2228,6 +2230,8 @@ spec:
22282230
type: object
22292231
flush_grace_period:
22302232
type: integer
2233+
kms_key:
2234+
type: string
22312235
log-fifo-size:
22322236
type: integer
22332237
max_object_size:
@@ -2244,6 +2248,8 @@ spec:
22442248
type: string
22452249
retries:
22462250
type: integer
2251+
role:
2252+
type: string
22472253
secret_key:
22482254
properties:
22492255
mountFrom:
@@ -2281,6 +2287,8 @@ spec:
22812287
x-kubernetes-map-type: atomic
22822288
type: object
22832289
type: object
2290+
server_side_encryption:
2291+
type: string
22842292
storage_class:
22852293
type: string
22862294
template:

‎docs/configuration/plugins/syslogng-outputs/s3.md

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ Enable or disable compression.
7373

7474
Default: false
7575

76+
### content_type (string, optional) {#s3output-content_type}
77+
78+
The content-type of the HTTP request. Defaults to "application/octet-stream".
79+
80+
7681
### disk_buffer (*DiskBuffer, optional) {#s3output-disk_buffer}
7782

7883
This option enables putting outgoing messages into the disk buffer of the destination to avoid message loss in case of a system failure on the destination side. For details, see the [Syslog-ng DiskBuffer options](../disk_buffer/).
@@ -85,6 +90,11 @@ Set the number of seconds for flush period.
8590

8691
Default: 60
8792

93+
### kms_key (string, optional) {#s3output-kms_key}
94+
95+
The kms-key() used for server-side encryption. The value of the kms-key() parameter must be one of the following: The ID of a key. An alias of a key. In that case, make sure to add the alias/prefix, for example: kms-key("alias/log-archive") The ARN of a key. Available in AxoSyslog 4.8 and later.
96+
97+
8898
### log-fifo-size (int, optional) {#s3output-log-fifo-size}
8999

90100
The number of messages that the output queue can store.
@@ -127,11 +137,21 @@ Set the region option.
127137
The number of times syslog-ng OSE attempts to send a message to this destination. If syslog-ng OSE could not send a message, it will try again until the number of attempts reaches retries, then drops the message.
128138

129139

140+
### role (string, optional) {#s3output-role}
141+
142+
The role for the S3 server.
143+
144+
130145
### secret_key (*secret.Secret, optional) {#s3output-secret_key}
131146

132147
The secret_key for the S3 server.
133148

134149

150+
### server_side_encryption (string, optional) {#s3output-server_side_encryption}
151+
152+
You can use the server-side-encryption() and kms-key() options to configure encryption. Currently only server-side-encryption("aws:kms") is supported. Available in AxoSyslog 4.8 and later.
153+
154+
135155
### storage_class (string, optional) {#s3output-storage_class}
136156

137157
Set the storage_class option.

‎pkg/sdk/logging/model/syslogng/config/output_tests/s3_test.go

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ source "main_input" {
134134
};
135135
136136
destination "output_default_test-s3-out" {
137-
s3(url("http://localhost:9000") bucket("s3bucket") access_key("access-key-secret-value") secret_key("secret-key-value") object_key("${HOST}/my-logs") object_key_timestamp("timestamp") template("${MESSAGE}\n") compression(no) compresslevel(9) chunk_size(5) max_object_size(5000) upload_threads(8) max_pending_uploads(32) flush_grace_period(60) region("s3region") storage_class("STANDARD") canned_acl("s3-canned-acl") persist_name("output_default_test-s3-out"));
137+
s3(url("http://localhost:9000") bucket("s3bucket") access_key("access-key-secret-value") secret_key("secret-key-value") role("s3-role") object_key("${HOST}/my-logs") object_key_timestamp("timestamp") template("${MESSAGE}\n") compression(no) compresslevel(9) chunk_size(5) max_object_size(5000) upload_threads(8) max_pending_uploads(32) flush_grace_period(60) region("s3region") server_side_encryption("aws:kms") kms_key("kms-key") storage_class("STANDARD") canned_acl("s3-canned-acl") content_type("application/octet-stream") persist_name("output_default_test-s3-out"));
138138
};
139139
`)
140140

@@ -193,19 +193,23 @@ destination "output_default_test-s3-out" {
193193
},
194194
},
195195
},
196-
ObjectKey: "${HOST}/my-logs",
197-
ObjectKeyTimestamp: "timestamp",
198-
Template: "${MESSAGE}\n",
199-
Compression: config.NewFalse(),
200-
CompressLevel: 9,
201-
ChunkSize: 5,
202-
MaxObjectSize: 5000,
203-
UploadThreads: 8,
204-
MaxPendingUploads: 32,
205-
FlushGracePeriod: 60,
206-
Region: "s3region",
207-
StorageClass: "STANDARD",
208-
CannedAcl: "s3-canned-acl",
196+
Role: "s3-role",
197+
ObjectKey: "${HOST}/my-logs",
198+
ObjectKeyTimestamp: "timestamp",
199+
Template: "${MESSAGE}\n",
200+
Compression: config.NewFalse(),
201+
CompressLevel: 9,
202+
ChunkSize: 5,
203+
MaxObjectSize: 5000,
204+
UploadThreads: 8,
205+
MaxPendingUploads: 32,
206+
FlushGracePeriod: 60,
207+
Region: "s3region",
208+
ServerSideEncryption: "aws:kms",
209+
KmsKey: "kms-key",
210+
StorageClass: "STANDARD",
211+
CannedAcl: "s3-canned-acl",
212+
ContentType: "application/octet-stream",
209213
}},
210214
},
211215
},

‎pkg/sdk/logging/model/syslogng/output/s3.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ type S3Output struct {
7171
AccessKey *secret.Secret `json:"access_key,omitempty"`
7272
// The secret_key for the S3 server.
7373
SecretKey *secret.Secret `json:"secret_key,omitempty"`
74+
// The role for the S3 server.
75+
Role string `json:"role,omitempty"`
7476
// The object_key for the S3 server.
7577
ObjectKey string `json:"object_key,omitempty"`
7678
// Set object_key_timestamp
@@ -93,10 +95,22 @@ type S3Output struct {
9395
FlushGracePeriod int `json:"flush_grace_period,omitempty"`
9496
// Set the region option.
9597
Region string `json:"region,omitempty"`
98+
// You can use the server-side-encryption() and kms-key() options to configure encryption. Currently only server-side-encryption("aws:kms") is supported.
99+
// Available in AxoSyslog 4.8 and later.
100+
ServerSideEncryption string `json:"server_side_encryption,omitempty"`
101+
// The kms-key() used for server-side encryption. The value of the kms-key() parameter must be one of the following:
102+
// The ID of a key.
103+
// An alias of a key. In that case, make sure to add the alias/prefix, for example: kms-key("alias/log-archive")
104+
// The ARN of a key.
105+
// Available in AxoSyslog 4.8 and later.
106+
KmsKey string `json:"kms_key,omitempty"`
96107
// Set the storage_class option.
97108
StorageClass string `json:"storage_class,omitempty"`
98109
// Set the canned_acl option.
99110
CannedAcl string `json:"canned_acl,omitempty"`
111+
// The content-type of the HTTP request.
112+
// Defaults to "application/octet-stream".
113+
ContentType string `json:"content_type,omitempty"`
100114
// The number of messages that the output queue can store.
101115
LogFIFOSize int `json:"log-fifo-size,omitempty"`
102116
// Persistname

0 commit comments

Comments
 (0)