Skip to content

Commit 46ef309

Browse files
sumanthravipatisumanthravipati
sumanthravipati
and
sumanthravipati
authored
MLE-11466: Log Collection changes commit (#22)
* MLE-11466: Log Collection changes commit * MLE-11466: Log Collection changes commit * MLE-11466: Review Comments changes commit --------- Co-authored-by: sumanthravipati <[email protected]>
1 parent f9c13fd commit 46ef309

File tree

11 files changed

+544
-2
lines changed

11 files changed

+544
-2
lines changed

‎api/v1alpha1/common_types.go‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,19 @@ type AdminAuth struct {
4040
AdminPassword *string `json:"adminPassword,omitempty"`
4141
WalletPassword *string `json:"walletPassword,omitempty"`
4242
}
43+
44+
type LogCollection struct {
45+
Enabled bool `json:"enabled,omitempty"`
46+
Image string `json:"image,omitempty"`
47+
Resources *corev1.ResourceRequirements `json:"resources,omitempty"`
48+
Files LogFilesConfig `json:"files,omitempty"`
49+
Outputs string `json:"outputs,omitempty"`
50+
}
51+
52+
type LogFilesConfig struct {
53+
ErrorLogs bool `json:"errorLogs,omitempty"`
54+
AccessLogs bool `json:"accessLogs,omitempty"`
55+
RequestLogs bool `json:"requestLogs,omitempty"`
56+
CrashLogs bool `json:"crashLogs,omitempty"`
57+
AuditLogs bool `json:"auditLogs,omitempty"`
58+
}

‎api/v1alpha1/marklogicgroup_types.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,9 @@ type MarklogicGroupSpec struct {
6565
// +kubebuilder:default:={enabled: false, initialDelaySeconds: 10, timeoutSeconds: 5, periodSeconds: 30, successThreshold: 1, failureThreshold: 3}
6666
ReadinessProbe ContainerProbe `json:"readinessProbe,omitempty"`
6767

68+
// +kubebuilder:default:={enabled: false, image: "fluent/fluent-bit:3.1.1", resources: {requests: {cpu: "100m", memory: "200Mi"}, limits: {cpu: "200m", memory: "500Mi"}}, files: {errorLogs: true, accessLogs: true, requestLogs: true}, outputs: "stdout"}
69+
LogCollection *LogCollection `json:"logCollection,omitempty"`
70+
6871
// +kubebuilder:default:={name: "Default", enableXdqpSsl: true}
6972
GroupConfig GroupConfig `json:"groupConfig,omitempty"`
7073
License *License `json:"license,omitempty"`

‎api/v1alpha1/zz_generated.deepcopy.go‎

Lines changed: 41 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎config/crd/bases/database.marklogic.com_marklogicclusters.yaml‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1283,6 +1283,100 @@ spec:
12831283
minimum: 0
12841284
type: integer
12851285
type: object
1286+
logCollection:
1287+
default:
1288+
enabled: false
1289+
files:
1290+
accessLogs: true
1291+
errorLogs: true
1292+
requestLogs: true
1293+
image: fluent/fluent-bit:3.1.1
1294+
outputs: stdout
1295+
resources:
1296+
limits:
1297+
cpu: 200m
1298+
memory: 500Mi
1299+
requests:
1300+
cpu: 100m
1301+
memory: 200Mi
1302+
properties:
1303+
enabled:
1304+
type: boolean
1305+
files:
1306+
properties:
1307+
accessLogs:
1308+
type: boolean
1309+
auditLogs:
1310+
type: boolean
1311+
crashLogs:
1312+
type: boolean
1313+
errorLogs:
1314+
type: boolean
1315+
requestLogs:
1316+
type: boolean
1317+
type: object
1318+
image:
1319+
type: string
1320+
outputs:
1321+
type: string
1322+
resources:
1323+
description: ResourceRequirements describes the compute
1324+
resource requirements.
1325+
properties:
1326+
claims:
1327+
description: |-
1328+
Claims lists the names of resources, defined in spec.resourceClaims,
1329+
that are used by this container.
1330+
1331+
1332+
This is an alpha field and requires enabling the
1333+
DynamicResourceAllocation feature gate.
1334+
1335+
1336+
This field is immutable. It can only be set for containers.
1337+
items:
1338+
description: ResourceClaim references one entry
1339+
in PodSpec.ResourceClaims.
1340+
properties:
1341+
name:
1342+
description: |-
1343+
Name must match the name of one entry in pod.spec.resourceClaims of
1344+
the Pod where this field is used. It makes that resource available
1345+
inside a container.
1346+
type: string
1347+
required:
1348+
- name
1349+
type: object
1350+
type: array
1351+
x-kubernetes-list-map-keys:
1352+
- name
1353+
x-kubernetes-list-type: map
1354+
limits:
1355+
additionalProperties:
1356+
anyOf:
1357+
- type: integer
1358+
- type: string
1359+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
1360+
x-kubernetes-int-or-string: true
1361+
description: |-
1362+
Limits describes the maximum amount of compute resources allowed.
1363+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
1364+
type: object
1365+
requests:
1366+
additionalProperties:
1367+
anyOf:
1368+
- type: integer
1369+
- type: string
1370+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
1371+
x-kubernetes-int-or-string: true
1372+
description: |-
1373+
Requests describes the minimum amount of compute resources required.
1374+
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
1375+
otherwise to an implementation-defined value. Requests cannot exceed Limits.
1376+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
1377+
type: object
1378+
type: object
1379+
type: object
12861380
name:
12871381
type: string
12881382
networkPolicy:

‎config/crd/bases/database.marklogic.com_marklogicgroups.yaml‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1257,6 +1257,99 @@ spec:
12571257
minimum: 0
12581258
type: integer
12591259
type: object
1260+
logCollection:
1261+
default:
1262+
enabled: false
1263+
files:
1264+
accessLogs: true
1265+
errorLogs: true
1266+
requestLogs: true
1267+
image: fluent/fluent-bit:3.1.1
1268+
outputs: stdout
1269+
resources:
1270+
limits:
1271+
cpu: 200m
1272+
memory: 500Mi
1273+
requests:
1274+
cpu: 100m
1275+
memory: 200Mi
1276+
properties:
1277+
enabled:
1278+
type: boolean
1279+
files:
1280+
properties:
1281+
accessLogs:
1282+
type: boolean
1283+
auditLogs:
1284+
type: boolean
1285+
crashLogs:
1286+
type: boolean
1287+
errorLogs:
1288+
type: boolean
1289+
requestLogs:
1290+
type: boolean
1291+
type: object
1292+
image:
1293+
type: string
1294+
outputs:
1295+
type: string
1296+
resources:
1297+
description: ResourceRequirements describes the compute resource
1298+
requirements.
1299+
properties:
1300+
claims:
1301+
description: |-
1302+
Claims lists the names of resources, defined in spec.resourceClaims,
1303+
that are used by this container.
1304+
1305+
1306+
This is an alpha field and requires enabling the
1307+
DynamicResourceAllocation feature gate.
1308+
1309+
1310+
This field is immutable. It can only be set for containers.
1311+
items:
1312+
description: ResourceClaim references one entry in PodSpec.ResourceClaims.
1313+
properties:
1314+
name:
1315+
description: |-
1316+
Name must match the name of one entry in pod.spec.resourceClaims of
1317+
the Pod where this field is used. It makes that resource available
1318+
inside a container.
1319+
type: string
1320+
required:
1321+
- name
1322+
type: object
1323+
type: array
1324+
x-kubernetes-list-map-keys:
1325+
- name
1326+
x-kubernetes-list-type: map
1327+
limits:
1328+
additionalProperties:
1329+
anyOf:
1330+
- type: integer
1331+
- type: string
1332+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
1333+
x-kubernetes-int-or-string: true
1334+
description: |-
1335+
Limits describes the maximum amount of compute resources allowed.
1336+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
1337+
type: object
1338+
requests:
1339+
additionalProperties:
1340+
anyOf:
1341+
- type: integer
1342+
- type: string
1343+
pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
1344+
x-kubernetes-int-or-string: true
1345+
description: |-
1346+
Requests describes the minimum amount of compute resources required.
1347+
If Requests is omitted for a container, it defaults to Limits if that is explicitly specified,
1348+
otherwise to an implementation-defined value. Requests cannot exceed Limits.
1349+
More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/
1350+
type: object
1351+
type: object
1352+
type: object
12601353
name:
12611354
type: string
12621355
networkPolicy:

‎config/samples/marklogicgroup.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,24 @@ spec:
2121
updateStrategy: OnDelete
2222
groupConfig:
2323
name: "node"
24+
logCollection:
25+
enabled: true
26+
image: fluent/fluent-bit:3.1.1
27+
files:
28+
errorLogs: true
29+
accessLogs: true
30+
requestLogs: true
31+
crashLogs: true
32+
auditLogs: true
33+
outputs: |-
34+
[OUTPUT]
35+
name loki
36+
match *
37+
host loki.loki.svc.cluster.local
38+
port 3100
39+
labels job=fluent-bit
40+
http_user admin
41+
http_passwd admin
2442
2543
# resources:
2644
# requests:

‎internal/controller/marklogicgroup_controller_test.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ const resourceHugepageValue = int64(104857600)
5252
var typeNamespaceName = types.NamespacedName{Name: Name, Namespace: Namespace}
5353

5454
const imageName = "progressofficial/marklogic-db:11.3.0-ubi-rootless"
55+
const fluentBitImage = "fluent/fluent-bit:3.1.1"
5556

5657
var groupConfig = databasev1alpha1.GroupConfig{
5758
Name: "dnode",
@@ -96,6 +97,7 @@ var _ = Describe("MarkLogicGroup controller", func() {
9697
ClusterDomain: "cluster.local",
9798
TopologySpreadConstraints: []corev1.TopologySpreadConstraint{{MaxSkew: 2, TopologyKey: "kubernetes.io/hostname", WhenUnsatisfiable: corev1.ScheduleAnyway}},
9899
Affinity: &corev1.Affinity{PodAffinity: &corev1.PodAffinity{PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{{PodAffinityTerm: corev1.PodAffinityTerm{TopologyKey: "kubernetes.io/hostname"}, Weight: 100}}}},
100+
LogCollection: &databasev1alpha1.LogCollection{Enabled: true, Image: "fluent/fluent-bit:3.1.1", Files: databasev1alpha1.LogFilesConfig{ErrorLogs: true, AccessLogs: true, RequestLogs: true, CrashLogs: true, AuditLogs: true}, Outputs: "stdout"},
99101
},
100102
}
101103
Expect(k8sClient.Create(ctx, mlGroup)).Should(Succeed())
@@ -128,6 +130,8 @@ var _ = Describe("MarkLogicGroup controller", func() {
128130
Expect(createdCR.Spec.TopologySpreadConstraints[0].TopologyKey).Should(Equal("kubernetes.io/hostname"))
129131
Expect(createdCR.Spec.TopologySpreadConstraints[0].WhenUnsatisfiable).Should(Equal(corev1.ScheduleAnyway))
130132
Expect(createdCR.Spec.Affinity.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight).Should(Equal(int32(100)))
133+
Expect(createdCR.Spec.LogCollection.Enabled).Should(Equal(true))
134+
Expect(createdCR.Spec.LogCollection.Image).Should(Equal(fluentBitImage))
131135

132136
// Validating if StatefulSet is created successfully
133137
sts := &appsv1.StatefulSet{}
@@ -136,6 +140,7 @@ var _ = Describe("MarkLogicGroup controller", func() {
136140
return err == nil
137141
}, timeout, interval).Should(BeTrue())
138142
Expect(sts.Spec.Template.Spec.Containers[0].Image).Should(Equal(imageName))
143+
Expect(sts.Spec.Template.Spec.Containers[1].Image).Should(Equal(fluentBitImage))
139144
Expect(sts.Spec.Replicas).Should(Equal(&replicas))
140145
Expect(sts.Name).Should(Equal(Name))
141146
Expect(sts.Spec.PodManagementPolicy).Should(Equal(appsv1.ParallelPodManagement))

‎pkg/k8sutil/common.go‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
package k8sutil
22

33
import (
4+
"math/rand"
5+
"time"
6+
47
databasev1alpha1 "github.com/marklogic/marklogic-kubernetes-operator/api/v1alpha1"
58
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
6-
"math/rand"
79
"sigs.k8s.io/controller-runtime/pkg/client"
8-
"time"
910
)
1011

1112
// generateTypeMeta generates the TyeMeta
@@ -52,6 +53,13 @@ func getMarkLogicLabels(name string) map[string]string {
5253
}
5354
}
5455

56+
func getFluentBitLabels(name string) map[string]string {
57+
return map[string]string{
58+
"app.kubernetes.io/name": "fluent-bit",
59+
"app.kubernetes.io/instance": name,
60+
}
61+
}
62+
5563
func marklogicClusterAsOwner(cr *databasev1alpha1.MarklogicCluster) metav1.OwnerReference {
5664
trueVar := true
5765
return metav1.OwnerReference{

0 commit comments

Comments
 (0)