Skip to content

Commit fb843be

Browse files
pepovpepov
authored
Merge pull request #1332 from kube-logging/fluentbit-multiagent
fluentbit multiagent
2 parents bd76c8f + cd480c8 commit fb843be

File tree

19 files changed

+284
-99
lines changed

19 files changed

+284
-99
lines changed

‎.github/workflows/e2e.yaml

Lines changed: 27 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ env:
1414

1515
jobs:
1616
build:
17-
name: Build
17+
name: Go end2end tests
1818
runs-on: ubuntu-latest
1919

2020
steps:
@@ -34,8 +34,16 @@ jobs:
3434
- name: Run e2e tests
3535
run: make test-e2e
3636

37+
- name: Archive Test Results
38+
if: always()
39+
uses: actions/upload-artifact@v3
40+
with:
41+
name: go-e2e-test-cluster-logs
42+
path: build/_test
43+
retention-days: 5
44+
3745
basic_flow:
38-
name: Basic flow
46+
name: Shell script tests with chart install
3947
runs-on: ubuntu-latest
4048

4149
steps:
@@ -58,6 +66,9 @@ jobs:
5866
- name: Checkout code
5967
uses: actions/checkout@v3
6068

69+
- name: Stern
70+
run: make stern && test -x bin/stern
71+
6172
- name: Build controller image
6273
run: make docker-build IMG='controller:local'
6374

@@ -88,3 +99,17 @@ jobs:
8899
89100
- name: Test
90101
run: hack/test.sh
102+
103+
- name: Print last 10k kubernetes logs from default and logging namespaces
104+
if: always()
105+
run: |
106+
mkdir -p build/_test
107+
bin/stern -n default,logging ".*" --tail 100000 --no-follow > build/_test/cluster.logs
108+
109+
- name: Archive Test Results
110+
if: always()
111+
uses: actions/upload-artifact@v3
112+
with:
113+
name: script-e2e-test-cluster-logs
114+
path: build/_test
115+
retention-days: 5

‎Makefile

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,8 +156,8 @@ test: generate fmt vet manifests ${ENVTEST_BINARY_ASSETS} ${KUBEBUILDER} ## Run
156156
ENVTEST_BINARY_ASSETS=${ENVTEST_BINARY_ASSETS} go test ./controllers/extensions/... ./pkg/... -coverprofile cover.out
157157

158158
.PHONY: test-e2e
159-
test-e2e: ${KIND} docker-build generate fmt vet manifests ## Run E2E tests
160-
cd e2e && LOGGING_OPERATOR_IMAGE="${IMG}" go test -timeout ${E2E_TEST_TIMEOUT} ./...
159+
test-e2e: ${KIND} docker-build generate fmt vet manifests stern ## Run E2E tests
160+
cd e2e && LOGGING_OPERATOR_IMAGE="${IMG}" PROJECT_DIR="$(PWD)" go test -v -timeout ${E2E_TEST_TIMEOUT} ./...
161161

162162
.PHONY: tidy
163163
tidy: ## Tidy Go modules
@@ -229,6 +229,9 @@ ${SETUP_ENVTEST}: VERSION := latest
229229
${SETUP_ENVTEST}: | ${BIN}
230230
GOBIN=${BIN} go install ${IMPORT_PATH}@${VERSION}
231231

232+
stern: | ${BIN}
233+
GOBIN=${BIN} go install github.com/stern/stern@latest
234+
232235
${ENVTEST_BIN_DIR}: | ${BIN}
233236
mkdir -p $@
234237

‎config/crd/bases/logging.banzaicloud.io_fluentbits.yaml renamed to ‎config/crd/bases/logging.banzaicloud.io_fluentbitagents.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@ metadata:
66
annotations:
77
controller-gen.kubebuilder.io/version: v0.6.0
88
creationTimestamp: null
9-
name: fluentbits.logging.banzaicloud.io
9+
name: fluentbitagents.logging.banzaicloud.io
1010
spec:
1111
group: logging.banzaicloud.io
1212
names:
1313
categories:
1414
- logging-all
1515
kind: FluentbitAgent
1616
listKind: FluentbitAgentList
17-
plural: fluentbits
17+
plural: fluentbitagents
1818
singular: fluentbitagent
1919
scope: Cluster
2020
versions:

‎config/samples/fluentbit-multi.yaml

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
apiVersion: logging.banzaicloud.io/v1beta1
2+
kind: Logging
3+
metadata:
4+
name: multi
5+
spec:
6+
fluentd: {}
7+
# # the legacy fluentbit spec should now be used separately
8+
# fluentbit:
9+
# positiondb:
10+
# hostPath:
11+
# path: ""
12+
# bufferStorageVolume:
13+
# hostPath:
14+
# path: ""
15+
controlNamespace: default
16+
---
17+
apiVersion: logging.banzaicloud.io/v1beta1
18+
kind: FluentbitAgent
19+
metadata:
20+
# important to use the same name as the logging agent does
21+
name: multi
22+
spec:
23+
positiondb:
24+
hostPath:
25+
path: ""
26+
bufferStorageVolume:
27+
hostPath:
28+
path: ""
29+
---
30+
apiVersion: logging.banzaicloud.io/v1beta1
31+
kind: FluentbitAgent
32+
metadata:
33+
name: multi-2
34+
spec:
35+
positiondb:
36+
hostPath:
37+
path: ""
38+
bufferStorageVolume:
39+
hostPath:
40+
path: ""
41+
---
42+
apiVersion: logging.banzaicloud.io/v1beta1
43+
kind: Flow
44+
metadata:
45+
name: all
46+
spec:
47+
filters:
48+
- stdout: {}
49+
match:
50+
- select: {}
51+
localOutputRefs:
52+
- "null"
53+
---
54+
apiVersion: logging.banzaicloud.io/v1beta1
55+
kind: Output
56+
metadata:
57+
name: "null"
58+
spec:
59+
nullout: {}

‎controllers/logging/logging_controller.go

Lines changed: 22 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ type LoggingReconciler struct {
8383

8484
// Reconcile logging resources
8585
func (r *LoggingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
86-
log := r.Log.WithValues("logging", req.NamespacedName)
86+
log := r.Log.WithValues("logging", req.Name)
8787

8888
var logging loggingv1beta1.Logging
8989
if err := r.Client.Get(ctx, req.NamespacedName, &logging); err != nil {
@@ -186,35 +186,38 @@ func (r *LoggingReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
186186
}
187187

188188
switch len(loggingResources.Fluentbits) {
189-
case 1:
190-
if logging.Spec.FluentbitSpec != nil {
191-
return ctrl.Result{}, errors.New("fluentbit has to be removed from the logging resource before the new FluentbitAgent can be reconciled")
192-
}
193-
reconcilers = append(reconcilers, fluentbit.New(
194-
r.Client,
195-
r.Log,
196-
&logging,
197-
reconcilerOpts,
198-
&loggingResources.Fluentbits[0].Spec,
199-
loggingDataProvider,
200-
loggingv1beta1.NewStandaloneFluentbitNameProvider(&loggingResources.Fluentbits[0]),
201-
).Reconcile)
202189
case 0:
203190
// check for legacy definition
204191
log.Info("WARNING fluentbit definition inside the Logging resource is deprecated and will be removed in the next major release")
205192
if logging.Spec.FluentbitSpec != nil {
193+
nameProvider := loggingv1beta1.NewLegacyFluentbitNameProvider(&logging)
206194
reconcilers = append(reconcilers, fluentbit.New(
207195
r.Client,
208-
r.Log,
196+
log.WithName("fluentbit-legacy"),
209197
&logging,
210198
reconcilerOpts,
211199
logging.Spec.FluentbitSpec,
212200
loggingDataProvider,
213-
loggingv1beta1.NewLegacyFluentbitNameProvider(&logging),
201+
nameProvider,
214202
).Reconcile)
215203
}
216204
default:
217-
return ctrl.Result{}, errors.New("cannot handle more than one FluentbitAgent for the same Logging resource")
205+
if logging.Spec.FluentbitSpec != nil {
206+
return ctrl.Result{}, errors.New("fluentbit has to be removed from the logging resource before the new FluentbitAgent can be reconciled")
207+
}
208+
l := log.WithName("fluentbit")
209+
for _, f := range loggingResources.Fluentbits {
210+
f := f
211+
reconcilers = append(reconcilers, fluentbit.New(
212+
r.Client,
213+
l.WithValues("fluentbitagent", f.Name),
214+
&logging,
215+
reconcilerOpts,
216+
&f.Spec,
217+
loggingDataProvider,
218+
loggingv1beta1.NewStandaloneFluentbitNameProvider(&f),
219+
).Reconcile)
220+
}
218221
}
219222

220223
if len(logging.Spec.NodeAgents) > 0 || len(loggingResources.NodeAgents) > 0 {
@@ -431,11 +434,8 @@ func SetupLoggingWithManager(mgr ctrl.Manager, logger logr.Logger) *ctrl.Builder
431434
logger.Info("processing NodeAgent CRDs is explicitly disabled (enable: ENABLE_NODEAGENT_CRD=1)")
432435
builder.Watches(&source.Kind{Type: &loggingv1beta1.NodeAgent{}}, requestMapper)
433436
}
434-
// TODO remove with the next major release
435-
if os.Getenv("ENABLE_FLUENTBIT_CRD") != "" {
436-
logger.Info("processing FluentbitAgent CRDs is explicitly disabled (enable: ENABLE_FLUENTBIT_CRD=1)")
437-
builder.Watches(&source.Kind{Type: &loggingv1beta1.FluentbitAgent{}}, requestMapper)
438-
}
437+
438+
builder.Watches(&source.Kind{Type: &loggingv1beta1.FluentbitAgent{}}, requestMapper)
439439

440440
fluentd.RegisterWatches(builder)
441441
fluentbit.RegisterWatches(builder)

‎e2e/charts/logging-operator/crds/logging.banzaicloud.io_fluentbits.yaml renamed to ‎e2e/charts/logging-operator/crds/logging.banzaicloud.io_fluentbitagents.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,15 @@ metadata:
66
annotations:
77
controller-gen.kubebuilder.io/version: v0.6.0
88
creationTimestamp: null
9-
name: fluentbits.logging.banzaicloud.io
9+
name: fluentbitagents.logging.banzaicloud.io
1010
spec:
1111
group: logging.banzaicloud.io
1212
names:
1313
categories:
1414
- logging-all
1515
kind: FluentbitAgent
1616
listKind: FluentbitAgentList
17-
plural: fluentbits
17+
plural: fluentbitagents
1818
singular: fluentbitagent
1919
scope: Cluster
2020
versions:

‎e2e/common/cluster.go

Lines changed: 59 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,16 @@ package common
1616

1717
import (
1818
"context"
19+
"os"
20+
"os/exec"
21+
"strings"
1922
"testing"
2023

2124
"emperror.dev/errors"
25+
"github.com/spf13/cast"
26+
"github.com/stretchr/testify/assert"
2227
"github.com/stretchr/testify/require"
28+
"k8s.io/client-go/rest"
2329
"k8s.io/client-go/tools/clientcmd"
2430
"sigs.k8s.io/controller-runtime/pkg/cluster"
2531

@@ -31,9 +37,17 @@ const defaultClusterName = "e2e-test"
3137
type Cluster interface {
3238
cluster.Cluster
3339
LoadImages(images ...string) error
40+
Cleanup() error
41+
PrintLogs(config PrintLogConfig) error
3442
}
3543

36-
func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Option) {
44+
type PrintLogConfig struct {
45+
Namespaces []string
46+
FilePath string
47+
Limit int
48+
}
49+
50+
func WithCluster(t *testing.T, fn func(*testing.T, Cluster), beforeCleanup func(*testing.T, Cluster) error, opts ...cluster.Option) {
3751
cluster, err := GetTestCluster(defaultClusterName, opts...)
3852
require.NoError(t, err)
3953

@@ -43,6 +57,8 @@ func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Opt
4357
}()
4458

4559
defer func() {
60+
assert.NoError(t, beforeCleanup(t, cluster))
61+
assert.NoError(t, cluster.Cleanup())
4662
cancel()
4763
require.NoError(t, DeleteTestCluster(defaultClusterName))
4864
}()
@@ -51,22 +67,38 @@ func WithCluster(t *testing.T, fn func(*testing.T, Cluster), opts ...cluster.Opt
5167
}
5268

5369
func GetTestCluster(clusterName string, opts ...cluster.Option) (Cluster, error) {
54-
kubeconfig, err := KindClusterKubeconfig(clusterName)
70+
var err error
71+
var kubeconfig []byte
72+
var kubeconfigFile *os.File
73+
var clientCfg clientcmd.ClientConfig
74+
var restCfg *rest.Config
75+
var c cluster.Cluster
76+
77+
kubeconfig, err = KindClusterKubeconfig(clusterName)
5578
if err != nil {
5679
return nil, errors.WrapIfWithDetails(err, "getting kubeconfig of kind cluster", "clusterName", clusterName)
5780
}
58-
clientCfg, err := clientcmd.NewClientConfigFromBytes(kubeconfig)
81+
kubeconfigFile, err = os.CreateTemp("", "kind-kind-kubeconfig")
82+
if err != nil {
83+
return nil, errors.WrapIfWithDetails(err, "unable to create temp file for kubeconfig", "clusterName", clusterName)
84+
}
85+
err = os.WriteFile(kubeconfigFile.Name(), kubeconfig, os.FileMode(0600))
86+
if err != nil {
87+
return nil, errors.WrapIfWithDetails(err, "failed to write kubeconfig", "clusterName", clusterName, "path", kubeconfigFile.Name())
88+
}
89+
clientCfg, err = clientcmd.NewClientConfigFromBytes(kubeconfig)
5990
if err != nil {
6091
return nil, errors.WrapIfWithDetails(err, "creating client config from kubeconfig bytes", "kubeconfig", kubeconfig)
6192
}
62-
restCfg, err := clientCfg.ClientConfig()
93+
restCfg, err = clientCfg.ClientConfig()
6394
if err != nil {
6495
return nil, errors.WrapIfWithDetails(err, "creating rest config from client config", "cfg", clientCfg)
6596
}
66-
c, err := cluster.New(restCfg, opts...)
97+
c, err = cluster.New(restCfg, opts...)
6798
return &kindCluster{
68-
Cluster: c,
69-
clusterName: clusterName,
99+
Cluster: c,
100+
kubeconfigFilePath: kubeconfigFile.Name(),
101+
clusterName: clusterName,
70102
}, errors.WrapIfWithDetails(err, "creating cluster with rest config", "cfg", restCfg)
71103
}
72104

@@ -78,7 +110,26 @@ func DeleteTestCluster(clusterName string) error {
78110

79111
type kindCluster struct {
80112
cluster.Cluster
81-
clusterName string
113+
kubeconfigFilePath string
114+
clusterName string
115+
}
116+
117+
func (c kindCluster) PrintLogs(config PrintLogConfig) error {
118+
cmd := exec.Command("stern", "-n", strings.Join(config.Namespaces, ","), ".*", "--no-follow", "--tail", cast.ToString(config.Limit), "--kubeconfig", c.kubeconfigFilePath)
119+
f, err := os.Create(config.FilePath)
120+
if err != nil {
121+
return err
122+
}
123+
defer f.Close()
124+
125+
cmd.Stdout = f
126+
cmd.Stderr = os.Stderr
127+
128+
return cmd.Run()
129+
}
130+
131+
func (c kindCluster) Cleanup() error {
132+
return os.Remove(c.kubeconfigFilePath)
82133
}
83134

84135
func (c kindCluster) LoadImages(images ...string) error {

‎e2e/go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ require (
77
github.com/cisco-open/operator-tools v0.29.0
88
github.com/go-logr/logr v1.2.4
99
github.com/kube-logging/logging-operator/pkg/sdk v0.9.1
10+
github.com/spf13/cast v1.5.0
1011
github.com/stretchr/testify v1.8.2
1112
k8s.io/api v0.26.4
1213
k8s.io/apiextensions-apiserver v0.26.4
@@ -56,7 +57,6 @@ require (
5657
github.com/prometheus/client_model v0.3.0 // indirect
5758
github.com/prometheus/common v0.37.0 // indirect
5859
github.com/prometheus/procfs v0.8.0 // indirect
59-
github.com/spf13/cast v1.5.0 // indirect
6060
github.com/spf13/pflag v1.0.5 // indirect
6161
github.com/wayneashleyberry/terminal-dimensions v1.0.0 // indirect
6262
go.uber.org/atomic v1.7.0 // indirect

0 commit comments

Comments
 (0)