Merge branch 'main' into K8SPSMDB-1413

Author: gkech

.e2eignore

@@ -9,5 +9,4 @@ operator.png
 kubernetes.svg
 release_versions
 .github/**
-.snyk
 .e2eignore

.github/workflows/scan.yml

@@ -30,27 +30,10 @@ jobs:
           export DOCKER_DEFAULT_PLATFORM='linux/arm64'
           ./e2e-tests/build
 
-      - name: Run Snyk vulnerability scanner image (linux/arm64)
-        uses: snyk/actions/docker@master
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-arm64'
-          args: --platform=linux/arm64 --severity-threshold=high --exclude-base-image-vulns --file=./build/Dockerfile -fail-on=upgradable
-
       - name: Build an image from Dockerfile (linux/amd64)
         run: |
           export IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-amd64
           export DOCKER_PUSH=0
           export DOCKER_SQUASH=0
           export DOCKER_DEFAULT_PLATFORM='linux/amd64'
           ./e2e-tests/build
-
-      - name: Run Snyk vulnerability scanner image (linux/amd64)
-        uses: snyk/actions/docker@master
-        env:
-          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
-        with:
-          image: '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.sha }}-amd64'
-          args: --platform=linux/amd64 --severity-threshold=high --exclude-base-image-vulns --file=./build/Dockerfile -fail-on=upgradable
-

.snyk

@@ -1,21 +1,100 @@
-#!/bin/sh
+#!/bin/bash
 set -e
-set -o xtrace
 
-export PATH="$PATH":/opt/fluent-bit/bin
+export PATH="$PATH:/opt/fluent-bit/bin"
 
-if [ "$1" = 'logrotate' ]; then
+LOGROTATE_SCHEDULE="${LOGROTATE_SCHEDULE:-0 0 * * *}"
+
+is_logrotate_config_invalid() {
+	local config_file="$1"
+	if [ -z "$config_file" ] || [ ! -f "$config_file" ]; then
+		return 1
+	fi
+	# Specifying -d runs in debug mode, so even in case of errors, it will exit with 0.
+	# We need to check the output for "error" but skip those lines that are related to the missing logrotate.status file.
+	# Filter out logrotate.status lines first, then check for remaining errors
+	(
+		set +e
+		logrotate -d "$config_file" 2>&1 | grep -v "logrotate.status" | grep -qi "error"
+	)
+	return $?
+}
+
+run_logrotate() {
+	local logrotate_status_file="/data/db/logs/logrotate.status"
+	local logrotate_conf_file="/opt/percona/logcollector/logrotate/logrotate.conf"
+	local logrotate_additional_conf_files=()
+	local conf_d_dir="/opt/percona/logcollector/logrotate/conf.d"
+
+	# Check if mongodb.conf exists and validate it
+	if [ -f "$conf_d_dir/mongodb.conf" ]; then
+		logrotate_conf_file="$conf_d_dir/mongodb.conf"
+		if is_logrotate_config_invalid "$logrotate_conf_file"; then
+			echo "ERROR: Logrotate configuration is invalid, fallback to default configuration"
+			logrotate_conf_file="/opt/percona/logcollector/logrotate/logrotate.conf"
+		fi
+	fi
+
+	# Process all .conf files in conf.d directory (excluding mongodb.conf which is already handled)
+	if [ -d "$conf_d_dir" ]; then
+		for conf_file in "$conf_d_dir"/*.conf; do
+			# Check if glob matched any files (if no .conf files exist, the glob returns itself)
+			[ -f "$conf_file" ] || continue
+			# Skip mongodb.conf as it's already processed above
+			[ "$(basename "$conf_file")" = "mongodb.conf" ] && continue
+			if is_logrotate_config_invalid "$conf_file"; then
+				echo "ERROR: Logrotate configuration file $conf_file is invalid, it will be ignored"
+			else
+				logrotate_additional_conf_files+=("$conf_file")
+			fi
+		done
+	fi
+	# Ensure logrotate can run with current UID
 	if [[ $EUID != 1001 ]]; then
 		# logrotate requires UID in /etc/passwd
 		sed -e "s^x:1001:^x:$EUID:^" /etc/passwd >/tmp/passwd
 		cat /tmp/passwd >/etc/passwd
 		rm -rf /tmp/passwd
 	fi
-	exec go-cron "0 0 * * *" sh -c "logrotate -s /data/db/logs/logrotate.status /opt/percona/logcollector/logrotate/logrotate.conf;"
-else
-	if [ "$1" = 'fluent-bit' ]; then
-		fluentbit_opt+='-c /opt/percona/logcollector/fluentbit/fluentbit.conf'
-	fi
 
-	exec "$@" $fluentbit_opt
-fi
+	local logrotate_cmd="logrotate -s \"$logrotate_status_file\" \"$logrotate_conf_file\""
+	for additional_conf in "${logrotate_additional_conf_files[@]}"; do
+		logrotate_cmd="$logrotate_cmd \"$additional_conf\""
+	done
+
+	set -o xtrace
+	exec go-cron "$LOGROTATE_SCHEDULE" sh -c "$logrotate_cmd"
+}
+
+run_fluentbit() {
+	local fluentbit_opt=(-c /opt/percona/logcollector/fluentbit/fluentbit.conf)
+	mkdir -p /tmp/fluentbit/custom
+	set +e
+	local fluentbit_conf_dir="/opt/percona/logcollector/fluentbit/custom"
+	for conf_file in $fluentbit_conf_dir/*.conf; do
+		[ -f "$conf_file" ] || continue
+		if ! fluent-bit --dry-run -c "$conf_file" >/dev/null 2>&1; then
+			echo "ERROR: Fluentbit configuration file $conf_file is invalid, it will be ignored"
+		else
+			cp "$conf_file" /tmp/fluentbit/custom/
+		fi
+	done
+	touch /tmp/fluentbit/custom/default.conf || true
+
+	set -e
+	set -o xtrace
+	exec "$@" "${fluentbit_opt[@]}"
+}
+
+case "$1" in
+	logrotate)
+		run_logrotate
+		;;
+	fluent-bit)
+		run_fluentbit "$@"
+		;;
+	*)
+		echo "Invalid argument: $1"
+		exit 1
+		;;
+esac

build/logcollector/entrypoint.sh

@@ -1,2 +1,2 @@
 @INCLUDE fluentbit_*.conf
-@INCLUDE custom/*.conf
+@INCLUDE /tmp/fluentbit/custom/*.conf

build/logcollector/fluentbit/fluentbit.conf

@@ -29,25 +29,27 @@ var (
 )
 
 func Dial(ctx context.Context, conf *Config) (mongo.Client, error) {
-	if err := conf.configureTLS(); err != nil {
+	log := logf.FromContext(ctx).WithName("Dial")
+	ctx = logf.IntoContext(ctx, log)
+
+	if err := conf.configureTLS(ctx); err != nil {
 		return nil, errors.Wrap(err, "configure TLS")
 	}
 
-	log := logf.FromContext(ctx)
 	log.V(1).Info("Connecting to mongodb", "hosts", conf.Hosts, "ssl", conf.SSL.Enabled, "ssl_insecure", conf.SSL.Insecure)
 
 	if conf.Username != "" && conf.Password != "" {
 		log.V(1).Info("Enabling authentication for session", "user", conf.Username)
 	}
 
-	cl, err := mongo.Dial(&conf.Config)
+	cl, err := mongo.Dial(ctx, &conf.Config)
 	if err != nil {
 		cfg := conf.Config
 		cfg.Direct = true
 		cfg.ReplSetName = ""
-		cl, err = mongo.Dial(&cfg)
+		cl, err = mongo.Dial(ctx, &cfg)
 		if err != nil {
-			return nil, errors.Wrap(err, "filed to dial mongo")
+			return nil, errors.Wrap(err, "failed to dial mongo")
 		}
 	}
 

cmd/mongodb-healthcheck/db/db.go

@@ -15,6 +15,7 @@
 package db
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"os"
@@ -40,8 +41,8 @@ func (sc *SSLConfig) loadCaCertificate() (*x509.CertPool, error) {
 	return certificates, nil
 }
 
-func (cnf *Config) configureTLS() error {
-	log := logf.Log
+func (cnf *Config) configureTLS(ctx context.Context) error {
+	log := logf.FromContext(ctx).WithName("configureTLS")
 
 	if !cnf.SSL.Enabled {
 		return nil
@@ -72,7 +73,7 @@ func (cnf *Config) configureTLS() error {
 			return errors.Wrapf(err, "check if file with name %s exists", cnf.SSL.CAFile)
 		}
 
-		log.V(1).Info("Loading SSL/TLS Certificate Authority: %s", "ca", cnf.SSL.CAFile)
+		log.V(1).Info("Loading SSL/TLS Certificate Authority", "ca", cnf.SSL.CAFile)
 		ca, err := cnf.SSL.loadCaCertificate()
 		if err != nil {
 			return errors.Wrapf(err, "load client CAs from %s", cnf.SSL.CAFile)

cmd/mongodb-healthcheck/db/ssl.go

@@ -16,7 +16,7 @@ func TestSSLNotEnabled(t *testing.T) {
 		},
 	}
 
-	if err := cfg.configureTLS(); err != nil {
+	if err := cfg.configureTLS(t.Context()); err != nil {
 		t.Fatalf("TLS configuration failed: %s", err)
 	}
 
@@ -32,7 +32,7 @@ func TestSSLEnabled(t *testing.T) {
 		},
 	}
 
-	if err := cfg.configureTLS(); err != nil {
+	if err := cfg.configureTLS(t.Context()); err != nil {
 		t.Fatalf("TLS configuration failed: %s", err)
 	}
 
@@ -49,7 +49,7 @@ func TestPEMKeyFileDoesNotExists(t *testing.T) {
 		},
 	}
 
-	err := cfg.configureTLS()
+	err := cfg.configureTLS(t.Context())
 	if err == nil {
 		t.Fatal("Expected TLS config to fail, but it returned no error")
 	}
@@ -71,7 +71,7 @@ func TestCAFileDoesNotExists(t *testing.T) {
 		},
 	}
 
-	err := cfg.configureTLS()
+	err := cfg.configureTLS(t.Context())
 	if err == nil {
 		t.Fatal("Expected TLS config to fail, but it returned no error")
 	}

cmd/mongodb-healthcheck/db/ssl_test.go

@@ -16,12 +16,9 @@ package healthcheck
 
 import (
 	"context"
-	"encoding/json"
 
-	v "github.com/hashicorp/go-version"
 	"github.com/pkg/errors"
 	"go.mongodb.org/mongo-driver/bson"
-	"go.mongodb.org/mongo-driver/bson/primitive"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	"github.com/percona/percona-server-mongodb-operator/cmd/mongodb-healthcheck/db"
@@ -32,6 +29,7 @@ var ErrNoReplsetConfigStr = "(NotYetInitialized) no replset config has been rece
 
 func HealthCheckMongosLiveness(ctx context.Context, cnf *db.Config) (err error) {
 	log := logf.FromContext(ctx).WithName("HealthCheckMongosLiveness")
+	ctx = logf.IntoContext(ctx, log)
 
 	client, err := db.Dial(ctx, cnf)
 	if err != nil {
@@ -58,6 +56,7 @@ func HealthCheckMongosLiveness(ctx context.Context, cnf *db.Config) (err error)
 
 func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelaySeconds int64) (_ *mongo.MemberState, err error) {
 	log := logf.FromContext(ctx).WithName("HealthCheckMongodLiveness")
+	ctx = logf.IntoContext(ctx, log)
 
 	client, err := db.Dial(ctx, cnf)
 	if err != nil {
@@ -74,50 +73,14 @@ func HealthCheckMongodLiveness(ctx context.Context, cnf *db.Config, startupDelay
 		return nil, errors.Wrap(err, "get isMaster response")
 	}
 
-	buildInfo, err := client.RSBuildInfo(ctx)
+	rsStatus, err := client.RSStatus(ctx)
 	if err != nil {
-		return nil, errors.Wrap(err, "get buildInfo response")
-	}
-
-	replSetStatusCommand := bson.D{{Key: "replSetGetStatus", Value: 1}}
-	mongoVersion := v.Must(v.NewVersion(buildInfo.Version))
-	if mongoVersion.Compare(v.Must(v.NewVersion("4.2.1"))) < 0 {
-		// https://docs.mongodb.com/manual/reference/command/replSetGetStatus/#syntax
-		replSetStatusCommand = append(replSetStatusCommand, primitive.E{Key: "initialSync", Value: 1})
-	}
-
-	res := client.Database("admin").RunCommand(ctx, replSetStatusCommand)
-	if res.Err() != nil {
-		// if we come this far, it means db connection was successful
-		// standalone mongod nodes in an unmanaged cluster doesn't need
-		// to die before they added to a replset
-		if res.Err().Error() == ErrNoReplsetConfigStr {
+		if err.Error() == ErrNoReplsetConfigStr {
 			state := mongo.MemberStateUnknown
-			log.V(1).Info("replSetGetStatus failed", "err", res.Err().Error(), "state", state)
+			log.V(1).Info("replSetGetStatus failed", "err", err.Error(), "state", state)
 			return &state, nil
 		}
-		return nil, errors.Wrap(res.Err(), "get replsetGetStatus response")
-	}
-
-	// this is a workaround to fix decoding of empty interfaces
-	// https://jira.mongodb.org/browse/GODRIVER-988
-	rsStatus := ReplSetStatus{}
-	tempResult := bson.M{}
-	err = res.Decode(&tempResult)
-	if err != nil {
-		return nil, errors.Wrap(err, "decode replsetGetStatus response")
-	}
-
-	if err == nil {
-		result, err := json.Marshal(tempResult)
-		if err != nil {
-			return nil, errors.Wrap(err, "marshal temp result")
-		}
-
-		err = json.Unmarshal(result, &rsStatus)
-		if err != nil {
-			return nil, errors.Wrap(err, "unmarshal temp result")
-		}
+		return nil, errors.Wrap(err, "get replSetGetStatus response")
 	}
 
 	oplogRs := OplogRs{}
@@ -156,15 +119,12 @@ type OplogRs struct {
 	StorageSize      int64 `bson:"storageSize" json:"storageSize"`
 }
 
-type ReplSetStatus struct {
-	InitialSyncStatus InitialSyncStatus `bson:"initialSyncStatus" json:"initialSyncStatus"`
-	mongo.Status      `bson:",inline"`
-}
-
-type InitialSyncStatus interface{}
-
-func CheckState(rs ReplSetStatus, startupDelaySeconds int64, oplogSize int64) error {
-	uptime := rs.GetSelf().Uptime
+func CheckState(rs mongo.Status, startupDelaySeconds int64, oplogSize int64) error {
+	self := rs.GetSelf()
+	if self == nil {
+		return errors.New("self member is not found")
+	}
+	uptime := self.Uptime
 
 	switch rs.MyState {
 	case mongo.MemberStatePrimary, mongo.MemberStateSecondary, mongo.MemberStateArbiter: