Merge branch 'main' into PMM-9870-fix-collstats-indexsizes-metrics

Author: BupycHuk

.github/workflows/go.yml

@@ -43,13 +43,22 @@ jobs:
         with:
           go-version-file: ${{ github.workspace }}/go.mod
 
-      - name: Test
+      - name: Run tests with code coverage
         run: |
           TEST_MONGODB_IMAGE=${{ matrix.image }} make test-cluster
           sleep 10
-          make test-race
+          make test-cover
           make test-cluster-clean
 
+
+      - name: Upload coverage results
+        uses: codecov/codecov-action@v3
+        with:
+          file: cover.out
+          flags: agent
+          env_vars: GO_VERSION,TEST_MONGODB_IMAGE
+          fail_ci_if_error: false
+
       - name: Run debug commands on failure
         if: ${{ failure() }}
         run: |

.github/workflows/scorecard.yml

@@ -0,0 +1,48 @@
+name: Scorecard
+on:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: "24 3 * * 1"
+  push:
+    branches:
+      - main
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+        with:
+          persist-credentials: false
+
+      - name: Run analysis
+        uses: ossf/scorecard-action@0864cf19026789058feabb7e87baa5f140aac736 # v2.3.1
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          publish_results: true
+
+      - name: Upload results
+        uses: actions/upload-artifact@97a0fba1372883ab732affbe8f94b823f91727db # v3.pre.node20
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard (optional).
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@v3
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -1,9 +1,13 @@
 .dist
 .env
+
 .vscode/
 .idea
+
 bin
 build
 dist
+
+cover.out
 mongodb_exporter
 .DS_Store

Makefile

@@ -50,12 +50,12 @@ define TEST_ENV
 	TEST_MONGODB_S1_PRIMARY_PORT=$(TEST_MONGODB_S1_PRIMARY_PORT) \
 	TEST_MONGODB_S1_SECONDARY1_PORT=$(TEST_MONGODB_S1_SECONDARY1_PORT) \
 	TEST_MONGODB_S1_SECONDARY2_PORT=$(TEST_MONGODB_S1_SECONDARY2_PORT) \
-	TEST_MONGODB_S1_ARTBITER_PORT=$(TEST_MONGODB_S1_ARBITER_PORT) \
+	TEST_MONGODB_S1_ARBITER_PORT=$(TEST_MONGODB_S1_ARBITER_PORT) \
 	TEST_MONGODB_S2_RS=$(TEST_MONGODB_S2_RS) \
 	TEST_MONGODB_S2_PRIMARY_PORT=$(TEST_MONGODB_S2_PRIMARY_PORT) \
 	TEST_MONGODB_S2_SECONDARY1_PORT=$(TEST_MONGODB_S2_SECONDARY1_PORT) \
 	TEST_MONGODB_S2_SECONDARY2_PORT=$(TEST_MONGODB_S2_SECONDARY2_PORT) \
-	TEST_MONGODB_S2_ARTBITER_PORT=$(TEST_MONGODB_S2_ARBITER_PORT) \
+	TEST_MONGODB_S2_ARBITER_PORT=$(TEST_MONGODB_S2_ARBITER_PORT) \
 	TEST_MONGODB_CONFIGSVR_RS=$(TEST_MONGODB_CONFIGSVR_RS) \
 	TEST_MONGODB_CONFIGSVR1_PORT=$(TEST_MONGODB_CONFIGSVR1_PORT) \
 	TEST_MONGODB_CONFIGSVR2_PORT=$(TEST_MONGODB_CONFIGSVR2_PORT) \
@@ -67,7 +67,7 @@ endef
 env:
 	@echo $(TEST_ENV) | tr ' ' '\n' >.env
 
-init:                       ## Install linters.
+init:                       ## Install linters
 	cd tools && go generate -x -tags=tools
 
 build:                      ## Compile using plain go build
@@ -82,32 +82,34 @@ release:                      ## Build the binaries using goreleaser
 
 FILES = $(shell find . -type f -name '*.go' -not -path "./vendor/*")
 
-format:                     ## Format source code.
+format:                     ## Format source code
 	go mod tidy
 	bin/gofumpt -l -w $(FILES)
 	bin/gci write --section Standard --section Default --section "Prefix(github.com/percona/mongodb_exporter)" .
 
 check:                      ## Run checks/linters
 	bin/golangci-lint run
 
-check-license:              ## Check license in headers.
+check-license:              ## Check license in headers
 	@go run .github/check-license.go
 
-help:                       ## Display this help message.
+help:                       ## Display this help message
 	@echo "Please use \`make <target>\` where <target> is one of:"
 	@grep '^[a-zA-Z]' $(MAKEFILE_LIST) | \
 	awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n", $$1, $$2}'
 
-test: env                   ## Run all tests.
+test: env                   ## Run all tests
 	go test -v -count 1 -timeout 30s ./...
 
-test-race: env              ## Run all tests with race flag.
+test-race: env              ## Run all tests with race flag
 	go test -race -v -timeout 30s ./...
 
+test-cover: env              ## Run tests and collect cross-package coverage information
+	go test -race -timeout 30s -coverprofile=cover.out -covermode=atomic -coverpkg=./... ./...
+
 test-cluster: env           ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster
 	docker compose up --build -d
 	./docker/scripts/setup-pbm.sh
 
 test-cluster-clean: env     ## Stops MongoDB test cluster.
-	docker compose down --remove-orphans
-
+	docker compose down --remove-orphans --volumes

README.md

@@ -3,7 +3,9 @@
 [![Build Status](https://github.com/percona/mongodb_exporter/actions/workflows/go.yml/badge.svg?branch=main)](https://github.com/percona/mongodb_exporter/actions/workflows/go.yml?query=branch%3Amain)
 [![codecov.io Code Coverage](https://img.shields.io/codecov/c/github/percona/mongodb_exporter.svg?maxAge=2592000)](https://codecov.io/github/percona/mongodb_exporter?branch=main)
 [![Go Report Card](https://goreportcard.com/badge/github.com/percona/mongodb_exporter)](https://goreportcard.com/report/github.com/percona/mongodb_exporter)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/percona/mongodb_exporter/badge)](https://scorecard.dev/viewer/?uri=github.com/percona/mongodb_exporter)
 [![CLA assistant](https://cla-assistant.percona.com/readme/badge/percona/mongodb_exporter)](https://cla-assistant.percona.com/percona/mongodb_exporter)
+[![Forum](https://img.shields.io/badge/Forum-join-brightgreen)](https://forums.percona.com/)
 
 
 This is the new MongoDB exporter implementation that handles ALL metrics exposed by MongoDB monitoring commands.

docker-compose.yml

@@ -176,7 +176,7 @@ services:
         container_name: "mongo-cnf-2"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
-            - "${TEST_MONGODB_CONFIGSVR1_PORT:-17007}:27017"
+            - "${TEST_MONGODB_CONFIGSVR2_PORT:-17008}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
         networks:
             - cnf-serv
@@ -185,7 +185,7 @@ services:
         container_name: "mongo-cnf-3"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
-            - "${TEST_MONGODB_CONFIGSVR2_PORT:-17008}:27017"
+            - "${TEST_MONGODB_CONFIGSVR3_PORT:-17009}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
         networks:
             - cnf-serv
@@ -194,7 +194,7 @@ services:
         container_name: "mongo-cnf-1"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
-            - "${TEST_MONGODB_CONFIGSVR3_PORT:-17009}:27017"
+            - "${TEST_MONGODB_CONFIGSVR1_PORT:-17007}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
         networks:
             - cnf-serv

docker/scripts/setup.sh

@@ -100,6 +100,7 @@ function general_servers() {
 EOF
 }
 
+
 case $1 in
     cnf_servers)
         cnf_servers

exporter/diagnostic_data_collector.go

@@ -117,7 +117,7 @@ func (d *diagnosticDataCollector) collect(ch chan<- prometheus.Metric) {
 		debugResult(logger, m)
 
 		// MongoDB 8.0 splits the diagnostic data into multiple blocks, so we need to merge them
-		if d.buildInfo.VersionArray[0] >= 8 { //nolint:gomnd
+		if _, ok := m["common"]; ok {
 			b := bson.M{}
 			for _, mv := range m {
 				block, ok := mv.(bson.M)

main.go

@@ -276,18 +276,36 @@ func parseURIList(uriList []string, logger *logrus.Logger, splitCluster bool) []
 	return URIs
 }
 
+// buildURIManually builds the URI manually by checking if the user and password are supplied
+func buildURIManually(uri string, user string, password string) string {
+	uriArray := strings.SplitN(uri, "://", 2) //nolint:mnd
+	prefix := uriArray[0] + "://"
+	uri = uriArray[1]
+
+	// IF user@pass not contained in uri AND custom user and pass supplied in arguments
+	// DO concat a new uri with user and pass arguments value
+	if !strings.Contains(uri, "@") && user != "" && password != "" {
+		// add user and pass to the uri
+		uri = fmt.Sprintf("%s:%s@%s", user, password, uri)
+	}
+
+	// add back prefix after adding the user and pass
+	uri = prefix + uri
+
+	return uri
+}
+
 func buildURI(uri string, user string, password string, log *logrus.Logger) string {
 	defaultPrefix := "mongodb://" // default prefix
-	matchRegexp := regexp.MustCompile(`^mongodb(\+srv)?://`)
 
-	// Split the uri defaultPrefix if there is any
-	if !matchRegexp.MatchString(uri) {
+	if !strings.HasPrefix(uri, defaultPrefix) && !strings.HasPrefix(uri, "mongodb+srv://") {
 		uri = defaultPrefix + uri
 	}
 	parsedURI, err := url.Parse(uri)
 	if err != nil {
-		log.Fatalf("Failed to parse URI %s: %v", uri, err)
-		return uri
+		// PMM generates URI with escaped path to socket file, so url.Parse fails
+		// in this case we build URI manually
+		return buildURIManually(uri, user, password)
 	}
 
 	if parsedURI.User == nil && user != "" && password != "" {

main_test.go

@@ -227,6 +227,48 @@ func TestBuildURI(t *testing.T) {
 			newPassword: "yyy?!#$%^&*()_+",
 			expect:      "mongodb://xxx%3F%21%23$%25%5E&%2A%28%29_+:yyy%3F%21%23$%25%5E&%2A%28%[email protected]",
 		},
+		{
+			situation:   "path to socket",
+			origin:      "mongodb:///tmp/mongodb-27017.sock",
+			newUser:     "",
+			newPassword: "",
+			expect:      "mongodb:///tmp/mongodb-27017.sock",
+		},
+		{
+			situation:   "path to socket with params",
+			origin:      "mongodb://username:s3cur3%20p%40$$w0r4.@%2Fvar%2Frun%2Fmongodb%2Fmongodb.sock/database?connectTimeoutMS=1000&directConnection=true&serverSelectionTimeoutMS=1000",
+			newUser:     "",
+			newPassword: "",
+			expect:      "mongodb://username:s3cur3%20p%40$$w0r4.@%2Fvar%2Frun%2Fmongodb%2Fmongodb.sock/database?connectTimeoutMS=1000&directConnection=true&serverSelectionTimeoutMS=1000",
+		},
+		{
+			situation:   "path to socket with auth",
+			origin:      "mongodb://xxx:yyy@/tmp/mongodb-27017.sock",
+			newUser:     "",
+			newPassword: "",
+			expect:      "mongodb://xxx:yyy@/tmp/mongodb-27017.sock",
+		},
+		{
+			situation:   "path to socket with auth and user params",
+			origin:      "mongodb:///tmp/mongodb-27017.sock",
+			newUser:     "xxx",
+			newPassword: "yyy",
+			expect:      "mongodb://xxx:yyy@/tmp/mongodb-27017.sock",
+		},
+		{
+			situation:   "path to socket without prefix",
+			origin:      "/tmp/mongodb-27017.sock",
+			newUser:     "",
+			newPassword: "",
+			expect:      "mongodb:///tmp/mongodb-27017.sock",
+		},
+		{
+			situation:   "path to socket without prefix with auth",
+			origin:      "/tmp/mongodb-27017.sock",
+			newUser:     "xxx",
+			newPassword: "yyy",
+			expect:      "mongodb://xxx:yyy@/tmp/mongodb-27017.sock",
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.situation, func(t *testing.T) {