Merge branch 'main' into PMM-7-use-image-from-env-for-encrypted

Author: BupycHuk

.github/workflows/go.yml

@@ -32,7 +32,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version-file: ${{ github.workspace }}/go.mod
 

.github/workflows/lint.yml

@@ -23,7 +23,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version-file: ${{ github.workspace }}/go.mod
 

.github/workflows/release.yml

@@ -18,7 +18,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version-file: ${{ github.workspace }}/go.mod
 

Makefile

@@ -104,7 +104,7 @@ test-race: env              ## Run all tests with race flag.
 	go test -race -v -timeout 30s ./...
 
 test-cluster: env           ## Starts MongoDB test cluster. Use env var TEST_MONGODB_IMAGE to set flavor and version. Example: TEST_MONGODB_IMAGE=mongo:3.6 make test-cluster
-	docker compose up -d --wait
+	docker compose up -d
 
 test-cluster-clean: env     ## Stops MongoDB test cluster.
 	docker compose down --remove-orphans

README.md

@@ -128,6 +128,17 @@ Usage example: `db.setProfilingLevel(2)`
 |1| The profiler collects data for operations that take longer than the value of `slowms` or that match a filter.<br> When a filter is set: <ul><li> The `slowms` and `sampleRate` options are not used for profiling.</li><li>The profiler only captures operations that match the filter.</li></ul>
 |2|The profiler collects data for all operations.|
 
+#### Enabling shards metrics gathering
+When shard metrics collection is enabled by `--collector.shards`, the exporter will expose metrics related to sharded Mongo. 
+Example, if shards collector is enabled:
+```
+# HELP mongodb_shards_collection_chunks_count sharded collection chunks.
+# TYPE mongodb_shards_collection_chunks_count counter
+mongodb_shards_collection_chunks_count{collection="system.sessions",database="config",shard="rs1"} 250
+mongodb_shards_collection_chunks_count{collection="system.sessions",database="config",shard="rs2"} 250
+```
+You can see shard name, it's collection, database and count.
+
 #### Cluster role labels
 The exporter sets some topology labels in all metrics.
 The labels are:

REFERENCE.md

@@ -27,5 +27,6 @@
 |--collector.collstats-limit=0|Disable collstats, dbstats, topmetrics and indexstats collector if there are more than \<n\> collections. 0=No limit|
 |--collector.profile-time-ts=30|Set time for scrape slow queries| This interval must be synchronized with the Prometheus scrape interval|
 |--collector.profile|Enable collecting metrics from profile|
+|--collector.shards|Enable collecting metrics related to Mongo shards|
 |--metrics.overridedescendingindex| Enable descending index name override to replace -1 with _DESC ||
 |--version|Show version and exit|

docker-compose.yml

@@ -6,30 +6,39 @@ services:
         ports:
             - "${TEST_MONGODB_S1_PRIMARY_PORT:-17001}:27017"
         command: mongod --replSet rs1 --shardsvr --port 27017  --oplogSize 16
-        links:
-            - mongo-1-2:mongo-1-2
-            - mongo-1-3:mongo-1-3
+        networks:
+            - rs1
+        depends_on:
+          - "mongo-1-2"
+          - "mongo-1-3"
+          - "mongo-1-arbiter"
 
     mongo-1-2:
         container_name: "mongo-1-2"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S1_SECONDARY1_PORT:-17002}:27017"
         command: mongod --replSet rs1 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs1
 
     mongo-1-3:
         container_name: "mongo-1-3"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S1_SECONDARY2_PORT:-17003}:27017"
         command: mongod --replSet rs1 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs1
 
     mongo-1-arbiter:
         container_name: "mongo-1-arbiter"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S1_ARBITER:-17011}:27017"
         command: mongod --replSet rs1 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs1
 
     mongo-rs1-setup:
         container_name: "mongo-rs1-setup"
@@ -39,11 +48,6 @@ services:
             - "mongo-1-2"
             - "mongo-1-3"
             - "mongo-1-arbiter"
-        links:
-            - mongo-1-1:mongo-1-1
-            - mongo-1-2:mongo-1-2
-            - mongo-1-3:mongo-1-3
-            - mongo-1-arbiter:mongo-1-arbiter
         volumes:
             - ./docker/scripts:/scripts
         environment:
@@ -52,38 +56,46 @@ services:
             - MONGO3=mongo-1-3
             - ARBITER=mongo-1-arbiter
             - RS=rs1
+            - VERSION=${TEST_MONGODB_IMAGE}
         entrypoint: [ "/scripts/setup.sh" ]
+        networks:
+            - rs1
 
     mongo-2-2:
         container_name: "mongo-2-2"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S2_PRIMARY_PORT:-17004}:27017"
         command: mongod --replSet rs2 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs2
 
     mongo-2-3:
         container_name: "mongo-2-3"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S2_SECONDARY1_PORT:-17005}:27017"
         command: mongod --replSet rs2 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs2
 
     mongo-2-1:
         container_name: "mongo-2-1"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S2_SECONDARY2_PORT:-17006}:27017"
         command: mongod --replSet rs2 --shardsvr --port 27017  --oplogSize 16
-        links:
-            - mongo-2-2:mongo-2-2
-            - mongo-2-3:mongo-2-3
+        networks:
+            - rs2
 
     mongo-2-arbiter:
         container_name: "mongo-2-arbiter"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_S2_ARBITER:-17012}:27017"
         command: mongod --replSet rs1 --shardsvr --port 27017  --oplogSize 16
+        networks:
+            - rs2
 
     mongo-rs2-setup:
         container_name: "mongo-rs2-setup"
@@ -93,19 +105,18 @@ services:
             - "mongo-2-2"
             - "mongo-2-3"
             - "mongo-2-arbiter"
-        links:
-            - mongo-2-1:mongo-2-1
-            - mongo-2-2:mongo-2-2
-            - mongo-2-3:mongo-2-3
         volumes:
             - ./docker/scripts:/scripts
         environment:
-            - MONGO1=mongo-2-1
-            - MONGO2=mongo-2-2
+            - MONGO1=mongo-2-2
+            - MONGO2=mongo-2-1
             - MONGO3=mongo-2-3
             - ARBITER=mongo-2-arbiter
             - RS=rs2
+            - VERSION=${TEST_MONGODB_IMAGE}
         entrypoint: [ "/scripts/setup.sh" ]
+        networks:
+            - rs2
 
     # Config servers
     mongo-cnf-2:
@@ -114,23 +125,29 @@ services:
         ports:
             - "${TEST_MONGODB_CONFIGSVR1_PORT:-17007}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
+        networks:
+            - cnf-serv
 
     mongo-cnf-3:
         container_name: "mongo-cnf-3"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_CONFIGSVR2_PORT:-17008}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
+        networks:
+            - cnf-serv
 
     mongo-cnf-1:
         container_name: "mongo-cnf-1"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_CONFIGSVR3_PORT:-17009}:27017"
         command: mongod --dbpath /data/db --replSet cnf-serv --configsvr --port 27017 --oplogSize 16
-        links:
-            - mongo-cnf-2:mongo-cnf-2
-            - mongo-cnf-3:mongo-cnf-3
+        networks:
+            - cnf-serv
+        depends_on:
+            - mongo-cnf-2
+            - mongo-cnf-3
 
     mongo-cnf-setup:
         container_name: "mongo-cnf-setup"
@@ -139,10 +156,6 @@ services:
             - "mongo-cnf-1"
             - "mongo-cnf-2"
             - "mongo-cnf-3"
-        links:
-            - mongo-cnf-1:mongo-cnf-1
-            - mongo-cnf-2:mongo-cnf-2
-            - mongo-cnf-3:mongo-cnf-3
         volumes:
             - ./docker/scripts:/scripts
         environment:
@@ -151,36 +164,34 @@ services:
             - MONGO3=mongo-cnf-3
             - RS=cnf-serv
             - PORT=27017
+            - VERSION=${TEST_MONGODB_IMAGE}
         entrypoint: [ "/scripts/setup.sh","cnf_servers" ]
+        networks:
+            - cnf-serv
 
     mongos:
         container_name: "mongos"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
             - "${TEST_MONGODB_MONGOS_PORT:-17000}:27017"
+        networks:
+            - mongo-shard
+            - rs1
+            - rs2
+            - cnf-serv
         depends_on:
             - "mongo-rs1-setup"
             - "mongo-rs2-setup"
             - "mongo-cnf-setup"
-        external_links:
-            - mongo-cnf-1:mongo-cnf-1
-            - mongo-cnf-2:mongo-cnf-2
-            - mongo-cnf-3:mongo-cnf-3
-            - mongo-1-1:mongo-1-1
-            - mongo-1-2:mongo-1-2
-            - mongo-1-3:mongo-1-3
-            - mongo-2-1:mongo-2-1
-            - mongo-2-2:mongo-2-2
-            - mongo-2-3:mongo-2-3
         command: mongos --configdb cnf-serv/mongo-cnf-1:27017,mongo-cnf-2:27017,mongo-cnf-3:27017 --port 27017 --bind_ip 0.0.0.0
 
     mongo-shard-setup:
         container_name: "mongo-shard-setup"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         depends_on:
             - "mongos"
-        links:
-            - mongos:mongos
+        networks:
+            - mongo-shard
         volumes:
             - ./docker/scripts:/scripts
         environment:
@@ -197,6 +208,7 @@ services:
             - PORT1=27017
             - PORT2=27017
             - PORT3=27017
+            - VERSION=${TEST_MONGODB_IMAGE}
         entrypoint: [ "/scripts/init-shard.sh" ]
         restart: on-failure:20
 
@@ -208,6 +220,7 @@ services:
         command: mongod --port 27017  --oplogSize 16
 
     standalone-encrypted:
+        user: root
         container_name: "standalone-encrypted"
         image: ${TEST_MONGODB_IMAGE:-mongo:4.4}
         ports:
@@ -216,3 +229,9 @@ services:
             - ./docker/secret/mongodb_secrets.txt:/secret/mongodb_secrets.txt
             - ./docker/scripts:/scripts
         command: /scripts/run-mongodb-encrypted.sh
+
+networks:
+    rs1:
+    rs2:
+    cnf-serv:
+    mongo-shard:

docker/scripts/init-shard.sh

@@ -1,4 +1,16 @@
 #!/bin/bash 
+# `mongosh` is used starting from MongoDB 5.x
+MONGODB_CLIENT="mongosh --quiet"
+PARSED=(${VERSION//:/ })
+MONGODB_VERSION=${PARSED[1]}
+MONGODB_VENDOR=${PARSED[0]}
+if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "4" ]; then
+  MONGODB_CLIENT="mongo"
+fi
+if [ "`echo ${MONGODB_VERSION} | cut -c 1`" = "5" ] && [ ${MONGODB_VENDOR} == "percona/percona-server-mongodb" ]; then
+  MONGODB_CLIENT="mongo"
+fi
+echo "MongoDB vendor, client and version: ${MONGODB_VENDOR} ${MONGODB_CLIENT} ${MONGODB_VERSION}"
 
 mongodb1=`getent hosts ${MONGOS} | awk '{ print $1 }'`
 
@@ -17,16 +29,20 @@ mongodb33=`getent hosts ${MONGO33} | awk '{ print $1 }'`
 port=${PORT:-27017}
 
 echo "Waiting for startup.."
-until mongo --host ${mongodb1}:${port} --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do
+until ${MONGODB_CLIENT} --host ${mongodb1}:${port} --eval 'quit(db.runCommand({ ping: 1 }).ok ? 0 : 2)' &>/dev/null; do
   printf '.'
   sleep 1
 done
 
 echo "Started.."
 
 echo init-shard.sh time now: `date +"%T" `
-mongo --host ${mongodb1}:${port} <<EOF
+${MONGODB_CLIENT} --host ${mongodb1}:${port} <<EOF
    sh.addShard( "${RS1}/${mongodb11}:${PORT1},${mongodb12}:${PORT2},${mongodb13}:${PORT3}" );
    sh.addShard( "${RS2}/${mongodb21}:${PORT1},${mongodb22}:${PORT2},${mongodb23}:${PORT3}" );
+   use test;
+   db.createCollection("shard");
+   sh.enableSharding("test");
+   sh.shardCollection( "test.shard", { id: "hashed" }, false, { numInitialChunks: 500, collation: { locale: "simple" }} );
    sh.status();
 EOF