SERVER-103741 Profiler collection metrics (#40142)

GitOrigin-RevId: feff19d43493819b330bed1844768299e8ae6b89

Author: louiswilliams

jstests/noPassthrough/profiler_stats.js

@@ -0,0 +1,31 @@
+/**
+ * Tests for serverStatus profiler stats.
+ */
+const conn = MongoRunner.runMongod();
+assert.neq(null, conn, "mongod was unable to start up");
+const db = conn.getDB(jsTest.name());
+const coll = db[jsTest.name()];
+
+const nDocs = 32;
+coll.drop();
+assert.commandWorked(db.createCollection(jsTest.name()));
+
+for (let i = 0; i < nDocs; i++) {
+    assert.commandWorked(coll.insert({a: i}));
+}
+
+// Enable profiling and run a query that implicitly creates the profile collection.
+assert.commandWorked(db.setProfilingLevel(2, 0));
+assert.eq(nDocs, coll.find({}).itcount());
+
+// Turn off the profiler to avoid incrementing the stats further.
+assert.commandWorked(db.setProfilingLevel(0, 0));
+
+let profilerEntries = db.system.profile.find({}).itcount();
+assert.eq(1, profilerEntries);
+
+let profilerWriteStats = db.serverStatus().profiler;
+assert.eq(0, profilerWriteStats.activeWriters);
+assert.eq(profilerWriteStats.totalWrites, profilerEntries);
+
+MongoRunner.stopMongod(conn);

jstests/noPassthrough/server_status_catalog_stats.js

@@ -8,7 +8,14 @@
 (function() {
 "use strict";
 
-const replSet = new ReplSetTest({nodes: 2});
+// Construct a replica set where the same node remains primary across restarts, because the
+// system.profile collection is not replicated.
+const replSet = new ReplSetTest({
+    nodes: [
+        {},
+        {rsConfig: {priority: 0}},
+    ]
+});
 replSet.startSet();
 replSet.initiate();
 
@@ -26,6 +33,7 @@ assertCatalogStats(db1, (stats) => {
     assert.eq(0, stats.capped);
     assert.eq(0, stats.clustered);
     assert.eq(0, stats.collections);
+    assert.eq(0, stats.systemProfile);
     assert.eq(0, stats.timeseries);
     assert.eq(0, stats.views);
     internalCollectionsAtStart = stats.internalCollections;
@@ -39,14 +47,25 @@ assert.commandWorked(
 assert.commandWorked(db1.createCollection('view', {viewOn: 'coll', pipeline: []}));
 assert.commandWorked(db1.createCollection('ts', {timeseries: {timeField: 't'}}));
 
+// A system.views and system.buckets collection should have been created.
+let internalCollectionsCreated = 2;
+
+// Create the profile collection.
+assert.commandWorked(db1.setProfilingLevel(2, 0));
+assert.eq(1, db1.coll.find({}).itcount());
+internalCollectionsCreated += 1;
+
+// Turn off profiler to avoid creating extra collections.
+assert.commandWorked(db1.setProfilingLevel(0, 100));
+
 assertCatalogStats(db1, (stats) => {
     assert.eq(1, stats.capped);
     assert.eq(1, stats.clustered);
     assert.eq(3, stats.collections);
+    assert.eq(1, stats.systemProfile);
     assert.eq(1, stats.timeseries);
     assert.eq(1, stats.views);
-    // A system.views and system.buckets collection should have been created.
-    assert.eq(internalCollectionsAtStart + 2, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
@@ -56,10 +75,11 @@ assertCatalogStats(db1, (stats) => {
     assert.eq(1, stats.capped);
     assert.eq(1, stats.clustered);
     assert.eq(3, stats.collections);
+    assert.eq(1, stats.systemProfile);
     assert.eq(1, stats.timeseries);
     assert.eq(1, stats.views);
     // An system.views and system.buckets collection should have been created.
-    assert.eq(internalCollectionsAtStart + 2, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
@@ -70,14 +90,17 @@ assert.commandWorked(
 assert.commandWorked(db2.createCollection('view', {viewOn: 'coll', pipeline: []}));
 assert.commandWorked(db2.createCollection('ts', {timeseries: {timeField: 't'}}));
 
+// An system.views and system.buckets collection should have been created.
+internalCollectionsCreated += 2;
+
 assertCatalogStats(db1, (stats) => {
     assert.eq(2, stats.capped);
     assert.eq(2, stats.clustered);
     assert.eq(6, stats.collections);
+    assert.eq(1, stats.systemProfile);
     assert.eq(2, stats.timeseries);
     assert.eq(2, stats.views);
-    // An system.views and system.buckets collection should have been created.
-    assert.eq(internalCollectionsAtStart + 4, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
@@ -92,9 +115,10 @@ assertCatalogStats(db1, (stats) => {
     assert.eq(2, stats.capped);
     assert.eq(2, stats.clustered);
     assert.eq(6, stats.collections);
+    assert.eq(1, stats.systemProfile);
     assert.eq(2, stats.timeseries);
     assert.eq(2, stats.views);
-    assert.eq(internalCollectionsAtStart + 4, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
@@ -104,39 +128,49 @@ assert(db1.clustered.drop());
 assert(db1.view.drop());
 assert(db1.ts.drop());
 
+// The system.buckets collection will be dropped, but not system.views.
+internalCollectionsCreated -= 1;
+
 assertCatalogStats(db1, (stats) => {
     assert.eq(1, stats.capped);
     assert.eq(1, stats.clustered);
     assert.eq(3, stats.collections);
+    assert.eq(1, stats.systemProfile);
     assert.eq(1, stats.timeseries);
     assert.eq(1, stats.views);
-    // The system.views collection will stick around
-    assert.eq(internalCollectionsAtStart + 3, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
 db1.dropDatabase();
 
+// The system.views and system.profile collections should be dropped.
+internalCollectionsCreated -= 2;
+
 assertCatalogStats(db1, (stats) => {
     assert.eq(1, stats.capped);
     assert.eq(3, stats.collections);
+    // The system.profile collection should be dropped.
+    assert.eq(0, stats.systemProfile);
     assert.eq(1, stats.timeseries);
     assert.eq(1, stats.views);
-    // The system.views collection should be dropped
-    assert.eq(internalCollectionsAtStart + 2, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 
 db2.dropDatabase();
 
+// The system.views and system.buckets collections should be dropped.
+internalCollectionsCreated -= 2;
+
 assertCatalogStats(db1, (stats) => {
     assert.eq(0, stats.capped);
     assert.eq(0, stats.clustered);
     assert.eq(0, stats.collections);
+    assert.eq(0, stats.systemProfile);
     assert.eq(0, stats.timeseries);
     assert.eq(0, stats.views);
-    // The system.views collection should be dropped
-    assert.eq(internalCollectionsAtStart, stats.internalCollections);
+    assert.eq(internalCollectionsAtStart + internalCollectionsCreated, stats.internalCollections);
     assert.eq(internalViewsAtStart, stats.internalViews);
 });
 

src/mongo/db/catalog/catalog_stats.cpp

@@ -65,6 +65,7 @@ class CatalogStatsSSS : public ServerStatusSection {
         int timeseriesExtendedRange = 0;
         int csfle = 0;
         int queryableEncryption = 0;
+        int systemProfile = 0;
 
         void toBson(BSONObjBuilder* builder) const {
             builder->append("collections", collections);
@@ -79,6 +80,7 @@ class CatalogStatsSSS : public ServerStatusSection {
             }
             builder->append("csfle", csfle);
             builder->append("queryableEncryption", queryableEncryption);
+            builder->append("systemProfile", systemProfile);
         }
     };
 
@@ -95,6 +97,7 @@ class CatalogStatsSSS : public ServerStatusSection {
         stats.timeseriesExtendedRange = requiresTimeseriesExtendedRangeSupport.load();
         stats.csfle = catalogStats.csfle;
         stats.queryableEncryption = catalogStats.queryableEncryption;
+        stats.systemProfile = catalogStats.systemProfile;
 
         const auto viewCatalogDbNames = catalog->getViewCatalogDbNames(opCtx);
         for (const auto& dbName : viewCatalogDbNames) {

src/mongo/db/catalog/collection_catalog.cpp

@@ -2256,19 +2256,27 @@ void CollectionCatalog::_registerCollection(OperationContext* opCtx,
     }
 
 
-    if (!nss.isOnInternalDb() && !nss.isSystem()) {
-        _stats.userCollections += 1;
-        if (coll->isCapped()) {
-            _stats.userCapped += 1;
-        }
-        if (coll->isClustered()) {
-            _stats.userClustered += 1;
-        }
-        if (coll->getCollectionOptions().encryptedFieldConfig) {
-            _stats.queryableEncryption += 1;
+    if (!nss.isOnInternalDb()) {
+        if (!nss.isSystem()) {
+            _stats.userCollections += 1;
+            if (coll->isCapped()) {
+                _stats.userCapped += 1;
+            }
+            if (coll->isClustered()) {
+                _stats.userClustered += 1;
+            }
+            if (coll->getCollectionOptions().encryptedFieldConfig) {
+                _stats.queryableEncryption += 1;
+            }
+            if (isCSFLE1Validator(coll->getValidatorDoc())) {
+                _stats.csfle += 1;
+            }
+        } else {
+            _stats.internal += 1;
         }
-        if (isCSFLE1Validator(coll->getValidatorDoc())) {
-            _stats.csfle += 1;
+
+        if (nss.isSystemDotProfile()) {
+            _stats.systemProfile += 1;
         }
     } else {
         _stats.internal += 1;
@@ -2327,19 +2335,27 @@ std::shared_ptr<Collection> CollectionCatalog::deregisterCollection(
         _pushCatalogIdForNSSAndUUID(ns, uuid, boost::none, commitTime);
     }
 
-    if (!ns.isOnInternalDb() && !ns.isSystem()) {
-        _stats.userCollections -= 1;
-        if (coll->isCapped()) {
-            _stats.userCapped -= 1;
-        }
-        if (coll->isClustered()) {
-            _stats.userClustered -= 1;
-        }
-        if (coll->getCollectionOptions().encryptedFieldConfig) {
-            _stats.queryableEncryption -= 1;
+    if (!ns.isOnInternalDb()) {
+        if (!ns.isSystem()) {
+            _stats.userCollections -= 1;
+            if (coll->isCapped()) {
+                _stats.userCapped -= 1;
+            }
+            if (coll->isClustered()) {
+                _stats.userClustered -= 1;
+            }
+            if (coll->getCollectionOptions().encryptedFieldConfig) {
+                _stats.queryableEncryption -= 1;
+            }
+            if (isCSFLE1Validator(coll->getValidatorDoc())) {
+                _stats.csfle -= 1;
+            }
+        } else {
+            _stats.internal -= 1;
         }
-        if (isCSFLE1Validator(coll->getValidatorDoc())) {
-            _stats.csfle -= 1;
+
+        if (ns.isSystemDotProfile()) {
+            _stats.systemProfile -= 1;
         }
     } else {
         _stats.internal -= 1;

src/mongo/db/catalog/collection_catalog.h

@@ -670,6 +670,8 @@ class CollectionCatalog {
         int csfle = 0;
         // Queryable Encryption collections on non-internal databases
         int queryableEncryption = 0;
+        // <>.system.profile collections on non-internal databases. Counted as `internal`.
+        int systemProfile = 0;
     };
 
     /**

src/mongo/db/introspect.cpp

@@ -34,6 +34,7 @@
 #include "mongo/db/auth/authorization_session.h"
 #include "mongo/db/catalog/collection_write_path.h"
 #include "mongo/db/client.h"
+#include "mongo/db/commands/server_status.h"
 #include "mongo/db/concurrency/exception_util.h"
 #include "mongo/db/concurrency/lock_state.h"
 #include "mongo/db/curop.h"
@@ -52,6 +53,32 @@ using std::endl;
 using std::string;
 using std::unique_ptr;
 
+namespace {
+
+AtomicWord<int64_t> profilerWritesTotal{0};
+AtomicWord<int64_t> profilerWritesActive{0};
+
+class ProfilerSection : public ServerStatusSection {
+public:
+    ProfilerSection() : ServerStatusSection("profiler") {}
+
+    ~ProfilerSection() override = default;
+
+    bool includeByDefault() const override {
+        return true;
+    }
+
+    BSONObj generateSection(OperationContext* opCtx,
+                            const BSONElement& configElement) const override {
+        BSONObjBuilder bob;
+        bob.append("totalWrites", profilerWritesTotal.loadRelaxed());
+        bob.append("activeWriters", profilerWritesActive.loadRelaxed());
+        return bob.obj();
+    }
+} profilerSection;
+
+}  // namespace
+
 void profile(OperationContext* opCtx, NetworkOp op) {
     // Initialize with 1kb at start in order to avoid realloc later
     BufBuilder profileBufBuilder(1024);
@@ -106,6 +133,10 @@ void profile(OperationContext* opCtx, NetworkOp op) {
         });
         AlternativeClientRegion acr(newClient);
         const auto dbProfilingNS = NamespaceString::makeSystemDotProfileNamespace(ns.dbName());
+
+        profilerWritesActive.fetchAndAddRelaxed(1);
+        ON_BLOCK_EXIT([&] { profilerWritesActive.fetchAndAddRelaxed(-1); });
+
         AutoGetCollection autoColl(newCtx.get(), dbProfilingNS, MODE_IX);
         Database* const db = autoColl.getDb();
         if (!db) {
@@ -126,6 +157,7 @@ void profile(OperationContext* opCtx, NetworkOp op) {
         uassertStatusOK(collection_internal::insertDocument(
             newCtx.get(), coll, InsertStatement(p), nullOpDebug, false));
         wuow.commit();
+        profilerWritesTotal.fetchAndAddRelaxed(1);
     } catch (const AssertionException& assertionEx) {
         LOGV2_WARNING(20703,
                       "Caught Assertion while trying to profile {operation} against "