PMM-9312 default collectors (#396)

* release-0.30.0

* PMM-9312 Added new filters for discovering mode

* Added flags for collstats & indexstats

* Update main.go

Co-authored-by: Nurlan Moldomurov <[email protected]>

* Changes for CR

* Updates for CR

* Ran format

* Fixed top collector flag

* Updated filters

* PMM-9312 Fix MongoFilter (#399)

Co-authored-by: Nurlan Moldomurov <[email protected]>

Author: percona-csalguero

Makefile

@@ -110,4 +110,4 @@ test-cluster: env           ## Starts MongoDB test cluster. Use env var TEST_MON
 	docker-compose up -d
 
 test-cluster-clean: env     ## Stops MongoDB test cluster.
-	docker-compose down
+	docker-compose down --remove-orphans

exporter/collstats_collector.go

@@ -44,7 +44,7 @@ func (d *collstatsCollector) Collect(ch chan<- prometheus.Metric) {
 	collections := d.collections
 
 	if d.discoveringMode {
-		namespaces, err := listAllCollections(d.ctx, d.client, d.collections)
+		namespaces, err := listAllCollections(d.ctx, d.client, d.collections, systemDBs)
 		if err != nil {
 			d.logger.Errorf("cannot auto discover databases and collections")
 

exporter/common.go

@@ -2,6 +2,8 @@ package exporter
 
 import (
 	"context"
+	"sort"
+	"strings"
 
 	"github.com/AlekSi/pointer"
 	"github.com/pkg/errors"
@@ -13,81 +15,133 @@ import (
 
 var systemDBs = []string{"admin", "config", "local"} //nolint:gochecknoglobals
 
-func listCollections(ctx context.Context, client *mongo.Client, collections []string, database string) ([]string, error) {
+func listCollections(ctx context.Context, client *mongo.Client, database string, filterInNamespaces []string) ([]string, error) {
 	filter := bson.D{} // Default=empty -> list all collections
 
 	// if there is a filter with the list of collections we want, create a filter like
 	// $or: {
 	//     {"$regex": "collection1"},
 	//     {"$regex": "collection2"},
 	// }
-	if len(collections) > 0 {
+	if len(filterInNamespaces) > 0 {
 		matchExpressions := []bson.D{}
 
-		for _, collection := range collections {
-			matchExpressions = append(matchExpressions,
-				bson.D{{Key: "name", Value: primitive.Regex{Pattern: collection, Options: "i"}}})
+		for _, namespace := range filterInNamespaces {
+			parts := strings.Split(namespace, ".") // db.collection.name.with.dots
+			if len(parts) > 1 {
+				// The part before the first dot is the database name.
+				// The rest is the collection name and it can have dots. We need to rebuild it.
+				collection := strings.Join(parts[1:], ".")
+				matchExpressions = append(matchExpressions,
+					bson.D{{Key: "name", Value: primitive.Regex{Pattern: collection, Options: "i"}}})
+			}
 		}
 
-		filter = bson.D{{Key: "$or", Value: matchExpressions}}
+		if len(matchExpressions) > 0 {
+			filter = bson.D{{Key: "$or", Value: matchExpressions}}
+		}
 	}
 
-	databases, err := client.Database(database).ListCollectionNames(ctx, filter)
+	collections, err := client.Database(database).ListCollectionNames(ctx, filter)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot get the list of collections for discovery")
 	}
 
-	return databases, nil
+	return collections, nil
 }
 
-func databases(ctx context.Context, client *mongo.Client, exclude []string) ([]string, error) {
+// databases returns the list of databases matching the filters.
+// - filterInNamespaces: Include only the database names matching the any of the regular expressions in this list.
+//                       Case will be ignored because the function will automatically add the ignore case
+//                       flag to the regular expression.
+// - exclude: List of databases to be excluded. Useful to ignore system databases.
+func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []string, exclude []string) ([]string, error) {
 	opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)}
+
+	filter := bson.D{}
+
+	if excludeFilter := makeExcludeFilter(exclude); excludeFilter != nil {
+		filter = append(filter, *excludeFilter)
+	}
+
+	if namespacesFilter := makeDBsFilter(filterInNamespaces); namespacesFilter != nil {
+		filter = append(filter, *namespacesFilter)
+	}
+
+	dbNames, err := client.ListDatabaseNames(ctx, filter, opts)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot get the database names list")
+	}
+
+	return dbNames, nil
+}
+
+func makeExcludeFilter(exclude []string) *primitive.E {
 	filterExpressions := []bson.D{}
 	for _, dbname := range exclude {
 		filterExpressions = append(filterExpressions,
 			bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}},
 		)
 	}
 
-	filter := bson.D{{Key: "$and", Value: filterExpressions}}
+	if len(filterExpressions) == 0 {
+		return nil
+	}
 
-	dbNames, err := client.ListDatabaseNames(ctx, filter, opts)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot get the database names list")
+	return &primitive.E{Key: "$and", Value: filterExpressions}
+}
+
+// makeDBsFilter creates a filter to list all databases or only the ones in the specified
+// namespaces list. Namespaces have the form of <db>.<collection> and the collection name
+// can have a dot. Example: db1.collection.one -> db: db1, collection: collection.one
+// db1, db2.col2, db3.col.one will produce [db1, db2, db3].
+func makeDBsFilter(filterInNamespaces []string) *primitive.E {
+	dbs := []string{}
+	if len(dbs) == 0 {
+		return nil
 	}
 
-	return dbNames, nil
+	for _, namespace := range filterInNamespaces {
+		parts := strings.Split(namespace, ".")
+		dbs = append(dbs, parts[0])
+	}
+
+	return &primitive.E{Key: "name", Value: primitive.E{Key: "$in", Value: dbs}}
 }
 
-func listAllCollections(ctx context.Context, client *mongo.Client, filter []string) (map[string][]string, error) {
+func listAllCollections(ctx context.Context, client *mongo.Client, filterInNamespaces []string, excludeDBs []string) (map[string][]string, error) {
 	namespaces := make(map[string][]string)
-	// exclude system databases
-	dbnames, err := databases(ctx, client, systemDBs)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot get the list of all collections in the server")
-	}
 
-	for _, dbname := range dbnames {
-		colls, err := listCollections(ctx, client, filter, dbname)
+	for _, namespace := range filterInNamespaces {
+		parts := strings.Split(namespace, ".")
+		dbname := parts[0]
+
+		colls, err := listCollections(ctx, client, dbname, []string{namespace})
 		if err != nil {
 			return nil, errors.Wrapf(err, "cannot list the collections for %q", dbname)
 		}
-		namespaces[dbname] = colls
+
+		if _, ok := namespaces[dbname]; !ok {
+			namespaces[dbname] = colls
+		} else {
+			namespaces[dbname] = append(namespaces[dbname], colls...)
+		}
+		sort.Strings(namespaces[dbname]) // make it testeable.
 	}
 
 	return namespaces, nil
 }
 
-func allCollectionsCount(ctx context.Context, client *mongo.Client, filter []string) (int, error) {
-	databases, err := databases(ctx, client, systemDBs)
+func nonSystemCollectionsCount(ctx context.Context, client *mongo.Client, includeNamespaces []string, filterInCollections []string) (int, error) {
+	databases, err := databases(ctx, client, includeNamespaces, systemDBs)
 	if err != nil {
 		return 0, errors.Wrap(err, "cannot retrieve the collection names for count collections")
 	}
 
 	var count int
 
 	for _, dbname := range databases {
-		colls, err := listCollections(ctx, client, filter, dbname)
+		colls, err := listCollections(ctx, client, dbname, filterInCollections)
 		if err != nil {
 			return 0, errors.Wrap(err, "cannot get collections count")
 		}

exporter/common_test.go

@@ -19,36 +19,54 @@ func TestListCollections(t *testing.T) {
 
 	client := tu.DefaultTestClient(ctx, t)
 
-	databases := []string{"testdb01", "testdb02"}
-	collections := []string{"col01", "col02", "colxx", "colyy"}
+	inDBs := []string{"testdb01", "testdb02"}
+	inColls := []string{"col01", "col02", "colxx", "colyy"}
 
 	defer func() {
-		for _, dbname := range databases {
+		for _, dbname := range inDBs {
 			client.Database(dbname).Drop(ctx) //nolint:errcheck
 		}
 	}()
 
-	for _, dbname := range databases {
-		for _, coll := range collections {
+	for _, dbname := range inDBs {
+		for _, coll := range inColls {
 			for j := 0; j < 10; j++ {
 				_, err := client.Database(dbname).Collection(coll).InsertOne(ctx, bson.M{"f1": j, "f2": "2"})
 				assert.NoError(t, err)
 			}
 		}
 	}
 
-	want := []string{"col01", "col02", "colxx"}
-	collections, err := listCollections(ctx, client, []string{"col0", "colx"}, databases[0])
-	sort.Strings(collections)
+	want := []string{"admin", "config", "local", "testdb01", "testdb02"}
+	allDBs, err := databases(ctx, client, nil, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, want, allDBs)
+
+	want = []string{"col01", "col02", "colxx"}
+	inNameSpaces := []string{inDBs[0] + ".col0", inDBs[0] + ".colx"}
+	colls, err := listCollections(ctx, client, inDBs[0], inNameSpaces)
+	sort.Strings(colls)
+
+	assert.NoError(t, err)
+	assert.Equal(t, want, colls)
+
+	// Advanced filtering test
+	wantNS := map[string][]string{
+		"testdb01": {"col01", "col02", "colxx", "colyy"},
+		"testdb02": {"col01", "col02"},
+	}
 
+	// List all collections in testdb01 (inDBs[0]) but only col01 and col02 from testdb02.
+	filterInNameSpaces := []string{inDBs[0], inDBs[1] + ".col01", inDBs[1] + ".col02"}
+	namespaces, err := listAllCollections(ctx, client, filterInNameSpaces, systemDBs)
 	assert.NoError(t, err)
-	assert.Equal(t, want, collections)
+	assert.Equal(t, wantNS, namespaces)
 
-	count, err := allCollectionsCount(ctx, client, nil)
+	count, err := nonSystemCollectionsCount(ctx, client, nil, nil)
 	assert.NoError(t, err)
-	assert.True(t, count > 8)
+	assert.True(t, count == 8)
 
-	count, err = allCollectionsCount(ctx, client, []string{"col0", "colx"})
+	count, err = nonSystemCollectionsCount(ctx, client, nil, []string{inDBs[0] + ".col0", inDBs[0] + ".colx"})
 	assert.NoError(t, err)
 	assert.Equal(t, 6, count)
 }

exporter/dbstats_collector.go

@@ -42,8 +42,10 @@ func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) {
 	dbNames, err := d.client.ListDatabaseNames(d.ctx, bson.M{})
 	if err != nil {
 		d.logger.Errorf("Failed to get database names: %s", err)
+
 		return
 	}
+
 	d.logger.Debugf("getting stats for databases: %v", dbNames)
 	for _, db := range dbNames {
 		var dbStats bson.M
@@ -52,6 +54,7 @@ func (d *dbstatsCollector) Collect(ch chan<- prometheus.Metric) {
 		err := r.Decode(&dbStats)
 		if err != nil {
 			d.logger.Errorf("Failed to get $dbstats for database %s: %s", db, err)
+
 			continue
 		}
 

exporter/diagnostic_data_collector_test.go

@@ -164,6 +164,9 @@ func addTestData(ctx context.Context, client *mongo.Client, count int) error {
 	}
 
 	if err = mongo.WithSession(ctx, session, func(sc mongo.SessionContext) error {
+		ctx, cancel := context.WithCancel(ctx)
+		defer cancel()
+
 		for i := 0; i < count; i++ {
 			dbName := fmt.Sprintf("testdb_%06d", i)
 			doc := bson.D{{Key: "field1", Value: "value 1"}}
@@ -173,7 +176,7 @@ func addTestData(ctx context.Context, client *mongo.Client, count int) error {
 			}
 		}
 
-		if err = session.CommitTransaction(sc); err != nil {
+		if err = session.CommitTransaction(ctx); err != nil {
 			return errors.Wrap(err, "cannot commit add test data transaction")
 		}