PBM-1455: oplog restore tests with tc (#1075)

Use testcontainers for unit tests with PSMDB:
* Add testcontainers dependency
* Add tests for oplog db logic using testcontainers
* Use PSMDB as tc image

Author: boris-ilijic

go.mod

@@ -18,50 +18,72 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/spf13/cobra v1.8.1
 	github.com/spf13/viper v1.19.0
+	github.com/testcontainers/testcontainers-go v0.34.0
+	github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0
 	go.mongodb.org/mongo-driver v1.17.1
 	golang.org/x/mod v0.19.0
 	golang.org/x/sync v0.10.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
+	dario.cat/mergo v1.0.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
-	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20231202071711-9a357b53e9c9 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
+	github.com/containerd/containerd v1.7.18 // indirect
 	github.com/containerd/log v0.1.0 // indirect
+	github.com/containerd/platforms v0.2.1 // indirect
+	github.com/cpuguy83/dockercfg v0.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/distribution/reference v0.5.0 // indirect
+	github.com/distribution/reference v0.6.0 // indirect
 	github.com/docker/go-connections v0.5.0 // indirect
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/docker-image-spec v1.3.1 // indirect
+	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/sys/sequential v0.5.0 // indirect
+	github.com/moby/sys/user v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect
 	github.com/montanaflynn/stats v0.7.1 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
 	github.com/sagikazarmark/locafero v0.4.0 // indirect
 	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.23.12 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.9.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
 	go.opentelemetry.io/otel v1.24.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect
@@ -72,11 +94,10 @@ require (
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 // indirect
-	golang.org/x/net v0.25.0 // indirect
+	golang.org/x/net v0.26.0 // indirect
 	golang.org/x/sys v0.28.0 // indirect
 	golang.org/x/term v0.27.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	gotest.tools/v3 v3.5.1 // indirect
 )

go.sum

@@ -0,0 +1,98 @@
+package oplog
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/modules/mongodb"
+	"go.mongodb.org/mongo-driver/bson"
+	"go.mongodb.org/mongo-driver/mongo"
+	"go.mongodb.org/mongo-driver/mongo/options"
+)
+
+var mClient *mongo.Client
+
+func TestMain(m *testing.M) {
+	ctx := context.Background()
+	mongodbContainer, err := mongodb.Run(ctx, "perconalab/percona-server-mongodb:7")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error while creating mongo test container: %v", err)
+		return
+	}
+	defer func() {
+		if err := testcontainers.TerminateContainer(mongodbContainer); err != nil {
+			fmt.Fprintf(os.Stderr, "failed to terminate container: %s", err)
+		}
+	}()
+
+	connStr, err := mongodbContainer.ConnectionString(ctx)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "conn string error: %v", err)
+		return
+	}
+	mClient, err = mongo.Connect(ctx, options.Client().ApplyURI(connStr))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "mongo client connect error: %v", err)
+		return
+	}
+
+	m.Run()
+}
+
+func TestGetUUIDForNSv2(t *testing.T) {
+	t.Run("uuid for existing collection", func(t *testing.T) {
+		db := newMDB(mClient)
+
+		tDB, tColl := "my_test_db", "my_test_coll"
+		err := mClient.Database(tDB).CreateCollection(context.Background(), tColl)
+		if err != nil {
+			t.Errorf("create collection err: %v", err)
+		}
+
+		uuid, err := db.getUUIDForNS(context.Background(), fmt.Sprintf("%s.%s", tDB, tColl))
+		if err != nil {
+			t.Errorf("got err=%v", err)
+		}
+		if uuid.IsZero() {
+			t.Error("expected to get uuid for collection")
+		}
+	})
+
+	t.Run("uuid for not existing collection", func(t *testing.T) {
+		db := newMDB(mClient)
+
+		tDB, tColl := "xDB", "yColl"
+		uuid, err := db.getUUIDForNS(context.Background(), fmt.Sprintf("%s.%s", tDB, tColl))
+		if err != nil {
+			t.Errorf("got err=%v", err)
+		}
+		if !uuid.IsZero() {
+			t.Errorf("expected to get zero value for uuid for not existing collection, got=%v", uuid)
+		}
+	})
+}
+
+func TestApplyOps(t *testing.T) {
+	db := newMDB(mClient)
+
+	tDB, tColl := "tAODB", "dAOColl"
+	if _, err := mClient.Database(tDB).Collection(tColl).InsertOne(context.Background(), bson.D{}); err != nil {
+		t.Errorf("insert doc err: %v", err)
+	}
+	iOps := createInsertSimpleOp(t, fmt.Sprintf("%s.%s", tDB, tColl))
+
+	err := db.applyOps([]any{iOps})
+	if err != nil {
+		t.Fatalf("error when using applyOps, err=%v", err)
+	}
+	cnt, err := mClient.Database(tDB).Collection(tColl).CountDocuments(context.Background(), bson.D{})
+	if err != nil {
+		t.Fatalf("error when counting docs within new collection, err=%v", err)
+	}
+	if cnt != 2 {
+		t.Fatalf("wrong number of docs in new collection, got=%d, want=1", cnt)
+	}
+}

pbm/oplog/db_tc_test.go

@@ -427,6 +427,62 @@ func createInsertOp(t *testing.T, ns string) *db.Oplog {
 	return replaceNsWithinOpEntry(t, iOpJSON, ns)
 }
 
+func createInsertSimpleOp(t *testing.T, ns string) *db.Oplog {
+	t.Helper()
+	iOpJSON := `
+		{
+		  "op": "i",
+		  "ns": "db.coll",
+		  "o": {
+			"_id": {
+			  "$oid": "6747008178d82a2b1134a2b8"
+			},
+			"d": {
+			  "$numberInt": "6"
+			},
+			"desc": "doc-6"
+		  },
+		  "o2": {
+			"_id": {
+			  "$oid": "6747008178d82a2b1134a2b8"
+			}
+		  },
+		  "stmtId": {
+			"$numberInt": "0"
+		  },
+		  "ts": {
+			"$timestamp": {
+			  "t": 1732706433,
+			  "i": 1
+			}
+		  },
+		  "t": {
+			"$numberLong": "2"
+		  },
+		  "v": {
+			"$numberLong": "2"
+		  },
+		  "wall": {
+			"$date": {
+			  "$numberLong": "1732706433987"
+			}
+		  },
+		  "prevOpTime": {
+			"ts": {
+			  "$timestamp": {
+				"t": 0,
+				"i": 0
+			  }
+			},
+			"t": {
+			  "$numberLong": "-1"
+			}
+		  }
+		}`
+
+	return replaceNsWithinOpEntry(t, iOpJSON, ns)
+}
+
 func createUpdateOp(t *testing.T, ns string) *db.Oplog {
 	t.Helper()
 

pbm/oplog/restore_test.go

@@ -0,0 +1,12 @@
+version = 1
+
+test_patterns = [
+  "*_test.go"
+]
+
+[[analyzers]]
+name = "go"
+enabled = true
+
+  [analyzers.meta]
+  import_path = "dario.cat/mergo"

vendor/dario.cat/mergo/.deepsource.toml

@@ -0,0 +1,33 @@
+#### joe made this: http://goel.io/joe
+
+#### go ####
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/
+
+#### vim ####
+# Swap
+[._]*.s[a-v][a-z]
+[._]*.sw[a-p]
+[._]s[a-v][a-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags

vendor/dario.cat/mergo/.gitignore

@@ -0,0 +1,12 @@
+language: go
+arch:
+    - amd64
+    - ppc64le
+install:
+  - go get -t
+  - go get golang.org/x/tools/cmd/cover
+  - go get github.com/mattn/goveralls
+script:
+  - go test -race -v ./...
+after_script:
+  - $HOME/gopath/bin/goveralls -service=travis-ci -repotoken $COVERALLS_TOKEN

vendor/dario.cat/mergo/.travis.yml

@@ -0,0 +1,46 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at [email protected]. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version]
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/