Skip to content

Commit 0157f87

Browse files
sandraromanchenkosandraromanchenko
authored
Merge pull request #1068 from percona/release-2.8.0
Release 2.8.0
2 parents 94ad374 + 15b51a8 commit 0157f87

File tree

178 files changed

+4982
-2838
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

178 files changed

+4982
-2838
lines changed

‎.github/workflows/ci.yml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ on:
2828
jobs:
2929
test:
3030
runs-on: ubuntu-latest
31-
timeout-minutes: 120
31+
timeout-minutes: 180
3232
strategy:
3333
fail-fast: false
3434
matrix:
35-
psmdb: ["5.0", "6.0", "7.0"]
35+
psmdb: ["6.0", "7.0", "8.0"]
3636
test: [logical, physical, incremental, external]
3737
env:
3838
PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
@@ -49,7 +49,7 @@ jobs:
4949

5050
- name: Setup environment with PSMDB ${{ matrix.psmdb }} for PBM PR/branch ${{ github.event.pull_request.title || env.PR_NUMBER || env.PBM_BRANCH }}
5151
run: |
52-
PSMDB=percona/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
52+
PSMDB=perconalab/percona-server-mongodb:${{ matrix.psmdb }} docker compose build
5353
docker compose up -d
5454
working-directory: psmdb-testing/pbm-functional/pytest
5555

‎.github/workflows/codecov.yml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: 'codecov'
2+
3+
on:
4+
push:
5+
branches:
6+
- dev
7+
8+
jobs:
9+
go-test:
10+
name: runner / go-test
11+
runs-on: ubuntu-latest
12+
steps:
13+
- uses: actions/checkout@v4
14+
- uses: actions/setup-go@v4
15+
with:
16+
go-version: "1.22"
17+
- name: test
18+
run: go test -v ./... -covermode=atomic -coverprofile=cover.out
19+
20+
- name: upload coverage report
21+
uses: codecov/codecov-action@v4
22+
with:
23+
file: cover.out
24+
fail_ci_if_error: false
25+
token: ${{ secrets.CODECOV_TOKEN }}

‎.github/workflows/reviewdog.yml

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,14 @@ jobs:
1010
with:
1111
go-version: "1.22"
1212
- name: test
13-
run: go test -v ./...
13+
run: go test -v ./... -covermode=atomic -coverprofile=cover.out
14+
15+
- name: upload coverage report
16+
uses: codecov/codecov-action@v4
17+
with:
18+
file: cover.out
19+
fail_ci_if_error: false
20+
token: ${{ secrets.CODECOV_TOKEN }}
1421

1522
shellcheck:
1623
name: runner / shellcheck

‎CODEOWNERS

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
* @defbin @boris-ilijic
1+
* @boris-ilijic @veceraj

‎Makefile

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,3 +112,20 @@ install-stest-static:
112112
$(ENVS_STATIC) go install -ldflags="$(LDFLAGS_STATIC)" $(BUILD_FLAGS) ./cmd/pbm-speed-test
113113
install-static-entrypoint:
114114
$(ENVS_STATIC) go install -ldflags="$(LDFLAGS_STATIC)" $(BUILD_FLAGS) ./cmd/pbm-agent-entrypoint
115+
116+
# BUILD WITH COVERAGE PROFILING
117+
build-cover: build-pbm-cover build-agent-cover build-stest-cover
118+
build-pbm-cover:
119+
$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm ./cmd/pbm
120+
build-agent-cover:
121+
$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm-agent ./cmd/pbm-agent
122+
build-stest-cover:
123+
$(ENVS) go build -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) -o ./bin/pbm-speed-test ./cmd/pbm-speed-test
124+
125+
install-cover: install-pbm-cover install-agent-cover install-stest-cover
126+
install-pbm-cover:
127+
$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm
128+
install-agent-cover:
129+
$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm-agent
130+
install-stest-cover:
131+
$(ENVS) go install -cover -ldflags="$(LDFLAGS)" $(BUILD_FLAGS) ./cmd/pbm-speed-test

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Percona Backup for MongoDB
2-
[![Go Report Card](https://goreportcard.com/badge/github.com/percona/percona-backup-mongodb)](https://goreportcard.com/report/github.com/percona/percona-backup-mongodb) [![codecov](https://codecov.io/gh/percona/percona-backup-mongodb/branch/master/graph/badge.svg?token=TiuOmTfp2p)](https://codecov.io/gh/percona/percona-backup-mongodb) [![CLA assistant](https://cla-assistant.percona.com/readme/badge/percona/percona-backup-mongodb)](https://cla-assistant.percona.com/percona/percona-backup-mongodb)
2+
[![Go Report Card](https://goreportcard.com/badge/github.com/percona/percona-backup-mongodb)](https://goreportcard.com/report/github.com/percona/percona-backup-mongodb) [![CLA assistant](https://cla-assistant.percona.com/readme/badge/percona/percona-backup-mongodb)](https://cla-assistant.percona.com/percona/percona-backup-mongodb)
33

44
![PBM logo](backup-mongo.jpeg)
55

‎cmd/pbm-agent/agent.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@ func (a *Agent) storStatus(
456456
return topo.SubsysStatus{OK: true}
457457
}
458458

459-
stg, err := util.GetStorage(ctx, a.leadConn, log)
459+
stg, err := util.GetStorage(ctx, a.leadConn, a.brief.Me, log)
460460
if err != nil {
461461
return topo.SubsysStatus{Err: fmt.Sprintf("unable to get storage: %v", err)}
462462
}

‎cmd/pbm-agent/delete.go

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,6 @@ import (
1111

1212
"github.com/percona/percona-backup-mongodb/pbm/backup"
1313
"github.com/percona/percona-backup-mongodb/pbm/config"
14-
"github.com/percona/percona-backup-mongodb/pbm/connect"
1514
"github.com/percona/percona-backup-mongodb/pbm/ctrl"
1615
"github.com/percona/percona-backup-mongodb/pbm/errors"
1716
"github.com/percona/percona-backup-mongodb/pbm/lock"
@@ -97,7 +96,7 @@ func (a *Agent) Delete(ctx context.Context, d *ctrl.DeleteBackupCmd, opid ctrl.O
9796
}
9897

9998
l.Info("deleting backups older than %v", t)
100-
err = backup.DeleteBackupBefore(ctx, a.leadConn, t, bcpType)
99+
err = backup.DeleteBackupBefore(ctx, a.leadConn, t, bcpType, nodeInfo.Me)
101100
if err != nil {
102101
l.Error("deleting: %v", err)
103102
return
@@ -107,7 +106,7 @@ func (a *Agent) Delete(ctx context.Context, d *ctrl.DeleteBackupCmd, opid ctrl.O
107106
ctx := log.SetLogEventToContext(ctx, l)
108107

109108
l.Info("deleting backup")
110-
err := backup.DeleteBackup(ctx, a.leadConn, d.Backup)
109+
err := backup.DeleteBackup(ctx, a.leadConn, d.Backup, nodeInfo.Me)
111110
if err != nil {
112111
l.Error("deleting: %v", err)
113112
return
@@ -188,7 +187,7 @@ func (a *Agent) DeletePITR(ctx context.Context, d *ctrl.DeletePITRCmd, opid ctrl
188187

189188
ts := primitive.Timestamp{T: uint32(t.Unix())}
190189
l.Info("deleting pitr chunks older than %v", t)
191-
err = deletePITRImpl(ctx, a.leadConn, ts)
190+
err = a.deletePITRImpl(ctx, ts)
192191
if err != nil {
193192
l.Error("deleting: %v", err)
194193
return
@@ -260,7 +259,7 @@ func (a *Agent) Cleanup(ctx context.Context, d *ctrl.CleanupCmd, opid ctrl.OPID,
260259
l.Error("get config: %v", err)
261260
}
262261

263-
stg, err := util.StorageFromConfig(&cfg.Storage, l)
262+
stg, err := util.StorageFromConfig(&cfg.Storage, a.brief.Me, l)
264263
if err != nil {
265264
l.Error("get storage: " + err.Error())
266265
}
@@ -298,16 +297,16 @@ func (a *Agent) Cleanup(ctx context.Context, d *ctrl.CleanupCmd, opid ctrl.OPID,
298297
l.Error(err.Error())
299298
}
300299

301-
err = resync.Resync(ctx, a.leadConn, &cfg.Storage)
300+
err = resync.Resync(ctx, a.leadConn, &cfg.Storage, a.brief.Me)
302301
if err != nil {
303302
l.Error("storage resync: " + err.Error())
304303
}
305304
}
306305

307-
func deletePITRImpl(ctx context.Context, conn connect.Client, ts primitive.Timestamp) error {
306+
func (a *Agent) deletePITRImpl(ctx context.Context, ts primitive.Timestamp) error {
308307
l := log.LogEventFromContext(ctx)
309308

310-
r, err := backup.MakeCleanupInfo(ctx, conn, ts)
309+
r, err := backup.MakeCleanupInfo(ctx, a.leadConn, ts)
311310
if err != nil {
312311
return errors.Wrap(err, "get pitr chunks")
313312
}
@@ -316,15 +315,15 @@ func deletePITRImpl(ctx context.Context, conn connect.Client, ts primitive.Times
316315
return nil
317316
}
318317

319-
stg, err := util.GetStorage(ctx, conn, l)
318+
stg, err := util.GetStorage(ctx, a.leadConn, a.brief.Me, l)
320319
if err != nil {
321320
return errors.Wrap(err, "get storage")
322321
}
323322

324-
return deleteChunks(ctx, conn, stg, r.Chunks)
323+
return a.deleteChunks(ctx, stg, r.Chunks)
325324
}
326325

327-
func deleteChunks(ctx context.Context, m connect.Client, stg storage.Storage, chunks []oplog.OplogChunk) error {
326+
func (a *Agent) deleteChunks(ctx context.Context, stg storage.Storage, chunks []oplog.OplogChunk) error {
328327
l := log.LogEventFromContext(ctx)
329328

330329
for _, chnk := range chunks {
@@ -333,7 +332,7 @@ func deleteChunks(ctx context.Context, m connect.Client, stg storage.Storage, ch
333332
return errors.Wrapf(err, "delete pitr chunk '%s' (%v) from storage", chnk.FName, chnk)
334333
}
335334

336-
_, err = m.PITRChunksCollection().DeleteOne(
335+
_, err = a.leadConn.PITRChunksCollection().DeleteOne(
337336
ctx,
338337
bson.D{
339338
{"rs", chnk.RS},

‎cmd/pbm-agent/main.go

Lines changed: 46 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,20 @@ func main() {
4949
versionFormat = versionCmd.Flag("format", "Output format <json or \"\">").
5050
Default("").
5151
String()
52+
53+
logPath = pbmCmd.Flag("log-path", "Path to file").
54+
Envar("LOG_PATH").
55+
Default("/dev/stderr").
56+
String()
57+
logJSON = pbmCmd.Flag("log-json", "Enable JSON output").
58+
Envar("LOG_JSON").
59+
Bool()
60+
logLevel = pbmCmd.Flag(
61+
"log-level",
62+
"Minimal log level based on severity level: D, I, W, E or F, low to high. Choosing one includes higher levels too.").
63+
Envar("LOG_LEVEL").
64+
Default(log.D).
65+
Enum(log.D, log.I, log.W, log.E, log.F)
5266
)
5367

5468
cmd, err := pbmCmd.DefaultEnvars().Parse(os.Args[1:])
@@ -74,19 +88,26 @@ func main() {
7488

7589
hidecreds()
7690

77-
fmt.Print(perconaSquadNotice)
91+
logOpts := &log.Opts{
92+
LogPath: *logPath,
93+
LogLevel: *logLevel,
94+
LogJSON: *logJSON,
95+
}
96+
l := log.NewWithOpts(nil, "", "", logOpts).NewDefaultEvent()
7897

79-
err = runAgent(url, *dumpConns)
80-
stdlog.Println("Exit:", err)
98+
err = runAgent(url, *dumpConns, logOpts)
8199
if err != nil {
100+
l.Error("Exit: %v", err)
82101
os.Exit(1)
83102
}
103+
l.Info("Exit: <nil>")
84104
}
85105

86-
func runAgent(mongoURI string, dumpConns int) error {
87-
mtLog.SetDateFormat(log.LogTimeFormat)
88-
mtLog.SetVerbosity(&options.Verbosity{VLevel: mtLog.DebugLow})
89-
106+
func runAgent(
107+
mongoURI string,
108+
dumpConns int,
109+
logOpts *log.Opts,
110+
) error {
90111
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill)
91112
defer cancel()
92113

@@ -95,16 +116,33 @@ func runAgent(mongoURI string, dumpConns int) error {
95116
return errors.Wrap(err, "connect to PBM")
96117
}
97118

119+
err = setupNewDB(ctx, leadConn)
120+
if err != nil {
121+
return errors.Wrap(err, "setup pbm collections")
122+
}
123+
98124
agent, err := newAgent(ctx, leadConn, mongoURI, dumpConns)
99125
if err != nil {
100126
return errors.Wrap(err, "connect to the node")
101127
}
102128

103-
logger := log.New(agent.leadConn.LogCollection(), agent.brief.SetName, agent.brief.Me)
129+
logger := log.NewWithOpts(
130+
agent.leadConn,
131+
agent.brief.SetName,
132+
agent.brief.Me,
133+
logOpts)
104134
defer logger.Close()
105135

106136
ctx = log.SetLoggerToContext(ctx, logger)
107137

138+
mtLog.SetDateFormat(log.LogTimeFormat)
139+
mtLog.SetVerbosity(&options.Verbosity{VLevel: mtLog.DebugLow})
140+
mtLog.SetWriter(logger)
141+
142+
logger.Printf(perconaSquadNotice)
143+
logger.Printf("log options: log-path=%s, log-level:%s, log-json:%t",
144+
logOpts.LogPath, logOpts.LogLevel, logOpts.LogJSON)
145+
108146
canRunSlicer := true
109147
if err := agent.CanStart(ctx); err != nil {
110148
if errors.Is(err, ErrArbiterNode) || errors.Is(err, ErrDelayedNode) {
@@ -114,11 +152,6 @@ func runAgent(mongoURI string, dumpConns int) error {
114152
}
115153
}
116154

117-
err = setupNewDB(ctx, agent.leadConn)
118-
if err != nil {
119-
return errors.Wrap(err, "setup pbm collections")
120-
}
121-
122155
agent.showIncompatibilityWarning(ctx)
123156

124157
if canRunSlicer {

‎cmd/pbm-agent/oplog.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ func (a *Agent) OplogReplay(ctx context.Context, r *ctrl.ReplayCmd, opID ctrl.OP
7777
}
7878

7979
l.Info("oplog replay started")
80-
rr := restore.New(a.leadConn, a.nodeConn, a.brief, cfg, r.RSMap, 0)
80+
rr := restore.New(a.leadConn, a.nodeConn, a.brief, cfg, r.RSMap, 0, 1)
8181
err = rr.ReplayOplog(ctx, r, opID, l)
8282
if err != nil {
8383
if errors.Is(err, restore.ErrNoDataForShard) {

0 commit comments

Comments
 (0)