Skip to content

Commit 59f10ac

Browse files
robertmutuhaihe
robertmu
authored and
tuhaihe
committed
Merge gpbackup-s3-plugin-archive into plugins/s3plugin directory
* Integrate S3 storage plugin functionality into main repository * Add gpbackup_s3_plugin executable target to Makefile * Update build system to include S3 plugin compilation and installation * Include comprehensive S3 plugin configuration and usage documentation * Add unit tests for S3 plugin functionality The S3 plugin allows users to backup and restore Cloudberry and Greenplum databases to Amazon S3 compatible storage systems. This integration eliminates the need for a separate plugin repository and simplifies deployment and maintenance. Previously located at: https://github.com/greenplum-db/gpbackup-s3-plugin-archive
1 parent d1059fe commit 59f10ac

File tree

10 files changed

+1548
-8
lines changed

10 files changed

+1548
-8
lines changed

‎Makefile‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,17 @@ endif
88
BACKUP=gpbackup
99
RESTORE=gprestore
1010
HELPER=gpbackup_helper
11+
S3PLUGIN=gpbackup_s3_plugin
1112
BIN_DIR=$(shell echo $${GOPATH:-~/go} | awk -F':' '{ print $$1 "/bin"}')
1213
GINKGO_FLAGS := -r --keep-going --randomize-suites --randomize-all --no-color
1314
GIT_VERSION := $(shell git describe --tags | perl -pe 's/(.*)-([0-9]*)-(g[0-9a-f]*)/\1+dev.\2.\3/')
1415
BACKUP_VERSION_STR=github.com/apache/cloudberry-backup/backup.version=$(GIT_VERSION)
1516
RESTORE_VERSION_STR=github.com/apache/cloudberry-backup/restore.version=$(GIT_VERSION)
1617
HELPER_VERSION_STR=github.com/apache/cloudberry-backup/helper.version=$(GIT_VERSION)
18+
S3PLUGIN_VERSION_STR=github.com/apache/cloudberry-backup/plugins/s3plugin.version=$(GIT_VERSION)
1719

1820
# note that /testutils is not a production directory, but has unit tests to validate testing tools
19-
SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/
21+
SUBDIRS_HAS_UNIT=backup/ filepath/ history/ helper/ options/ report/ restore/ toc/ utils/ testutils/ plugins/s3plugin/
2022
SUBDIRS_ALL=$(SUBDIRS_HAS_UNIT) integration/ end_to_end/
2123
GOLANG_LINTER=$(GOPATH)/bin/golangci-lint
2224
GINKGO=$(GOPATH)/bin/ginkgo
@@ -26,6 +28,7 @@ DEBUG=-gcflags=all="-N -l"
2628

2729
CUSTOM_BACKUP_DIR ?= "/tmp"
2830
helper_path ?= $(BIN_DIR)/$(HELPER)
31+
s3plugin_path ?= $(BIN_DIR)/$(S3PLUGIN)
2932

3033
# Prefer gpsync as the newer utility, fall back to gpscp if not present (older installs)
3134
ifeq (, $(shell which gpsync))
@@ -82,26 +85,29 @@ build : $(GOSQLITE)
8285
CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) --ldflags '-X $(BACKUP_VERSION_STR)'
8386
CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) --ldflags '-X $(RESTORE_VERSION_STR)'
8487
CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) --ldflags '-X $(HELPER_VERSION_STR)'
88+
CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) --ldflags '-X $(S3PLUGIN_VERSION_STR)'
8589

8690
debug :
8791
CGO_ENABLED=1 $(GO_BUILD) -tags '$(BACKUP)' -o $(BIN_DIR)/$(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)" $(DEBUG)
8892
CGO_ENABLED=1 $(GO_BUILD) -tags '$(RESTORE)' -o $(BIN_DIR)/$(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)" $(DEBUG)
8993
CGO_ENABLED=1 $(GO_BUILD) -tags '$(HELPER)' -o $(BIN_DIR)/$(HELPER) -ldflags "-X $(HELPER_VERSION_STR)" $(DEBUG)
94+
CGO_ENABLED=1 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(BIN_DIR)/$(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)" $(DEBUG)
9095

9196
build_linux :
9297
env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(BACKUP)' -o $(BACKUP) -ldflags "-X $(BACKUP_VERSION_STR)"
9398
env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(RESTORE)' -o $(RESTORE) -ldflags "-X $(RESTORE_VERSION_STR)"
9499
env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(HELPER)' -o $(HELPER) -ldflags "-X $(HELPER_VERSION_STR)"
100+
env GOOS=linux GOARCH=amd64 $(GO_BUILD) -tags '$(S3PLUGIN)' -o $(S3PLUGIN) -ldflags "-X $(S3PLUGIN_VERSION_STR)"
95101

96102
install :
97103
cp $(BIN_DIR)/$(BACKUP) $(BIN_DIR)/$(RESTORE) $(GPHOME)/bin
98104
@psql -X -t -d template1 -c 'select distinct hostname from gp_segment_configuration where content != -1' > /tmp/seg_hosts 2>/dev/null; \
99105
if [ $$? -eq 0 ]; then \
100-
$(COPYUTIL) -f /tmp/seg_hosts $(helper_path) =:$(GPHOME)/bin/$(HELPER); \
106+
$(COPYUTIL) -f /tmp/seg_hosts $(helper_path) $(s3plugin_path) =:$(GPHOME)/bin/; \
101107
if [ $$? -eq 0 ]; then \
102-
echo 'Successfully copied gpbackup_helper to $(GPHOME) on all segments'; \
108+
echo 'Successfully copied gpbackup_helper and gpbackup_s3_plugin to $(GPHOME) on all segments'; \
103109
else \
104-
echo 'Failed to copy gpbackup_helper to $(GPHOME)'; \
110+
echo 'Failed to copy gpbackup_helper and gpbackup_s3_plugin to $(GPHOME)'; \
105111
exit 1; \
106112
fi; \
107113
else \
@@ -112,7 +118,7 @@ install :
112118

113119
clean :
114120
# Build artifacts
115-
rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER)
121+
rm -f $(BIN_DIR)/$(BACKUP) $(BACKUP) $(BIN_DIR)/$(RESTORE) $(RESTORE) $(BIN_DIR)/$(HELPER) $(HELPER) $(BIN_DIR)/$(S3PLUGIN) $(S3PLUGIN)
116122
# Test artifacts
117123
rm -rf /tmp/go-build* /tmp/gexec_artifacts* /tmp/ginkgo*
118124
docker stop s3-minio # stop minio before removing its data directories

‎go.mod‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,32 @@ go 1.21
55
require (
66
github.com/DATA-DOG/go-sqlmock v1.5.0
77
github.com/apache/cloudberry-go-libs v1.0.12-0.20250910014224-fc376e8a1056
8+
github.com/aws/aws-sdk-go v1.44.257
89
github.com/blang/semver v3.5.1+incompatible
910
github.com/blang/vfs v1.0.0
11+
github.com/inhies/go-bytesize v0.0.0-20220417184213-4913239db9cf
1012
github.com/jackc/pgconn v1.14.3
1113
github.com/jmoiron/sqlx v1.3.5
1214
github.com/klauspost/compress v1.15.15
1315
github.com/lib/pq v1.10.7
1416
github.com/mattn/go-sqlite3 v1.14.19
1517
github.com/nightlyone/lockfile v1.0.0
18+
github.com/olekukonko/tablewriter v0.0.5
1619
github.com/onsi/ginkgo/v2 v2.13.0
1720
github.com/onsi/gomega v1.27.10
1821
github.com/pkg/errors v0.9.1
1922
github.com/sergi/go-diff v1.3.1
2023
github.com/spf13/cobra v1.6.1
2124
github.com/spf13/pflag v1.0.5
25+
github.com/urfave/cli v1.22.13
2226
golang.org/x/sys v0.18.0
2327
golang.org/x/tools v0.12.0
2428
gopkg.in/cheggaaa/pb.v1 v1.0.28
2529
gopkg.in/yaml.v2 v2.4.0
2630
)
2731

2832
require (
33+
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
2934
github.com/fatih/color v1.14.1 // indirect
3035
github.com/go-logr/logr v1.2.4 // indirect
3136
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
@@ -39,9 +44,11 @@ require (
3944
github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
4045
github.com/jackc/pgtype v1.14.0 // indirect
4146
github.com/jackc/pgx/v4 v4.18.2 // indirect
47+
github.com/jmespath/go-jmespath v0.4.0 // indirect
4248
github.com/mattn/go-colorable v0.1.13 // indirect
4349
github.com/mattn/go-runewidth v0.0.13 // indirect
4450
github.com/rivo/uniseg v0.2.0 // indirect
51+
github.com/russross/blackfriday/v2 v2.1.0 // indirect
4552
golang.org/x/crypto v0.21.0 // indirect
4653
golang.org/x/mod v0.12.0 // indirect
4754
golang.org/x/net v0.23.0 // indirect

0 commit comments

Comments
 (0)