Skip to content

Commit 75af9a2

Browse files
committed
Amazon S3 backup support
Uses amazonka-s3, which auto discovers credentials. Did some manual testing: ``` glean --tier test --db-root ~/glean backup myrepo/0 s3:mybucket/jade-test glean --tier test --db-root ~/glean2 restore s3:mybucket/jade-test/myrepo.0 ``` Configuration ~/.config/glean/server/test, though I think this doesn't actually matter? ``` { "backup": { "allowed": ["myrepo"], "location": "s3:mybucket/jade-test" }, "restore": { "enabled": true } } ``` This feature really needs some documentation (especially given I don't know if fb expected anyone external to immediately go and use it so it's probably in a wiki somewhere?); I would appreciate the docs that exist getting released, or I can maybe write some.
1 parent 59789d8 commit 75af9a2

File tree

6 files changed

+511
-24
lines changed

6 files changed

+511
-24
lines changed

glean.cabal.in

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ flag opt
9393
flag benchmarks
9494
default: False
9595

96+
-- Support for backups to Amazon S3
97+
flag s3-support
98+
default: True
99+
96100
-- run tests that require clang
97101
flag clang-tests
98102
default: True
@@ -666,6 +670,30 @@ library db
666670
glean:stubs,
667671
glean:rocksdb,
668672

673+
library db-backup-s3
674+
import: fb-haskell, deps
675+
visibility: private
676+
hs-source-dirs: glean/db/backup/s3
677+
678+
exposed-modules: Glean.Database.Backup.S3
679+
build-depends:
680+
glean:util,
681+
glean:if-internal-hs,
682+
glean:if-glean-hs,
683+
glean:db,
684+
amazonka,
685+
amazonka-s3,
686+
conduit,
687+
unliftio,
688+
microlens,
689+
http-client,
690+
691+
if flag(s3-support)
692+
buildable: True
693+
else
694+
buildable: False
695+
696+
669697
-- Backend API, and a few things built on top
670698
library backend-api
671699
import: fb-haskell, fb-cpp, deps
@@ -712,6 +740,7 @@ library backend-local
712740
Glean.Backend.Local
713741
Glean.Backend.Logging
714742
Glean.Dump
743+
715744
build-depends:
716745
glean:angle,
717746
glean:backend-api,
@@ -1138,6 +1167,10 @@ executable glean-server
11381167
haskeline >=0.7.3 && <0.9,
11391168
json
11401169

1170+
if flag(s3-support)
1171+
cpp-options: -DENABLE_S3=1
1172+
build-depends: glean:db-backup-s3
1173+
11411174
library shell-lib
11421175
import: fb-haskell, fb-cpp, deps
11431176
hs-source-dirs: glean/shell
@@ -1266,6 +1299,10 @@ executable glean
12661299
split,
12671300
Glob,
12681301

1302+
if flag(s3-support)
1303+
cpp-options: -DENABLE_S3=1
1304+
build-depends: glean:db-backup-s3
1305+
12691306
executable glean-hyperlink
12701307
import: fb-haskell, fb-cpp, deps, exe
12711308
hs-source-dirs: glean/demo
@@ -2160,6 +2197,27 @@ test-suite backup
21602197
glean:stubs,
21612198
glean:util
21622199

2200+
test-suite backup-s3
2201+
import: test
2202+
type: exitcode-stdio-1.0
2203+
main-is: BackupTestS3.hs
2204+
ghc-options: -main-is BackupTestS3
2205+
build-depends:
2206+
glean:config,
2207+
glean:core,
2208+
glean:db,
2209+
glean:if-glean-hs,
2210+
glean:if-internal-hs,
2211+
glean:schema,
2212+
glean:stubs,
2213+
glean:util,
2214+
glean:db-backup-s3
2215+
2216+
if flag(s3-support)
2217+
buildable: True
2218+
else
2219+
buildable: False
2220+
21632221
test-suite catalog
21642222
import: test
21652223
type: exitcode-stdio-1.0

0 commit comments

Comments
 (0)