Skip to content

Commit 5da86a7

Browse files
committed
Amazon S3 backup support
Uses amazonka-s3, which auto discovers credentials. Did some manual testing: ``` glean --tier test --db-root ~/glean backup myrepo/0 s3:mybucket/jade-test glean --tier test --db-root ~/glean2 restore s3:mybucket/jade-test/myrepo.0 ``` Configuration ~/.config/glean/server/test, though I think this doesn't actually matter? ``` { "backup": { "allowed": ["myrepo"], "location": "s3:mybucket/jade-test" }, "restore": { "enabled": true } } ``` This feature really needs some documentation (especially given I don't know if fb expected anyone external to immediately go and use it so it's probably in a wiki somewhere?); I would appreciate the docs that exist getting released, or I can maybe write some.
1 parent e8e9d44 commit 5da86a7

File tree

6 files changed

+511
-24
lines changed

6 files changed

+511
-24
lines changed

glean.cabal.in

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ flag opt
8484
flag benchmarks
8585
default: False
8686

87+
-- Support for backups to Amazon S3
88+
flag s3-support
89+
default: True
90+
8791
-- run tests that require clang
8892
flag clang-tests
8993
default: True
@@ -646,6 +650,30 @@ library db
646650
glean:stubs,
647651
glean:rocksdb,
648652

653+
library db-backup-s3
654+
import: fb-haskell, deps
655+
visibility: private
656+
hs-source-dirs: glean/db/backup/s3
657+
658+
exposed-modules: Glean.Database.Backup.S3
659+
build-depends:
660+
glean:util,
661+
glean:if-internal-hs,
662+
glean:if-glean-hs,
663+
glean:db,
664+
amazonka,
665+
amazonka-s3,
666+
conduit,
667+
unliftio,
668+
microlens,
669+
http-client,
670+
671+
if flag(s3-support)
672+
buildable: True
673+
else
674+
buildable: False
675+
676+
649677
-- Backend API, and a few things built on top
650678
library backend-api
651679
import: fb-haskell, fb-cpp, deps
@@ -692,6 +720,7 @@ library backend-local
692720
Glean.Backend.Local
693721
Glean.Backend.Logging
694722
Glean.Dump
723+
695724
build-depends:
696725
glean:angle,
697726
glean:backend-api,
@@ -1118,6 +1147,10 @@ executable glean-server
11181147
haskeline >=0.7.3 && <0.9,
11191148
json
11201149

1150+
if flag(s3-support)
1151+
cpp-options: -DENABLE_S3=1
1152+
build-depends: glean:db-backup-s3
1153+
11211154
library shell-lib
11221155
import: fb-haskell, fb-cpp, deps
11231156
hs-source-dirs: glean/shell
@@ -1246,6 +1279,10 @@ executable glean
12461279
split,
12471280
Glob,
12481281

1282+
if flag(s3-support)
1283+
cpp-options: -DENABLE_S3=1
1284+
build-depends: glean:db-backup-s3
1285+
12491286
executable glean-hyperlink
12501287
import: fb-haskell, fb-cpp, deps, exe
12511288
hs-source-dirs: glean/demo
@@ -2140,6 +2177,27 @@ test-suite backup
21402177
glean:stubs,
21412178
glean:util
21422179

2180+
test-suite backup-s3
2181+
import: test
2182+
type: exitcode-stdio-1.0
2183+
main-is: BackupTestS3.hs
2184+
ghc-options: -main-is BackupTestS3
2185+
build-depends:
2186+
glean:config,
2187+
glean:core,
2188+
glean:db,
2189+
glean:if-glean-hs,
2190+
glean:if-internal-hs,
2191+
glean:schema,
2192+
glean:stubs,
2193+
glean:util,
2194+
glean:db-backup-s3
2195+
2196+
if flag(s3-support)
2197+
buildable: True
2198+
else
2199+
buildable: False
2200+
21432201
test-suite catalog
21442202
import: test
21452203
type: exitcode-stdio-1.0

0 commit comments

Comments
 (0)