Skip to content

Commit 79c5dfe

Browse files
committed
Amazon S3 backup support
Uses amazonka-s3, which auto discovers credentials. Did some manual testing: ``` glean --tier test --db-root ~/glean backup myrepo/0 s3:mybucket/jade-test glean --tier test --db-root ~/glean2 restore s3:mybucket/jade-test/myrepo.0 ``` Configuration ~/.config/glean/server/test, though I think this doesn't actually matter? ``` { "backup": { "allowed": ["myrepo"], "location": "s3:mybucket/jade-test" }, "restore": { "enabled": true } } ``` This feature really needs some documentation (especially given I don't know if fb expected anyone external to immediately go and use it so it's probably in a wiki somewhere?); I would appreciate the docs that exist getting released, or I can maybe write some.
1 parent 9e62ac3 commit 79c5dfe

File tree

5 files changed

+509
-23
lines changed

5 files changed

+509
-23
lines changed

glean.cabal.in

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,10 @@ flag opt
8484
flag benchmarks
8585
default: False
8686

87+
-- Support for backups to Amazon S3
88+
flag s3-support
89+
default: True
90+
8791
-- run tests that require clang
8892
flag clang-tests
8993
default: True
@@ -645,6 +649,30 @@ library db
645649
glean:stubs,
646650
glean:rocksdb,
647651

652+
library db-backup-s3
653+
import: fb-haskell, deps
654+
visibility: private
655+
hs-source-dirs: glean/db/backup/s3
656+
657+
exposed-modules: Glean.Database.Backup.S3
658+
build-depends:
659+
glean:util,
660+
glean:if-internal-hs,
661+
glean:if-glean-hs,
662+
glean:db,
663+
amazonka,
664+
amazonka-s3,
665+
conduit,
666+
unliftio,
667+
microlens,
668+
http-client,
669+
670+
if flag(s3-support)
671+
buildable: True
672+
else
673+
buildable: False
674+
675+
648676
-- Backend API, and a few things built on top
649677
library backend-api
650678
import: fb-haskell, fb-cpp, deps
@@ -691,6 +719,7 @@ library backend-local
691719
Glean.Backend.Local
692720
Glean.Backend.Logging
693721
Glean.Dump
722+
694723
build-depends:
695724
glean:angle,
696725
glean:backend-api,
@@ -1116,6 +1145,10 @@ executable glean-server
11161145
haskeline >=0.7.3 && <0.9,
11171146
json
11181147

1148+
if flag(s3-support)
1149+
cpp-options: -DENABLE_S3=1
1150+
build-depends: glean:db-backup-s3
1151+
11191152
library shell-lib
11201153
import: fb-haskell, fb-cpp, deps
11211154
hs-source-dirs: glean/shell
@@ -1244,6 +1277,10 @@ executable glean
12441277
split,
12451278
Glob,
12461279

1280+
if flag(s3-support)
1281+
cpp-options: -DENABLE_S3=1
1282+
build-depends: glean:db-backup-s3
1283+
12471284
executable glean-hyperlink
12481285
import: fb-haskell, fb-cpp, deps, exe
12491286
hs-source-dirs: glean/demo
@@ -2131,6 +2168,27 @@ test-suite backup
21312168
glean:stubs,
21322169
glean:util
21332170

2171+
test-suite backup-s3
2172+
import: test
2173+
type: exitcode-stdio-1.0
2174+
main-is: BackupTestS3.hs
2175+
ghc-options: -main-is BackupTestS3
2176+
build-depends:
2177+
glean:config,
2178+
glean:core,
2179+
glean:db,
2180+
glean:if-glean-hs,
2181+
glean:if-internal-hs,
2182+
glean:schema,
2183+
glean:stubs,
2184+
glean:util,
2185+
glean:db-backup-s3
2186+
2187+
if flag(s3-support)
2188+
buildable: True
2189+
else
2190+
buildable: False
2191+
21342192
test-suite catalog
21352193
import: test
21362194
type: exitcode-stdio-1.0

0 commit comments

Comments
 (0)