Skip to content

Commit f5a9ff2

Browse files
jhoblittjhoblitt
authored
Merge pull request #1763 from lsst-it/IT-5635/s3nd
 (site/*/cluster/*-ccs) deploy s3nd service
2 parents 9955799 + 9c3d27f commit f5a9ff2

22 files changed

+351
-21
lines changed

‎hieradata/common.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -575,6 +575,9 @@ restic::enable_backup: true
575575
restic::host: "s3.us-east-1.amazonaws.com"
576576

577577
s3daemon_default_image: "ghcr.io/lsst-dm/s3daemon:sha-57e1aa9"
578+
s3nd_default_image: "ghcr.io/lsst-dm/s3nd:sha-76c43ff"
579+
s3daemon::env:
580+
AWS_REGION: "lfa"
578581

579582
pi::config::reboot: false # XXX this should be removed when pi role/profile refactoring is complete
580583
pi::cmdline::reboot: false # XXX this should be removed when pi role/profile refactoring is complete

‎hieradata/site/cp/cluster/auxtel-ccs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,18 @@ s3daemon::instances:
55
env:
66
S3_ENDPOINT_URL: "https://s3.cp.lsst.org"
77
S3DAEMON_PORT: 15570
8+
cp-latiss-s3nd:
9+
image: "%{alias('s3nd_default_image')}"
10+
env:
11+
S3ND_ENDPOINT_URL: "https://s3.cp.lsst.org"
12+
S3ND_PORT: 15571
813
s3dfrgw-latiss:
914
image: "%{alias('s3daemon_default_image')}"
1015
env:
1116
S3_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
1217
S3DAEMON_PORT: 15580
18+
s3dfrgw-latiss-s3nd:
19+
image: "%{alias('s3nd_default_image')}"
20+
env:
21+
S3ND_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
22+
S3ND_PORT: 15581

‎hieradata/site/cp/cluster/comcam-ccs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,18 @@ s3daemon::instances:
77
env:
88
S3_ENDPOINT_URL: "https://s3.cp.lsst.org"
99
S3DAEMON_PORT: 15570
10+
cp-comcam-s3nd:
11+
image: "%{alias('s3nd_default_image')}"
12+
env:
13+
S3ND_ENDPOINT_URL: "https://s3.cp.lsst.org"
14+
S3ND_PORT: 15571
1015
sdfembs3-comcam:
1116
image: "%{alias('s3daemon_default_image')}"
1217
env:
1318
S3_ENDPOINT_URL: "https://sdfembs3.sdf.slac.stanford.edu"
1419
S3DAEMON_PORT: 15580
20+
sdfembs3-comcam-s3nd:
21+
image: "%{alias('s3nd_default_image')}"
22+
env:
23+
S3ND_ENDPOINT_URL: "https://sdfembs3.sdf.slac.stanford.edu"
24+
S3ND_PORT: 15581

‎hieradata/site/cp/cluster/lsstcam-ccs.yaml‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,32 @@ s3daemon::instances:
1919
env:
2020
S3_ENDPOINT_URL: "https://s3.cp.lsst.org"
2121
S3DAEMON_PORT: 15570
22+
S3DAEMON_HOST: "0.0.0.0" # allow connections from other nodes for testing
23+
cp-lsstcam-s3nd:
24+
image: "%{alias('s3nd_default_image')}"
25+
env:
26+
S3ND_ENDPOINT_URL: "https://s3.cp.lsst.org"
27+
S3ND_PORT: 15571
28+
S3ND_HOST: "" # allow connections from other nodes for testing
29+
S3ND_UPLOAD_TIMEOUT: "3s"
30+
S3ND_QUEUE_TIMEOUT: "30s"
31+
S3ND_UPLOAD_MAX_PARALLEL: "25"
32+
S3ND_UPLOAD_TRIES: "5"
2233
sdfembs3-lsstcam:
2334
image: "%{alias('s3daemon_default_image')}"
2435
env:
2536
S3_ENDPOINT_URL: "https://sdfembs3.sdf.slac.stanford.edu"
2637
S3DAEMON_PORT: 15580
38+
S3DAEMON_HOST: "0.0.0.0" # allow connections from other nodes for testing
39+
sdfembs3-lsstcam-s3nd:
40+
image: "%{alias('s3nd_default_image')}"
41+
env:
42+
S3ND_ENDPOINT_URL: "https://sdfembs3.sdf.slac.stanford.edu"
43+
S3ND_PORT: 15581
44+
S3ND_HOST: "" # allow connections from other nodes for testing
45+
S3ND_UPLOAD_TIMEOUT: "10s"
46+
S3ND_QUEUE_TIMEOUT: "30s"
47+
S3ND_UPLOAD_MAX_PARALLEL: "25"
48+
S3ND_UPLOAD_TRIES: "5"
49+
S3ND_UPLOAD_PARTSIZE: "50Mi" # larger than test file size
50+
S3ND_UPLOAD_BWLIMIT: "160Mi"

‎hieradata/site/ls/cluster/auxtel-ccs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,8 +25,18 @@ s3daemon::instances:
2525
env:
2626
S3_ENDPOINT_URL: "https://s3.ls.lsst.org"
2727
S3DAEMON_PORT: 15570
28+
ls-latiss-s3nd:
29+
image: "%{alias('s3nd_default_image')}"
30+
env:
31+
S3ND_ENDPOINT_URL: "https://s3.ls.lsst.org"
32+
S3ND_PORT: 15571
2833
s3dfrgw-latiss:
2934
image: "%{alias('s3daemon_default_image')}"
3035
env:
3136
S3_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
3237
S3DAEMON_PORT: 15580
38+
s3dfrgw-latiss-s3nd:
39+
image: "%{alias('s3nd_default_image')}"
40+
env:
41+
S3ND_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
42+
S3ND_PORT: 15581

‎hieradata/site/ls/cluster/lsstcam-ccs.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,3 +19,30 @@ s3daemon::instances:
1919
env:
2020
S3_ENDPOINT_URL: "https://s3.ls.lsst.org"
2121
S3DAEMON_PORT: 15570
22+
S3DAEMON_HOST: "0.0.0.0" # allow connections from other nodes for testing
23+
ls-lsstcam-s3nd: # untuned because s3.ls.lsst.org is currently low performance
24+
image: "%{alias('s3nd_default_image')}"
25+
env:
26+
S3ND_ENDPOINT_URL: "https://s3.ls.lsst.org"
27+
S3ND_PORT: 15571
28+
S3ND_HOST: "" # allow connections from other nodes for testing
29+
s3dfrgw-lsstcam:
30+
image: "%{alias('s3daemon_default_image')}"
31+
env:
32+
S3_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
33+
S3DAEMON_PORT: 15580
34+
S3DAEMON_HOST: "0.0.0.0" # allow connections from other nodes for testing
35+
# needs to be tuned for s3dfrgw; tests in 2024 shows s3dfrgw as having very
36+
# slow performance compared to sdfembs3
37+
s3dfrgw-lsstcam-s3nd:
38+
image: "%{alias('s3nd_default_image')}"
39+
env:
40+
S3ND_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
41+
S3ND_PORT: 15581
42+
S3ND_HOST: "" # allow connections from other nodes for testing
43+
S3ND_UPLOAD_TIMEOUT: "10s"
44+
S3ND_QUEUE_TIMEOUT: "30s"
45+
S3ND_UPLOAD_MAX_PARALLEL: "25"
46+
S3ND_UPLOAD_TRIES: "5"
47+
S3ND_UPLOAD_PARTSIZE: "50Mi" # larger than test file size
48+
S3ND_UPLOAD_BWLIMIT: "160Mi"

‎hieradata/site/tu/cluster/auxtel-ccs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,18 @@ s3daemon::instances:
1919
env:
2020
S3_ENDPOINT_URL: "https://s3.tu.lsst.org"
2121
S3DAEMON_PORT: 15570
22+
tu-latiss-s3nd:
23+
image: "%{alias('s3nd_default_image')}"
24+
env:
25+
S3ND_ENDPOINT_URL: "https://s3.tu.lsst.org"
26+
S3ND_PORT: 15571
2227
s3dfrgw-latiss:
2328
image: "%{alias('s3daemon_default_image')}"
2429
env:
2530
S3_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
2631
S3DAEMON_PORT: 15580
32+
s3dfrgw-latiss-s3nd:
33+
image: "%{alias('s3nd_default_image')}"
34+
env:
35+
S3ND_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
36+
S3ND_PORT: 15581

‎hieradata/site/tu/cluster/comcam-ccs.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,18 @@ s3daemon::instances:
1919
env:
2020
S3_ENDPOINT_URL: "https://s3.tu.lsst.org"
2121
S3DAEMON_PORT: 15570
22+
tu-comcam-s3nd:
23+
image: "%{alias('s3nd_default_image')}"
24+
env:
25+
S3ND_ENDPOINT_URL: "https://s3.tu.lsst.org"
26+
S3ND_PORT: 15571
2227
s3dfrgw-comcam:
2328
image: "%{alias('s3daemon_default_image')}"
2429
env:
2530
S3_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
2631
S3DAEMON_PORT: 15580
32+
s3dfrgw-comcam-s3nd:
33+
image: "%{alias('s3nd_default_image')}"
34+
env:
35+
S3ND_ENDPOINT_URL: "https://s3dfrgw.slac.stanford.edu"
36+
S3ND_PORT: 15581

‎spec/fixtures/hieradata/site/cp/cluster/auxtel-ccs.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ s3daemon::instances:
33
cp-latiss:
44
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
55
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
6+
cp-latiss-s3nd:
7+
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
8+
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
69
s3dfrgw-latiss:
710
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
811
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
12+
s3dfrgw-latiss-s3nd:
13+
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
14+
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"

‎spec/fixtures/hieradata/site/cp/cluster/comcam-ccs.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,12 @@ s3daemon::instances:
33
cp-comcam:
44
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
55
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
6+
cp-comcam-s3nd:
7+
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
8+
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
69
sdfembs3-comcam:
710
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
811
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"
12+
sdfembs3-comcam-s3nd:
13+
aws_access_key_id: "%{alias('s3daemon_test::aws_access_key_id')}"
14+
aws_secret_access_key: "%{alias('s3daemon_test::aws_secret_access_key')}"

0 commit comments

Comments
 (0)