Merge pull request #2244 from neicnordic/feature/multiple-backends-applications-impl

Feature/multiple backends applications impl

Author: jbygdell

.github/integration/scripts/charts/dependencies.sh

@@ -77,15 +77,58 @@ helm install --namespace default nfs-ganesha nfs-ganesha-server-and-external-pro
 kubectl create namespace minio
 kubectl apply -f .github/integration/scripts/charts/dependencies.yaml
 
+
+values_file=".github/integration/scripts/charts/values.yaml"
+if [ "$1" == "local" ]; then
+  values_file=/tmp/values.yaml
+  cp .github/integration/scripts/charts/values.yaml /tmp/values.yaml
+fi
+
 if [ "$2" == "s3" ]; then
-        ## S3 storage backend
-        MINIO_ACCESS="$(random-string)"
-        export MINIO_ACCESS
-        MINIO_SECRET="$(random-string)"
-        export MINIO_SECRET
-        helm install minio minio/minio \
-                --namespace minio \
-                --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi
+  if [ "$3" = true ] ; then
+    # Sleep to give cert issuer time to issue certs so we can create an secret with format minio expects
+    sleep 5
+
+    kubectl -n minio create secret generic minio-tls \
+      --from-file=public.crt=<(kubectl -n minio get secret minio-cert -o jsonpath='{.data.tls\.crt}' | base64 -d) \
+      --from-file=private.key=<(kubectl -n minio get secret minio-cert -o jsonpath='{.data.tls\.key}' | base64 -d)
+
+    ## S3 storage backend
+    MINIO_ACCESS="$(random-string)"
+    export MINIO_ACCESS
+    MINIO_SECRET="$(random-string)"
+    export MINIO_SECRET
+    helm install minio minio/minio \
+            --namespace minio \
+            --set tls.enabled=true,tls.certSecret=minio-tls,rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi
+
+    yq -i '
+.global.archive.s3Url = "https://minio.minio" |
+.global.backupArchive.s3Url = "https://minio.minio" |
+.global.inbox.s3Url = "https://minio.minio" |
+.global.s3Inbox.url = "https://minio.minio" |
+.global.sync.destination.s3Url = "https://minio.minio"
+' "$values_file"
+
+  else
+    ## S3 storage backend
+    MINIO_ACCESS="$(random-string)"
+    export MINIO_ACCESS
+    MINIO_SECRET="$(random-string)"
+    export MINIO_SECRET
+    helm install minio minio/minio \
+            --namespace minio \
+            --set rootUser="$MINIO_ACCESS",rootPassword="$MINIO_SECRET",persistence.enabled=false,mode=standalone,resources.requests.memory=128Mi
+
+    yq -i '
+.global.archive.s3Url = "http://minio.minio" |
+.global.backupArchive.s3Url = "http://minio.minio" |
+.global.inbox.s3Url = "http://minio.minio" |
+.global.s3Inbox.url = "http://minio.minio" |
+.global.sync.destination.s3Url = "http://minio.minio"
+' "$values_file"
+
+  fi
 fi
 
 PGPASSWORD="$(random-string)"
@@ -97,12 +140,6 @@ export MQPASSWORD
 TEST_TOKEN="$(bash .github/integration/scripts/sign_jwt.sh ES256 "$dir/jwt.key")"
 export TEST_TOKEN
 
-values_file=".github/integration/scripts/charts/values.yaml"
-if [ "$1" == "local" ]; then
-        values_file=/tmp/values.yaml
-        cp .github/integration/scripts/charts/values.yaml /tmp/values.yaml
-fi
-
 ## update values file with all credentials
 if [ "$2" == "federated" ]; then
         yq -i '.global.schemaType = "federated"' "$values_file"
@@ -116,10 +153,13 @@ yq -i '
 .global.broker.password = strenv(MQPASSWORD) |
 .global.c4gh.privateKeys[0].passphrase = strenv(C4GHPASSPHRASE) |
 .global.db.password = strenv(PGPASSWORD) |
+.global.db.admin.password = strenv(PGPASSWORD) |
 .global.inbox.s3AccessKey = strenv(MINIO_ACCESS) |
 .global.inbox.s3SecretKey = strenv(MINIO_SECRET) |
-.global.sync.destination.accessKey = strenv(MINIO_ACCESS) |
-.global.sync.destination.secretKey = strenv(MINIO_SECRET) |
+.global.s3Inbox.accessKey = strenv(MINIO_ACCESS) |
+.global.s3Inbox.secretKey = strenv(MINIO_SECRET) |
+.global.sync.destination.s3AccessKey = strenv(MINIO_ACCESS) |
+.global.sync.destination.s3SecretKey = strenv(MINIO_SECRET) |
 .releasetest.secrets.accessToken = strenv(TEST_TOKEN)
 ' "$values_file"
 

.github/integration/scripts/charts/dependencies.yaml

@@ -50,6 +50,7 @@ spec:
   dnsNames:
     - localhost
     - minio
+    - minio.minio
     - minio.minio.svc
     - minio.minio.svc.cluster.local
   ipAddresses:

.github/integration/scripts/charts/values.yaml

@@ -22,20 +22,33 @@ global:
     storageType: s3
     s3AccessKey: PLACEHOLDER_VALUE
     s3SecretKey: PLACEHOLDER_VALUE
-    s3Url: "http://minio.minio"
+    s3Url: PLACEHOLDER_VALUE
     s3Bucket: "archive"
     s3Port: 9000
-    s3ReadyPath: "/minio/health/ready"
     existingClaim: archive-pvc
   backupArchive:
     storageType: "s3"
     s3AccessKey: PLACEHOLDER_VALUE
     s3SecretKey: PLACEHOLDER_VALUE
-    s3Url: "http://minio.minio"
+    s3Url: PLACEHOLDER_VALUE
     s3Bucket: "backup"
     s3Port: 9000
-    s3ReadyPath: "/minio/health/ready"
     existingClaim: backup-pvc
+  inbox:
+    storageType: s3
+    s3AccessKey: PLACEHOLDER_VALUE
+    s3SecretKey: PLACEHOLDER_VALUE
+    s3Url: PLACEHOLDER_VALUE
+    s3Port: 9000
+    s3Bucket: "inbox"
+    existingClaim: inbox-pvc
+  s3Inbox:
+    accessKey: PLACEHOLDER_VALUE
+    secretKey: PLACEHOLDER_VALUE
+    url: PLACEHOLDER_VALUE
+    port: 9000
+    bucket: "inbox"
+    readyPath: "/minio/health/ready"
   auth:
     jwtSecret: jwk
     jwtAlg: ES256
@@ -65,6 +78,9 @@ global:
     host: "postgres-sda-db"
     user: "postgres"
     password: PLACEHOLDER_VALUE
+    admin:
+      username: "postgres"
+      password: PLACEHOLDER_VALUE
   doa:
     enabled: false
   download:
@@ -80,15 +96,6 @@ global:
     jwkPath: "/jwks"
     id: DfCieZLuBU
     secret: DfCieZLuBU
-  inbox:
-    storageType: s3
-    s3AccessKey: PLACEHOLDER_VALUE
-    s3SecretKey: PLACEHOLDER_VALUE
-    s3Url: http://minio.minio
-    s3Port: 9000
-    s3Bucket: "inbox"
-    s3ReadyPath: "/minio/health/ready"
-    existingClaim: inbox-pvc
   reencrypt:
     host: pipeline-sda-svc-reencrypt
     port: 50051
@@ -99,14 +106,12 @@ global:
     brokerQueue: "mapping_stream"
     centerPrefix: "SYNC"
     destination:
-      storageType: "s3"
-      url: "http://minio.minio"
-      port: 9000
-      readypath: "/minio/health/ready"
-      accessKey: PLACEHOLDER_VALUE
-      secretKey: PLACEHOLDER_VALUE
-      bucket: "sync"
-      region: "us-east-1"
+      s3Url: PLACEHOLDER_VALUE
+      s3Port: 9000
+      s3AccessKey: PLACEHOLDER_VALUE
+      s3SecretKey: PLACEHOLDER_VALUE
+      s3BucketPrefix: "sync"
+      s3Region: "us-east-1"
     remote:
       host: "http://remote-sync"
       port: "8080"

.github/integration/sda/config.yaml

@@ -3,15 +3,54 @@ log:
   level: "debug"
 api:
   rbacFile: /rbac.json
-archive:
-  type: s3
-  url: "http://s3"
-  port: 9000
-  readypath: "/minio/health/ready"
-  accessKey: "access"
-  secretKey: "secretKey"
-  bucket: "archive"
+
+storage:
+  archive:
+    s3:
+    - endpoint: "http://s3:9000"
+      access_key: "access"
+      secret_key: "secretKey"
+      bucket_prefix: "archive"
+      region: "us-east-1"
+      disable_https: true
+      max_buckets: 10
+      max_objects: 3
+  backup:
+    s3:
+    - endpoint: "http://s3:9000"
+      access_key: "access"
+      secret_key: "secretKey"
+      bucket_prefix: "backup"
+      region: "us-east-1"
+      disable_https: true
+  inbox:
+    s3:
+    - endpoint: "http://s3:9000"
+      access_key: "access"
+      secret_key: "secretKey"
+      bucket_prefix: "inbox"
+      region: "us-east-1"
+      disable_https: true
+  sync:
+    s3:
+    - endpoint: "http://s3:9000"
+      access_key: "access"
+      secret_key: "secretKey"
+      bucket_prefix: "sync"
+      region: "us-east-1"
+      disable_https: true
+      max_buckets: 10
+      max_objects: 10
+
+s3inbox:
+  endpoint: "http://s3:9000"
+  access_key: "access"
+  secret_key: "secretKey"
+  bucket: "inbox"
   region: "us-east-1"
+  ready_path: "/minio/health/ready"
+
+location_broker.cache_ttl: 0
 
 grpc:
   host: "reencrypt"
@@ -33,25 +72,6 @@ auth:
   resignJwt:
   s3Inbox: "http://inbox:8000"
 
-backup:
-  type: s3
-  url: "http://s3"
-  port: 9000
-  readypath: "/minio/health/ready"
-  accessKey: "access"
-  secretKey: "secretKey"
-  bucket: "backup"
-  region: "us-east-1"
-inbox:
-  type: s3
-  url: "http://s3"
-  port: 9000
-  readypath: "/minio/health/ready"
-  accessKey: "access"
-  secretKey: "secretKey"
-  bucket: "inbox"
-  region: "us-east-1"
-
 broker:
   host: "rabbitmq"
   port: "5672"
@@ -99,15 +119,6 @@ sync:
     password: "pass"
     user: "user"
   centerPrefix: "SYNC"
-  destination:
-    type: "s3"
-    url: "http://s3"
-    port: 9000
-    readypath: "/minio/health/ready"
-    accessKey: "access"
-    secretKey: "secretKey"
-    bucket: "sync"
-    region: "us-east-1"
   remote:
     host: "http://sync-api"
     port: "8080"

.github/integration/tests/sda/20_ingest-verify_test.sh

@@ -80,4 +80,16 @@ if [ "$key_hashes" -eq 0 ]; then
 	exit 1
 fi
 
+num_in_archive_1="$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(id) FROM sda.files WHERE archive_location LIKE '%archive1'")"
+if [ "$num_in_archive_1" -ne 3 ]; then
+	echo "::error::Unexpected amount of files in archive bucket 1: $num_in_archive_1, expected 3"
+	exit 1
+fi
+
+num_in_archive_2="$(psql -U postgres -h postgres -d sda -At -c "SELECT COUNT(id) FROM sda.files WHERE archive_location LIKE '%archive2'")"
+if [ "$num_in_archive_2" -ne 1 ]; then
+	echo "::error::Unexpected amount of files in archive bucket 2: : $num_in_archive_2, expected 1"
+	exit 1
+fi
+
 echo "ingestion and verification test completed successfully"

.github/integration/tests/sda/21_cancel_test.sh

@@ -60,6 +60,47 @@ if [ "$(psql -U postgres -h postgres -d sda -At -c "select event from sda.file_e
     exit 1
 fi
 
+
+# check database to verify file archive location and path has been unset
+if [ "$(psql -U postgres -h postgres -d sda -At -c "SELECT 1 FROM sda.files WHERE id = '$CORRID' AND archive_file_path = '' AND archive_location IS NULL")" != "1" ]; then
+    echo "canceling file failed"
+    exit 1
+fi
+
+cat >/shared/direct <<EOD
+[default]
+access_key=access
+secret_key=secretKey
+check_ssl_certificate = False
+check_ssl_hostname = False
+encoding = UTF-8
+encrypt = False
+guess_mime_type = True
+host_base = s3:9000
+host_bucket = s3:9000
+human_readable_sizes = false
+multipart_chunk_size_mb = 50
+use_https = False
+socket_timeout = 30
+EOD
+
+# Verify that archived file is removed
+result=$(s3cmd -c direct ls s3://archive1/test_dummy.org/"$CORRID")
+if [ "$result" != "" ]; then
+    echo "file with id $CORRID was not removed from archive"
+    exit 1
+fi
+result=$(s3cmd -c direct ls s3://archive2/test_dummy.org/"$CORRID")
+if [ "$result" != "" ]; then
+    echo "file with id $CORRID was not removed from archive"
+    exit 1
+fi
+result=$(s3cmd -c direct ls s3://backup1/test_dummy.org/"$CORRID")
+if [ "$result" != "" ]; then
+    echo "file with id $CORRID was not removed from backup"
+    exit 1
+fi
+
 # re-ingest cancelled file
 ingest_payload=$(
     jq -r -c -n \

.github/integration/tests/sda/30_backup-finalize_test.sh

@@ -95,7 +95,7 @@ EOD
 # check DB for archive file names
 for file in NA12878.bam.c4gh NA12878.bai.c4gh NA12878_20k_b37.bam.c4gh NA12878_20k_b37.bai.c4gh; do
     archiveName=$(psql -U postgres -h postgres -d sda -At -c "SELECT archive_file_path from sda.files where submission_file_path = '$file';")
-    size=$(s3cmd -c direct ls s3://backup/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3)
+    size=$(s3cmd -c direct ls s3://backup1/"$archiveName" | tr -s ' ' | cut -d ' ' -f 3)
     if [ "$size" -eq 0 ]; then
         echo "Failed to get size of $file from backup site"
         exit 1

.github/integration/tests/sda/45_sync_test.sh

@@ -11,7 +11,7 @@ fi
 # check bucket for synced files
 for file in NA12878.bai NA12878_20k_b37.bai; do
     RETRY_TIMES=0
-    until [ "$(s3cmd -c direct ls s3://sync/"$file")" != "" ]; do
+    until [ "$(s3cmd -c direct ls s3://sync1/"$file")" != "" ]; do
         RETRY_TIMES=$((RETRY_TIMES + 1))
         if [ "$RETRY_TIMES" -eq 30 ]; then
             echo "::error::Time out while waiting for files to be synced"