save workspaceFiles in s3 service instead of localStorage

changed local storage saving to s3 saving
added s3 tests capabilities
added integration tests for mentionned changes

Author: Leopold-Cramer

api/kubernetes/helm-chart/templates/_helpers.tpl

@@ -108,13 +108,6 @@ E.g:
 {{- end }}
 {{- end }}
 
-{{/*
-Location of the persistence data
-*/}}
-{{- define "cosmotech-api.blobPersistencePath" -}}
-"/var/lib/cosmotech-api/data"
-{{- end }}
-
 {{- define "cosmotech-api.custom-rootca-path" -}}
 /mnt/cosmotech/certificates
 {{- end }}
@@ -197,8 +190,6 @@ csm:
       {{- else }}
       image-pull-secrets: []
       {{- end }}
-    blobPersistence:
-      path: {{ include "cosmotech-api.blobPersistencePath" . }}
     identityProvider:
       tls:
         bundle: {{ include "cosmotech-api.custom-rootca-bundle" . }}

api/kubernetes/helm-chart/templates/deployment.yaml

@@ -46,11 +46,6 @@ spec:
         - name: helm-config
           secret:
             secretName: {{ include "cosmotech-api.fullname" . }}
-        {{if .Values.persistence.enabled}}
-        - name: blob-storage
-          persistentVolumeClaim:
-            claimName: {{ include "cosmotech-api.fullname" . }}-blob-storage
-        {{end}}
         {{if .Values.api.tlsTruststore.enabled}}
         - name: custom-rootca
           secret:
@@ -90,11 +85,6 @@ spec:
             - mountPath: /config
               name: helm-config
               readOnly: true
-            {{if .Values.persistence.enabled}}
-            - mountPath: {{ include "cosmotech-api.blobPersistencePath" . }}
-              name: blob-storage
-              readOnly: false
-            {{end}}
             {{if .Values.api.tlsTruststore.enabled }}
             - mountPath: {{ include "cosmotech-api.custom-rootca-path" . | quote }}
               name: custom-rootca

api/kubernetes/helm-chart/templates/pvc.yaml

@@ -192,6 +192,11 @@ config:
           requests:
             # File storage minimal claim is 100Gi for Premium classes
             storage: 100Gi
+      s3:
+        endpointUrl: "http://s3-server:9000"
+        bucketName: "changeme"
+        accessKeyId: "changeme"
+        secretAccessKey: "changeme"
       authorization:
         allowed-tenants: []
       identityProvider:

api/kubernetes/helm-chart/values.yaml

@@ -0,0 +1,31 @@
+// Copyright (c) Cosmo Tech.
+// Licensed under the MIT license.
+package com.cosmotech.api.exceptions
+
+import java.net.URI
+import org.springframework.core.Ordered
+import org.springframework.core.annotation.Order
+import org.springframework.http.HttpStatus
+import org.springframework.http.ProblemDetail
+import org.springframework.web.bind.annotation.ExceptionHandler
+import org.springframework.web.bind.annotation.RestControllerAdvice
+import software.amazon.awssdk.awscore.exception.AwsServiceException
+
+@RestControllerAdvice
+@Order(Ordered.HIGHEST_PRECEDENCE)
+internal class AwsExceptionHandling : CsmExceptionHandling() {
+  private val httpStatusCodeTypePrefix = "https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/"
+
+  @ExceptionHandler
+  fun handleBlobStorageException(exception: AwsServiceException): ProblemDetail {
+    val status = HttpStatus.INTERNAL_SERVER_ERROR
+
+    val problemDetail = ProblemDetail.forStatus(status)
+    problemDetail.type = URI.create(httpStatusCodeTypePrefix + status.value())
+
+    if (exception.message != null) {
+      problemDetail.detail = exception.message
+    }
+    return problemDetail
+  }
+}

api/src/main/kotlin/com/cosmotech/api/exceptions/AwsExceptionHandling.kt

@@ -34,6 +34,19 @@ spring:
       enabled: false
     kubernetes:
       enabled: false
+    aws:
+      credentials:
+        access-key: ${csm.platform.s3.accessKeyId}
+        secret-key: ${csm.platform.s3.secretAccessKey}
+      s3:
+        # We don't need/have a region for our local S3 service but the AWS SDK requires one to be configured
+        region: ${csm.platform.s3.region}
+        # Enable path-style / disable DNS-style
+        # By default, and for AWS S3, the client crafts its URL with the bucket as sub-domain of the endpoint
+        # which is not how our current S3 implementation works as it expects the bucket in the path
+        # '<bucket_name>.<endpoint>/<object_key>' DNS vs Path '<endpoint>/<bucket_name>/<object_key>'
+        path-style-access-enabled: true
+        endpoint: ${csm.platform.s3.endpointUrl}
 
 management:
   endpoints:
@@ -119,8 +132,11 @@ csm:
         sender:
           username: "eventbus_sender_username"
           password: "eventbus_sender_password"
-    blobPersistence:
-      path: /tmp/cosmotech-api-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "cosmotech-api"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
     argo:
       base-uri: "https://localhost:2746"
       image-pull-secrets: []

api/src/main/resources/application.yml

@@ -61,7 +61,7 @@ val springWebVersion = "6.2.1"
 
 // Implementation
 val kotlinJvmTarget = 21
-val cosmotechApiCommonVersion = "2.1.0-SNAPSHOT"
+val cosmotechApiCommonVersion = "2.1.1-LCRA-store_workspace_files_in_seaweed_PROD-14290-SNAPSHOT"
 val jedisVersion = "4.4.6"
 val springOauthVersion = "6.4.2"
 val redisOmSpringVersion = "0.9.7"
@@ -79,6 +79,7 @@ val testNgVersion = "7.8.0"
 val testContainersRedisVersion = "1.6.4"
 val testContainersPostgreSQLVersion = "1.19.7"
 val commonCompressVersion = "1.27.1"
+val awsSpringVersion = "3.1.1"
 
 // Checks
 val detektVersion = "1.23.7"
@@ -140,6 +141,7 @@ allprojects {
   configurations { all { resolutionStrategy { force("com.redis.om:redis-om-spring:0.9.1") } } }
 
   repositories {
+    mavenLocal()
     maven {
       name = "GitHubPackages"
       url = uri("https://maven.pkg.github.com/Cosmo-Tech/cosmotech-api-common")
@@ -312,6 +314,9 @@ subprojects {
 
     implementation("org.json:json:$orgJsonVersion")
 
+    implementation(platform("io.awspring.cloud:spring-cloud-aws-dependencies:$awsSpringVersion"))
+    implementation("io.awspring.cloud:spring-cloud-aws-starter-s3:$awsSpringVersion")
+
     testImplementation(kotlin("test"))
     testImplementation(platform("org.junit:junit-bom:$jUnitBomVersion"))
     testImplementation("org.junit.jupiter:junit-jupiter")

build.gradle.kts

@@ -40,6 +40,15 @@ spring:
         keycloak:
           truststore:
             certificate: "classpath:[fill-this-value].pem" # certificate file
+  cloud:
+    aws:
+      credentials:
+        access-key: "s3_username"
+        secret-key: "s3_password"
+      s3:
+        region: "dummy"
+        path-style-access-enabled: true
+        endpoint: "http://localhost:9000"
 
 csm:
   platform:
@@ -72,6 +81,11 @@ csm:
       tokenUrl: "[fill-this-value]" # eg. https://kubernetes.cosmotech.com/keycloak/realms/brewery/protocol/openid-connect/token
     metrics:
       enabled: false
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "cosmotech-api"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"
     argo:
       base-uri: "http://localhost:2746"
       workflows:

config/application-dev.sample.yml

@@ -95,5 +95,8 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
-    blobPersistence:
-      path: /tmp/cosmotech-api-connector-test-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"

connector/src/integrationTest/resources/application-connector-test.yml

@@ -100,5 +100,8 @@ csm:
         default-page-size: 5
     rbac:
       enabled: true
-    blobPersistence:
-      path: /tmp/cosmotech-api-dataset-test-data
+    s3:
+      endpointUrl: "http://localhost:9000"
+      bucketName: "test-bucket"
+      accessKeyId: "s3_username"
+      secretAccessKey: "s3_password"