WD-696 change MinIO to RustFS as target S3, update deserialization and tests

Author: AndrewGnev

.env

@@ -5,6 +5,8 @@ S3_REGION=us-east-1
 S3_BUCKET=test-bucket
 MINIO_ACCESS_KEY=my_access_key
 MINIO_SECRET_KEY=my_secret_key
+RUSTFS_ACCESS_KEY=my_access_key
+RUSTFS_SECRET_KEY=my_secret_key
 JPG_TEST_OBJECT=100.jpg
 PNG_TEST_OBJECT=100.png
 GIF_TEST_OBJECT=100.gif

docker-compose.yaml

@@ -11,8 +11,22 @@ services:
     environment:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+    volumes:
+      - ./data:/data
+    command: 'server /data'
+
+  rustfs:
+    image: rustfs/rustfs
+    restart: unless-stopped
+    logging:
+      driver: "json-file"
+      options:
+        max-size: "20m"
+        max-file: "5"
     ports:
       - ${APP_HOST}:9000:9000
+    environment:
+      RUSTFS_ACCESS_KEY: ${RUSTFS_ACCESS_KEY}
+      RUSTFS_SECRET_KEY: ${RUSTFS_SECRET_KEY}
     volumes:
       - ./data:/data
-    command: 'server /data'

storage-service/src/main/kotlin/com/icerockdev/service/storage/s3/policy/dto/Principal.kt

@@ -2,10 +2,17 @@ package com.icerockdev.service.storage.s3.policy.dto
 
 import com.fasterxml.jackson.annotation.JsonInclude
 import com.fasterxml.jackson.annotation.JsonProperty
+import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.core.JsonProcessingException
+import com.fasterxml.jackson.databind.DeserializationContext
+import com.fasterxml.jackson.databind.JsonDeserializer
+import com.fasterxml.jackson.databind.JsonNode
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize
 
 @JsonInclude(JsonInclude.Include.NON_EMPTY)
 data class Principal(
     @JsonProperty("AWS")
+    @JsonDeserialize(using = PrincipalAwsDeserializer::class)
     val aws: List<String>,
     @JsonProperty("CanonicalUser")
     val canonicalUser: String?,
@@ -14,3 +21,20 @@ data class Principal(
     @JsonProperty("Service")
     val service: List<String> = emptyList(),
 )
+
+class PrincipalAwsDeserializer : JsonDeserializer<List<String>>() {
+    override fun deserialize(p: JsonParser?, ctxt: DeserializationContext?): List<String> {
+        try {
+            val path = p?.parsingContext?.pathAsPointer()
+            val node = p?.codec?.readTree<JsonNode>(p)
+            return when {
+                node?.isNull == true -> emptyList()
+                node?.isArray == true -> p.codec?.treeToValue(node, Array<String>::class.java)?.toList() ?: emptyList()
+                node?.isTextual == true -> p.codec?.treeToValue(node, String::class.java)?.let { listOf(it) } ?: emptyList()
+                else -> throw IllegalArgumentException("Principal deserialization error. Unexpected value: $node at path: $path")
+            }
+        } catch (e: JsonProcessingException) {
+            throw RuntimeException("Principal deserialization error happened at: ${e.location}", e)
+        }
+    }
+}

storage-service/src/test/kotlin/S3StorageTest.kt

@@ -338,7 +338,7 @@ class S3StorageTest {
 
         val jpgObject = storage.get(bucketName, jpgFileName)
 
-        assertEquals(metadata, jpgObject?.response()?.metadata())
+        assertTrue(jpgObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         // copy testing
         val copyFileName = storage.generateFileKey()
@@ -349,7 +349,7 @@ class S3StorageTest {
 
         val copyObject = storage.get(bucketName, copyFileName)
 
-        assertEquals(metadata, copyObject?.response()?.metadata())
+        assertTrue(copyObject?.response()?.metadata()?.entries?.containsAll(metadata.entries) ?: false)
 
         assertTrue {
             storage.deleteBucketWithObjects(bucketName)