feat: add HashingSource and HashingSink (#756)

Author: lauzadis

.changes/01b0c84d-bd45-4742-aad7-c3a1f9bd5f17.json

@@ -0,0 +1,5 @@
+{
+    "id": "01b0c84d-bd45-4742-aad7-c3a1f9bd5f17",
+    "type": "feature",
+    "description": "Add HashingSource and HashingSink"
+}

runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt

@@ -5,7 +5,6 @@
 package aws.smithy.kotlin.runtime.auth.awssigning
 
 import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials
-import aws.smithy.kotlin.runtime.hashing.HashFunction
 import aws.smithy.kotlin.runtime.hashing.HashSupplier
 import aws.smithy.kotlin.runtime.hashing.Sha256
 import aws.smithy.kotlin.runtime.hashing.hash
@@ -17,7 +16,6 @@ import aws.smithy.kotlin.runtime.http.request.toBuilder
 import aws.smithy.kotlin.runtime.http.util.encodeLabel
 import aws.smithy.kotlin.runtime.io.*
 import aws.smithy.kotlin.runtime.io.internal.SdkDispatchers
-import aws.smithy.kotlin.runtime.io.internal.SdkSinkObserver
 import aws.smithy.kotlin.runtime.time.TimestampFormat
 import aws.smithy.kotlin.runtime.util.*
 import aws.smithy.kotlin.runtime.util.text.*
@@ -179,15 +177,6 @@ internal class DefaultCanonicalizer(private val sha256Supplier: HashSupplier = :
     }
 }
 
-private class HashingSink(
-    val hash: HashFunction,
-    sink: SdkSink = SdkSink.blackhole(),
-) : SdkSinkObserver(sink) {
-    override fun observe(data: ByteArray, offset: Int, length: Int) {
-        hash.update(data, offset, length)
-    }
-}
-
 /** The number of bytes to read at a time during SHA256 calculation on streaming bodies. */
 private const val STREAM_CHUNK_BYTES = 16384 // 16KB
 

runtime/io/build.gradle.kts

@@ -29,6 +29,7 @@ kotlin {
         commonMain {
             dependencies {
                 implementation(project(":runtime:utils"))
+                implementation(project(":runtime:hashing"))
 
                 implementation("com.squareup.okio:okio:$okioVersion")
                 implementation("org.jetbrains.kotlinx:atomicfu:$atomicFuVersion")

runtime/io/common/src/aws/smithy/kotlin/runtime/io/HashingSink.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.io
+
+import aws.smithy.kotlin.runtime.hashing.HashFunction
+import aws.smithy.kotlin.runtime.io.internal.SdkSinkObserver
+import aws.smithy.kotlin.runtime.util.InternalApi
+
+/**
+ * A sink which hashes data as it is being written to
+ * @param hash The [HashFunction] to hash data with
+ * @param sink the [SdkSink] to hash
+ */
+@InternalApi
+public class HashingSink(private val hash: HashFunction, sink: SdkSink = SdkSink.blackhole()) : SdkSinkObserver(sink) {
+    override fun observe(data: ByteArray, offset: Int, length: Int) {
+        hash.update(data, offset, length)
+    }
+
+    /**
+     * Provides the digest as a ByteArray
+     * @return a ByteArray representing the contents of the hash
+     */
+    public fun digest(): ByteArray = hash.digest()
+}

runtime/io/common/src/aws/smithy/kotlin/runtime/io/HashingSource.kt

@@ -0,0 +1,28 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.io
+
+import aws.smithy.kotlin.runtime.hashing.HashFunction
+import aws.smithy.kotlin.runtime.io.internal.SdkSourceObserver
+import aws.smithy.kotlin.runtime.util.InternalApi
+
+/**
+ * A source which hashes data as it is being consumed
+ * @param hash The [HashFunction] to hash data with
+ * @param source the [SdkSource] to hash
+ */
+@InternalApi
+public class HashingSource(private val hash: HashFunction, source: SdkSource) : SdkSourceObserver(source) {
+    override fun observe(data: ByteArray, offset: Int, length: Int) {
+        hash.update(data, offset, length)
+    }
+
+    /**
+     * Provides the digest as a ByteArray
+     * @return a ByteArray representing the contents of the hash
+     */
+    public fun digest(): ByteArray = hash.digest()
+}

runtime/io/common/test/aws/smithy/kotlin/runtime/io/HashingSinkTest.kt

@@ -0,0 +1,59 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.io
+
+import aws.smithy.kotlin.runtime.hashing.*
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
+import kotlin.test.assertEquals
+
+class HashingSinkTest {
+    private fun getHashFunction(name: String): HashFunction = when (name) {
+        "crc32" -> Crc32()
+        "crc32c" -> Crc32c()
+        "md5" -> Md5()
+        "sha1" -> Sha1()
+        "sha256" -> Sha256()
+        else -> throw RuntimeException("HashFunction $name is not supported")
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = ["crc32", "crc32c", "md5", "sha1", "sha256"])
+    fun testHashingSinkDigest(hashFunctionName: String) = run {
+        val byteArray = ByteArray(19456) { 0xf }
+        val buffer = SdkBuffer()
+        buffer.write(byteArray)
+
+        val hashingSink = HashingSink(getHashFunction(hashFunctionName), SdkSink.blackhole())
+
+        val expectedHash = getHashFunction(hashFunctionName)
+
+        assertEquals(expectedHash.digest().decodeToString(), hashingSink.digest().decodeToString())
+        hashingSink.write(buffer, buffer.size)
+        expectedHash.update(byteArray)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSink.digest().decodeToString())
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = ["crc32", "crc32c", "md5", "sha1", "sha256"])
+    fun testHashingSinkPartialWrite(hashFunctionName: String) = run {
+        val byteArray = ByteArray(19456) { 0xf }
+        val buffer = SdkBuffer()
+        buffer.write(byteArray)
+
+        val hashingSink = HashingSink(getHashFunction(hashFunctionName), SdkSink.blackhole())
+        val expectedHash = getHashFunction(hashFunctionName)
+
+        assertEquals(expectedHash.digest().decodeToString(), hashingSink.digest().decodeToString())
+        hashingSink.write(buffer, 512)
+        expectedHash.update(byteArray, 0, 512)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSink.digest().decodeToString())
+
+        hashingSink.write(buffer, 512)
+        expectedHash.update(byteArray, 512, 512)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSink.digest().decodeToString())
+    }
+}

runtime/io/common/test/aws/smithy/kotlin/runtime/io/HashingSourceTest.kt

@@ -0,0 +1,61 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package aws.smithy.kotlin.runtime.io
+
+import aws.smithy.kotlin.runtime.hashing.*
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.ValueSource
+import kotlin.test.assertEquals
+
+class HashingSourceTest {
+    private fun getHashFunction(name: String): HashFunction = when (name) {
+        "crc32" -> Crc32()
+        "crc32c" -> Crc32c()
+        "md5" -> Md5()
+        "sha1" -> Sha1()
+        "sha256" -> Sha256()
+        else -> throw RuntimeException("HashFunction $name is not supported")
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = ["crc32", "crc32c", "md5", "sha1", "sha256"])
+    fun testHashingSourceDigest(hashFunctionName: String) = run {
+        val byteArray = ByteArray(19456) { 0xf }
+        val source = byteArray.source()
+        val hashingSource = HashingSource(getHashFunction(hashFunctionName), source)
+
+        val sink = SdkBuffer()
+
+        val expectedHash = getHashFunction(hashFunctionName)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSource.digest().decodeToString())
+
+        hashingSource.read(sink, 19456)
+        expectedHash.update(byteArray)
+
+        assertEquals(expectedHash.digest().decodeToString(), hashingSource.digest().decodeToString())
+    }
+
+    @ParameterizedTest
+    @ValueSource(strings = ["crc32", "crc32c", "md5", "sha1", "sha256"])
+    fun testHashingSourcePartialRead(hashFunctionName: String) = run {
+        val byteArray = ByteArray(19456) { 0xf }
+        val source = byteArray.source()
+        val hashingSource = HashingSource(getHashFunction(hashFunctionName), source)
+
+        val sink = SdkBuffer()
+
+        val expectedHash = getHashFunction(hashFunctionName)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSource.digest().decodeToString())
+
+        hashingSource.read(sink, 512)
+        expectedHash.update(byteArray, 0, 512)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSource.digest().decodeToString())
+
+        hashingSource.read(sink, 512)
+        expectedHash.update(byteArray, 512, 512)
+        assertEquals(expectedHash.digest().decodeToString(), hashingSource.digest().decodeToString())
+    }
+}