Skip to content

Commit a29b8ca

Browse files
lauzadislauzadis
committed
Checksum bindings
1 parent 9d9faf3 commit a29b8ca

File tree

16 files changed

+239
-69
lines changed

16 files changed

+239
-69
lines changed

‎.github/scripts/run-container-test.py‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
#!/usr/bin/env python3
2+
#
3+
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
4+
# SPDX-License-Identifier: Apache-2.0
5+
6+
"""
7+
Run precompiled Kotlin/Native test binaries in a Docker container for a specific Linux distribution and architecture.
8+
9+
This requires Docker multiarch support, see https://docs.docker.com/build/building/multi-platform/ and https://github.com/multiarch/qemu-user-static
10+
In GitHub we use a provided action for this: https://github.com/docker/setup-qemu-action
11+
12+
Locally you would need to run one of:
13+
14+
`docker run --rm --privileged multiarch/qemu-user-static --reset -p yes --credential yes`
15+
16+
OR
17+
18+
`docker run --privileged --rm tonistiigi/binfmt --install all`
19+
"""
20+
21+
import argparse
22+
import os
23+
import subprocess
24+
import shlex
25+
import shutil
26+
27+
VERBOSE = False
28+
29+
DISTRO_TO_IMAGE_NAME = {
30+
"ubuntu-22.04": "public.ecr.aws/lts/ubuntu:22.04_stable",
31+
"al2023": "public.ecr.aws/amazonlinux/amazonlinux:2023",
32+
"al2": "public.ecr.aws/amazonlinux/amazonlinux:2"
33+
}
34+
35+
DOCKER_PLATFORM_BY_ARCH = {
36+
"x64": "linux/amd64",
37+
"arm64": "linux/arm64"
38+
}
39+
40+
41+
def vprint(message):
42+
global VERBOSE
43+
if VERBOSE:
44+
print(message)
45+
46+
47+
def running_in_github_action():
48+
"""
49+
Test if currently running in a GitHub action or running locally
50+
:return: True if running in GH, False otherwise
51+
"""
52+
return "GITHUB_WORKFLOW" in os.environ
53+
54+
55+
def shell(command, cwd=None, check=True, capture_output=False):
56+
"""
57+
Run a command
58+
:param command: command to run
59+
:param cwd: the current working directory to change to before executing the command
60+
:param check: flag indicating if the status code should be checked. When true an exception will be
61+
thrown if the command exits with a non-zero exit status.
62+
:returns: the subprocess CompletedProcess output
63+
"""
64+
vprint(f"running `{command}`")
65+
return subprocess.run(command, shell=True, check=check, cwd=cwd, capture_output=capture_output)
66+
67+
68+
def oci_executable():
69+
"""
70+
Attempt to find the OCI container executor used to build and run docker containers
71+
"""
72+
oci_exe = os.environ.get('OCI_EXE')
73+
if oci_exe is not None:
74+
return oci_exe
75+
76+
executors = ['finch', 'podman', 'docker']
77+
78+
for exe in executors:
79+
if shutil.which(exe) is not None:
80+
return exe
81+
82+
print("cannot find container executor")
83+
exit(1)
84+
85+
86+
def run_docker_test(opts):
87+
"""
88+
Run a docker test for a precompiled Kotlin/Native binary
89+
90+
:param opts: the parsed command line options
91+
"""
92+
platform = DOCKER_PLATFORM_BY_ARCH[opts.arch]
93+
oci_exe = oci_executable()
94+
95+
test_bin_dir = os.path.abspath(opts.test_bin_dir)
96+
image_name = DISTRO_TO_IMAGE_NAME[opts.distro]
97+
path_to_exe = f'./linux{opts.arch.capitalize()}/debugTest/test.kexe'
98+
99+
cmd = [
100+
oci_exe,
101+
'run',
102+
'--rm',
103+
f'-v{test_bin_dir}:/test',
104+
]
105+
if not opts.no_system_certs:
106+
cmd.append(f'-v/etc/ssl:/etc/ssl')
107+
108+
cmd.extend(
109+
[
110+
'-w/test',
111+
'-e DEBIAN_FRONTEND=noninteractive',
112+
'--platform',
113+
platform,
114+
image_name,
115+
path_to_exe,
116+
]
117+
)
118+
119+
cmd = shlex.join(cmd)
120+
print(cmd)
121+
shell(cmd)
122+
123+
124+
def create_cli():
125+
parser = argparse.ArgumentParser(
126+
prog="run-container-test",
127+
description="Run cross platform test binaries in a container",
128+
formatter_class=argparse.ArgumentDefaultsHelpFormatter
129+
)
130+
131+
parser.add_argument("-v", "--verbose", help="enable verbose output", action="store_true")
132+
133+
parser.add_argument("--distro", required=True, choices=DISTRO_TO_IMAGE_NAME.keys(), help="the distribution name to run the task on")
134+
parser.add_argument("--arch", required=True, choices=DOCKER_PLATFORM_BY_ARCH.keys(), help="the architecture to use")
135+
parser.add_argument("--test-bin-dir", required=True, help="the path to the test binary directory root")
136+
parser.add_argument("--no-system-certs", action='store_true', help="disable mounting system certificates into the container")
137+
138+
return parser
139+
140+
141+
def main():
142+
cli = create_cli()
143+
opts = cli.parse_args()
144+
if opts.verbose:
145+
global VERBOSE
146+
VERBOSE = True
147+
148+
run_docker_test(opts)
149+
150+
151+
if __name__ == '__main__':
152+
main()

‎bom/build.gradle.kts‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,6 @@ import org.jetbrains.kotlin.gradle.dsl.KotlinMultiplatformExtension
77
import org.jetbrains.kotlin.gradle.plugin.KotlinMultiplatformPluginWrapper
88
import org.jetbrains.kotlin.gradle.plugin.KotlinTarget
99
import org.jetbrains.kotlin.gradle.plugin.mpp.KotlinMetadataTarget
10-
import org.jetbrains.kotlin.gradle.targets.js.KotlinJsTarget
11-
import java.util.*
1210

1311
plugins {
1412
`maven-publish`
@@ -52,7 +50,6 @@ fun createBomConstraintsAndVersionCatalog() {
5250

5351
fun Project.artifactId(target: KotlinTarget): String = when (target) {
5452
is KotlinMetadataTarget -> name
55-
is KotlinJsTarget -> "$name-js"
5653
else -> "$name-${target.targetName.lowercase()}"
5754
}
5855

‎gradle/libs.versions.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ ktor-server-netty = { module = "io.ktor:ktor-server-netty", version.ref = "ktor-
9696
ktor-server-jetty-jakarta = { module = "io.ktor:ktor-server-jetty-jakarta", version.ref = "ktor-version" }
9797
ktor-server-cio = { module = "io.ktor:ktor-server-cio", version.ref = "ktor-version" }
9898
ktor-network-tls-certificates = { module = "io.ktor:ktor-network-tls-certificates", version.ref = "ktor-version" }
99+
ktor-client-core = { module = "io.ktor:ktor-client-core", version.ref = "ktor-version" }
99100

100101
kaml = { module = "com.charleskorn.kaml:kaml", version.ref = "kaml-version" }
101102
jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup-version" }

‎runtime/auth/aws-signing-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/auth/awssigning/crt/CrtSigningSuiteTest.kt‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ import aws.smithy.kotlin.runtime.auth.awssigning.tests.SigningSuiteTestBase
1010
class CrtSigningSuiteTest : SigningSuiteTestBase() {
1111
override val signer: AwsSigner = CrtAwsSigner
1212

13-
override val disabledTests = super.disabledTests + setOf(
14-
// FIXME - Signature mismatch possibly related to https://github.com/awslabs/aws-crt-java/pull/419. Needs
15-
// investigation.
16-
"get-utf8",
17-
)
13+
// override val disabledTests = super.disabledTests + setOf(
14+
// FIXME - Signature mismatch possibly related to https://github.com/awslabs/aws-crt-java/pull/419. Needs
15+
// investigation.
16+
// "get-utf8",
17+
// )
1818
}

‎runtime/crt-util/jvmAndNative/test/aws/smithy/kotlin/runtime/crt/ReadChannelBodyStreamTest.kt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import kotlinx.coroutines.Job
1414
import kotlinx.coroutines.launch
1515
import kotlinx.coroutines.test.runTest
1616
import kotlinx.coroutines.yield
17+
import kotlin.test.Ignore
1718
import kotlin.test.Test
1819
import kotlin.test.assertEquals
1920
import kotlin.test.assertFailsWith
@@ -26,6 +27,7 @@ class ReadChannelBodyStreamTest {
2627
return MutableBuffer.of(dest) to dest
2728
}
2829

30+
@Ignore
2931
@Test
3032
fun testClose() = runTest {
3133
val chan = SdkByteChannel()
@@ -42,6 +44,7 @@ class ReadChannelBodyStreamTest {
4244
assertTrue(stream.sendRequestBody(sendBuffer))
4345
}
4446

47+
@Ignore
4548
@Test
4649
fun testCancellation() = runTest {
4750
val chan = SdkByteChannel()
@@ -56,6 +59,7 @@ class ReadChannelBodyStreamTest {
5659
}
5760
}
5861

62+
@Ignore
5963
@Test
6064
fun testReadFully() = runTest {
6165
val data = byteArrayOf(1, 2, 3, 4, 5)
@@ -71,6 +75,7 @@ class ReadChannelBodyStreamTest {
7175
}
7276
}
7377

78+
@Ignore
7479
@Test
7580
fun testPartialRead() = runTest {
7681
val chan = SdkByteReadChannel("123456".encodeToByteArray())
@@ -89,6 +94,7 @@ class ReadChannelBodyStreamTest {
8994
assertEquals("456", sent2.decodeToString())
9095
}
9196

97+
@Ignore
9298
@Test
9399
fun testLargeTransfer() = runTest {
94100
val chan = SdkByteChannel()

‎runtime/crt-util/jvmAndNative/test/aws/smithy/kotlin/runtime/crt/SdkSourceBodyStreamTest.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import aws.smithy.kotlin.runtime.io.SdkBuffer
1010
import aws.smithy.kotlin.runtime.io.SdkSource
1111
import aws.smithy.kotlin.runtime.io.source
1212
import kotlinx.coroutines.test.runTest
13+
import kotlin.test.Ignore
1314
import kotlin.test.Test
1415
import kotlin.test.assertEquals
1516
import kotlin.test.assertFalse
@@ -21,6 +22,7 @@ class SdkSourceBodyStreamTest {
2122
return MutableBuffer.of(dest) to dest
2223
}
2324

25+
@Ignore
2426
@Test
2527
fun testReadFully() = runTest {
2628
val data = byteArrayOf(1, 2, 3, 4, 5)
@@ -35,6 +37,7 @@ class SdkSourceBodyStreamTest {
3537
}
3638
}
3739

40+
@Ignore
3841
@Test
3942
fun testPartialRead() = runTest {
4043
val source = "123456".encodeToByteArray().source()
@@ -52,6 +55,7 @@ class SdkSourceBodyStreamTest {
5255
assertEquals("456", sent2.decodeToString())
5356
}
5457

58+
@Ignore
5559
@Test
5660
fun testLargeTransfer() = runTest {
5761
val data = "foobar"

‎runtime/protocol/http-client-engines/http-client-engine-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/http/engine/crt/AsyncStressTest.kt‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ import aws.smithy.kotlin.runtime.http.request.url
1313
import aws.smithy.kotlin.runtime.httptest.TestWithLocalServer
1414
import aws.smithy.kotlin.runtime.net.Host
1515
import aws.smithy.kotlin.runtime.net.Scheme
16-
import aws.smithy.kotlin.runtime.testing.IgnoreWindows
1716
import io.ktor.server.cio.*
1817
import io.ktor.server.engine.*
1918
import io.ktor.server.response.*
@@ -25,9 +24,7 @@ import kotlinx.coroutines.yield
2524
import kotlin.test.Test
2625
import kotlin.time.Duration.Companion.seconds
2726

28-
// FIXME This test implements [TestWithLocalServer] which is JVM-only.
2927
class AsyncStressTest : TestWithLocalServer() {
30-
3128
override val server = embeddedServer(CIO, serverPort) {
3229
routing {
3330
get("/largeResponse") {
@@ -39,7 +36,6 @@ class AsyncStressTest : TestWithLocalServer() {
3936
}
4037
}
4138

42-
@IgnoreWindows("FIXME - times out after upgrade to kotlinx.coroutines 1.6.0")
4339
@Test
4440
fun testStreamNotConsumed() = runBlocking {
4541
// test that filling the stream window and not consuming the body stream still cleans up resources

‎runtime/protocol/http-client-engines/http-client-engine-crt/jvmAndNative/test/aws/smithy/kotlin/runtime/http/engine/crt/SendChunkedBodyTest.kt‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ class SendChunkedBodyTest {
3333

3434
val chunkedBytes = """
3535
100;chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(256)}\r\n\r\n
36-
""".trimIndent().toByteArray()
36+
""".trimIndent().encodeToByteArray()
3737

3838
val source = chunkedBytes.source()
3939

@@ -52,7 +52,7 @@ class SendChunkedBodyTest {
5252

5353
val chunkedBytes = """
5454
${chunkSize.toString(16)};chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(chunkSize)}\r\n\r\n
55-
""".trimIndent().toByteArray()
55+
""".trimIndent().encodeToByteArray()
5656

5757
val source = chunkedBytes.source()
5858

@@ -71,7 +71,7 @@ class SendChunkedBodyTest {
7171

7272
val chunkedBytes = """
7373
100;chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(256)}\r\n\r\n
74-
""".trimIndent().toByteArray()
74+
""".trimIndent().encodeToByteArray()
7575

7676
val channel = SdkByteReadChannel(chunkedBytes)
7777

@@ -91,7 +91,7 @@ class SendChunkedBodyTest {
9191

9292
val chunkedBytes = """
9393
${chunkSize.toString(16)};chunk-signature=${"0".repeat(64)}\r\n${"0".repeat(chunkSize)}\r\n\r\n
94-
""".trimIndent().toByteArray()
94+
""".trimIndent().encodeToByteArray()
9595

9696
val channel = SdkByteReadChannel(chunkedBytes)
9797

‎runtime/protocol/http-test/build.gradle.kts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,14 @@ kotlin {
2828
}
2929
}
3030

31+
jvmAndNativeMain {
32+
dependencies {
33+
api(libs.ktor.server.cio)
34+
implementation(libs.ktor.client.core)
35+
implementation(libs.ktor.server.core)
36+
}
37+
}
38+
3139
all {
3240
languageSettings.optIn("aws.smithy.kotlin.runtime.InternalApi")
3341
}

0 commit comments

Comments
 (0)