From b1e4f1966adf69474870be8593e450c20a87a116 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Mon, 28 Mar 2022 10:59:34 +0200 Subject: [PATCH 001/146] Use SPM API diff checker (#572) ### Motivation: SPM has built in functionality to check the API of modules against a target git treeish. We can use this to simplify our `check_no_api_breakages.sh` script. Closes https://github.com/apple/swift-nio/issues/1239 ### Modifications: This PR, exchanges the direct calls to Swift's API checker with the new SPM `diagnose-api-breaking-changes` tool. This allows us to get rid of the manual module parsing, build invocations and result comparisons. ### Result: We are now using SPMs `diagnose-api-breaking-changes` to check for breaking changes. --- scripts/check_no_api_breakages.sh | 80 +++---------------------------- 1 file changed, 6 insertions(+), 74 deletions(-) diff --git a/scripts/check_no_api_breakages.sh b/scripts/check_no_api_breakages.sh index f380e2423..0fa3d4fb3 100755 --- a/scripts/check_no_api_breakages.sh +++ b/scripts/check_no_api_breakages.sh @@ -29,44 +29,15 @@ set -eu -# repodir -function all_modules() { - local repodir="$1" - ( - set -eu - cd "$repodir" - swift package dump-package | jq '.products | - map(select(.type | has("library") )) | - map(.name) | .[]' | tr -d '"' - ) -} - -# repodir tag output -function build_and_do() { - local repodir=$1 - local tag=$2 - local output=$3 - - ( - cd "$repodir" - git checkout -q "$tag" - swift build - while read -r module; do - swift api-digester -sdk "$sdk" -dump-sdk -module "$module" \ - -o "$output/$module.json" -I "$repodir/.build/debug" - done < <(all_modules "$repodir") - ) -} - function usage() { echo >&2 "Usage: $0 REPO-GITHUB-URL NEW-VERSION OLD-VERSIONS..." echo >&2 - echo >&2 "This script requires a Swift 5.1+ toolchain." + echo >&2 "This script requires a Swift 5.2+ toolchain." echo >&2 echo >&2 "Examples:" echo >&2 - echo >&2 "Check between master and tag 2.1.1 of swift-nio:" - echo >&2 " $0 https://github.com/apple/swift-nio master 2.1.1" + echo >&2 "Check between main and tag 2.1.1 of swift-nio:" + echo >&2 " $0 https://github.com/apple/swift-nio main 2.1.1" echo >&2 echo >&2 "Check between HEAD and commit 64cf63d7 using the provided toolchain:" echo >&2 " xcrun --toolchain org.swift.5120190702a $0 ../some-local-repo HEAD 64cf63d7" @@ -77,12 +48,6 @@ if [[ $# -lt 3 ]]; then exit 1 fi -sdk=/ -if [[ "$(uname -s)" == Darwin ]]; then - sdk=$(xcrun --show-sdk-path) -fi - -hash jq 2> /dev/null || { echo >&2 "ERROR: jq must be installed"; exit 1; } tmpdir=$(mktemp -d /tmp/.check-api_XXXXXX) repo_url=$1 new_tag=$2 @@ -91,46 +56,13 @@ shift 2 repodir="$tmpdir/repo" git clone "$repo_url" "$repodir" git -C "$repodir" fetch -q origin '+refs/pull/*:refs/remotes/origin/pr/*' -errors=0 +cd "$repodir" +git checkout -q "$new_tag" for old_tag in "$@"; do - mkdir "$tmpdir/api-old" - mkdir "$tmpdir/api-new" - echo "Checking public API breakages from $old_tag to $new_tag" - build_and_do "$repodir" "$new_tag" "$tmpdir/api-new/" - build_and_do "$repodir" "$old_tag" "$tmpdir/api-old/" - - for f in "$tmpdir/api-new"/*; do - f=$(basename "$f") - report="$tmpdir/$f.report" - if [[ ! -f "$tmpdir/api-old/$f" ]]; then - echo "NOTICE: NEW MODULE $f" - continue - fi - - echo -n "Checking $f... " - swift api-digester -sdk "$sdk" -diagnose-sdk \ - --input-paths "$tmpdir/api-old/$f" -input-paths "$tmpdir/api-new/$f" 2>&1 \ - > "$report" 2>&1 - - if ! shasum "$report" | grep -q afd2a1b542b33273920d65821deddc653063c700; then - echo ERROR - echo >&2 "==============================" - echo >&2 "ERROR: public API change in $f" - echo >&2 "==============================" - cat >&2 "$report" - errors=$(( errors + 1 )) - else - echo OK - fi - done - rm -rf "$tmpdir/api-new" "$tmpdir/api-old" + swift package diagnose-api-breaking-changes "$old_tag" done -if [[ "$errors" == 0 ]]; then - echo "OK, all seems good" -fi echo done -exit "$errors" From f50bf983ea3729010dd041d3b16fcdf99cf37ca0 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 11 Apr 2022 10:13:56 +0100 Subject: [PATCH 002/146] Tolerate the request stream being started after .finished (#577) Motivation The RequestBag intermediates between two different threads. This means it can get requests that were reasonable when they were made but have been superseded with newer information since then. These generally have to be tolerated. Unfortunately if we received a request to resume the request body stream _after_ the need for that stream has been invalidated, we could hit a crash. That's unnecessary, and we should tolerate it better. Modifications Tolerated receiving requests to resume body streaming in the finished state. Result Fewer crashes Fixes #576 --- .../RequestBag+StateMachine.swift | 6 ++- .../RequestBagTests+XCTest.swift | 1 + .../RequestBagTests.swift | 39 +++++++++++++++++++ 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index 456317f11..59beed926 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -127,7 +127,11 @@ extension RequestBag.StateMachine { return .none case .finished: - preconditionFailure("Invalid state: \(self.state)") + // If this task has been cancelled we may be in an error state. As a matter of + // defensive programming, we also tolerate receiving this notification if we've ended cleanly: + // while it shouldn't happen, nothing will go wrong if we just ignore it. + // All paths through this state machine should cancel our request body stream to get here anyway. + return .none case .modifying: preconditionFailure("Invalid state: \(self.state)") diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index b2919081c..0a8f850ad 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -31,6 +31,7 @@ extension RequestBagTests { ("testCancelFailsTaskAfterRequestIsSent", testCancelFailsTaskAfterRequestIsSent), ("testCancelFailsTaskWhenTaskIsQueued", testCancelFailsTaskWhenTaskIsQueued), ("testFailsTaskWhenTaskIsWaitingForMoreFromServer", testFailsTaskWhenTaskIsWaitingForMoreFromServer), + ("testChannelBecomingWritableDoesntCrashCancelledTask", testChannelBecomingWritableDoesntCrashCancelledTask), ("testHTTPUploadIsCancelledEvenThoughRequestSucceeds", testHTTPUploadIsCancelledEvenThoughRequestSucceeds), ("testRaceBetweenConnectionCloseAndDemandMoreData", testRaceBetweenConnectionCloseAndDemandMoreData), ] diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 2f2eb8cf5..ed50ae02d 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -336,6 +336,45 @@ final class RequestBagTests: XCTestCase { } } + func testChannelBecomingWritableDoesntCrashCancelledTask() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( + url: "https://swift.org", + body: .bytes([1, 2, 3, 4, 5]) + )) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + executor.runRequest(bag) + + // This simulates a race between the user cancelling the task (which invokes `RequestBag.cancel`) and the + // call to `resumeRequestBodyStream` (which comes from the `Channel` event loop and so may have to hop. + bag.cancel() + bag.resumeRequestBodyStream() + + XCTAssertEqual(executor.isCancelled, true) + XCTAssertThrowsError(try bag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .cancelled) + } + } + func testHTTPUploadIsCancelledEvenThoughRequestSucceeds() { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } From d2da15c31c256b0215f5f68a3faa7032e5ce89f9 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 11 Apr 2022 16:24:57 +0200 Subject: [PATCH 003/146] [HTTP2] Tolerate GoAway and Settings frames after connection close (#578) --- .../HTTP2/HTTP2IdleHandler.swift | 32 +++++++++++++++---- .../HTTP2IdleHandlerTests+XCTest.swift | 1 + .../HTTP2IdleHandlerTests.swift | 26 +++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift index 8978e1a86..c522b2425 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift @@ -168,7 +168,7 @@ extension HTTP2IdleHandler { mutating func settingsReceived(_ settings: HTTP2Settings) -> Action { switch self.state { - case .initialized, .closed: + case .initialized: preconditionFailure("Invalid state: \(self.state)") case .connected: @@ -188,12 +188,17 @@ extension HTTP2IdleHandler { case .closing: return .nothing + + case .closed: + // We may receive a Settings frame after we have called connection close, because of + // packages being delivered from the incoming buffer. + return .nothing } } mutating func goAwayReceived() -> Action { switch self.state { - case .initialized, .closed: + case .initialized: preconditionFailure("Invalid state: \(self.state)") case .connected: @@ -206,6 +211,11 @@ extension HTTP2IdleHandler { case .closing: return .notifyConnectionGoAwayReceived(close: false) + + case .closed: + // We may receive a GoAway frame after we have called connection close, because of + // packages being delivered from the incoming buffer. + return .nothing } } @@ -234,6 +244,9 @@ extension HTTP2IdleHandler { mutating func streamCreated() -> Action { switch self.state { + case .initialized, .connected: + preconditionFailure("Invalid state: \(self.state)") + case .active(var openStreams, let maxStreams): openStreams += 1 self.state = .active(openStreams: openStreams, maxStreams: maxStreams) @@ -246,13 +259,18 @@ extension HTTP2IdleHandler { self.state = .closing(openStreams: openStreams, maxStreams: maxStreams) return .nothing - case .initialized, .connected, .closed: - preconditionFailure("Invalid state: \(self.state)") + case .closed: + // We may receive a events after we have called connection close, because of + // internal races. We should just ignore these cases. + return .nothing } } mutating func streamClosed() -> Action { switch self.state { + case .initialized, .connected: + preconditionFailure("Invalid state: \(self.state)") + case .active(var openStreams, let maxStreams): openStreams -= 1 assert(openStreams >= 0) @@ -269,8 +287,10 @@ extension HTTP2IdleHandler { self.state = .closing(openStreams: openStreams, maxStreams: maxStreams) return .nothing - case .initialized, .connected, .closed: - preconditionFailure("Invalid state: \(self.state)") + case .closed: + // We may receive a events after we have called connection close, because of + // internal races. We should just ignore these cases. + return .nothing } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift index 5c7021e23..1b9558105 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift @@ -35,6 +35,7 @@ extension HTTP2IdleHandlerTests { ("testCloseEventWhileNoOpenStreams", testCloseEventWhileNoOpenStreams), ("testCloseEventWhileThereAreOpenStreams", testCloseEventWhileThereAreOpenStreams), ("testGoAwayWhileThereAreOpenStreams", testGoAwayWhileThereAreOpenStreams), + ("testReceiveSettingsAndGoAwayAfterClientSideClose", testReceiveSettingsAndGoAwayAfterClientSideClose), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift index 57560b659..355969c6a 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift @@ -225,6 +225,32 @@ class HTTP2IdleHandlerTests: XCTestCase { } } } + + func testReceiveSettingsAndGoAwayAfterClientSideClose() { + let delegate = MockHTTP2IdleHandlerDelegate() + let idleHandler = HTTP2IdleHandler(delegate: delegate, logger: Logger(label: "test")) + let embedded = EmbeddedChannel(handlers: [idleHandler]) + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + + let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)]))) + XCTAssertEqual(delegate.maxStreams, nil) + XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) + XCTAssertEqual(delegate.maxStreams, 10) + + XCTAssertTrue(embedded.isActive) + embedded.pipeline.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) + XCTAssertFalse(embedded.isActive) + + let newSettingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 20)]))) + XCTAssertEqual(delegate.maxStreams, 10) + XCTAssertNoThrow(try embedded.writeInbound(newSettingsFrame)) + XCTAssertEqual(delegate.maxStreams, 10, "Expected message to not be forwarded.") + + let goAwayFrame = HTTP2Frame(streamID: HTTP2StreamID(0), payload: .goAway(lastStreamID: 2, errorCode: .http11Required, opaqueData: nil)) + XCTAssertEqual(delegate.goAwayReceived, false) + XCTAssertNoThrow(try embedded.writeInbound(goAwayFrame)) + XCTAssertEqual(delegate.goAwayReceived, false, "Expected go away to not be forwarded.") + } } class MockHTTP2IdleHandlerDelegate: HTTP2IdleHandlerDelegate { From 0a2004b1c6e4d6002ccf98ffb38876de8fd7a712 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Mon, 11 Apr 2022 16:41:10 +0200 Subject: [PATCH 004/146] [HTTP1] Tolerate immediate write errors (#579) Same fix for HTTP/1 that landed for HTTP/2 in #558. ### Motivation `HTTP1ClientChannelHandler` currently does not tolerate immediate write errors. ### Changes Make `HTTP1ClientChannelHandler` resilient to failing writes. ### Result Less crashes in AHC HTTP/1. --- .../HTTP1/HTTP1ClientChannelHandler.swift | 63 +++++++++++++------ ...TTP1ClientChannelHandlerTests+XCTest.swift | 1 + .../HTTP1ClientChannelHandlerTests.swift | 63 +++++++++++++++++++ 3 files changed, 108 insertions(+), 19 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 15544dc4a..9d1a3b5fd 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -183,23 +183,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { private func run(_ action: HTTP1ConnectionStateMachine.Action, context: ChannelHandlerContext) { switch action { case .sendRequestHead(let head, startBody: let startBody): - if startBody { - context.write(self.wrapOutboundOut(.head(head)), promise: nil) - context.flush() - - self.request!.requestHeadSent() - self.request!.resumeRequestBodyStream() - } else { - context.write(self.wrapOutboundOut(.head(head)), promise: nil) - context.write(self.wrapOutboundOut(.end(nil)), promise: nil) - context.flush() - - self.request!.requestHeadSent() - - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) - } - } + self.sendRequestHead(head, startBody: startBody, context: context) case .sendBodyPart(let part): context.writeAndFlush(self.wrapOutboundOut(.body(part)), promise: nil) @@ -212,9 +196,13 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } case .pauseRequestBodyStream: + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet self.request!.pauseRequestBodyStream() case .resumeRequestBodyStream: + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet self.request!.resumeRequestBodyStream() case .fireChannelActive: @@ -239,15 +227,25 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { break case .forwardResponseHead(let head, let pauseRequestBodyStream): + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet self.request!.receiveResponseHead(head) - if pauseRequestBodyStream { - self.request!.pauseRequestBodyStream() + if pauseRequestBodyStream, let request = self.request { + // The above response head forward might lead the request to mark itself as + // cancelled, which in turn might pop the request of the handler. For this reason we + // must check if the request is still present here. + request.pauseRequestBodyStream() } case .forwardResponseBodyParts(let buffer): + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet self.request!.receiveResponseBodyParts(buffer) case .succeedRequest(let finalAction, let buffer): + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet + // The order here is very important... // We first nil our own task property! `taskCompleted` will potentially lead to // situations in which we get a new request right away. We should finish the task @@ -293,6 +291,33 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } } + private func sendRequestHead(_ head: HTTPRequestHead, startBody: Bool, context: ChannelHandlerContext) { + if startBody { + context.writeAndFlush(self.wrapOutboundOut(.head(head)), promise: nil) + + // The above write might trigger an error, which may lead to a call to `errorCaught`, + // which in turn, may fail the request and pop it from the handler. For this reason + // we must check if the request is still present here. + guard let request = self.request else { return } + request.requestHeadSent() + request.resumeRequestBodyStream() + } else { + context.write(self.wrapOutboundOut(.head(head)), promise: nil) + context.write(self.wrapOutboundOut(.end(nil)), promise: nil) + context.flush() + + // The above write might trigger an error, which may lead to a call to `errorCaught`, + // which in turn, may fail the request and pop it from the handler. For this reason + // we must check if the request is still present here. + guard let request = self.request else { return } + request.requestHeadSent() + + if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(timeoutAction, context: context) + } + } + } + private func runTimeoutAction(_ action: IdleReadStateMachine.Action, context: ChannelHandlerContext) { switch action { case .startIdleReadTimeoutTimer(let timeAmount): diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift index 8d28c15c4..86707520c 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift @@ -31,6 +31,7 @@ extension HTTP1ClientChannelHandlerTests { ("testIdleReadTimeout", testIdleReadTimeout), ("testIdleReadTimeoutIsCanceledIfRequestIsCanceled", testIdleReadTimeoutIsCanceledIfRequestIsCanceled), ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand), + ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index 62e42f94e..4769d2c7e 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -394,6 +394,69 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { XCTAssertEqual($0 as? HTTPClientError, .remoteConnectionClosed) } } + + func testWriteHTTPHeadFails() { + struct WriteError: Error, Equatable {} + + class FailWriteHandler: ChannelOutboundHandler { + typealias OutboundIn = HTTPClientRequestPart + typealias OutboundOut = HTTPClientRequestPart + + func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise?) { + let error = WriteError() + promise?.fail(error) + context.fireErrorCaught(error) + } + } + + let bodies: [HTTPClient.Body?] = [ + .none, + .some(.byteBuffer(ByteBuffer(string: "hello world"))), + ] + + for body in bodies { + let embedded = EmbeddedChannel() + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + XCTAssertNoThrow(try embedded.pipeline.syncOperations.addHandler(FailWriteHandler(), position: .after(testUtils.readEventHandler))) + + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: body)) + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = false + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + embedded.write(requestBag, promise: nil) + + // the handler only writes once the channel is writable + XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .none) + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? WriteError, WriteError()) + } + + XCTAssertEqual(embedded.isActive, false) + } + } } class TestBackpressureWriter { From 9d8cd9592719e8b958f42cb0b1ce1fe57c1ac32f Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 12 Apr 2022 13:40:16 +0200 Subject: [PATCH 005/146] [Redirect] Allow redirect response to have body (#580) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Motivation Currently, we donโ€™t consume the response body of redirect responses. Because of this requests, that receive a redirect response with response body, may hang indefinitely. ### Changes - Consume redirect response body if less than 3kb - Cancel redirect response if larger than 3kb ### Result Redirect responses are consumed. Fixes #574 --- .../RequestBag+StateMachine.swift | 79 +++++-- Sources/AsyncHTTPClient/RequestBag.swift | 52 +++-- .../RequestBagTests+XCTest.swift | 3 + .../RequestBagTests.swift | 193 ++++++++++++++++++ 4 files changed, 295 insertions(+), 32 deletions(-) diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index 59beed926..557af2af1 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -16,6 +16,16 @@ import struct Foundation.URL import NIOCore import NIOHTTP1 +extension HTTPClient { + /// The maximum body size allowed, before a redirect response is cancelled. 3KB. + /// + /// Why 3KB? We feel like this is a good compromise between potentially reusing the + /// connection in HTTP/1.1 mode (if we load all data from the redirect response we can + /// reuse the connection) and not being to wasteful in the amount of data that is thrown + /// away being transferred. + fileprivate static let maxBodySizeRedirectResponse = 1024 * 3 +} + extension RequestBag { struct StateMachine { fileprivate enum State { @@ -23,7 +33,7 @@ extension RequestBag { case queued(HTTPRequestScheduler) case executing(HTTPRequestExecutor, RequestStreamState, ResponseStreamState) case finished(error: Error?) - case redirected(HTTPResponseHead, URL) + case redirected(HTTPRequestExecutor, Int, HTTPResponseHead, URL) case modifying } @@ -259,11 +269,18 @@ extension RequestBag.StateMachine { } } + enum ReceiveResponseHeadAction { + case none + case forwardResponseHead(HTTPResponseHead) + case signalBodyDemand(HTTPRequestExecutor) + case redirect(HTTPRequestExecutor, RedirectHandler, HTTPResponseHead, URL) + } + /// The response head has been received. /// /// - Parameter head: The response' head /// - Returns: Whether the response should be forwarded to the delegate. Will be `false` if the request follows a redirect. - mutating func receiveResponseHead(_ head: HTTPResponseHead) -> Bool { + mutating func receiveResponseHead(_ head: HTTPResponseHead) -> ReceiveResponseHeadAction { switch self.state { case .initialized, .queued: preconditionFailure("How can we receive a response, if the request hasn't started yet.") @@ -276,16 +293,25 @@ extension RequestBag.StateMachine { status: head.status, responseHeaders: head.headers ) { - self.state = .redirected(head, redirectURL) - return false + // If we will redirect, we need to consume the response's body ASAP, to be able to + // reuse the existing connection. We will consume a response body, if the body is + // smaller than 3kb. + switch head.contentLength { + case .some(0...(HTTPClient.maxBodySizeRedirectResponse)), .none: + self.state = .redirected(executor, 0, head, redirectURL) + return .signalBodyDemand(executor) + case .some: + self.state = .finished(error: HTTPClientError.cancelled) + return .redirect(executor, self.redirectHandler!, head, redirectURL) + } } else { self.state = .executing(executor, requestState, .buffering(.init(), next: .askExecutorForMore)) - return true + return .forwardResponseHead(head) } case .redirected: preconditionFailure("This state can only be reached after we have received a HTTP head") case .finished(error: .some): - return false + return .none case .finished(error: .none): preconditionFailure("How can the request be finished without error, before receiving response head?") case .modifying: @@ -293,7 +319,14 @@ extension RequestBag.StateMachine { } } - mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ByteBuffer? { + enum ReceiveResponseBodyAction { + case none + case forwardResponsePart(ByteBuffer) + case signalBodyDemand(HTTPRequestExecutor) + case redirect(HTTPRequestExecutor, RedirectHandler, HTTPResponseHead, URL) + } + + mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponseBodyAction { switch self.state { case .initialized, .queued: preconditionFailure("How can we receive a response body part, if the request hasn't started yet.") @@ -312,17 +345,26 @@ extension RequestBag.StateMachine { currentBuffer.append(contentsOf: buffer) } self.state = .executing(executor, requestState, .buffering(currentBuffer, next: next)) - return nil + return .none case .executing(let executor, let requestState, .waitingForRemote): var buffer = buffer let first = buffer.removeFirst() self.state = .executing(executor, requestState, .buffering(buffer, next: .askExecutorForMore)) - return first - case .redirected: - // ignore body - return nil + return .forwardResponsePart(first) + case .redirected(let executor, var receivedBytes, let head, let redirectURL): + let partsLength = buffer.reduce(into: 0) { $0 += $1.readableBytes } + receivedBytes += partsLength + + if receivedBytes > HTTPClient.maxBodySizeRedirectResponse { + self.state = .finished(error: HTTPClientError.cancelled) + return .redirect(executor, self.redirectHandler!, head, redirectURL) + } else { + self.state = .redirected(executor, receivedBytes, head, redirectURL) + return .signalBodyDemand(executor) + } + case .finished(error: .some): - return nil + return .none case .finished(error: .none): preconditionFailure("How can the request be finished without error, before receiving response head?") case .modifying: @@ -368,7 +410,7 @@ extension RequestBag.StateMachine { self.state = .executing(executor, requestState, .buffering(newChunks, next: .eof)) return .consume(first) - case .redirected(let head, let redirectURL): + case .redirected(_, _, let head, let redirectURL): self.state = .finished(error: nil) return .redirect(self.redirectHandler!, head, redirectURL) @@ -529,3 +571,12 @@ extension RequestBag.StateMachine { } } } + +extension HTTPResponseHead { + var contentLength: Int? { + guard let header = self.headers.first(name: "content-length") else { + return nil + } + return Int(header) + } +} diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 9a40e9ff5..b4aeef0e7 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -196,33 +196,49 @@ final class RequestBag { self.task.eventLoop.assertInEventLoop() // runs most likely on channel eventLoop - let forwardToDelegate = self.state.receiveResponseHead(head) + switch self.state.receiveResponseHead(head) { + case .none: + break - guard forwardToDelegate else { return } + case .signalBodyDemand(let executor): + executor.demandResponseBodyStream(self) - self.delegate.didReceiveHead(task: self.task, head) - .hop(to: self.task.eventLoop) - .whenComplete { result in - // After the head received, let's start to consume body data - self.consumeMoreBodyData0(resultOfPreviousConsume: result) - } + case .redirect(let executor, let handler, let head, let newURL): + handler.redirect(status: head.status, to: newURL, promise: self.task.promise) + executor.cancelRequest(self) + + case .forwardResponseHead(let head): + self.delegate.didReceiveHead(task: self.task, head) + .hop(to: self.task.eventLoop) + .whenComplete { result in + // After the head received, let's start to consume body data + self.consumeMoreBodyData0(resultOfPreviousConsume: result) + } + } } private func receiveResponseBodyParts0(_ buffer: CircularBuffer) { self.task.eventLoop.assertInEventLoop() - let maybeForwardBuffer = self.state.receiveResponseBodyParts(buffer) + switch self.state.receiveResponseBodyParts(buffer) { + case .none: + break - guard let forwardBuffer = maybeForwardBuffer else { - return - } + case .signalBodyDemand(let executor): + executor.demandResponseBodyStream(self) - self.delegate.didReceiveBodyPart(task: self.task, forwardBuffer) - .hop(to: self.task.eventLoop) - .whenComplete { result in - // on task el - self.consumeMoreBodyData0(resultOfPreviousConsume: result) - } + case .redirect(let executor, let handler, let head, let newURL): + handler.redirect(status: head.status, to: newURL, promise: self.task.promise) + executor.cancelRequest(self) + + case .forwardResponsePart(let part): + self.delegate.didReceiveBodyPart(task: self.task, part) + .hop(to: self.task.eventLoop) + .whenComplete { result in + // on task el + self.consumeMoreBodyData0(resultOfPreviousConsume: result) + } + } } private func succeedRequest0(_ buffer: CircularBuffer?) { diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index 0a8f850ad..74c68fd1f 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -34,6 +34,9 @@ extension RequestBagTests { ("testChannelBecomingWritableDoesntCrashCancelledTask", testChannelBecomingWritableDoesntCrashCancelledTask), ("testHTTPUploadIsCancelledEvenThoughRequestSucceeds", testHTTPUploadIsCancelledEvenThoughRequestSucceeds), ("testRaceBetweenConnectionCloseAndDemandMoreData", testRaceBetweenConnectionCloseAndDemandMoreData), + ("testRedirectWith3KBBody", testRedirectWith3KBBody), + ("testRedirectWith4KBBodyAnnouncedInResponseHead", testRedirectWith4KBBodyAnnouncedInResponseHead), + ("testRedirectWith4KBBodyNotAnnouncedInResponseHead", testRedirectWith4KBBodyNotAnnouncedInResponseHead), ] } } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index ed50ae02d..c80f8846b 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -496,6 +496,199 @@ final class RequestBagTests: XCTestCase { XCTAssertNoThrow(try XCTUnwrap(delegate.backpressurePromise).succeed(())) XCTAssertEqual(delegate.hitDidReceiveResponse, 1) } + + func testRedirectWith3KBBody() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org")) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + var redirectTriggered = false + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + executor.runRequest(bag) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"])) + XCTAssertNil(delegate.backpressurePromise) + XCTAssertTrue(executor.signalledDemandForResponseBody) + executor.resetResponseStreamDemandSignal() + + // "foo" is forwarded for consumption. We expect the RequestBag to consume "foo" with the + // delegate and call demandMoreBody afterwards. + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseBodyParts([ByteBuffer(repeating: 0, count: 1024)]) + XCTAssertTrue(executor.signalledDemandForResponseBody) + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertNil(delegate.backpressurePromise) + executor.resetResponseStreamDemandSignal() + + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseBodyParts([ByteBuffer(repeating: 1, count: 1024)]) + XCTAssertTrue(executor.signalledDemandForResponseBody) + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertNil(delegate.backpressurePromise) + executor.resetResponseStreamDemandSignal() + + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.succeedRequest([ByteBuffer(repeating: 2, count: 1024)]) + XCTAssertFalse(executor.signalledDemandForResponseBody) + XCTAssertEqual(delegate.hitDidReceiveResponse, 0) + XCTAssertNil(delegate.backpressurePromise) + executor.resetResponseStreamDemandSignal() + + XCTAssertTrue(redirectTriggered) + } + + func testRedirectWith4KBBodyAnnouncedInResponseHead() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org")) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + var redirectTriggered = false + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + executor.runRequest(bag) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(4 * 1024)", "location": "https://swift.org/sswg"])) + XCTAssertNil(delegate.backpressurePromise) + XCTAssertFalse(executor.signalledDemandForResponseBody) + XCTAssertTrue(executor.isCancelled) + + XCTAssertTrue(redirectTriggered) + } + + func testRedirectWith4KBBodyNotAnnouncedInResponseHead() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org")) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + var redirectTriggered = false + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + executor.runRequest(bag) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"])) + XCTAssertNil(delegate.backpressurePromise) + XCTAssertTrue(executor.signalledDemandForResponseBody) + executor.resetResponseStreamDemandSignal() + + // "foo" is forwarded for consumption. We expect the RequestBag to consume "foo" with the + // delegate and call demandMoreBody afterwards. + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseBodyParts([ByteBuffer(repeating: 0, count: 2024)]) + XCTAssertTrue(executor.signalledDemandForResponseBody) + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertNil(delegate.backpressurePromise) + executor.resetResponseStreamDemandSignal() + + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertFalse(executor.isCancelled) + XCTAssertFalse(executor.signalledDemandForResponseBody) + bag.receiveResponseBodyParts([ByteBuffer(repeating: 1, count: 2024)]) + XCTAssertFalse(executor.signalledDemandForResponseBody) + XCTAssertTrue(executor.isCancelled) + XCTAssertEqual(delegate.hitDidReceiveBodyPart, 0) + XCTAssertNil(delegate.backpressurePromise) + executor.resetResponseStreamDemandSignal() + + XCTAssertTrue(redirectTriggered) + } } class UploadCountingDelegate: HTTPClientResponseDelegate { From 13e93d4d43c9dc012df4ab6178f8daa3b25b60a6 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Thu, 21 Apr 2022 08:50:42 +0200 Subject: [PATCH 006/146] Drop support for Swift 5.2 and 5.3 (#581) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As outlined in a [Swift forums post in November โ€™21](https://forums.swift.org/t/swiftnio-swift-version-support/53232), SwiftNIO will only support the latest non-patch Swift release and the 2 immediately prior non-patch versions. - drop support for Swift 5.2 and 5.3. - update CI for Swift 5.4 to run on bionic instead of focal to ensure that we still test bionic. --- .swiftformat | 2 +- CONTRIBUTING.md | 2 +- Package.swift | 2 +- docker/Dockerfile | 2 +- docker/docker-compose.1604.52.yaml | 18 ------------------ ...804.53.yaml => docker-compose.1804.54.yaml} | 8 ++++---- docker/docker-compose.2004.54.yaml | 18 ------------------ scripts/check_no_api_breakages.sh | 8 ++++---- 8 files changed, 12 insertions(+), 48 deletions(-) delete mode 100644 docker/docker-compose.1604.52.yaml rename docker/{docker-compose.1804.53.yaml => docker-compose.1804.54.yaml} (54%) delete mode 100644 docker/docker-compose.2004.54.yaml diff --git a/.swiftformat b/.swiftformat index dac9cd4d3..c26e226e3 100644 --- a/.swiftformat +++ b/.swiftformat @@ -1,6 +1,6 @@ # file options ---swiftversion 5.2 +--swiftversion 5.4 --exclude .build # format options diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6382fcd4d..3803bb618 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ A good AsyncHTTPClient patch is: 3. Documented, adding API documentation as needed to cover new functions and properties. 4. Accompanied by a great commit message, using our commit message template. -*Note* as of version 1.5.0 AsyncHTTPClient requires Swift 5.2. Earlier versions support as far back as Swift 5.0. +*Note* as of version 1.10.0 AsyncHTTPClient requires Swift 5.4. Earlier versions support as far back as Swift 5.0. ### Commit Message Template diff --git a/Package.swift b/Package.swift index e4dcc717c..5deb0de31 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.2 +// swift-tools-version:5.4 //===----------------------------------------------------------------------===// // // This source file is part of the AsyncHTTPClient open source project diff --git a/docker/Dockerfile b/docker/Dockerfile index 6395405c1..1cd4f2140 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -ARG swift_version=5.2 +ARG swift_version=5.4 ARG ubuntu_version=bionic ARG base_image=swift:$swift_version-$ubuntu_version FROM $base_image diff --git a/docker/docker-compose.1604.52.yaml b/docker/docker-compose.1604.52.yaml deleted file mode 100644 index 4f74bca9f..000000000 --- a/docker/docker-compose.1604.52.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:16.04-5.2 - build: - args: - ubuntu_version: "xenial" - swift_version: "5.2" - - test: - image: async-http-client:16.04-5.2 - environment: - - SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:16.04-5.2 diff --git a/docker/docker-compose.1804.53.yaml b/docker/docker-compose.1804.54.yaml similarity index 54% rename from docker/docker-compose.1804.53.yaml rename to docker/docker-compose.1804.54.yaml index e9e4e53dc..660429851 100644 --- a/docker/docker-compose.1804.53.yaml +++ b/docker/docker-compose.1804.54.yaml @@ -3,16 +3,16 @@ version: "3" services: runtime-setup: - image: async-http-client:18.04-5.3 + image: async-http-client:18.04-5.4 build: args: ubuntu_version: "bionic" - swift_version: "5.3" + swift_version: "5.4" test: - image: async-http-client:18.04-5.3 + image: async-http-client:18.04-5.4 environment: [] #- SANITIZER_ARG=--sanitize=thread shell: - image: async-http-client:18.04-5.3 + image: async-http-client:18.04-5.4 diff --git a/docker/docker-compose.2004.54.yaml b/docker/docker-compose.2004.54.yaml deleted file mode 100644 index 154540ccb..000000000 --- a/docker/docker-compose.2004.54.yaml +++ /dev/null @@ -1,18 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:20.04-5.4 - build: - args: - ubuntu_version: "focal" - swift_version: "5.4" - - test: - image: async-http-client:20.04-5.4 - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:20.04-5.4 diff --git a/scripts/check_no_api_breakages.sh b/scripts/check_no_api_breakages.sh index 0fa3d4fb3..2d7028617 100755 --- a/scripts/check_no_api_breakages.sh +++ b/scripts/check_no_api_breakages.sh @@ -3,7 +3,7 @@ ## ## This source file is part of the AsyncHTTPClient open source project ## -## Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +## Copyright (c) 2018-2022 Apple Inc. and the AsyncHTTPClient project authors ## Licensed under Apache License v2.0 ## ## See LICENSE.txt for license information @@ -32,12 +32,12 @@ set -eu function usage() { echo >&2 "Usage: $0 REPO-GITHUB-URL NEW-VERSION OLD-VERSIONS..." echo >&2 - echo >&2 "This script requires a Swift 5.2+ toolchain." + echo >&2 "This script requires a Swift 5.6+ toolchain." echo >&2 echo >&2 "Examples:" echo >&2 - echo >&2 "Check between main and tag 2.1.1 of swift-nio:" - echo >&2 " $0 https://github.com/apple/swift-nio main 2.1.1" + echo >&2 "Check between main and tag 1.9.0 of async-http-client:" + echo >&2 " $0 https://github.com/swift-server/async-http-client main 1.9.0" echo >&2 echo >&2 "Check between HEAD and commit 64cf63d7 using the provided toolchain:" echo >&2 " xcrun --toolchain org.swift.5120190702a $0 ../some-local-repo HEAD 64cf63d7" From 89b0da2bf87219f46ed0a40a3fd175a44fcfb0fc Mon Sep 17 00:00:00 2001 From: Christian Priebe Date: Fri, 22 Apr 2022 07:59:58 +0100 Subject: [PATCH 007/146] Add HTTPClientError shortDescription property (#583) This adds a shortDescription property to HTTPClientError which provides a short description of the error without associated values. --- Sources/AsyncHTTPClient/HTTPClient.swift | 70 ++++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 9301094ef..a6ee8956a 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -949,6 +949,76 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { return "HTTPClientError.\(String(describing: self.code))" } + /// Short description of the error that can be used in case a bounded set of error descriptions is expected, e.g. to + /// include in metric labels. For this reason the description must not contain associated values. + public var shortDescription: String { + // When adding new cases here, do *not* include dynamic (associated) values in the description. + switch self.code { + case .invalidURL: + return "Invalid URL" + case .emptyHost: + return "Empty host" + case .missingSocketPath: + return "Missing socket path" + case .alreadyShutdown: + return "Already shutdown" + case .emptyScheme: + return "Empty scheme" + case .unsupportedScheme: + return "Unsupported scheme" + case .readTimeout: + return "Read timeout" + case .remoteConnectionClosed: + return "Remote connection closed" + case .cancelled: + return "Cancelled" + case .identityCodingIncorrectlyPresent: + return "Identity coding incorrectly present" + case .chunkedSpecifiedMultipleTimes: + return "Chunked specified multiple times" + case .invalidProxyResponse: + return "Invalid proxy response" + case .contentLengthMissing: + return "Content length missing" + case .proxyAuthenticationRequired: + return "Proxy authentication required" + case .redirectLimitReached: + return "Redirect limit reached" + case .redirectCycleDetected: + return "Redirect cycle detected" + case .uncleanShutdown: + return "Unclean shutdown" + case .traceRequestWithBody: + return "Trace request with body" + case .invalidHeaderFieldNames: + return "Invalid header field names" + case .bodyLengthMismatch: + return "Body length mismatch" + case .writeAfterRequestSent: + return "Write after request sent" + case .incompatibleHeaders: + return "Incompatible headers" + case .connectTimeout: + return "Connect timeout" + case .socksHandshakeTimeout: + return "SOCKS handshake timeout" + case .httpProxyHandshakeTimeout: + return "HTTP proxy handshake timeout" + case .tlsHandshakeTimeout: + return "TLS handshake timeout" + case .serverOfferedUnsupportedApplicationProtocol: + return "Server offered unsupported application protocol" + case .requestStreamCancelled: + return "Request stream cancelled" + case .getConnectionFromPoolTimeout: + return "Get connection from pool timeout" + case .deadlineExceeded: + return "Deadline exceeded" + case .httpEndReceivedAfterHeadWith1xx: + return "HTTP end received after head with 1xx" + } + } + /// URL provided is invalid. public static let invalidURL = HTTPClientError(code: .invalidURL) /// URL does not contain host. From 3725095966681e679d971042dc548faf77b9c052 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 22 Apr 2022 10:37:00 +0200 Subject: [PATCH 008/146] Fix flaky `HTTPClientTests.testResponseDelayGet()` test (#584) --- .swiftformat | 1 + Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.swiftformat b/.swiftformat index c26e226e3..7b7c486ea 100644 --- a/.swiftformat +++ b/.swiftformat @@ -19,5 +19,6 @@ --disable redundantReturn --disable preferKeyPath --disable sortedSwitchCases +--disable numberFormatting # rules diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 6bb4dd9b4..a6eff950a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1216,9 +1216,9 @@ class HTTPClientTests: XCTestCase { method: .GET, headers: ["X-internal-delay": "2000"], body: nil) - let start = Date() - let response = try! self.defaultClient.execute(request: req).wait() - XCTAssertGreaterThan(Date().timeIntervalSince(start), 2) + let start = NIODeadline.now() + let response = try self.defaultClient.execute(request: req).wait() + XCTAssertGreaterThanOrEqual(.now() - start, .milliseconds(1_900 /* 1.9 seconds */ )) XCTAssertEqual(response.status, .ok) } From a586fba7ebccbc9c310fa79c2fc7ac5603561d6a Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 22 Apr 2022 10:47:30 +0200 Subject: [PATCH 009/146] Fix flaky `TransactionTests.testResponseStreamFails()` test (#582) --- .../TransactionTests.swift | 75 ++++++++++++++----- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 7e2c62a0d..d124eb089 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -41,7 +41,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return XCTFail("Expected to have a request here.") } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -78,7 +78,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -141,7 +141,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return } - var tuple: (Transaction, Task)! = Transaction.makeWithResultTask( + var tuple: (Transaction, Task)! = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -196,7 +196,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return XCTFail("Expected to have a request here.") } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -282,7 +282,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return XCTFail("Expected to have a request here.") } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: eventLoopGroup.next() ) @@ -324,7 +324,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return XCTFail("Expected to have a request here.") } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -366,7 +366,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return XCTFail("Expected to have a request here.") } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -397,7 +397,7 @@ final class TransactionTests: XCTestCase { func testResponseStreamFails() { #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } - XCTAsyncTest { + XCTAsyncTest(timeout: 30) { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -409,7 +409,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: embeddedEventLoop ) @@ -427,6 +427,7 @@ final class TransactionTests: XCTestCase { transaction.receiveResponseHead(responseHead) let response = try await responseTask.value + XCTAssertEqual(response.status, responseHead.status) XCTAssertEqual(response.headers, responseHead.headers) XCTAssertEqual(response.version, responseHead.version) @@ -438,6 +439,7 @@ final class TransactionTests: XCTestCase { XCTAssertNoThrow(try executor.receiveResponseDemand()) executor.resetResponseStreamDemandSignal() transaction.receiveResponseBodyParts([ByteBuffer(integer: 123)]) + let result = try await part1 XCTAssertEqual(result, ByteBuffer(integer: 123)) @@ -493,7 +495,7 @@ final class TransactionTests: XCTestCase { guard let preparedRequest = maybePreparedRequest else { return } - let (transaction, responseTask) = Transaction.makeWithResultTask( + let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, preferredEventLoop: eventLoopGroup.next() ) @@ -553,6 +555,45 @@ actor SharedIterator { } } +/// non fail-able promise that only supports one observer +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +fileprivate actor Promise { + private enum State { + case initialised + case fulfilled(Value) + } + + private var state: State = .initialised + + private var observer: CheckedContinuation? + + init() {} + + func fulfil(_ value: Value) { + switch self.state { + case .initialised: + self.state = .fulfilled(value) + self.observer?.resume(returning: value) + case .fulfilled: + preconditionFailure("\(Self.self) over fulfilled") + } + } + + var value: Value { + get async { + switch self.state { + case .initialised: + return await withCheckedContinuation { (continuation: CheckedContinuation) in + precondition(self.observer == nil, "\(Self.self) supports only one observer") + self.observer = continuation + } + case .fulfilled(let value): + return value + } + } + } +} + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction { fileprivate static func makeWithResultTask( @@ -561,9 +602,9 @@ extension Transaction { logger: Logger = Logger(label: "test"), connectionDeadline: NIODeadline = .distantFuture, preferredEventLoop: EventLoop - ) -> (Transaction, _Concurrency.Task) { - let transactionPromise = preferredEventLoop.makePromise(of: Transaction.self) - let result = Task { + ) async -> (Transaction, _Concurrency.Task) { + let transactionPromise = Promise() + let task = Task { try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in let transaction = Transaction( request: request, @@ -573,13 +614,13 @@ extension Transaction { preferredEventLoop: preferredEventLoop, responseContinuation: continuation ) - transactionPromise.succeed(transaction) + Task { + await transactionPromise.fulfil(transaction) + } } } - // the promise can never fail and it is therefore safe to force unwrap - let transaction = try! transactionPromise.futureResult.wait() - return (transaction, result) + return (await transactionPromise.value, task) } } #endif From 24425989dadab6d6e4167174791a23d4e2a6d0c3 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Tue, 26 Apr 2022 16:04:54 +0200 Subject: [PATCH 010/146] Call `didSendRequestPart` after the write has hit the socket (#566) ### Motivation Today `didSendRequestPart` is called after a request body part has been passed to the executor. However, this does not mean that the write hit the socket. Users may depend on this behavior to implement back-pressure. For this reason, we should only call this `didSendRequestPart` once the write was successful. ### Modification Pass a promise to the actual channel write and only call the delegate once that promise succeeds. ### Result The delegate method `didSendRequestPart` is only called after the write was successful. Fixes #565. Co-authored-by: Fabian Fett --- .../AsyncAwait/Transaction.swift | 8 +- .../HTTP1/HTTP1ClientChannelHandler.swift | 69 ++++++--- .../HTTP1/HTTP1ConnectionStateMachine.swift | 63 +++++--- .../HTTP2/HTTP2ClientRequestHandler.swift | 55 ++++--- .../HTTPExecutableRequest.swift | 4 +- .../HTTPRequestStateMachine.swift | 71 +++++---- Sources/AsyncHTTPClient/RequestBag.swift | 18 ++- .../HTTP1ConnectionStateMachineTests.swift | 54 +++++-- .../HTTPRequestStateMachineTests+XCTest.swift | 1 + .../HTTPRequestStateMachineTests.swift | 143 ++++++++++++------ .../Mocks/MockRequestExecutor.swift | 6 +- 11 files changed, 327 insertions(+), 165 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index c2ce52eeb..8830406b4 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -63,7 +63,7 @@ final class Transaction: @unchecked Sendable { switch writeAction { case .writeAndWait(let executor), .writeAndContinue(let executor): - executor.writeRequestBodyPart(.byteBuffer(byteBuffer), request: self) + executor.writeRequestBodyPart(.byteBuffer(byteBuffer), request: self, promise: nil) case .fail: // an error/cancellation has happened. we don't need to continue here @@ -105,14 +105,14 @@ final class Transaction: @unchecked Sendable { switch self.state.writeNextRequestPart() { case .writeAndContinue(let executor): self.stateLock.unlock() - executor.writeRequestBodyPart(.byteBuffer(part), request: self) + executor.writeRequestBodyPart(.byteBuffer(part), request: self, promise: nil) case .writeAndWait(let executor): try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in self.state.waitForRequestBodyDemand(continuation: continuation) self.stateLock.unlock() - executor.writeRequestBodyPart(.byteBuffer(part), request: self) + executor.writeRequestBodyPart(.byteBuffer(part), request: self, promise: nil) } case .fail: @@ -132,7 +132,7 @@ final class Transaction: @unchecked Sendable { break case .forwardStreamFinished(let executor, let succeedContinuation): - executor.finishRequestBodyStream(self) + executor.finishRequestBodyStream(self, promise: nil) succeedContinuation?.resume(returning: nil) } return diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 9d1a3b5fd..2a3bc9c27 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -185,11 +185,11 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { case .sendRequestHead(let head, startBody: let startBody): self.sendRequestHead(head, startBody: startBody, context: context) - case .sendBodyPart(let part): - context.writeAndFlush(self.wrapOutboundOut(.body(part)), promise: nil) + case .sendBodyPart(let part, let writePromise): + context.writeAndFlush(self.wrapOutboundOut(.body(part)), promise: writePromise) - case .sendRequestEnd: - context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + case .sendRequestEnd(let writePromise): + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { self.runTimeoutAction(timeoutAction, context: context) @@ -260,16 +260,25 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { switch finalAction { case .close: context.close(promise: nil) - case .sendRequestEnd: - context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + oldRequest.succeedRequest(buffer) + case .sendRequestEnd(let writePromise): + let writePromise = writePromise ?? context.eventLoop.makePromise(of: Void.self) + // We need to defer succeeding the old request to avoid ordering issues + writePromise.futureResult.whenComplete { result in + switch result { + case .success: + oldRequest.succeedRequest(buffer) + case .failure(let error): + oldRequest.fail(error) + } + } + + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) case .informConnectionIsIdle: self.connection.taskCompleted() - case .none: - break + oldRequest.succeedRequest(buffer) } - oldRequest.succeedRequest(buffer) - case .failRequest(let error, let finalAction): // see comment in the `succeedRequest` case. let oldRequest = self.request! @@ -277,17 +286,25 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) switch finalAction { - case .close: + case .close(let writePromise): context.close(promise: nil) - case .sendRequestEnd: - context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + writePromise?.fail(error) + oldRequest.fail(error) + case .informConnectionIsIdle: self.connection.taskCompleted() + oldRequest.fail(error) + + case .failWritePromise(let writePromise): + writePromise?.fail(error) + oldRequest.fail(error) + case .none: - break + oldRequest.fail(error) } - oldRequest.fail(error) + case .failSendBodyPart(let error, let writePromise), .failSendStreamFinished(let error, let writePromise): + writePromise?.fail(error) } } @@ -355,27 +372,29 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { // MARK: Private HTTPRequestExecutor - private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest) { + private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { guard self.request === request, let context = self.channelContext else { // Because the HTTPExecutableRequest may run in a different thread to our eventLoop, // calls from the HTTPExecutableRequest to our ChannelHandler may arrive here after // the request has been popped by the state machine or the ChannelHandler has been // removed from the Channel pipeline. This is a normal threading issue, noone has // screwed up. + promise?.fail(HTTPClientError.requestStreamCancelled) return } - let action = self.state.requestStreamPartReceived(data) + let action = self.state.requestStreamPartReceived(data, promise: promise) self.run(action, context: context) } - private func finishRequestBodyStream0(_ request: HTTPExecutableRequest) { + private func finishRequestBodyStream0(_ request: HTTPExecutableRequest, promise: EventLoopPromise?) { guard self.request === request, let context = self.channelContext else { // See code comment in `writeRequestBodyPart0` + promise?.fail(HTTPClientError.requestStreamCancelled) return } - let action = self.state.requestStreamFinished() + let action = self.state.requestStreamFinished(promise: promise) self.run(action, context: context) } @@ -405,22 +424,22 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } extension HTTP1ClientChannelHandler: HTTPRequestExecutor { - func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest) { + func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { - self.writeRequestBodyPart0(data, request: request) + self.writeRequestBodyPart0(data, request: request, promise: promise) } else { self.eventLoop.execute { - self.writeRequestBodyPart0(data, request: request) + self.writeRequestBodyPart0(data, request: request, promise: promise) } } } - func finishRequestBodyStream(_ request: HTTPExecutableRequest) { + func finishRequestBodyStream(_ request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { - self.finishRequestBodyStream0(request) + self.finishRequestBodyStream0(request, promise: promise) } else { self.eventLoop.execute { - self.finishRequestBodyStream0(request) + self.finishRequestBodyStream0(request, promise: promise) } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index 19825aec7..ecff7afc7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -28,21 +28,37 @@ struct HTTP1ConnectionStateMachine { enum Action { /// A action to execute, when we consider a request "done". - enum FinalStreamAction { + enum FinalSuccessfulStreamAction { /// Close the connection case close /// If the server has replied, with a status of 200...300 before all data was sent, a request is considered succeeded, /// as soon as we wrote the request end onto the wire. - case sendRequestEnd + /// + /// The promise is an optional write promise. + case sendRequestEnd(EventLoopPromise?) /// Inform an observer that the connection has become idle case informConnectionIsIdle + } + + /// A action to execute, when we consider a request "done". + enum FinalFailedStreamAction { + /// Close the connection + /// + /// The promise is an optional write promise. + case close(EventLoopPromise?) + /// Inform an observer that the connection has become idle + case informConnectionIsIdle + /// Fail the write promise + case failWritePromise(EventLoopPromise?) /// Do nothing. case none } case sendRequestHead(HTTPRequestHead, startBody: Bool) - case sendBodyPart(IOData) - case sendRequestEnd + case sendBodyPart(IOData, EventLoopPromise?) + case sendRequestEnd(EventLoopPromise?) + case failSendBodyPart(Error, EventLoopPromise?) + case failSendStreamFinished(Error, EventLoopPromise?) case pauseRequestBodyStream case resumeRequestBodyStream @@ -50,8 +66,8 @@ struct HTTP1ConnectionStateMachine { case forwardResponseHead(HTTPResponseHead, pauseRequestBodyStream: Bool) case forwardResponseBodyParts(CircularBuffer) - case failRequest(Error, FinalStreamAction) - case succeedRequest(FinalStreamAction, CircularBuffer) + case failRequest(Error, FinalFailedStreamAction) + case succeedRequest(FinalSuccessfulStreamAction, CircularBuffer) case read case close @@ -189,25 +205,25 @@ struct HTTP1ConnectionStateMachine { } } - mutating func requestStreamPartReceived(_ part: IOData) -> Action { + mutating func requestStreamPartReceived(_ part: IOData, promise: EventLoopPromise?) -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { preconditionFailure("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in - let action = requestStateMachine.requestStreamPartReceived(part) + let action = requestStateMachine.requestStreamPartReceived(part, promise: promise) state = .inRequest(requestStateMachine, close: close) return state.modify(with: action) } } - mutating func requestStreamFinished() -> Action { + mutating func requestStreamFinished(promise: EventLoopPromise?) -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { preconditionFailure("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in - let action = requestStateMachine.requestStreamFinished() + let action = requestStateMachine.requestStreamFinished(promise: promise) state = .inRequest(requestStateMachine, close: close) return state.modify(with: action) } @@ -377,10 +393,10 @@ extension HTTP1ConnectionStateMachine.State { return .pauseRequestBodyStream case .resumeRequestBodyStream: return .resumeRequestBodyStream - case .sendBodyPart(let part): - return .sendBodyPart(part) - case .sendRequestEnd: - return .sendRequestEnd + case .sendBodyPart(let part, let writePromise): + return .sendBodyPart(part, writePromise) + case .sendRequestEnd(let writePromise): + return .sendRequestEnd(writePromise) case .forwardResponseHead(let head, let pauseRequestBodyStream): return .forwardResponseHead(head, pauseRequestBodyStream: pauseRequestBodyStream) case .forwardResponseBodyParts(let parts): @@ -390,13 +406,13 @@ extension HTTP1ConnectionStateMachine.State { preconditionFailure("Invalid state: \(self)") } - let newFinalAction: HTTP1ConnectionStateMachine.Action.FinalStreamAction + let newFinalAction: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction switch finalAction { case .close: self = .closing newFinalAction = .close - case .sendRequestEnd: - newFinalAction = .sendRequestEnd + case .sendRequestEnd(let writePromise): + newFinalAction = .sendRequestEnd(writePromise) case .none: self = .idle newFinalAction = close ? .close : .informConnectionIsIdle @@ -410,9 +426,12 @@ extension HTTP1ConnectionStateMachine.State { case .idle: preconditionFailure("How can we fail a task, if we are idle") case .inRequest(_, close: let close): - if close || finalAction == .close { + if case .close(let promise) = finalAction { + self = .closing + return .failRequest(error, .close(promise)) + } else if close { self = .closing - return .failRequest(error, .close) + return .failRequest(error, .close(nil)) } else { self = .idle return .failRequest(error, .informConnectionIsIdle) @@ -433,6 +452,12 @@ extension HTTP1ConnectionStateMachine.State { case .wait: return .wait + + case .failSendBodyPart(let error, let writePromise): + return .failSendBodyPart(error, writePromise) + + case .failSendStreamFinished(let error, let writePromise): + return .failSendStreamFinished(error, writePromise) } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 8b2a50738..578b83029 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -148,11 +148,11 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // that the request is neither failed nor finished yet self.request!.pauseRequestBodyStream() - case .sendBodyPart(let data): - context.writeAndFlush(self.wrapOutboundOut(.body(data)), promise: nil) + case .sendBodyPart(let data, let writePromise): + context.writeAndFlush(self.wrapOutboundOut(.body(data)), promise: writePromise) - case .sendRequestEnd: - context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + case .sendRequestEnd(let writePromise): + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { self.runTimeoutAction(timeoutAction, context: context) @@ -185,7 +185,7 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // that the request is neither failed nor finished yet self.request!.receiveResponseBodyParts(parts) - case .failRequest(let error, _): + case .failRequest(let error, let finalAction): // We can force unwrap the request here, as we have just validated in the state machine, // that the request object is still present. self.request!.fail(error) @@ -195,7 +195,7 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // once the h2 stream is closed, it is released from the h2 multiplexer. The // HTTPRequestStateMachine may signal finalAction: .none in the error case (as this is // the right result for HTTP/1). In the h2 case we MUST always close. - self.runFinalAction(.close, context: context) + self.runFailedFinalAction(finalAction, context: context, error: error) case .succeedRequest(let finalAction, let finalParts): // We can force unwrap the request here, as we have just validated in the state machine, @@ -203,7 +203,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { self.request!.succeedRequest(finalParts) self.request = nil self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) - self.runFinalAction(finalAction, context: context) + self.runSuccessfulFinalAction(finalAction, context: context) + + case .failSendBodyPart(let error, let writePromise), .failSendStreamFinished(let error, let writePromise): + writePromise?.fail(error) } } @@ -234,13 +237,24 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } } - private func runFinalAction(_ action: HTTPRequestStateMachine.Action.FinalStreamAction, context: ChannelHandlerContext) { + private func runSuccessfulFinalAction(_ action: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, context: ChannelHandlerContext) { switch action { case .close: context.close(promise: nil) - case .sendRequestEnd: - context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + case .sendRequestEnd(let writePromise): + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) + + case .none: + break + } + } + + private func runFailedFinalAction(_ action: HTTPRequestStateMachine.Action.FinalFailedRequestAction, context: ChannelHandlerContext, error: Error) { + switch action { + case .close(let writePromise): + context.close(promise: nil) + writePromise?.fail(error) case .none: break @@ -281,27 +295,28 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // MARK: Private HTTPRequestExecutor - private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest) { + private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { guard self.request === request, let context = self.channelContext else { // Because the HTTPExecutableRequest may run in a different thread to our eventLoop, // calls from the HTTPExecutableRequest to our ChannelHandler may arrive here after // the request has been popped by the state machine or the ChannelHandler has been // removed from the Channel pipeline. This is a normal threading issue, noone has // screwed up. + promise?.fail(HTTPClientError.requestStreamCancelled) return } - let action = self.state.requestStreamPartReceived(data) + let action = self.state.requestStreamPartReceived(data, promise: promise) self.run(action, context: context) } - private func finishRequestBodyStream0(_ request: HTTPExecutableRequest) { + private func finishRequestBodyStream0(_ request: HTTPExecutableRequest, promise: EventLoopPromise?) { guard self.request === request, let context = self.channelContext else { // See code comment in `writeRequestBodyPart0` return } - let action = self.state.requestStreamFinished() + let action = self.state.requestStreamFinished(promise: promise) self.run(action, context: context) } @@ -327,22 +342,22 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } extension HTTP2ClientRequestHandler: HTTPRequestExecutor { - func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest) { + func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { - self.writeRequestBodyPart0(data, request: request) + self.writeRequestBodyPart0(data, request: request, promise: promise) } else { self.eventLoop.execute { - self.writeRequestBodyPart0(data, request: request) + self.writeRequestBodyPart0(data, request: request, promise: promise) } } } - func finishRequestBodyStream(_ request: HTTPExecutableRequest) { + func finishRequestBodyStream(_ request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { - self.finishRequestBodyStream0(request) + self.finishRequestBodyStream0(request, promise: promise) } else { self.eventLoop.execute { - self.finishRequestBodyStream0(request) + self.finishRequestBodyStream0(request, promise: promise) } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift index 2477e1154..d64ceedd6 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift @@ -180,12 +180,12 @@ protocol HTTPRequestExecutor { /// Writes a body part into the channel pipeline /// /// This method may be **called on any thread**. The executor needs to ensure thread safety. - func writeRequestBodyPart(_: IOData, request: HTTPExecutableRequest) + func writeRequestBodyPart(_: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) /// Signals that the request body stream has finished /// /// This method may be **called on any thread**. The executor needs to ensure thread safety. - func finishRequestBodyStream(_ task: HTTPExecutableRequest) + func finishRequestBodyStream(_ task: HTTPExecutableRequest, promise: EventLoopPromise?) /// Signals that more bytes from response body stream can be consumed. /// diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift index fa520a865..aafa3d28b 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift @@ -70,21 +70,34 @@ struct HTTPRequestStateMachine { } enum Action { - /// A action to execute, when we consider a request "done". - enum FinalStreamAction { + /// A action to execute, when we consider a successful request "done". + enum FinalSuccessfulRequestAction { /// Close the connection case close /// If the server has replied, with a status of 200...300 before all data was sent, a request is considered succeeded, /// as soon as we wrote the request end onto the wire. - case sendRequestEnd + /// + /// The promise is an optional write promise. + case sendRequestEnd(EventLoopPromise?) + /// Do nothing. This is action is used, if the request failed, before we the request head was written onto the wire. + /// This might happen if the request is cancelled, or the request failed the soundness check. + case none + } + + /// A action to execute, when we consider a failed request "done". + enum FinalFailedRequestAction { + /// Close the connection + case close(EventLoopPromise?) /// Do nothing. This is action is used, if the request failed, before we the request head was written onto the wire. /// This might happen if the request is cancelled, or the request failed the soundness check. case none } case sendRequestHead(HTTPRequestHead, startBody: Bool) - case sendBodyPart(IOData) - case sendRequestEnd + case sendBodyPart(IOData, EventLoopPromise?) + case sendRequestEnd(EventLoopPromise?) + case failSendBodyPart(Error, EventLoopPromise?) + case failSendStreamFinished(Error, EventLoopPromise?) case pauseRequestBodyStream case resumeRequestBodyStream @@ -92,8 +105,8 @@ struct HTTPRequestStateMachine { case forwardResponseHead(HTTPResponseHead, pauseRequestBodyStream: Bool) case forwardResponseBodyParts(CircularBuffer) - case failRequest(Error, FinalStreamAction) - case succeedRequest(FinalStreamAction, CircularBuffer) + case failRequest(Error, FinalFailedRequestAction) + case succeedRequest(FinalSuccessfulRequestAction, CircularBuffer) case read case wait @@ -212,7 +225,7 @@ struct HTTPRequestStateMachine { return .failRequest(error, .none) case .running: self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(nil)) case .finished, .failed: // ignore error @@ -254,14 +267,14 @@ struct HTTPRequestStateMachine { // we have received all necessary bytes. For this reason we forward the uncleanShutdown // error to the user. self.state = .failed(NIOSSLError.uncleanShutdown) - return .failRequest(NIOSSLError.uncleanShutdown, .close) + return .failRequest(NIOSSLError.uncleanShutdown, .close(nil)) case .waitForChannelToBecomeWritable, .running, .finished, .failed, .initialized, .modifying: return nil } } - mutating func requestStreamPartReceived(_ part: IOData) -> Action { + mutating func requestStreamPartReceived(_ part: IOData, promise: EventLoopPromise?) -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable, @@ -274,7 +287,7 @@ struct HTTPRequestStateMachine { // won't be interested. We expect that the producer has been informed to pause // producing. assert(producerState == .paused) - return .wait + return .failSendBodyPart(HTTPClientError.requestStreamCancelled, promise) case .running(.streaming(let expectedBodyLength, var sentBodyBytes, let producerState), let responseState): // We don't check the producer state here: @@ -290,7 +303,7 @@ struct HTTPRequestStateMachine { if let expected = expectedBodyLength, sentBodyBytes + part.readableBytes > expected { let error = HTTPClientError.bodyLengthMismatch self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(promise)) } sentBodyBytes += part.readableBytes @@ -303,10 +316,10 @@ struct HTTPRequestStateMachine { self.state = .running(requestState, responseState) - return .sendBodyPart(part) + return .sendBodyPart(part, promise) - case .failed: - return .wait + case .failed(let error): + return .failSendBodyPart(error, promise) case .finished: // A request may be finished, before we have send all parts. This might be the case if @@ -318,14 +331,14 @@ struct HTTPRequestStateMachine { // We may still receive something, here because of potential race conditions with the // producing thread. - return .wait + return .failSendBodyPart(HTTPClientError.requestStreamCancelled, promise) case .modifying: preconditionFailure("Invalid state: \(self.state)") } } - mutating func requestStreamFinished() -> Action { + mutating func requestStreamFinished(promise: EventLoopPromise?) -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable, @@ -336,11 +349,11 @@ struct HTTPRequestStateMachine { if let expected = expectedBodyLength, expected != sentBodyBytes { let error = HTTPClientError.bodyLengthMismatch self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(promise)) } self.state = .running(.endSent, .waitingForHead) - return .sendRequestEnd + return .sendRequestEnd(promise) case .running(.streaming(let expectedBodyLength, let sentBodyBytes, _), .receivingBody(let head, let streamState)): assert(head.status.code < 300) @@ -348,24 +361,24 @@ struct HTTPRequestStateMachine { if let expected = expectedBodyLength, expected != sentBodyBytes { let error = HTTPClientError.bodyLengthMismatch self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(promise)) } self.state = .running(.endSent, .receivingBody(head, streamState)) - return .sendRequestEnd + return .sendRequestEnd(promise) case .running(.streaming(let expectedBodyLength, let sentBodyBytes, _), .endReceived): if let expected = expectedBodyLength, expected != sentBodyBytes { let error = HTTPClientError.bodyLengthMismatch self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(promise)) } self.state = .finished - return .succeedRequest(.sendRequestEnd, .init()) + return .succeedRequest(.sendRequestEnd(promise), .init()) - case .failed: - return .wait + case .failed(let error): + return .failSendStreamFinished(error, promise) case .finished: // A request may be finished, before we have send all parts. This might be the case if @@ -377,7 +390,7 @@ struct HTTPRequestStateMachine { // We may still receive something, here because of potential race conditions with the // producing thread. - return .wait + return .failSendStreamFinished(HTTPClientError.requestStreamCancelled, promise) case .modifying: preconditionFailure("Invalid state: \(self.state)") @@ -398,7 +411,7 @@ struct HTTPRequestStateMachine { case .running: let error = HTTPClientError.cancelled self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(nil)) case .finished: return .wait @@ -597,7 +610,7 @@ struct HTTPRequestStateMachine { // the request is still uploading, we will not be able to finish the upload. For // this reason we can fail the request here. state = .failed(HTTPClientError.remoteConnectionClosed) - return .failRequest(HTTPClientError.remoteConnectionClosed, .close) + return .failRequest(HTTPClientError.remoteConnectionClosed, .close(nil)) } } @@ -670,7 +683,7 @@ struct HTTPRequestStateMachine { case .running(.endSent, .waitingForHead), .running(.endSent, .receivingBody): let error = HTTPClientError.readTimeout self.state = .failed(error) - return .failRequest(error, .close) + return .failRequest(error, .close(nil)) case .running(.endSent, .endReceived): preconditionFailure("Invalid state. This state should be: .finished") diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index b4aeef0e7..dbef802e9 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -154,8 +154,11 @@ final class RequestBag { return self.task.eventLoop.makeFailedFuture(error) case .write(let part, let writer, let future): - writer.writeRequestBodyPart(part, request: self) - self.delegate.didSendRequestPart(task: self.task, part) + let promise = self.task.eventLoop.makePromise(of: Void.self) + promise.futureResult.whenSuccess { + self.delegate.didSendRequestPart(task: self.task, part) + } + writer.writeRequestBodyPart(part, request: self, promise: promise) return future } } @@ -168,11 +171,12 @@ final class RequestBag { switch action { case .none: break - case .forwardStreamFinished(let writer, let promise): - writer.finishRequestBodyStream(self) - promise?.succeed(()) - - self.delegate.didSendRequest(task: self.task) + case .forwardStreamFinished(let writer, let writerPromise): + let promise = writerPromise ?? self.task.eventLoop.makePromise(of: Void.self) + promise.futureResult.whenSuccess { + self.delegate.didSendRequest(task: self.task) + } + writer.finishRequestBodyStream(self, promise: promise) case .forwardStreamFailureAndFailTask(let writer, let error, let promise): writer.cancelRequest(self) diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index c8ad3d510..55014f8c6 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -32,22 +32,22 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) let part2 = IOData.byteBuffer(ByteBuffer(bytes: [2])) let part3 = IOData.byteBuffer(ByteBuffer(bytes: [3])) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) // oh the channel reports... we should slow down producing... XCTAssertEqual(state.writabilityChanged(writable: false), .pauseRequestBodyStream) // but we issued a .produceMoreRequestBodyData before... Thus, we must accept more produced // data - XCTAssertEqual(state.requestStreamPartReceived(part2), .sendBodyPart(part2)) + XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .sendBodyPart(part2, nil)) // however when we have put the data on the channel, we should not issue further // .produceMoreRequestBodyData events // once we receive a writable event again, we can allow the producer to produce more data XCTAssertEqual(state.writabilityChanged(writable: true), .resumeRequestBodyStream) - XCTAssertEqual(state.requestStreamPartReceived(part3), .sendBodyPart(part3)) - XCTAssertEqual(state.requestStreamFinished(), .sendRequestEnd) + XCTAssertEqual(state.requestStreamPartReceived(part3, promise: nil), .sendBodyPart(part3, nil)) + XCTAssertEqual(state.requestStreamFinished(promise: nil), .sendRequestEnd(nil)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -186,9 +186,9 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) - XCTAssertEqual(state.requestCancelled(closeConnection: false), .failRequest(HTTPClientError.cancelled, .close)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) + XCTAssertEqual(state.requestCancelled(closeConnection: false), .failRequest(HTTPClientError.cancelled, .close(nil))) } func testCancelRequestIsIgnoredWhenConnectionIsIdle() { @@ -241,7 +241,7 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.body(ByteBuffer(string: "Hello world!\n"))), .wait) XCTAssertEqual(state.channelRead(.body(ByteBuffer(string: "Foo Bar!\n"))), .wait) let decompressionError = NIOHTTPDecompression.DecompressionError.limit - XCTAssertEqual(state.errorHappened(decompressionError), .failRequest(decompressionError, .close)) + XCTAssertEqual(state.errorHappened(decompressionError), .failRequest(decompressionError, .close(nil))) } func testConnectionIsClosedAfterSwitchingProtocols() { @@ -295,8 +295,8 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { case (.sendRequestHead(let lhsHead, let lhsStartBody), .sendRequestHead(let rhsHead, let rhsStartBody)): return lhsHead == rhsHead && lhsStartBody == rhsStartBody - case (.sendBodyPart(let lhsData), .sendBodyPart(let rhsData)): - return lhsData == rhsData + case (.sendBodyPart(let lhsData, let lhsPromise), .sendBodyPart(let rhsData, let rhsPromise)): + return lhsData == rhsData && lhsPromise?.futureResult == rhsPromise?.futureResult case (.sendRequestEnd, .sendRequestEnd): return true @@ -332,3 +332,35 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { } } } + +extension HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction: Equatable { + public static func == (lhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction, rhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction) -> Bool { + switch (lhs, rhs) { + case (.close, .close): + return true + case (sendRequestEnd(let lhsPromise), sendRequestEnd(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult + case (informConnectionIsIdle, informConnectionIsIdle): + return true + default: + return false + } + } +} + +extension HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction: Equatable { + public static func == (lhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction, rhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction) -> Bool { + switch (lhs, rhs) { + case (.close(let lhsPromise), .close(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult + case (.informConnectionIsIdle, .informConnectionIsIdle): + return true + case (.failWritePromise(let lhsPromise), .failWritePromise(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult + case (.none, .none): + return true + default: + return false + } + } +} diff --git a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift index b54865fd8..ad85bd71e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift @@ -30,6 +30,7 @@ extension HTTPRequestStateMachineTests { ("testPOSTContentLengthIsTooLong", testPOSTContentLengthIsTooLong), ("testPOSTContentLengthIsTooShort", testPOSTContentLengthIsTooShort), ("testRequestBodyStreamIsCancelledIfServerRespondsWith301", testRequestBodyStreamIsCancelledIfServerRespondsWith301), + ("testStreamPartReceived_whenCancelled", testStreamPartReceived_whenCancelled), ("testRequestBodyStreamIsCancelledIfServerRespondsWith301WhileWriteBackpressure", testRequestBodyStreamIsCancelledIfServerRespondsWith301WhileWriteBackpressure), ("testRequestBodyStreamIsContinuedIfServerRespondsWith200", testRequestBodyStreamIsContinuedIfServerRespondsWith200), ("testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200", testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200), diff --git a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift index ab55345c9..a68d58aa0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift @@ -14,6 +14,7 @@ @testable import AsyncHTTPClient import NIOCore +import NIOEmbedded import NIOHTTP1 import NIOSSL import XCTest @@ -42,22 +43,22 @@ class HTTPRequestStateMachineTests: XCTestCase { let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) let part2 = IOData.byteBuffer(ByteBuffer(bytes: [2])) let part3 = IOData.byteBuffer(ByteBuffer(bytes: [3])) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) // oh the channel reports... we should slow down producing... XCTAssertEqual(state.writabilityChanged(writable: false), .pauseRequestBodyStream) // but we issued a .produceMoreRequestBodyData before... Thus, we must accept more produced // data - XCTAssertEqual(state.requestStreamPartReceived(part2), .sendBodyPart(part2)) + XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .sendBodyPart(part2, nil)) // however when we have put the data on the channel, we should not issue further // .produceMoreRequestBodyData events // once we receive a writable event again, we can allow the producer to produce more data XCTAssertEqual(state.writabilityChanged(writable: true), .resumeRequestBodyStream) - XCTAssertEqual(state.requestStreamPartReceived(part3), .sendBodyPart(part3)) - XCTAssertEqual(state.requestStreamFinished(), .sendRequestEnd) + XCTAssertEqual(state.requestStreamPartReceived(part3, promise: nil), .sendBodyPart(part3, nil)) + XCTAssertEqual(state.requestStreamFinished(promise: nil), .sendRequestEnd(nil)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -74,9 +75,9 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) - state.requestStreamPartReceived(part1).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close) + state.requestStreamPartReceived(part1, promise: nil).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close(nil)) // if another error happens the new one is ignored XCTAssertEqual(state.errorHappened(HTTPClientError.remoteConnectionClosed), .wait) @@ -88,9 +89,9 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(8)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) - state.requestStreamFinished().assertFailRequest(HTTPClientError.bodyLengthMismatch, .close) + state.requestStreamFinished(promise: nil).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close(nil)) } func testRequestBodyStreamIsCancelledIfServerRespondsWith301() { @@ -99,22 +100,31 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) - XCTAssertEqual(state.requestStreamPartReceived(part), .sendBodyPart(part)) + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .movedPermanently) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: true)) XCTAssertEqual(state.writabilityChanged(writable: false), .wait) XCTAssertEqual(state.writabilityChanged(writable: true), .wait) - XCTAssertEqual(state.requestStreamPartReceived(part), .wait, + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init())) - XCTAssertEqual(state.requestStreamPartReceived(part), .wait, + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") - XCTAssertEqual(state.requestStreamFinished(), .wait, + XCTAssertEqual(state.requestStreamFinished(promise: nil), .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300") + } + + func testStreamPartReceived_whenCancelled() { + var state = HTTPRequestStateMachine(isChannelWritable: false) + let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) + + XCTAssertEqual(state.requestCancelled(), .failRequest(HTTPClientError.cancelled, .none)) + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.cancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") } @@ -124,22 +134,22 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) - XCTAssertEqual(state.requestStreamPartReceived(part), .sendBodyPart(part)) + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) XCTAssertEqual(state.writabilityChanged(writable: false), .pauseRequestBodyStream) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .movedPermanently) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) XCTAssertEqual(state.writabilityChanged(writable: true), .wait) - XCTAssertEqual(state.requestStreamPartReceived(part), .wait, + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init())) - XCTAssertEqual(state.requestStreamPartReceived(part), .wait, + XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") - XCTAssertEqual(state.requestStreamFinished(), .wait, + XCTAssertEqual(state.requestStreamFinished(promise: nil), .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), "Expected to drop all stream data after having received a response head, with status >= 300") } @@ -149,7 +159,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) @@ -157,10 +167,12 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.end(nil)), .forwardResponseBodyParts(.init())) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) let part2 = IOData.byteBuffer(ByteBuffer(bytes: 8...11)) - XCTAssertEqual(state.requestStreamPartReceived(part2), .sendBodyPart(part2)) - XCTAssertEqual(state.requestStreamFinished(), .succeedRequest(.sendRequestEnd, .init())) + XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .sendBodyPart(part2, nil)) + XCTAssertEqual(state.requestStreamFinished(promise: nil), .succeedRequest(.sendRequestEnd(nil), .init())) + + XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil)) } func testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200() { @@ -169,17 +181,17 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) let part2 = IOData.byteBuffer(ByteBuffer(bytes: 8...11)) - XCTAssertEqual(state.requestStreamPartReceived(part2), .sendBodyPart(part2)) - XCTAssertEqual(state.requestStreamFinished(), .sendRequestEnd) + XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .sendBodyPart(part2, nil)) + XCTAssertEqual(state.requestStreamFinished(promise: nil), .sendRequestEnd(nil)) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init())) } @@ -190,7 +202,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) @@ -198,8 +210,8 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.end(nil)), .forwardResponseBodyParts(.init())) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) - state.requestStreamFinished().assertFailRequest(HTTPClientError.bodyLengthMismatch, .close) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) + state.requestStreamFinished(promise: nil).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) } @@ -209,15 +221,15 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) - XCTAssertEqual(state.requestStreamPartReceived(part0), .sendBodyPart(part0)) + XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) - XCTAssertEqual(state.requestStreamPartReceived(part1), .sendBodyPart(part1)) - state.requestStreamFinished().assertFailRequest(HTTPClientError.bodyLengthMismatch, .close) + XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) + state.requestStreamFinished(promise: nil).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close(nil)) XCTAssertEqual(state.channelRead(.end(nil)), .wait) } @@ -366,7 +378,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) - state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close) + state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) } func testRemoteSuddenlyClosesTheConnection() { @@ -374,8 +386,8 @@ class HTTPRequestStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/", headers: .init([("content-length", "4")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) - state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close) - XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(.init(bytes: 1...3))), .wait) + state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) + XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(.init(bytes: 1...3)), promise: nil), .failSendBodyPart(HTTPClientError.cancelled, nil)) } func testReadTimeoutLeadsToFailureWithEverythingAfterBeingIgnored() { @@ -388,7 +400,7 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) let part0 = ByteBuffer(bytes: 0...3) XCTAssertEqual(state.channelRead(.body(part0)), .wait) - state.idleReadTimeoutTriggered().assertFailRequest(HTTPClientError.readTimeout, .close) + state.idleReadTimeoutTriggered().assertFailRequest(HTTPClientError.readTimeout, .close(nil)) XCTAssertEqual(state.channelRead(.body(ByteBuffer(bytes: 4...7))), .wait) XCTAssertEqual(state.channelRead(.body(ByteBuffer(bytes: 8...11))), .wait) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) @@ -441,7 +453,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) - state.errorHappened(HTTPParserError.invalidChunkSize).assertFailRequest(HTTPParserError.invalidChunkSize, .close) + state.errorHappened(HTTPParserError.invalidChunkSize).assertFailRequest(HTTPParserError.invalidChunkSize, .close(nil)) XCTAssertEqual(state.requestCancelled(), .wait, "A cancellation that happens to late is ignored") } @@ -486,7 +498,7 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) let part1: ByteBuffer = .init(string: "foo") - XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(part1)), .sendBodyPart(.byteBuffer(part1))) + XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(part1), promise: nil), .sendBodyPart(.byteBuffer(part1), nil)) let responseHead = HTTPResponseHead(version: .http1_0, status: .ok) let body = ByteBuffer(string: "foo bar") XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -495,7 +507,7 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.read(), .read) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.channelRead(.body(body)), .wait) - state.channelRead(.end(nil)).assertFailRequest(HTTPClientError.remoteConnectionClosed, .close) + state.channelRead(.end(nil)).assertFailRequest(HTTPClientError.remoteConnectionClosed, .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) } @@ -510,7 +522,7 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelRead(.body(body)), .wait) - state.errorHappened(NIOSSLError.uncleanShutdown).assertFailRequest(NIOSSLError.uncleanShutdown, .close) + state.errorHappened(NIOSSLError.uncleanShutdown).assertFailRequest(NIOSSLError.uncleanShutdown, .close(nil)) XCTAssertEqual(state.channelRead(.end(nil)), .wait) XCTAssertEqual(state.channelInactive(), .wait) } @@ -532,7 +544,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) - state.errorHappened(ArbitraryError()).assertFailRequest(ArbitraryError(), .close) + state.errorHappened(ArbitraryError()).assertFailRequest(ArbitraryError(), .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) } @@ -550,7 +562,7 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.channelRead(.body(body)), .wait) XCTAssertEqual(state.channelReadComplete(), .forwardResponseBodyParts([body])) XCTAssertEqual(state.errorHappened(NIOSSLError.uncleanShutdown), .wait) - state.errorHappened(HTTPParserError.invalidEOFState).assertFailRequest(HTTPParserError.invalidEOFState, .close) + state.errorHappened(HTTPParserError.invalidEOFState).assertFailRequest(HTTPParserError.invalidEOFState, .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) } @@ -656,11 +668,11 @@ extension HTTPRequestStateMachine.Action: Equatable { case (.sendRequestHead(let lhsHead, let lhsStartBody), .sendRequestHead(let rhsHead, let rhsStartBody)): return lhsHead == rhsHead && lhsStartBody == rhsStartBody - case (.sendBodyPart(let lhsData), .sendBodyPart(let rhsData)): - return lhsData == rhsData + case (.sendBodyPart(let lhsData, let lhsPromise), .sendBodyPart(let rhsData, let rhsPromise)): + return lhsData == rhsData && lhsPromise?.futureResult == rhsPromise?.futureResult - case (.sendRequestEnd, .sendRequestEnd): - return true + case (.sendRequestEnd(let lhsPromise), .sendRequestEnd(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult case (.pauseRequestBodyStream, .pauseRequestBodyStream): return true @@ -685,6 +697,45 @@ extension HTTPRequestStateMachine.Action: Equatable { case (.wait, .wait): return true + case (.failSendBodyPart(let lhsError as HTTPClientError, let lhsPromise), .failSendBodyPart(let rhsError as HTTPClientError, let rhsPromise)): + return lhsError == rhsError && lhsPromise?.futureResult == rhsPromise?.futureResult + + case (.failSendStreamFinished(let lhsError as HTTPClientError, let lhsPromise), .failSendStreamFinished(let rhsError as HTTPClientError, let rhsPromise)): + return lhsError == rhsError && lhsPromise?.futureResult == rhsPromise?.futureResult + + default: + return false + } + } +} + +extension HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction: Equatable { + public static func == (lhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, rhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction) -> Bool { + switch (lhs, rhs) { + case (.close, close): + return true + + case (.sendRequestEnd(let lhsPromise), .sendRequestEnd(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult + + case (.none, .none): + return true + + default: + return false + } + } +} + +extension HTTPRequestStateMachine.Action.FinalFailedRequestAction: Equatable { + public static func == (lhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction, rhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction) -> Bool { + switch (lhs, rhs) { + case (.close(let lhsPromise), close(let rhsPromise)): + return lhsPromise?.futureResult == rhsPromise?.futureResult + + case (.none, .none): + return true + default: return false } @@ -694,7 +745,7 @@ extension HTTPRequestStateMachine.Action: Equatable { extension HTTPRequestStateMachine.Action { fileprivate func assertFailRequest( _ expectedError: Error, - _ expectedFinalStreamAction: HTTPRequestStateMachine.Action.FinalStreamAction, + _ expectedFinalStreamAction: HTTPRequestStateMachine.Action.FinalFailedRequestAction, file: StaticString = #file, line: UInt = #line ) where Error: Swift.Error & Equatable { diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift b/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift index b5b67c809..b37ce8fa3 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift @@ -184,12 +184,14 @@ extension MockRequestExecutor: HTTPRequestExecutor { // this should always be called twice. When we receive the first call, the next call to produce // data is already scheduled. If we call pause here, once, after the second call new subsequent // calls should not be scheduled. - func writeRequestBodyPart(_ part: IOData, request: HTTPExecutableRequest) { + func writeRequestBodyPart(_ part: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { self.writeNextRequestPart(.body(part), request: request) + promise?.succeed(()) } - func finishRequestBodyStream(_ request: HTTPExecutableRequest) { + func finishRequestBodyStream(_ request: HTTPExecutableRequest, promise: EventLoopPromise?) { self.writeNextRequestPart(.endOfStream, request: request) + promise?.succeed(()) } private func writeNextRequestPart(_ part: RequestParts, request: HTTPExecutableRequest) { From 3fcd67061f17d105282701c8ebd32467e458ceb6 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 1 Jun 2022 14:13:47 +0100 Subject: [PATCH 011/146] Improve errors and testing using NIOTS (#588) Motivation Currently error reporting with NIO Transport Services is often sub-par. This occurs because the Network.framework connections may enter the waiting state until the network connectivity state changes. We were not watching for the user event that contains the error in that state, so if we timed out in that state we'd just give a generic timeout error, instead of telling the user anything more detailed. Additionally, several of our tests assume that failure will be fast, but in NIO Transport Services we will enter that .waiting state. This is reasonable, as changed network connections may make a connection that was not succeeding suddenly viable. However, it's inconvenient for testing, where we're mostly interested in confirming that the error path works as expected. Modifications - Add an observer of the WaitingForConnectivity event that records it into our state machine for later reporting. - Add support for disabling waiting for connectivity for testing purposes. - Add annotations to several tests to stop them waiting for connectivity. Results Faster tests, better coverage, better errors for our users. Co-authored-by: David Nadoba --- .../HTTPConnectionPool+Factory.swift | 71 ++++++++++++++----- .../ConnectionPool/HTTPConnectionPool.swift | 10 +++ ...HTTPConnectionPool+HTTP1StateMachine.swift | 6 ++ ...HTTPConnectionPool+HTTP2StateMachine.swift | 5 ++ .../HTTPConnectionPool+StateMachine.swift | 8 +++ Sources/AsyncHTTPClient/HTTPClient.swift | 5 ++ .../NWWaitingHandler.swift | 41 +++++++++++ .../HTTP2ConnectionTests.swift | 4 ++ .../HTTPClient+SOCKSTests.swift | 19 +++-- .../HTTPClientInternalTests.swift | 4 +- .../HTTPClientNIOTSTests+XCTest.swift | 1 + .../HTTPClientNIOTSTests.swift | 38 ++++++++-- .../HTTPClientTests.swift | 36 ++++++++-- .../HTTPConnectionPool+FactoryTests.swift | 23 ++++++ 14 files changed, 240 insertions(+), 31 deletions(-) create mode 100644 Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 4a3338697..1444df9bb 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -47,6 +47,7 @@ protocol HTTPConnectionRequester { func http1ConnectionCreated(_: HTTP1Connection) func http2ConnectionCreated(_: HTTP2Connection, maximumStreams: Int) func failedToCreateHTTPConnection(_: HTTPConnectionPool.Connection.ID, error: Error) + func waitingForConnectivity(_: HTTPConnectionPool.Connection.ID, error: Error) } extension HTTPConnectionPool.ConnectionFactory { @@ -62,7 +63,7 @@ extension HTTPConnectionPool.ConnectionFactory { var logger = logger logger[metadataKey: "ahc-connection-id"] = "\(connectionID)" - self.makeChannel(connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger).whenComplete { result in + self.makeChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger).whenComplete { result in switch result { case .success(.http1_1(let channel)): do { @@ -104,13 +105,15 @@ extension HTTPConnectionPool.ConnectionFactory { case http2(Channel) } - func makeHTTP1Channel( + func makeHTTP1Channel( + requester: Requester, connectionID: HTTPConnectionPool.Connection.ID, deadline: NIODeadline, eventLoop: EventLoop, logger: Logger ) -> EventLoopFuture { self.makeChannel( + requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, @@ -137,7 +140,8 @@ extension HTTPConnectionPool.ConnectionFactory { } } - func makeChannel( + func makeChannel( + requester: Requester, connectionID: HTTPConnectionPool.Connection.ID, deadline: NIODeadline, eventLoop: EventLoop, @@ -150,6 +154,7 @@ extension HTTPConnectionPool.ConnectionFactory { case .socks: channelFuture = self.makeSOCKSProxyChannel( proxy, + requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, @@ -158,6 +163,7 @@ extension HTTPConnectionPool.ConnectionFactory { case .http: channelFuture = self.makeHTTPProxyChannel( proxy, + requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, @@ -165,7 +171,7 @@ extension HTTPConnectionPool.ConnectionFactory { ) } } else { - channelFuture = self.makeNonProxiedChannel(deadline: deadline, eventLoop: eventLoop, logger: logger) + channelFuture = self.makeNonProxiedChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger) } // let's map `ChannelError.connectTimeout` into a `HTTPClientError.connectTimeout` @@ -179,16 +185,18 @@ extension HTTPConnectionPool.ConnectionFactory { } } - private func makeNonProxiedChannel( + private func makeNonProxiedChannel( + requester: Requester, + connectionID: HTTPConnectionPool.Connection.ID, deadline: NIODeadline, eventLoop: EventLoop, logger: Logger ) -> EventLoopFuture { switch self.key.scheme { case .http, .httpUnix, .unix: - return self.makePlainChannel(deadline: deadline, eventLoop: eventLoop).map { .http1_1($0) } + return self.makePlainChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop).map { .http1_1($0) } case .https, .httpsUnix: - return self.makeTLSChannel(deadline: deadline, eventLoop: eventLoop, logger: logger).flatMapThrowing { + return self.makeTLSChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger).flatMapThrowing { channel, negotiated in try self.matchALPNToHTTPVersion(negotiated, channel: channel) @@ -196,13 +204,19 @@ extension HTTPConnectionPool.ConnectionFactory { } } - private func makePlainChannel(deadline: NIODeadline, eventLoop: EventLoop) -> EventLoopFuture { + private func makePlainChannel( + requester: Requester, + connectionID: HTTPConnectionPool.Connection.ID, + deadline: NIODeadline, + eventLoop: EventLoop + ) -> EventLoopFuture { precondition(!self.key.scheme.usesTLS, "Unexpected scheme") - return self.makePlainBootstrap(deadline: deadline, eventLoop: eventLoop).connect(target: self.key.connectionTarget) + return self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop).connect(target: self.key.connectionTarget) } - private func makeHTTPProxyChannel( + private func makeHTTPProxyChannel( _ proxy: HTTPClient.Configuration.Proxy, + requester: Requester, connectionID: HTTPConnectionPool.Connection.ID, deadline: NIODeadline, eventLoop: EventLoop, @@ -211,7 +225,7 @@ extension HTTPConnectionPool.ConnectionFactory { // A proxy connection starts with a plain text connection to the proxy server. After // the connection has been established with the proxy server, the connection might be // upgraded to TLS before we send our first request. - let bootstrap = self.makePlainBootstrap(deadline: deadline, eventLoop: eventLoop) + let bootstrap = self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop) return bootstrap.connect(host: proxy.host, port: proxy.port).flatMap { channel in let encoder = HTTPRequestEncoder() let decoder = ByteToMessageHandler(HTTPResponseDecoder(leftOverBytesStrategy: .dropBytes)) @@ -243,8 +257,9 @@ extension HTTPConnectionPool.ConnectionFactory { } } - private func makeSOCKSProxyChannel( + private func makeSOCKSProxyChannel( _ proxy: HTTPClient.Configuration.Proxy, + requester: Requester, connectionID: HTTPConnectionPool.Connection.ID, deadline: NIODeadline, eventLoop: EventLoop, @@ -253,7 +268,7 @@ extension HTTPConnectionPool.ConnectionFactory { // A proxy connection starts with a plain text connection to the proxy server. After // the connection has been established with the proxy server, the connection might be // upgraded to TLS before we send our first request. - let bootstrap = self.makePlainBootstrap(deadline: deadline, eventLoop: eventLoop) + let bootstrap = self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop) return bootstrap.connect(host: proxy.host, port: proxy.port).flatMap { channel in let socksConnectHandler = SOCKSClientHandler(targetAddress: SOCKSAddress(self.key.connectionTarget)) let socksEventHandler = SOCKSEventsHandler(deadline: deadline) @@ -331,14 +346,21 @@ extension HTTPConnectionPool.ConnectionFactory { } } - private func makePlainBootstrap(deadline: NIODeadline, eventLoop: EventLoop) -> NIOClientTCPBootstrapProtocol { + private func makePlainBootstrap( + requester: Requester, + connectionID: HTTPConnectionPool.Connection.ID, + deadline: NIODeadline, + eventLoop: EventLoop + ) -> NIOClientTCPBootstrapProtocol { #if canImport(Network) if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { return tsBootstrap + .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) .connectTimeout(deadline - NIODeadline.now()) .channelInitializer { channel in do { try channel.pipeline.syncOperations.addHandler(HTTPClient.NWErrorHandler()) + try channel.pipeline.syncOperations.addHandler(NWWaitingHandler(requester: requester, connectionID: connectionID)) return channel.eventLoop.makeSucceededVoidFuture() } catch { return channel.eventLoop.makeFailedFuture(error) @@ -355,9 +377,17 @@ extension HTTPConnectionPool.ConnectionFactory { preconditionFailure("No matching bootstrap found") } - private func makeTLSChannel(deadline: NIODeadline, eventLoop: EventLoop, logger: Logger) -> EventLoopFuture<(Channel, String?)> { + private func makeTLSChannel( + requester: Requester, + connectionID: HTTPConnectionPool.Connection.ID, + deadline: NIODeadline, + eventLoop: EventLoop, + logger: Logger + ) -> EventLoopFuture<(Channel, String?)> { precondition(self.key.scheme.usesTLS, "Unexpected scheme") let bootstrapFuture = self.makeTLSBootstrap( + requester: requester, + connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger @@ -387,8 +417,13 @@ extension HTTPConnectionPool.ConnectionFactory { return channelFuture } - private func makeTLSBootstrap(deadline: NIODeadline, eventLoop: EventLoop, logger: Logger) - -> EventLoopFuture { + private func makeTLSBootstrap( + requester: Requester, + connectionID: HTTPConnectionPool.Connection.ID, + deadline: NIODeadline, + eventLoop: EventLoop, + logger: Logger + ) -> EventLoopFuture { var tlsConfig = self.tlsConfiguration switch self.clientConfiguration.httpVersion.configuration { case .automatic: @@ -408,11 +443,13 @@ extension HTTPConnectionPool.ConnectionFactory { options -> NIOClientTCPBootstrapProtocol in tsBootstrap + .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) .connectTimeout(deadline - NIODeadline.now()) .tlsOptions(options) .channelInitializer { channel in do { try channel.pipeline.syncOperations.addHandler(HTTPClient.NWErrorHandler()) + try channel.pipeline.syncOperations.addHandler(NWWaitingHandler(requester: requester, connectionID: connectionID)) // we don't need to set a TLS deadline for NIOTS connections, since the // TLS handshake is part of the TS connection bootstrap. If the TLS // handshake times out the complete connection creation will be failed. diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 764ad2093..49e755733 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -467,6 +467,16 @@ extension HTTPConnectionPool: HTTPConnectionRequester { $0.failedToCreateNewConnection(error, connectionID: connectionID) } } + + func waitingForConnectivity(_ connectionID: HTTPConnectionPool.Connection.ID, error: Error) { + self.logger.debug("waiting for connectivity", metadata: [ + "ahc-error": "\(error)", + "ahc-connection-id": "\(connectionID)", + ]) + self.modifyStateAndRunActions { + $0.waitingForConnectivity(error, connectionID: connectionID) + } + } } extension HTTPConnectionPool: HTTP1ConnectionDelegate { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift index 6b3f7352e..d654f5a87 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift @@ -241,6 +241,12 @@ extension HTTPConnectionPool { } } + mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { + self.lastConnectFailure = error + + return .init(request: .none, connection: .none) + } + mutating func connectionCreationBackoffDone(_ connectionID: Connection.ID) -> Action { switch self.lifecycleState { case .running: diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index d3e6fbdcd..06fc36ad0 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -406,6 +406,11 @@ extension HTTPConnectionPool { return .init(request: .none, connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop)) } + mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { + self.lastConnectFailure = error + return .init(request: .none, connection: .none) + } + mutating func connectionCreationBackoffDone(_ connectionID: Connection.ID) -> Action { // The naming of `failConnection` is a little confusing here. All it does is moving the // connection state from `.backingOff` to `.closed` here. It also returns the diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift index 4d912633c..61e57941a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift @@ -211,6 +211,14 @@ extension HTTPConnectionPool { }) } + mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { + self.state.modify(http1: { http1 in + http1.waitingForConnectivity(error, connectionID: connectionID) + }, http2: { http2 in + http2.waitingForConnectivity(error, connectionID: connectionID) + }) + } + mutating func connectionCreationBackoffDone(_ connectionID: Connection.ID) -> Action { self.state.modify(http1: { http1 in http1.connectionCreationBackoffDone(connectionID) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index a6ee8956a..45b2ce0ff 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -655,6 +655,10 @@ public class HTTPClient { /// is set to `.automatic` by default which will use HTTP/2 if run over https and the server supports it, otherwise HTTP/1 public var httpVersion: HTTPVersion + /// Whether `HTTPClient` will let Network.framework sit in the `.waiting` state awaiting new network changes, or fail immediately. Defaults to `true`, + /// which is the recommended setting. Only set this to `false` when attempting to trigger a particular error path. + public var networkFrameworkWaitForConnectivity: Bool + public init( tlsConfiguration: TLSConfiguration? = nil, redirectConfiguration: RedirectConfiguration? = nil, @@ -671,6 +675,7 @@ public class HTTPClient { self.proxy = proxy self.decompression = decompression self.httpVersion = .automatic + self.networkFrameworkWaitForConnectivity = true } public init(tlsConfiguration: TLSConfiguration? = nil, diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift b/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift new file mode 100644 index 000000000..3474a8821 --- /dev/null +++ b/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Network) +import Network +import NIOCore +import NIOHTTP1 +import NIOTransportServices + +@available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) +final class NWWaitingHandler: ChannelInboundHandler { + typealias InboundIn = Any + typealias InboundOut = Any + + private var requester: Requester + private let connectionID: HTTPConnectionPool.Connection.ID + + init(requester: Requester, connectionID: HTTPConnectionPool.Connection.ID) { + self.requester = requester + self.connectionID = connectionID + } + + func userInboundEventTriggered(context: ChannelHandlerContext, event: Any) { + if let waitingEvent = event as? NIOTSNetworkEvents.WaitingForConnectivity { + self.requester.waitingForConnectivity(self.connectionID, error: HTTPClient.NWErrorHandler.translateError(waitingEvent.transientError)) + } + context.fireUserInboundEventTriggered(event) + } +} +#endif diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index fab866867..bcdaf1af2 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -405,6 +405,10 @@ extension TestConnectionCreator: HTTPConnectionRequester { } wrapper.fail(error) } + + func waitingForConnectivity(_: HTTPConnectionPool.Connection.ID, error: Swift.Error) { + preconditionFailure("TODO") + } } class TestHTTP2ConnectionDelegate: HTTP2ConnectionDelegate { diff --git a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift index 5fdc5ac61..7cef6b58a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift @@ -90,8 +90,10 @@ class HTTPClientSOCKSTests: XCTestCase { } func testProxySOCKSBogusAddress() throws { + var config = HTTPClient.Configuration(proxy: .socksServer(host: "127.0..")) + config.networkFrameworkWaitForConnectivity = false let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .socksServer(host: "127.0.."))) + configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -102,8 +104,11 @@ class HTTPClientSOCKSTests: XCTestCase { // there is no socks server, so we should fail func testProxySOCKSFailureNoServer() throws { let localHTTPBin = HTTPBin() + var config = HTTPClient.Configuration(proxy: .socksServer(host: "localhost", port: localHTTPBin.port)) + config.networkFrameworkWaitForConnectivity = false + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .socksServer(host: "localhost", port: localHTTPBin.port))) + configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -113,8 +118,11 @@ class HTTPClientSOCKSTests: XCTestCase { // speak to a server that doesn't speak SOCKS func testProxySOCKSFailureInvalidServer() throws { + var config = HTTPClient.Configuration(proxy: .socksServer(host: "localhost")) + config.networkFrameworkWaitForConnectivity = false + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .socksServer(host: "localhost"))) + configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -124,8 +132,11 @@ class HTTPClientSOCKSTests: XCTestCase { // test a handshake failure with a misbehaving server func testProxySOCKSMisbehavingServer() throws { let socksBin = try MockSOCKSServer(expectedURL: "/socks/test", expectedResponse: "it works!", misbehave: true) + var config = HTTPClient.Configuration(proxy: .socksServer(host: "localhost", port: socksBin.port)) + config.networkFrameworkWaitForConnectivity = false + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .socksServer(host: "localhost", port: socksBin.port))) + configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index eb8d523bb..492bb4c35 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -429,7 +429,9 @@ class HTTPClientInternalTests: XCTestCase { let el2 = elg.next() let httpBin = HTTPBin(.refuse) - let client = HTTPClient(eventLoopGroupProvider: .shared(elg)) + var config = HTTPClient.Configuration() + config.networkFrameworkWaitForConnectivity = false + let client = HTTPClient(eventLoopGroupProvider: .shared(elg), configuration: config) defer { XCTAssertNoThrow(try client.syncShutdown()) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift index cc33f6aee..77f4298ba 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift @@ -27,6 +27,7 @@ extension HTTPClientNIOTSTests { return [ ("testCorrectEventLoopGroup", testCorrectEventLoopGroup), ("testTLSFailError", testTLSFailError), + ("testConnectionFailsFastError", testConnectionFailsFastError), ("testConnectionFailError", testConnectionFailError), ("testTLSVersionError", testTLSVersionError), ("testTrustRootCertificateLoadFail", testTrustRootCertificateLoadFail), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 172ee89ba..cc114cb9a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -16,6 +16,7 @@ #if canImport(Network) import Network #endif +import NIOConcurrencyHelpers import NIOCore import NIOPosix import NIOSSL @@ -54,7 +55,10 @@ class HTTPClientNIOTSTests: XCTestCase { guard isTestingNIOTS() else { return } let httpBin = HTTPBin(.http1_1(ssl: true)) - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) + var config = HTTPClient.Configuration() + config.networkFrameworkWaitForConnectivity = false + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: config) defer { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) XCTAssertNoThrow(try httpBin.shutdown()) @@ -75,9 +79,32 @@ class HTTPClientNIOTSTests: XCTestCase { #endif } + func testConnectionFailsFastError() { + guard isTestingNIOTS() else { return } + #if canImport(Network) + let httpBin = HTTPBin(.http1_1(ssl: false)) + var config = HTTPClient.Configuration() + config.networkFrameworkWaitForConnectivity = false + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: config) + + defer { + XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) + } + + let port = httpBin.port + XCTAssertNoThrow(try httpBin.shutdown()) + + XCTAssertThrowsError(try httpClient.get(url: "http://localhost:\(port)/get").wait()) { + XCTAssertTrue($0 is NWError) + } + #endif + } + func testConnectionFailError() { guard isTestingNIOTS() else { return } - let httpBin = HTTPBin(.http1_1(ssl: true)) + let httpBin = HTTPBin(.http1_1(ssl: false)) let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(100)))) @@ -89,7 +116,7 @@ class HTTPClientNIOTSTests: XCTestCase { let port = httpBin.port XCTAssertNoThrow(try httpBin.shutdown()) - XCTAssertThrowsError(try httpClient.get(url: "https://localhost:\(port)/get").wait()) { + XCTAssertThrowsError(try httpClient.get(url: "http://localhost:\(port)/get").wait()) { XCTAssertEqual($0 as? HTTPClientError, .connectTimeout) } } @@ -102,9 +129,12 @@ class HTTPClientNIOTSTests: XCTestCase { tlsConfig.certificateVerification = .none tlsConfig.minimumTLSVersion = .tlsv11 tlsConfig.maximumTLSVersion = .tlsv1 + + var clientConfig = HTTPClient.Configuration(tlsConfiguration: tlsConfig) + clientConfig.networkFrameworkWaitForConnectivity = false let httpClient = HTTPClient( eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(tlsConfiguration: tlsConfig) + configuration: clientConfig ) defer { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index a6eff950a..de110fdb5 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1157,12 +1157,21 @@ class HTTPClientTests: XCTestCase { } func testStressGetHttpsSSLError() throws { + var config = HTTPClient.Configuration() + config.networkFrameworkWaitForConnectivity = false + + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: config) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + } + let request = try Request(url: "https://localhost:\(self.defaultHTTPBin.port)/wait", method: .GET) let tasks = (1...100).map { _ -> HTTPClient.Task in - self.defaultClient.execute(request: request, delegate: TestHTTPDelegate()) + localClient.execute(request: request, delegate: TestHTTPDelegate()) } - let results = try EventLoopFuture.whenAllComplete(tasks.map { $0.futureResult }, on: self.defaultClient.eventLoopGroup.next()).wait() + let results = try EventLoopFuture.whenAllComplete(tasks.map { $0.futureResult }, on: localClient.eventLoopGroup.next()).wait() for result in results { switch result { @@ -1786,7 +1795,12 @@ class HTTPClientTests: XCTestCase { XCTAssertThrowsError(try localClient.get(url: "http://localhost:\(port)").wait()) { error in if isTestingNIOTS() { - XCTAssertEqual(error as? HTTPClientError, .connectTimeout) + #if canImport(Network) + // We can't be more specific than this. + XCTAssertTrue(error is HTTPClient.NWTLSError || error is HTTPClient.NWPOSIXError) + #else + XCTFail("Impossible condition") + #endif } else { XCTAssert(error is NIOConnectionError, "Unexpected error: \(error)") } @@ -2749,6 +2763,8 @@ class HTTPClientTests: XCTestCase { if isTestingNIOTS() { // If we are using Network.framework, we set the connect timeout down very low here // because on NIOTS a failing TLS handshake manifests as a connect timeout. + // Note that we do this here to prove that we correctly manifest the underlying error: + // DO NOT CHANGE THIS TO DISABLE WAITING FOR CONNECTIVITY. timeout.connect = .milliseconds(100) } @@ -2763,7 +2779,12 @@ class HTTPClientTests: XCTestCase { XCTAssertThrowsError(try task.wait()) { error in if isTestingNIOTS() { - XCTAssertEqual(error as? HTTPClientError, .connectTimeout) + #if canImport(Network) + // We can't be more specific than this. + XCTAssertTrue(error is HTTPClient.NWTLSError) + #else + XCTFail("Impossible condition") + #endif } else { switch error as? NIOSSLError { case .some(.handshakeFailed(.sslError(_))): break @@ -2815,7 +2836,12 @@ class HTTPClientTests: XCTestCase { XCTAssertThrowsError(try task.wait()) { error in if isTestingNIOTS() { - XCTAssertEqual(error as? HTTPClientError, .connectTimeout) + #if canImport(Network) + // We can't be more specific than this. + XCTAssertTrue(error is HTTPClient.NWTLSError) + #else + XCTFail("Impossible condition") + #endif } else { switch error as? NIOSSLError { case .some(.handshakeFailed(.sslError(_))): break diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift index b13ff3d18..3cfb25e03 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift @@ -46,6 +46,7 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { ) XCTAssertThrowsError(try factory.makeChannel( + requester: ExplodingRequester(), connectionID: 1, deadline: .now() - .seconds(1), eventLoop: group.next(), @@ -81,6 +82,7 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { ) XCTAssertThrowsError(try factory.makeChannel( + requester: ExplodingRequester(), connectionID: 1, deadline: .now() + .seconds(1), eventLoop: group.next(), @@ -116,6 +118,7 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { ) XCTAssertThrowsError(try factory.makeChannel( + requester: ExplodingRequester(), connectionID: 1, deadline: .now() + .seconds(1), eventLoop: group.next(), @@ -153,6 +156,7 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { ) XCTAssertThrowsError(try factory.makeChannel( + requester: ExplodingRequester(), connectionID: 1, deadline: .now() + .seconds(1), eventLoop: group.next(), @@ -171,3 +175,22 @@ class NeverrespondServerHandler: ChannelInboundHandler { // do nothing } } + +/// A `HTTPConnectionRequester` that will fail a test if any of its methods are ever called. +final class ExplodingRequester: HTTPConnectionRequester { + func http1ConnectionCreated(_: HTTP1Connection) { + XCTFail("http1ConnectionCreated called unexpectedly") + } + + func http2ConnectionCreated(_: HTTP2Connection, maximumStreams: Int) { + XCTFail("http2ConnectionCreated called unexpectedly") + } + + func failedToCreateHTTPConnection(_: HTTPConnectionPool.Connection.ID, error: Error) { + XCTFail("failedToCreateHTTPConnection called unexpectedly") + } + + func waitingForConnectivity(_: HTTPConnectionPool.Connection.ID, error: Error) { + XCTFail("waitingForConnectivity called unexpectedly") + } +} From 2483e08ffbc57439b4ea82118e3f28824c41e2bf Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 7 Jun 2022 12:31:57 +0100 Subject: [PATCH 012/146] Fix crash when receiving 2xx response before stream is complete. (#591) Motivation It's totally acceptable for a HTTP server to respond before a request upload has completed. If the response is an error, we should abort the upload (and we do), but if the response is a 2xx we should probably just finish the upload. In this case it turns out we'll actually hit a crash when we attempt to deliver an empty body message. his is no good! Once that bug was fixed it revealed another: while we'd attempted to account for this case, we hadn't tested it, and so it turns out that shutdown would hang. As a result, I've also cleaned that up. Modifications - Tolerate empty circular buffers of bytes when streaming an upload. - Notify the connection that the task is complete when we're done. Result Fewer crashes and hangs. --- .../HTTP1/HTTP1ClientChannelHandler.swift | 4 ++ .../RequestBag+StateMachine.swift | 12 +++-- .../HTTPClientTestUtils.swift | 31 +++++++++++ .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 54 +++++++++++++++++++ 5 files changed, 98 insertions(+), 4 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 2a3bc9c27..97f850c33 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -267,6 +267,10 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { writePromise.futureResult.whenComplete { result in switch result { case .success: + // If our final action was `sendRequestEnd`, that means we've already received + // the complete response. As a result, once we've uploaded all the body parts + // we need to tell the pool that the connection is idle. + self.connection.taskCompleted() oldRequest.succeedRequest(buffer) case .failure(let error): oldRequest.fail(error) diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index 557af2af1..63cb15758 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -347,10 +347,14 @@ extension RequestBag.StateMachine { self.state = .executing(executor, requestState, .buffering(currentBuffer, next: next)) return .none case .executing(let executor, let requestState, .waitingForRemote): - var buffer = buffer - let first = buffer.removeFirst() - self.state = .executing(executor, requestState, .buffering(buffer, next: .askExecutorForMore)) - return .forwardResponsePart(first) + if buffer.count > 0 { + var buffer = buffer + let first = buffer.removeFirst() + self.state = .executing(executor, requestState, .buffering(buffer, next: .askExecutorForMore)) + return .forwardResponsePart(first) + } else { + return .none + } case .redirected(let executor, var receivedBytes, let head, let redirectURL): let partsLength = buffer.reduce(into: 0) { $0 += $1.readableBytes } receivedBytes += partsLength diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 230c91a2b..63a1cf540 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -1253,6 +1253,37 @@ class HTTPEchoHandler: ChannelInboundHandler { } } +final class HTTP200DelayedHandler: ChannelInboundHandler { + typealias InboundIn = HTTPServerRequestPart + typealias OutboundOut = HTTPServerResponsePart + + var pendingBodyParts: Int? + + init(bodyPartsBeforeResponse: Int) { + self.pendingBodyParts = bodyPartsBeforeResponse + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let request = self.unwrapInboundIn(data) + switch request { + case .head: + break + case .body: + if let pendingBodyParts = self.pendingBodyParts { + if pendingBodyParts > 0 { + self.pendingBodyParts = pendingBodyParts - 1 + } else { + self.pendingBodyParts = nil + context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok))), promise: nil) + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + } + } + case .end: + break + } + } +} + private let cert = """ -----BEGIN CERTIFICATE----- MIICmDCCAYACCQCPC8JDqMh1zzANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJ1 diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 7eb532cf9..d6fe77e47 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -129,6 +129,7 @@ extension HTTPClientTests { ("testSSLHandshakeErrorPropagationDelayedClose", testSSLHandshakeErrorPropagationDelayedClose), ("testWeCloseConnectionsWhenConnectionCloseSetByServer", testWeCloseConnectionsWhenConnectionCloseSetByServer), ("testBiDirectionalStreaming", testBiDirectionalStreaming), + ("testBiDirectionalStreamingEarly200", testBiDirectionalStreamingEarly200), ("testSynchronousHandshakeErrorReporting", testSynchronousHandshakeErrorReporting), ("testFileDownloadChunked", testFileDownloadChunked), ("testCloseWhileBackpressureIsExertedIsFine", testCloseWhileBackpressureIsExertedIsFine), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index de110fdb5..0e5ccf63f 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -2940,6 +2940,60 @@ class HTTPClientTests: XCTestCase { XCTAssertNil(try delegate.next().wait()) } + // In this test, we test that a request can continue to stream its body after the response head and end + // was received where the end is a 200. + func testBiDirectionalStreamingEarly200() { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + let writeEL = eventLoopGroup.next() + let delegateEL = eventLoopGroup.next() + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) + defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } + + let delegate = ResponseStreamDelegate(eventLoop: delegateEL) + + let body: HTTPClient.Body = .stream { writer in + let finalPromise = writeEL.makePromise(of: Void.self) + + func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + // always invoke from the wrong el to test thread safety + writeEL.preconditionInEventLoop() + + if index >= 30 { + return finalPromise.succeed(()) + } + + let sent = ByteBuffer(integer: index) + writer.write(.byteBuffer(sent)).whenComplete { result in + switch result { + case .success: + writeEL.execute { + writeLoop(writer, index: index + 1) + } + + case .failure(let error): + finalPromise.fail(error) + } + } + } + + writeEL.execute { + writeLoop(writer, index: 0) + } + + return finalPromise.futureResult + } + + let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) + let future = httpClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: delegateEL)) + XCTAssertNoThrow(try future.wait()) + XCTAssertNil(try delegate.next().wait()) + } + func testSynchronousHandshakeErrorReporting() throws { // This only affects cases where we use NIOSSL. guard !isTestingNIOTS() else { return } From 0f21b44d1ad5227ccbaa073aa40cd37eb8bbc337 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 9 Jun 2022 19:35:05 +0200 Subject: [PATCH 013/146] =?UTF-8?q?Use=20a=20local=20TCP=20server=20that?= =?UTF-8?q?=20doesn=E2=80=99t=20accept=20connections=20on=20macOS=20for=20?= =?UTF-8?q?`testConnectTimeout()`=20(#592)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use a local TCP server that doesnโ€™t accept connections on macOS for `testConnectTimeout()` * fix linting --- .../HTTPClientTests.swift | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 0e5ccf63f..726809415 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -640,7 +640,35 @@ class HTTPClientTests: XCTestCase { } } - func testConnectTimeout() { + func testConnectTimeout() throws { + #if os(Linux) + // 198.51.100.254 is reserved for documentation only and therefore should not accept any TCP connection + let url = "http://198.51.100.254/get" + #else + // on macOS we can use the TCP backlog behaviour when the queue is full to simulate a non reachable server. + // this makes this test a bit more stable if `198.51.100.254` actually responds to connection attempt. + // The backlog behaviour on Linux can not be used to simulate a non-reachable server. + // Linux sends a `SYN/ACK` back even if the `backlog` queue is full as it has two queues. + // The second queue is not limit by `ChannelOptions.backlog` but by `/proc/sys/net/ipv4/tcp_max_syn_backlog`. + + let serverChannel = try ServerBootstrap(group: self.serverGroup) + .serverChannelOption(ChannelOptions.backlog, value: 1) + .serverChannelOption(ChannelOptions.autoRead, value: false) + .bind(host: "127.0.0.1", port: 0) + .wait() + defer { + XCTAssertNoThrow(try serverChannel.close().wait()) + } + let port = serverChannel.localAddress!.port! + let firstClientChannel = try ClientBootstrap(group: self.serverGroup) + .connect(host: "127.0.0.1", port: port) + .wait() + defer { + XCTAssertNoThrow(try firstClientChannel.close().wait()) + } + let url = "http://localhost:\(port)/get" + #endif + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150)))) @@ -648,9 +676,8 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try httpClient.syncShutdown()) } - // This must throw as 198.51.100.254 is reserved for documentation only - XCTAssertThrowsError(try httpClient.get(url: "http://198.51.100.254/get").wait()) { - XCTAssertEqual($0 as? HTTPClientError, .connectTimeout) + XCTAssertThrowsError(try httpClient.get(url: url).wait()) { + XCTAssertEqualTypeAndValue($0, HTTPClientError.connectTimeout) } } From 062989efd890c43391c822f6666f050279f28bd7 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Fri, 17 Jun 2022 11:52:29 +0100 Subject: [PATCH 014/146] Correctly reset our state after .sendEnd (#597) Motivation In some cases, the last thing that happens in a request-response pair is that we send our HTTP/1.1 .end. This can happen when the peer has sent an early response, before we have finished uploading our body. When it does, we need to be diligent about cleaning up our connection state. Unfortunately, there was an edge in the HTTP1ConnectionStateMachine that processed .succeedRequest but that did not transition the state into either .idle or .closing. That was an error, and needed to be fixed. Modifications Transition to .idle when we're returning .succeedRequest(.sendRequestEnd). Result Fewer crashes --- .../HTTP1/HTTP1ConnectionStateMachine.swift | 1 + .../HTTPClientTestUtils.swift | 6 ++- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 54 +++++++++++++++++++ 4 files changed, 61 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index ecff7afc7..f0aff762c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -412,6 +412,7 @@ extension HTTP1ConnectionStateMachine.State { self = .closing newFinalAction = .close case .sendRequestEnd(let writePromise): + self = .idle newFinalAction = .sendRequestEnd(writePromise) case .none: self = .idle diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 63a1cf540..f2cc7b1d8 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -1267,7 +1267,11 @@ final class HTTP200DelayedHandler: ChannelInboundHandler { let request = self.unwrapInboundIn(data) switch request { case .head: - break + // Once we have received one response, all further requests are responded to immediately. + if self.pendingBodyParts == nil { + context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok))), promise: nil) + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + } case .body: if let pendingBodyParts = self.pendingBodyParts { if pendingBodyParts > 0 { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index d6fe77e47..a709cf2d6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -130,6 +130,7 @@ extension HTTPClientTests { ("testWeCloseConnectionsWhenConnectionCloseSetByServer", testWeCloseConnectionsWhenConnectionCloseSetByServer), ("testBiDirectionalStreaming", testBiDirectionalStreaming), ("testBiDirectionalStreamingEarly200", testBiDirectionalStreamingEarly200), + ("testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests", testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests), ("testSynchronousHandshakeErrorReporting", testSynchronousHandshakeErrorReporting), ("testFileDownloadChunked", testFileDownloadChunked), ("testCloseWhileBackpressureIsExertedIsFine", testCloseWhileBackpressureIsExertedIsFine), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 726809415..e5d935fb9 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3021,6 +3021,60 @@ class HTTPClientTests: XCTestCase { XCTAssertNil(try delegate.next().wait()) } + // This test is identical to the one above, except that we send another request immediately after. This is a regression + // test for https://github.com/swift-server/async-http-client/issues/595. + func testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests() { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + let writeEL = eventLoopGroup.next() + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) + defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } + + let body: HTTPClient.Body = .stream { writer in + let finalPromise = writeEL.makePromise(of: Void.self) + + func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + // always invoke from the wrong el to test thread safety + writeEL.preconditionInEventLoop() + + if index >= 30 { + return finalPromise.succeed(()) + } + + let sent = ByteBuffer(integer: index) + writer.write(.byteBuffer(sent)).whenComplete { result in + switch result { + case .success: + writeEL.execute { + writeLoop(writer, index: index + 1) + } + + case .failure(let error): + finalPromise.fail(error) + } + } + } + + writeEL.execute { + writeLoop(writer, index: 0) + } + + return finalPromise.futureResult + } + + let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) + let future = httpClient.execute(request: request) + XCTAssertNoThrow(try future.wait()) + + // Try another request + let future2 = httpClient.execute(request: request) + XCTAssertNoThrow(try future2.wait()) + } + func testSynchronousHandshakeErrorReporting() throws { // This only affects cases where we use NIOSSL. guard !isTestingNIOTS() else { return } From ac34f6debc929ace3548b9f7c510ac2e9add3ad8 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Fri, 17 Jun 2022 12:27:03 +0100 Subject: [PATCH 015/146] Correctly handle Connection: close with streaming (#598) Motivation When users stream their bodies they may still want to send Connection: close headers and terminate the connection early. This should work properly. Unfortunately it became clear that we didn't correctly pass the information that the connection needed to be closed. As a result, we'd inappropriately re-use the connection, potentially causing unnecessary HTTP errors. Modifications Signal whether the connection needs to be closed when the final connection action is to send .end. Results We behave better with streaming uploads. --- .../HTTP1/HTTP1ClientChannelHandler.swift | 14 +++-- .../HTTP1/HTTP1ConnectionStateMachine.swift | 7 ++- .../HTTP1ConnectionStateMachineTests.swift | 4 +- .../HTTPClientTestUtils.swift | 26 +++++++++ .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 54 +++++++++++++++++++ 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 97f850c33..affe4770c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -261,16 +261,22 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { case .close: context.close(promise: nil) oldRequest.succeedRequest(buffer) - case .sendRequestEnd(let writePromise): + case .sendRequestEnd(let writePromise, let shouldClose): let writePromise = writePromise ?? context.eventLoop.makePromise(of: Void.self) // We need to defer succeeding the old request to avoid ordering issues - writePromise.futureResult.whenComplete { result in + writePromise.futureResult.hop(to: context.eventLoop).whenComplete { result in switch result { case .success: // If our final action was `sendRequestEnd`, that means we've already received // the complete response. As a result, once we've uploaded all the body parts - // we need to tell the pool that the connection is idle. - self.connection.taskCompleted() + // we need to tell the pool that the connection is idle or, if we were asked to + // close when we're done, send the close. Either way, we then succeed the request + if shouldClose { + context.close(promise: nil) + } else { + self.connection.taskCompleted() + } + oldRequest.succeedRequest(buffer) case .failure(let error): oldRequest.fail(error) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index f0aff762c..e7258611c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -35,7 +35,10 @@ struct HTTP1ConnectionStateMachine { /// as soon as we wrote the request end onto the wire. /// /// The promise is an optional write promise. - case sendRequestEnd(EventLoopPromise?) + /// + /// `shouldClose` records whether we have attached a Connection: close header to this request, and so the connection should + /// be terminated + case sendRequestEnd(EventLoopPromise?, shouldClose: Bool) /// Inform an observer that the connection has become idle case informConnectionIsIdle } @@ -413,7 +416,7 @@ extension HTTP1ConnectionStateMachine.State { newFinalAction = .close case .sendRequestEnd(let writePromise): self = .idle - newFinalAction = .sendRequestEnd(writePromise) + newFinalAction = .sendRequestEnd(writePromise, shouldClose: close) case .none: self = .idle newFinalAction = close ? .close : .informConnectionIsIdle diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index 55014f8c6..fd771aca0 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -338,8 +338,8 @@ extension HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction: Equata switch (lhs, rhs) { case (.close, .close): return true - case (sendRequestEnd(let lhsPromise), sendRequestEnd(let rhsPromise)): - return lhsPromise?.futureResult == rhsPromise?.futureResult + case (sendRequestEnd(let lhsPromise, let lhsShouldClose), sendRequestEnd(let rhsPromise, let rhsShouldClose)): + return lhsPromise?.futureResult == rhsPromise?.futureResult && lhsShouldClose == rhsShouldClose case (informConnectionIsIdle, informConnectionIsIdle): return true default: diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index f2cc7b1d8..c99facc3f 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -1017,6 +1017,32 @@ internal final class CloseWithoutClosingServerHandler: ChannelInboundHandler { } } +final class ExpectClosureServerHandler: ChannelInboundHandler { + typealias InboundIn = HTTPServerRequestPart + typealias OutboundOut = HTTPServerResponsePart + + private let onClosePromise: EventLoopPromise + + init(onClosePromise: EventLoopPromise) { + self.onClosePromise = onClosePromise + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + switch self.unwrapInboundIn(data) { + case .head: + let head = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "0"]) + context.write(self.wrapOutboundOut(.head(head)), promise: nil) + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + case .body, .end: + () + } + } + + func channelInactive(context: ChannelHandlerContext) { + self.onClosePromise.succeed(()) + } +} + struct EventLoopFutureTimeoutError: Error {} extension EventLoopFuture { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index a709cf2d6..603c1aa9c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -131,6 +131,7 @@ extension HTTPClientTests { ("testBiDirectionalStreaming", testBiDirectionalStreaming), ("testBiDirectionalStreamingEarly200", testBiDirectionalStreamingEarly200), ("testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests", testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests), + ("testCloseConnectionAfterEarly2XXWhenStreaming", testCloseConnectionAfterEarly2XXWhenStreaming), ("testSynchronousHandshakeErrorReporting", testSynchronousHandshakeErrorReporting), ("testFileDownloadChunked", testFileDownloadChunked), ("testCloseWhileBackpressureIsExertedIsFine", testCloseWhileBackpressureIsExertedIsFine), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index e5d935fb9..8f2c7c1aa 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3075,6 +3075,60 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try future2.wait()) } + // This test validates that we correctly close the connection after our body completes when we've streamed a + // body and received the 2XX response _before_ we finished our stream. + func testCloseConnectionAfterEarly2XXWhenStreaming() { + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + + let onClosePromise = eventLoopGroup.next().makePromise(of: Void.self) + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in ExpectClosureServerHandler(onClosePromise: onClosePromise) } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let writeEL = eventLoopGroup.next() + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) + defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } + + let body: HTTPClient.Body = .stream { writer in + let finalPromise = writeEL.makePromise(of: Void.self) + + func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + // always invoke from the wrong el to test thread safety + writeEL.preconditionInEventLoop() + + if index >= 30 { + return finalPromise.succeed(()) + } + + let sent = ByteBuffer(integer: index) + writer.write(.byteBuffer(sent)).whenComplete { result in + switch result { + case .success: + writeEL.execute { + writeLoop(writer, index: index + 1) + } + + case .failure(let error): + finalPromise.fail(error) + } + } + } + + writeEL.execute { + writeLoop(writer, index: 0) + } + + return finalPromise.futureResult + } + + let headers = HTTPHeaders([("Connection", "close")]) + let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", headers: headers, body: body) + let future = httpClient.execute(request: request) + XCTAssertNoThrow(try future.wait()) + XCTAssertNoThrow(try onClosePromise.futureResult.wait()) + } + func testSynchronousHandshakeErrorReporting() throws { // This only affects cases where we use NIOSSL. guard !isTestingNIOTS() else { return } From 9a8553e8aa00b6a98f60bbacef33221f56673509 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Fri, 17 Jun 2022 17:09:19 +0100 Subject: [PATCH 016/146] Correctly close the connection if sendEnd fails (#599) Motivation If we receive an early HTTP response, the last action on a HTTP/1.1 connection is to send the .end message. While we had an error handling path in the code, it wasn't tested, and when executed it would end up leaking the connection by failing to close it _or_ return it to the pool. This patch fixes the issue by appropriately terminating the connection and adding a test. Modifications Add a test Terminate the connection if sendEnd fails Result Fewer connection leaks --- .../HTTP1/HTTP1ClientChannelHandler.swift | 1 + ...TTP1ClientChannelHandlerTests+XCTest.swift | 1 + .../HTTP1ClientChannelHandlerTests.swift | 85 +++++++++++++++++++ 3 files changed, 87 insertions(+) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index affe4770c..ac92e4bc8 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -279,6 +279,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { oldRequest.succeedRequest(buffer) case .failure(let error): + context.close(promise: nil) oldRequest.fail(error) } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift index 86707520c..66c1a48d1 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift @@ -32,6 +32,7 @@ extension HTTP1ClientChannelHandlerTests { ("testIdleReadTimeoutIsCanceledIfRequestIsCanceled", testIdleReadTimeoutIsCanceledIfRequestIsCanceled), ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand), ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), + ("testHandlerClosesChannelIfLastActionIsSendEndAndItFails", testHandlerClosesChannelIfLastActionIsSendEndAndItFails), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index 4769d2c7e..f97580372 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -457,6 +457,75 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { XCTAssertEqual(embedded.isActive, false) } } + + func testHandlerClosesChannelIfLastActionIsSendEndAndItFails() { + let embedded = EmbeddedChannel() + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + testWriter.start(writer: writer) + })) + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + XCTAssertNoThrow(try embedded.pipeline.addHandler(FailEndHandler(), position: .first).wait()) + + // Execute the request and we'll receive the head. + testWriter.writabilityChanged(true) + testUtils.connection.executeRequest(requestBag) + XCTAssertNoThrow(try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .POST) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + XCTAssertEqual($0.headers.first(name: "content-length"), "10") + }) + // We're going to immediately send the response head and end. + let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) + XCTAssertNoThrow(try embedded.writeInbound(HTTPClientResponsePart.head(responseHead))) + embedded.read() + + // Send the end and confirm the connection is still live. + XCTAssertNoThrow(try embedded.writeInbound(HTTPClientResponsePart.end(nil))) + XCTAssertEqual(testUtils.connectionDelegate.hitConnectionClosed, 0) + XCTAssertEqual(testUtils.connectionDelegate.hitConnectionReleased, 0) + + // Ok, now we can process some reads. We expect 5 reads, but we do _not_ expect an .end, because + // the `FailEndHandler` is going to fail it. + embedded.embeddedEventLoop.run() + XCTAssertEqual(testWriter.written, 5) + for _ in 0..<5 { + XCTAssertNoThrow(try embedded.receiveBodyAndVerify { + XCTAssertEqual($0.readableBytes, 2) + }) + } + + embedded.embeddedEventLoop.run() + XCTAssertNil(try embedded.readOutbound(as: HTTPClientRequestPart.self)) + + // We should have seen the connection close, and the request is complete. + XCTAssertEqual(testUtils.connectionDelegate.hitConnectionClosed, 1) + XCTAssertEqual(testUtils.connectionDelegate.hitConnectionReleased, 0) + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { error in + XCTAssertTrue(error is FailEndHandler.Error) + } + } } class TestBackpressureWriter { @@ -636,3 +705,19 @@ class ReadEventHitHandler: ChannelOutboundHandler { context.read() } } + +final class FailEndHandler: ChannelOutboundHandler { + typealias OutboundIn = HTTPClientRequestPart + typealias OutboundOut = HTTPClientRequestPart + + struct Error: Swift.Error {} + + func write(context: ChannelHandlerContext, data: NIOAny, promise: EventLoopPromise?) { + if case .end = self.unwrapOutboundIn(data) { + // We fail this. + promise?.fail(Self.Error()) + } else { + context.write(data, promise: promise) + } + } +} From 794dc9d42720af97cedd395e8cd2add9173ffd9a Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 17 Jun 2022 20:32:33 +0200 Subject: [PATCH 017/146] Add `testSelfSignedCertificateIsRejectedWithCorrectError` (#594) --- Package.swift | 4 ++ .../NIOTransportServices/NWErrorHandler.swift | 6 +-- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 40 ++++++++++++++ ...onnectionPool+HTTP2StateMachineTests.swift | 2 +- .../Resources/self_signed_cert.pem | 27 ++++++++++ .../Resources/self_signed_key.pem | 52 +++++++++++++++++++ 7 files changed, 128 insertions(+), 4 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/Resources/self_signed_cert.pem create mode 100644 Tests/AsyncHTTPClientTests/Resources/self_signed_key.pem diff --git a/Package.swift b/Package.swift index 5deb0de31..20484832d 100644 --- a/Package.swift +++ b/Package.swift @@ -61,6 +61,10 @@ let package = Package( .product(name: "NIOHTTP2", package: "swift-nio-http2"), .product(name: "NIOSOCKS", package: "swift-nio-extras"), .product(name: "Logging", package: "swift-log"), + ], + resources: [ + .copy("Resources/self_signed_cert.pem"), + .copy("Resources/self_signed_key.pem"), ] ), ] diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift index 4334bb9f9..f732c37f4 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift @@ -60,7 +60,7 @@ extension HTTPClient { } #endif - class NWErrorHandler: ChannelInboundHandler { + final class NWErrorHandler: ChannelInboundHandler { typealias InboundIn = HTTPClientResponsePart func errorCaught(context: ChannelHandlerContext, error: Error) { @@ -73,9 +73,9 @@ extension HTTPClient { if let error = error as? NWError { switch error { case .tls(let status): - return NWTLSError(status, reason: error.localizedDescription) + return NWTLSError(status, reason: String(describing: error)) case .posix(let errorCode): - return NWPOSIXError(errorCode, reason: error.localizedDescription) + return NWPOSIXError(errorCode, reason: String(describing: error)) default: return error } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 603c1aa9c..3975036ea 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -78,6 +78,7 @@ extension HTTPClientTests { ("testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose", testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose), ("testStressGetHttps", testStressGetHttps), ("testStressGetHttpsSSLError", testStressGetHttpsSSLError), + ("testSelfSignedCertificateIsRejectedWithCorrectError", testSelfSignedCertificateIsRejectedWithCorrectError), ("testFailingConnectionIsReleased", testFailingConnectionIsReleased), ("testResponseDelayGet", testResponseDelayGet), ("testIdleTimeoutNoReuse", testIdleTimeoutNoReuse), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8f2c7c1aa..ece09c52d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1229,6 +1229,46 @@ class HTTPClientTests: XCTestCase { } } + func testSelfSignedCertificateIsRejectedWithCorrectError() throws { + /// key + cert was created with the follwing command: + /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' + let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! + let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let configuration = TLSConfiguration.makeServerConfiguration( + certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, + privateKey: .file(keyPath) + ) + let sslContext = try NIOSSLContext(configuration: configuration) + + let server = ServerBootstrap(group: serverGroup) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + } + let serverChannel = try server.bind(host: "localhost", port: 0).wait() + defer { XCTAssertNoThrow(try serverChannel.close().wait()) } + let port = serverChannel.localAddress!.port! + + var config = HTTPClient.Configuration() + config.timeout.connect = .seconds(2) + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) + defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(port)").wait()) { error in + #if canImport(Network) + guard let nwTLSError = error as? HTTPClient.NWTLSError else { + XCTFail("could not cast \(error) of type \(type(of: error)) to \(HTTPClient.NWTLSError.self)") + return + } + XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") + #else + guard let sslError = error as? NIOSSLError, + case .handshakeFailed(.sslError) = sslError else { + XCTFail("unexpected error \(error)") + return + } + #endif + } + } + func testFailingConnectionIsReleased() { let localHTTPBin = HTTPBin(.refuse) let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index 825ffc9b3..2574d3da2 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -1242,7 +1242,7 @@ func XCTAssertEqualTypeAndValue( let lhs = try lhs() let rhs = try rhs() guard let lhsAsRhs = lhs as? Right else { - XCTFail("could not cast \(lhs) of type \(Right.self) to \(Left.self)") + XCTFail("could not cast \(lhs) of type \(type(of: lhs)) to \(type(of: rhs))") return } XCTAssertEqual(lhsAsRhs, rhs) diff --git a/Tests/AsyncHTTPClientTests/Resources/self_signed_cert.pem b/Tests/AsyncHTTPClientTests/Resources/self_signed_cert.pem new file mode 100644 index 000000000..20b46f355 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/self_signed_cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEpjCCAo4CCQCeTmfiTQcJrzANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDDAls +b2NhbGhvc3QwIBcNMjIwNjE0MTI1NDQ4WhgPMjI5NjAzMjgxMjU0NDhaMBQxEjAQ +BgNVBAMMCWxvY2FsaG9zdDCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIB +AK16gPDwP/Xbaf36x5BNd6yHDxCPIIJP4JLfMEuozwLE0YRqwmZOuklb4jUbAXf7 +u9u24ANrC4XS6VVWkfPdugokAUkaKPpwkV4GOiMCXeSDjDiLt1dYxlbp+MLV78a5 +oUDbCAqfFKebIgv1oiK+L6/p818eAHSBWEXXMhTeBDEQAIpJLTG88iVu6r3fMJeH +FbMWuPmAajmx2AEGmwD1x6+NHZLJv1zaufa7j0sHADraagXnfKn6rkLn1is6QFu4 +v7xaNlEwsRCYbh0nrtCtEdJIqnEHc0GCu/gnw5GE3CuRG3FYBTZStIF7d9h+XZQB +ky/YEWSGw9DXFBbebOZugopvl91qaZLqo6Wg0J8qCodgFtJHOSVMq/SAOBmKyw+b +7FYZbj4tQKpuuhwCN+gwEveTy+BK+zGY/sVzPwR8PNjpCgT/HiOBM7dNt4+2r9pY +Ld/mcMvakgRzM4Iqqntem9ltuckZev0TRjdrIylVWsAlNYVXm4ncMLkbzxFkv5Gb +AlhAuTwxyFkIo0M7+GS4lXCZ2bX2umJ0DTl3/NGJserFdkOhvHZSHHC9BzDBysmc +SejX/cGOFQ8O3sFeJdVMGlO64dU482O0FbBcLHmTLXWR4t8dlhrzJuXZ4X6WtHqY +83RwyD1gacYRZnT0eL+Z7XGrO1/qypji1RNaFIaGUt7DAgMBAAEwDQYJKoZIhvcN +AQELBQADggIBAIigOuEVirgqXoUMStTwYObs/DcNIPEugn9gAq9Lt1cr6fm7CvhG +AupxoJTbKLHQX6FegvFSA+4Kt3KYXX9Qi9SJF3Vr4zOhV0q203d4Aui6Lamo5Yye +nhbzzXuDSIyxpaWPFRC2RqCA6+hV8/Ar9Bx0TCI4NQxWxQEPerwqzqWCuTbViccw +WzlwRD2AHibaQaCbpzXg9lOX0fRJHoSM3exYQd91pDoSoL3f/EV3I/czssq+10M8 +F4GhE4bQjaKD7jL5U59dlvfy73nLAzzxzsxsFuYTAgzZwDg586sdbrqqFjzjoZ9A +dF8NuVYkHyFDQkpe66e1isNZi7eFdSjeVmj8llp4b6in59ik7ZS7arzGOxhZZzmv +Jf3nfE4hJzMS/4GJsKMdtcI+6K+hMi6Yt9OoPh82SQ2q8gK4QSWWrwAKuQ4F4UeO +pgiWBryKrkOXlGARBbsR/ZDhlqyAskeGuhIpEY5NLCByFfQ5KlcrX+n4TVLRZMvb +/7PZqboGgU+CUVawm/suPAs8jOlFQOzrxWQPRfWVvFII62ABgozS8N/xZ/WbgTVj +kOtWj85NpaBSCUliIY/7z1FkjpMZO8Kds45WQzAq4YChDLZGbgV0MkyXqO/LEYFJ +zqGOP1yGxVcKxu6t8Xh0hL6JPFmKWiMEWVrd1wut6NAIu6WNftmWZX6J +-----END CERTIFICATE----- diff --git a/Tests/AsyncHTTPClientTests/Resources/self_signed_key.pem b/Tests/AsyncHTTPClientTests/Resources/self_signed_key.pem new file mode 100644 index 000000000..8811c2d81 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/self_signed_key.pem @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCteoDw8D/122n9 ++seQTXeshw8QjyCCT+CS3zBLqM8CxNGEasJmTrpJW+I1GwF3+7vbtuADawuF0ulV +VpHz3boKJAFJGij6cJFeBjojAl3kg4w4i7dXWMZW6fjC1e/GuaFA2wgKnxSnmyIL +9aIivi+v6fNfHgB0gVhF1zIU3gQxEACKSS0xvPIlbuq93zCXhxWzFrj5gGo5sdgB +BpsA9cevjR2Syb9c2rn2u49LBwA62moF53yp+q5C59YrOkBbuL+8WjZRMLEQmG4d +J67QrRHSSKpxB3NBgrv4J8ORhNwrkRtxWAU2UrSBe3fYfl2UAZMv2BFkhsPQ1xQW +3mzmboKKb5fdammS6qOloNCfKgqHYBbSRzklTKv0gDgZissPm+xWGW4+LUCqbroc +AjfoMBL3k8vgSvsxmP7Fcz8EfDzY6QoE/x4jgTO3TbePtq/aWC3f5nDL2pIEczOC +Kqp7XpvZbbnJGXr9E0Y3ayMpVVrAJTWFV5uJ3DC5G88RZL+RmwJYQLk8MchZCKND +O/hkuJVwmdm19rpidA05d/zRibHqxXZDobx2UhxwvQcwwcrJnEno1/3BjhUPDt7B +XiXVTBpTuuHVOPNjtBWwXCx5ky11keLfHZYa8ybl2eF+lrR6mPN0cMg9YGnGEWZ0 +9Hi/me1xqztf6sqY4tUTWhSGhlLewwIDAQABAoICAApRcP3jrEo5RLKgieIhWW7f +kZvQh4R4r8jMkZjOb5Gglz2jA/EF2bqnRmsWMh4q0N+envBVG5hYFRzIS2IP3BLi +VVk9vxY2P88x259dcqw2zs5GMR923kUpIWylQN+3BspOvMm08IuPhJTlhUE/wqJZ +7enIZQqI7vEofYgUNHeelgmjlJaSwGxNjpTAg6lflYDTZykf5DGOTGSzOeDyvW/J +muqyKTmioND2Eu3JetAFUa0MObP6fwbntytXCaDq+ix/yR9HICD2kAYX6CPtR1QU +kl6qrMZGultmMhGjr1zAArvZGmZCwQ26hERSL8qv1UtRNKegBGGViVJa5GtIQ2dT +UmTWmWu/5gyxKvvjuqYl8Dub2/ZT0iGAsA6hGyUr+vpgjcNEZqsYhiEiQPi0g1sM +XyszytqG1F7JzXYgVzcdFA9L+eLD+i4nKD18TYTYHFGRmxwQ+HzHnetgDQ2gqRbB +XwT4lp643oNLMGyL+T0cQ7i1Hpq7Ko0S2FeXzzFe9B33uXbDvc0usier5qx2tgxc +zfgSqJjahfo4LCxhxvBWOup3U/sXNgyMCctr1qjpwGwLek+H1keOyv7FO9O6OgI1 +v5ZPFsJV7mK1fDLM/8QLDpUcUNnhPUfzsBdxKrjLfnZ8MPNczgv1GPzb4jsLvewf +g6ps8oBwnZDQVa6dMuyRAoIBAQDnTKRUsTMmFo01o0k90C8SwwE2x7Wry8r6vIIf +PMni3ZAS+zWFnu1zg82+83QpdvskntWM2iXS7nimmkXClCCFMDU/hYA9EsZtGIv6 ++xA6gYF0Xd3Qf9QrvhixOxHj3ixNyCeee3/9XUYln3ZfEx8cgCwHjPSIm3rOKI2M +PFnuG9xJ513sy6YCDrCdtb661E6bmsaMcIhu6S7At0njwnoL9aB617TSds5tFEr8 +74EW3D9epN01uUQ9MgZSXbzdQ82IswLps4a/k4wfDFp4qKpx7zOsoTSjA9il3fgW +QLhBXxnzTYYTvwxIgaW//fyqEL3p6t9zuYcjbORcrj7v8xIvAoIBAQDAASGjsSCA +hn03DXrI/atoXEC0htVwPwp4HTI0Z1/rOS0IrFBcX3CWx90Dr/clePHQGPk1yOO7 +oM83zumwggIOymtDhlTcCa77yN9x9AZMW3qPMF+mvAouUzItnlMrOjvfEnIWziWC +UsylBiV4/I6tf0zpH8zFYPNXq98fpv+UXyJDTW+YGBc2b2BwZZA6RdtFalqvunM7 +M8FIH8vSYEMR0YC47L2ceBJY/U9EQpsc6vuS7+CoXOH/WRb5v1z+a5O9sHWp8Rdc +Oh67B6v2feUT9TwhGUVF0L+ktW389e3N+VzPvbvICvRsOvo6+bceCJTszhNno00s +87bPyelaHXutAoIBAFtJ6onqri9YMz96RMv6wLl88Zu3UsKNWn1/rTO7AEtj+xsi +vssQINO4r5mv6Kb86L5ZWhuPdeI8cK4AsYvMftFSZ5G8lRKFuH8Scx0Jviv5NSjC +a2uBKDJjgsdgcv0mkQHZ/5kTUT6kc60htMxtdZgAFmCch17rTprTcppor23E3Trl +8DInZkvllFuKgc6nQKc1fSustoxfyC4TqTwVY6oYtdAGFr4CWhK/MaGGvcJSB0jJ +dO1hQ8eLWOdlS8dgnVxYmsu2KXavO1x9ua9pkmwJZrG5pla4i+dbJjFSNebHLCzU +6hgdDTIIyWxvSCuvE+Wg57R7AxU+Qxs5Qmnd280CggEAex4+m+BwnvmeQTb7jPZc +e0bsltX+90L1S6AtGT1QXF0Fa5JS1Wi9oXH3Xu3u5LBxHqdk5gAzR5UOSxL69pvn +BeT2cw4oTBBJjFp6LW/0ufHO3RJ/w0LApIPkoSvs2MM2sQv67HSzyKWfZBJU5QfN +1aLTholFnStV3tnu8TT8nf+C0PVOoZCREe7JQElf+n3g5NoV3KkKSuQdBEqfP/9K +Apr8l5f23eaAnV+Q/IxZOmnTd50pycwFft95xBvZXatNyUzlpltaR2FdY0DAHAcO +ZYXTUMYLjYEV4mAUbyijnHhR80QOrW+Y2+3VlwuZSEDofhCGkOY+Dp0YlJU8dPSC +4QKCAQEA3qlwsjJT8Vv+Sx2sZySDNtey/00yjxe4TckdH8dWTC22G38/ppbazlp/ +YVqFUgteoo5FI60HKa0+jLKQZpCIH6JgpL3yq81Obl0wRkrSNePRTL1Ikiff8P2j +bowFpbIdJLgDco1opJpDgTOz2mB7HlHu6RyoKjiVrNA/EOoks1Uljxdth6h/5ctr +rLn8dnz2sTtwxcUsOpyFcFQ2qaWJvSg+bF7JPPzMrpQfCR1qVWa43Kl8KlcWSKaq +ITpglIBY+h3F2GygAAcnpfkXde381Iw89y7TFd2LxWQR98zhnbJWF2JmuuPDtVRv ++HYZkcyQcpDwfC+2NOWOU7NQj+IDIA== +-----END PRIVATE KEY----- From b3583ba7ff3771a0adfe9d6f98ee8c1580cc7386 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 24 Jun 2022 09:38:35 +0200 Subject: [PATCH 018/146] Fix flaky Network.framework `testConnectionFailError` test (#600) we may recieve a posix `ECONNREFUSED` too --- Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index cc114cb9a..3b659a14a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -104,6 +104,7 @@ class HTTPClientNIOTSTests: XCTestCase { func testConnectionFailError() { guard isTestingNIOTS() else { return } + #if canImport(Network) let httpBin = HTTPBin(.http1_1(ssl: false)) let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(timeout: .init(connect: .milliseconds(100), @@ -117,8 +118,15 @@ class HTTPClientNIOTSTests: XCTestCase { XCTAssertNoThrow(try httpBin.shutdown()) XCTAssertThrowsError(try httpClient.get(url: "http://localhost:\(port)/get").wait()) { - XCTAssertEqual($0 as? HTTPClientError, .connectTimeout) + if let httpClientError = $0 as? HTTPClientError { + XCTAssertEqual(httpClientError, .connectTimeout) + } else if let posixError = $0 as? HTTPClient.NWPOSIXError { + XCTAssertEqual(posixError.errorCode, .ECONNREFUSED) + } else { + XCTFail("unexpected error \($0)") + } } + #endif } func testTLSVersionError() { From 14fa6d944d88e5702ba630f8a8843966331fd944 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 1 Jul 2022 10:39:38 +0200 Subject: [PATCH 019/146] Report last connection error if request deadline is exceeded (#601) --- .../ConnectionPool/HTTPConnectionPool.swift | 2 - ...HTTPConnectionPool+HTTP1StateMachine.swift | 6 +- ...HTTPConnectionPool+HTTP2StateMachine.swift | 6 +- .../HTTPConnectionPool+StateMachine.swift | 1 - Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- .../RequestBag+StateMachine.swift | 85 ++++++++++++++----- Sources/AsyncHTTPClient/RequestBag.swift | 48 ++++++++--- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 41 +++++++++ .../HTTPConnectionPool+HTTP1StateTests.swift | 9 +- ...onnectionPool+HTTP2StateMachineTests.swift | 6 +- .../HTTPConnectionPool+StateTestUtils.swift | 2 - .../Mocks/MockRequestQueuer.swift | 8 +- .../RequestBagTests+XCTest.swift | 1 + .../RequestBagTests.swift | 38 +++++++++ 15 files changed, 205 insertions(+), 51 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 49e755733..5a0b2708e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -147,8 +147,6 @@ final class HTTPConnectionPool { self.unlocked = Unlocked(connection: .none, request: .none) switch stateMachineAction.request { - case .cancelRequestTimeout(let requestID): - self.locked.request = .cancelRequestTimeout(requestID) case .executeRequest(let request, let connection, cancelTimeout: let cancelTimeout): if cancelTimeout { self.locked.request = .cancelRequestTimeout(request.id) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift index d654f5a87..2cd667bb3 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift @@ -323,9 +323,11 @@ extension HTTPConnectionPool { mutating func cancelRequest(_ requestID: Request.ID) -> Action { // 1. check requests in queue - if self.requests.remove(requestID) != nil { + if let request = self.requests.remove(requestID) { + // Use the last connection error to let the user know why the request was never scheduled + let error = self.lastConnectFailure ?? HTTPClientError.cancelled return .init( - request: .cancelRequestTimeout(requestID), + request: .failRequest(request, error, cancelTimeout: true), connection: .none ) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index 06fc36ad0..d517d82e6 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -444,9 +444,11 @@ extension HTTPConnectionPool { mutating func cancelRequest(_ requestID: Request.ID) -> Action { // 1. check requests in queue - if self.requests.remove(requestID) != nil { + if let request = self.requests.remove(requestID) { + // Use the last connection error to let the user know why the request was never scheduled + let error = self.lastConnectFailure ?? HTTPClientError.cancelled return .init( - request: .cancelRequestTimeout(requestID), + request: .failRequest(request, error, cancelTimeout: true), connection: .none ) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift index 61e57941a..63f3e5a9a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift @@ -61,7 +61,6 @@ extension HTTPConnectionPool { case failRequestsAndCancelTimeouts([Request], Error) case scheduleRequestTimeout(for: Request, on: EventLoop) - case cancelRequestTimeout(Request.ID) case none } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 45b2ce0ff..1f08fb41d 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -606,7 +606,7 @@ public class HTTPClient { var deadlineSchedule: Scheduled? if let deadline = deadline { deadlineSchedule = taskEL.scheduleTask(deadline: deadline) { - requestBag.fail(HTTPClientError.deadlineExceeded) + requestBag.deadlineExceeded() } task.promise.futureResult.whenComplete { _ in diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index 63cb15758..a2a90749a 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -31,6 +31,10 @@ extension RequestBag { fileprivate enum State { case initialized case queued(HTTPRequestScheduler) + /// if the deadline was exceeded while in the `.queued(_:)` state, + /// we wait until the request pool fails the request with a potential more descriptive error message, + /// if a connection failure has occured while the request was queued. + case deadlineExceededWhileQueued case executing(HTTPRequestExecutor, RequestStreamState, ResponseStreamState) case finished(error: Error?) case redirected(HTTPRequestExecutor, Int, HTTPResponseHead, URL) @@ -90,13 +94,23 @@ extension RequestBag.StateMachine { self.state = .queued(scheduler) } - mutating func willExecuteRequest(_ executor: HTTPRequestExecutor) -> Bool { + enum WillExecuteRequestAction { + case cancelExecuter(HTTPRequestExecutor) + case failTaskAndCancelExecutor(Error, HTTPRequestExecutor) + case none + } + + mutating func willExecuteRequest(_ executor: HTTPRequestExecutor) -> WillExecuteRequestAction { switch self.state { case .initialized, .queued: self.state = .executing(executor, .initialized, .initialized) - return true + return .none + case .deadlineExceededWhileQueued: + let error: Error = HTTPClientError.deadlineExceeded + self.state = .finished(error: error) + return .failTaskAndCancelExecutor(error, executor) case .finished(error: .some): - return false + return .cancelExecuter(executor) case .executing, .redirected, .finished(error: .none), .modifying: preconditionFailure("Invalid state: \(self.state)") } @@ -110,7 +124,7 @@ extension RequestBag.StateMachine { mutating func resumeRequestBodyStream() -> ResumeProducingAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("A request stream can only be resumed, if the request was started") case .executing(let executor, .initialized, .initialized): @@ -150,7 +164,7 @@ extension RequestBag.StateMachine { mutating func pauseRequestBodyStream() { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("A request stream can only be paused, if the request was started") case .executing(let executor, let requestState, let responseState): switch requestState { @@ -185,7 +199,7 @@ extension RequestBag.StateMachine { mutating func writeNextRequestPart(_ part: IOData, taskEventLoop: EventLoop) -> WriteAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Invalid state: \(self.state)") case .executing(let executor, let requestState, let responseState): switch requestState { @@ -231,7 +245,7 @@ extension RequestBag.StateMachine { mutating func finishRequestBodyStream(_ result: Result) -> FinishAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Invalid state: \(self.state)") case .executing(let executor, let requestState, let responseState): switch requestState { @@ -282,7 +296,7 @@ extension RequestBag.StateMachine { /// - Returns: Whether the response should be forwarded to the delegate. Will be `false` if the request follows a redirect. mutating func receiveResponseHead(_ head: HTTPResponseHead) -> ReceiveResponseHeadAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("How can we receive a response, if the request hasn't started yet.") case .executing(let executor, let requestState, let responseState): guard case .initialized = responseState else { @@ -328,7 +342,7 @@ extension RequestBag.StateMachine { mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponseBodyAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("How can we receive a response body part, if the request hasn't started yet.") case .executing(_, _, .initialized): preconditionFailure("If we receive a response body, we must have received a head before") @@ -385,7 +399,7 @@ extension RequestBag.StateMachine { mutating func succeedRequest(_ newChunks: CircularBuffer?) -> ReceiveResponseEndAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("How can we receive a response body part, if the request hasn't started yet.") case .executing(_, _, .initialized): preconditionFailure("If we receive a response body, we must have received a head before") @@ -447,7 +461,7 @@ extension RequestBag.StateMachine { private mutating func failWithConsumptionError(_ error: Error) -> ConsumeAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Invalid state: \(self.state)") case .executing(_, _, .initialized): preconditionFailure("Invalid state: Must have received response head, before this method is called for the first time") @@ -482,7 +496,7 @@ extension RequestBag.StateMachine { private mutating func consumeMoreBodyData() -> ConsumeAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Invalid state: \(self.state)") case .executing(_, _, .initialized): @@ -532,8 +546,33 @@ extension RequestBag.StateMachine { } } + enum DeadlineExceededAction { + case cancelScheduler(HTTPRequestScheduler?) + case fail(FailAction) + } + + mutating func deadlineExceeded() -> DeadlineExceededAction { + switch self.state { + case .queued(let queuer): + /// We do not fail the request immediately because we want to give the scheduler a chance of throwing a better error message + /// We therefore depend on the scheduler failing the request after we cancel the request. + self.state = .deadlineExceededWhileQueued + return .cancelScheduler(queuer) + + case .initialized, + .deadlineExceededWhileQueued, + .executing, + .finished, + .redirected, + .modifying: + /// if we are not in the queued state, we can fail early by just calling down to `self.fail(_:)` + /// which does the appropriate state transition for us. + return .fail(self.fail(HTTPClientError.deadlineExceeded)) + } + } + enum FailAction { - case failTask(HTTPRequestScheduler?, HTTPRequestExecutor?) + case failTask(Error, HTTPRequestScheduler?, HTTPRequestExecutor?) case cancelExecutor(HTTPRequestExecutor) case none } @@ -542,31 +581,39 @@ extension RequestBag.StateMachine { switch self.state { case .initialized: self.state = .finished(error: error) - return .failTask(nil, nil) + return .failTask(error, nil, nil) case .queued(let queuer): self.state = .finished(error: error) - return .failTask(queuer, nil) + return .failTask(error, queuer, nil) case .executing(let executor, let requestState, .buffering(_, next: .eof)): self.state = .executing(executor, requestState, .buffering(.init(), next: .error(error))) return .cancelExecutor(executor) case .executing(let executor, _, .buffering(_, next: .askExecutorForMore)): self.state = .finished(error: error) - return .failTask(nil, executor) + return .failTask(error, nil, executor) case .executing(let executor, _, .buffering(_, next: .error(_))): // this would override another error, let's keep the first one return .cancelExecutor(executor) case .executing(let executor, _, .initialized): self.state = .finished(error: error) - return .failTask(nil, executor) + return .failTask(error, nil, executor) case .executing(let executor, _, .waitingForRemote): self.state = .finished(error: error) - return .failTask(nil, executor) + return .failTask(error, nil, executor) case .redirected: self.state = .finished(error: error) - return .failTask(nil, nil) + return .failTask(error, nil, nil) case .finished(.none): // An error occurred after the request has finished. Ignore... return .none + case .deadlineExceededWhileQueued: + // if we just get a `HTTPClientError.cancelled` we can use the original cancellation reason + // to give a more descriptive error to the user. + if (error as? HTTPClientError) == .cancelled { + return .failTask(HTTPClientError.deadlineExceeded, nil, nil) + } + // otherwise we already had an intermediate connection error which we should present to the user instead + return .failTask(error, nil, nil) case .finished(.some(_)): // this might happen, if the stream consumer has failed... let's just drop the data return .none diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index dbef802e9..4ec7004c1 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -81,8 +81,16 @@ final class RequestBag { private func willExecuteRequest0(_ executor: HTTPRequestExecutor) { self.task.eventLoop.assertInEventLoop() - if !self.state.willExecuteRequest(executor) { - return executor.cancelRequest(self) + let action = self.state.willExecuteRequest(executor) + switch action { + case .cancelExecuter(let executor): + executor.cancelRequest(self) + case .failTaskAndCancelExecutor(let error, let executor): + self.delegate.didReceiveError(task: self.task, error) + self.task.fail(with: error, delegateType: Delegate.self) + executor.cancelRequest(self) + case .none: + break } } @@ -320,8 +328,12 @@ final class RequestBag { let action = self.state.fail(error) + self.executeFailAction0(action) + } + + private func executeFailAction0(_ action: RequestBag.StateMachine.FailAction) { switch action { - case .failTask(let scheduler, let executor): + case .failTask(let error, let scheduler, let executor): scheduler?.cancelRequest(self) executor?.cancelRequest(self) self.failTask0(error) @@ -331,6 +343,28 @@ final class RequestBag { break } } + + func deadlineExceeded0() { + self.task.eventLoop.assertInEventLoop() + let action = self.state.deadlineExceeded() + + switch action { + case .cancelScheduler(let scheduler): + scheduler?.cancelRequest(self) + case .fail(let failAction): + self.executeFailAction0(failAction) + } + } + + func deadlineExceeded() { + if self.task.eventLoop.inEventLoop { + self.deadlineExceeded0() + } else { + self.task.eventLoop.execute { + self.deadlineExceeded0() + } + } + } } extension RequestBag: HTTPSchedulableRequest { @@ -457,12 +491,6 @@ extension RequestBag: HTTPExecutableRequest { extension RequestBag: HTTPClientTaskDelegate { func cancel() { - if self.task.eventLoop.inEventLoop { - self.fail0(HTTPClientError.cancelled) - } else { - self.task.eventLoop.execute { - self.fail0(HTTPClientError.cancelled) - } - } + self.fail(HTTPClientError.cancelled) } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 3975036ea..b3a13486c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -79,6 +79,7 @@ extension HTTPClientTests { ("testStressGetHttps", testStressGetHttps), ("testStressGetHttpsSSLError", testStressGetHttpsSSLError), ("testSelfSignedCertificateIsRejectedWithCorrectError", testSelfSignedCertificateIsRejectedWithCorrectError), + ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), ("testFailingConnectionIsReleased", testFailingConnectionIsReleased), ("testResponseDelayGet", testResponseDelayGet), ("testIdleTimeoutNoReuse", testIdleTimeoutNoReuse), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index ece09c52d..02c60d177 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1269,6 +1269,47 @@ class HTTPClientTests: XCTestCase { } } + func testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded() throws { + /// key + cert was created with the follwing command: + /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' + let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! + let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let configuration = TLSConfiguration.makeServerConfiguration( + certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, + privateKey: .file(keyPath) + ) + let sslContext = try NIOSSLContext(configuration: configuration) + + let server = ServerBootstrap(group: serverGroup) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + } + let serverChannel = try server.bind(host: "localhost", port: 0).wait() + defer { XCTAssertNoThrow(try serverChannel.close().wait()) } + let port = serverChannel.localAddress!.port! + + var config = HTTPClient.Configuration() + config.timeout.connect = .seconds(3) + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) + defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + + XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(port)", deadline: .now() + .seconds(2)).wait()) { error in + #if canImport(Network) + guard let nwTLSError = error as? HTTPClient.NWTLSError else { + XCTFail("could not cast \(error) of type \(type(of: error)) to \(HTTPClient.NWTLSError.self)") + return + } + XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") + #else + guard let sslError = error as? NIOSSLError, + case .handshakeFailed(.sslError) = sslError else { + XCTFail("unexpected error \(error)") + return + } + #endif + } + } + func testFailingConnectionIsReleased() { let localHTTPBin = HTTPBin(.refuse) let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift index 49a6fb574..7f59fd4e1 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift @@ -21,6 +21,7 @@ import XCTest class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { func testCreatingAndFailingConnections() { + struct SomeError: Error, Equatable {} let elg = EmbeddedEventLoopGroup(loops: 4) defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } @@ -65,8 +66,6 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // fail all connection attempts while let randomConnectionID = connections.randomStartingConnection() { - struct SomeError: Error, Equatable {} - XCTAssertNoThrow(try connections.failConnectionCreation(randomConnectionID)) let action = state.failedToCreateNewConnection(SomeError(), connectionID: randomConnectionID) @@ -86,9 +85,9 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // cancel all queued requests while let request = queuer.timeoutRandomRequest() { - let cancelAction = state.cancelRequest(request) + let cancelAction = state.cancelRequest(request.0) XCTAssertEqual(cancelAction.connection, .none) - XCTAssertEqual(cancelAction.request, .cancelRequestTimeout(request)) + XCTAssertEqual(cancelAction.request, .failRequest(.init(request.1), SomeError(), cancelTimeout: true)) } // connection backoff done @@ -184,7 +183,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // 2. cancel request let cancelAction = state.cancelRequest(request.id) - XCTAssertEqual(cancelAction.request, .cancelRequestTimeout(request.id)) + XCTAssertEqual(cancelAction.request, .failRequest(request, HTTPClientError.cancelled, cancelTimeout: true)) XCTAssertEqual(cancelAction.connection, .none) // 3. request timeout triggers to late diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index 2574d3da2..e42a98ac7 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -212,7 +212,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // 2. cancel request let cancelAction = state.cancelRequest(request.id) - XCTAssertEqual(cancelAction.request, .cancelRequestTimeout(request.id)) + XCTAssertEqual(cancelAction.request, .failRequest(request, HTTPClientError.cancelled, cancelTimeout: true)) XCTAssertEqual(cancelAction.connection, .none) // 3. request timeout triggers to late @@ -1242,9 +1242,9 @@ func XCTAssertEqualTypeAndValue( let lhs = try lhs() let rhs = try rhs() guard let lhsAsRhs = lhs as? Right else { - XCTFail("could not cast \(lhs) of type \(type(of: lhs)) to \(type(of: rhs))") + XCTFail("could not cast \(lhs) of type \(type(of: lhs)) to \(type(of: rhs))", file: file, line: line) return } - XCTAssertEqual(lhsAsRhs, rhs) + XCTAssertEqual(lhsAsRhs, rhs, file: file, line: line) }(), file: file, line: line) } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift index cb67837d7..0ffdeebd8 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift @@ -126,8 +126,6 @@ extension HTTPConnectionPool.StateMachine.RequestAction: Equatable { return lhsReqs.elementsEqual(rhsReqs, by: { $0 == $1 }) case (.scheduleRequestTimeout(for: let lhsReq, on: let lhsEL), .scheduleRequestTimeout(for: let rhsReq, on: let rhsEL)): return lhsReq == rhsReq && lhsEL === rhsEL - case (.cancelRequestTimeout(let lhsReqID), .cancelRequestTimeout(let rhsReqID)): - return lhsReqID == rhsReqID case (.none, .none): return true default: diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift b/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift index e81f1ed0a..520b51875 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift @@ -82,11 +82,11 @@ struct MockRequestQueuer { return waiter.request } - mutating func timeoutRandomRequest() -> RequestID? { - guard let waiterID = self.waiters.randomElement().map(\.0) else { + mutating func timeoutRandomRequest() -> (RequestID, HTTPSchedulableRequest)? { + guard let waiter = self.waiters.randomElement() else { return nil } - self.waiters.removeValue(forKey: waiterID) - return waiterID + self.waiters.removeValue(forKey: waiter.key) + return (waiter.key, waiter.value.request) } } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index 74c68fd1f..19de474c2 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -28,6 +28,7 @@ extension RequestBagTests { ("testWriteBackpressureWorks", testWriteBackpressureWorks), ("testTaskIsFailedIfWritingFails", testTaskIsFailedIfWritingFails), ("testCancelFailsTaskBeforeRequestIsSent", testCancelFailsTaskBeforeRequestIsSent), + ("testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart", testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart), ("testCancelFailsTaskAfterRequestIsSent", testCancelFailsTaskAfterRequestIsSent), ("testCancelFailsTaskWhenTaskIsQueued", testCancelFailsTaskWhenTaskIsQueued), ("testFailsTaskWhenTaskIsWaitingForMoreFromServer", testFailsTaskWhenTaskIsWaitingForMoreFromServer), diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index c80f8846b..b896aca0a 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -228,6 +228,44 @@ final class RequestBagTests: XCTestCase { } } + func testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org")) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + XCTAssert(bag.eventLoop === embeddedEventLoop) + + let queuer = MockTaskQueuer() + bag.requestWasQueued(queuer) + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + XCTAssertEqual(queuer.hitCancelCount, 0) + bag.deadlineExceeded() + XCTAssertEqual(queuer.hitCancelCount, 1) + + bag.willExecuteRequest(executor) + XCTAssertTrue(executor.isCancelled, "The request bag, should call cancel immediately on the executor") + XCTAssertThrowsError(try bag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .deadlineExceeded) + } + } + func testCancelFailsTaskAfterRequestIsSent() { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } From 1af18d2f95a24358b9e0001d9597a4646d2555ca Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 4 Jul 2022 11:02:09 +0100 Subject: [PATCH 020/146] Use 5.7 nightlies (#593) Co-authored-by: David Nadoba --- docker/docker-compose.2004.57.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.2004.57.yaml b/docker/docker-compose.2004.57.yaml index 16c564482..bf6dd6e97 100644 --- a/docker/docker-compose.2004.57.yaml +++ b/docker/docker-compose.2004.57.yaml @@ -6,7 +6,7 @@ services: image: async-http-client:20.04-5.7 build: args: - base_image: "swiftlang/swift:nightly-main-focal" + base_image: "swiftlang/swift:nightly-5.7-focal" test: image: async-http-client:20.04-5.7 From 2adca4b0038a7ef81ea0838c5feeda9439c52b64 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 13 Jul 2022 13:48:24 +0100 Subject: [PATCH 021/146] Use `swift-atomics` instead of `NIOAtomics` (#603) `NIOAtomics` was deprecated in https://github.com/apple/swift-nio/pull/2204 in favor of `swift-atomics` https://github.com/apple/swift-atomics --- Package.swift | 3 ++ .../HTTPConnectionPool+Manager.swift | 7 +++-- .../HTTPClientTestUtils.swift | 25 ++++++++------- .../HTTPClientTests.swift | 31 ++++++++++--------- .../HTTPConnectionPool+StateTestUtils.swift | 5 +-- 5 files changed, 39 insertions(+), 32 deletions(-) diff --git a/Package.swift b/Package.swift index 20484832d..a9ea2ba7f 100644 --- a/Package.swift +++ b/Package.swift @@ -27,6 +27,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), ], targets: [ .target(name: "CAsyncHTTPClient"), @@ -46,6 +47,7 @@ let package = Package( .product(name: "NIOSOCKS", package: "swift-nio-extras"), .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), ] ), .testTarget( @@ -61,6 +63,7 @@ let package = Package( .product(name: "NIOHTTP2", package: "swift-nio-http2"), .product(name: "NIOSOCKS", package: "swift-nio-extras"), .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), ], resources: [ .copy("Resources/self_signed_cert.pem"), diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift index 1a1760908..8500c59da 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Atomics import Logging import NIOConcurrencyHelpers import NIOCore @@ -165,14 +166,14 @@ extension HTTPConnectionPool.Connection.ID { static var globalGenerator = Generator() struct Generator { - private let atomic: NIOAtomic + private let atomic: ManagedAtomic init() { - self.atomic = .makeAtomic(value: 0) + self.atomic = .init(0) } func next() -> Int { - return self.atomic.add(1) + return self.atomic.loadThenWrappingIncrement(ordering: .relaxed) } } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index c99facc3f..3a7f1fe90 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import AsyncHTTPClient +import Atomics import Foundation import Logging import NIOConcurrencyHelpers @@ -351,7 +352,7 @@ internal final class HTTPBin where private let mode: Mode private let sslContext: NIOSSLContext? private var serverChannel: Channel! - private let isShutdown: NIOAtomic = .makeAtomic(value: false) + private let isShutdown = ManagedAtomic(false) private let handlerFactory: (Int) -> (RequestHandler) init( @@ -376,7 +377,7 @@ internal final class HTTPBin where self.activeConnCounterHandler = ConnectionsCountHandler() - let connectionIDAtomic = NIOAtomic.makeAtomic(value: 0) + let connectionIDAtomic = ManagedAtomic(0) self.serverChannel = try! ServerBootstrap(group: self.group) .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEADDR), value: 1) @@ -384,7 +385,7 @@ internal final class HTTPBin where channel.pipeline.addHandler(self.activeConnCounterHandler) }.childChannelInitializer { channel in do { - let connectionID = connectionIDAtomic.add(1) + let connectionID = connectionIDAtomic.loadThenWrappingIncrement(ordering: .relaxed) if case .refuse = mode { throw HTTPBinError.refusedConnection @@ -572,12 +573,12 @@ internal final class HTTPBin where } func shutdown() throws { - self.isShutdown.store(true) + self.isShutdown.store(true, ordering: .relaxed) try self.group.syncShutdownGracefully() } deinit { - assert(self.isShutdown.load(), "HTTPBin not shutdown before deinit") + assert(self.isShutdown.load(ordering: .relaxed), "HTTPBin not shutdown before deinit") } } @@ -946,24 +947,24 @@ internal final class HTTPBinHandler: ChannelInboundHandler { final class ConnectionsCountHandler: ChannelInboundHandler { typealias InboundIn = Channel - private let activeConns = NIOAtomic.makeAtomic(value: 0) - private let createdConns = NIOAtomic.makeAtomic(value: 0) + private let activeConns = ManagedAtomic(0) + private let createdConns = ManagedAtomic(0) var createdConnections: Int { - self.createdConns.load() + self.createdConns.load(ordering: .relaxed) } var currentlyActiveConnections: Int { - self.activeConns.load() + self.activeConns.load(ordering: .relaxed) } func channelRead(context: ChannelHandlerContext, data: NIOAny) { let channel = self.unwrapInboundIn(data) - _ = self.activeConns.add(1) - _ = self.createdConns.add(1) + _ = self.activeConns.loadThenWrappingIncrement(ordering: .relaxed) + _ = self.createdConns.loadThenWrappingIncrement(ordering: .relaxed) channel.closeFuture.whenComplete { _ in - _ = self.activeConns.sub(1) + _ = self.activeConns.loadThenWrappingDecrement(ordering: .relaxed) } context.fireChannelRead(data) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 02c60d177..610df95c9 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// /* NOT @testable */ import AsyncHTTPClient // Tests that need @testable go into HTTPClientInternalTests.swift +import Atomics #if canImport(Network) import Network #endif @@ -1790,16 +1791,16 @@ class HTTPClientTests: XCTestCase { typealias InboundIn = HTTPServerRequestPart typealias OutboundOut = HTTPServerResponsePart - let requestNumber: NIOAtomic - let connectionNumber: NIOAtomic + let requestNumber: ManagedAtomic + let connectionNumber: ManagedAtomic - init(requestNumber: NIOAtomic, connectionNumber: NIOAtomic) { + init(requestNumber: ManagedAtomic, connectionNumber: ManagedAtomic) { self.requestNumber = requestNumber self.connectionNumber = connectionNumber } func channelActive(context: ChannelHandlerContext) { - _ = self.connectionNumber.add(1) + _ = self.connectionNumber.loadThenWrappingIncrement(ordering: .relaxed) } func channelRead(context: ChannelHandlerContext, data: NIOAny) { @@ -1809,7 +1810,7 @@ class HTTPClientTests: XCTestCase { case .head, .body: () case .end: - let last = self.requestNumber.add(1) + let last = self.requestNumber.loadThenWrappingIncrement(ordering: .relaxed) switch last { case 0, 2: context.write(self.wrapOutboundOut(.head(.init(version: .init(major: 1, minor: 1), status: .ok))), @@ -1824,8 +1825,8 @@ class HTTPClientTests: XCTestCase { } } - let requestNumber = NIOAtomic.makeAtomic(value: 0) - let connectionNumber = NIOAtomic.makeAtomic(value: 0) + let requestNumber = ManagedAtomic(0) + let connectionNumber = ManagedAtomic(0) let sharedStateServerHandler = ServerThatAcceptsThenRejects(requestNumber: requestNumber, connectionNumber: connectionNumber) var maybeServer: Channel? @@ -1854,19 +1855,19 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try client.syncShutdown()) } - XCTAssertEqual(0, sharedStateServerHandler.connectionNumber.load()) - XCTAssertEqual(0, sharedStateServerHandler.requestNumber.load()) + XCTAssertEqual(0, sharedStateServerHandler.connectionNumber.load(ordering: .relaxed)) + XCTAssertEqual(0, sharedStateServerHandler.requestNumber.load(ordering: .relaxed)) XCTAssertEqual(.ok, try client.get(url: url).wait().status) - XCTAssertEqual(1, sharedStateServerHandler.connectionNumber.load()) - XCTAssertEqual(1, sharedStateServerHandler.requestNumber.load()) + XCTAssertEqual(1, sharedStateServerHandler.connectionNumber.load(ordering: .relaxed)) + XCTAssertEqual(1, sharedStateServerHandler.requestNumber.load(ordering: .relaxed)) XCTAssertThrowsError(try client.get(url: url).wait().status) { error in XCTAssertEqual(.remoteConnectionClosed, error as? HTTPClientError) } - XCTAssertEqual(1, sharedStateServerHandler.connectionNumber.load()) - XCTAssertEqual(2, sharedStateServerHandler.requestNumber.load()) + XCTAssertEqual(1, sharedStateServerHandler.connectionNumber.load(ordering: .relaxed)) + XCTAssertEqual(2, sharedStateServerHandler.requestNumber.load(ordering: .relaxed)) XCTAssertEqual(.ok, try client.get(url: url).wait().status) - XCTAssertEqual(2, sharedStateServerHandler.connectionNumber.load()) - XCTAssertEqual(3, sharedStateServerHandler.requestNumber.load()) + XCTAssertEqual(2, sharedStateServerHandler.connectionNumber.load(ordering: .relaxed)) + XCTAssertEqual(3, sharedStateServerHandler.requestNumber.load(ordering: .relaxed)) } func testPoolClosesIdleConnections() { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift index 0ffdeebd8..53bba940c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// @testable import AsyncHTTPClient +import Atomics import Dispatch import NIOConcurrencyHelpers import NIOCore @@ -21,14 +22,14 @@ import NIOEmbedded /// An `EventLoopGroup` of `EmbeddedEventLoop`s. final class EmbeddedEventLoopGroup: EventLoopGroup { private let loops: [EmbeddedEventLoop] - private let index = NIOAtomic.makeAtomic(value: 0) + private let index = ManagedAtomic(0) internal init(loops: Int) { self.loops = (0.. EventLoop { - let index: Int = self.index.add(1) + let index: Int = self.index.loadThenWrappingIncrement(ordering: .relaxed) return self.loops[index % self.loops.count] } From 0527bbb466b8f48b36fe48933c425c42dac09358 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 3 Aug 2022 10:02:20 +0100 Subject: [PATCH 022/146] Remove the last remaining NIOAtomic (#607) Motivation Warnings aren't great, and NIOAtomic is deprecated. Modifications Replace the last use of NIOAtomic with ManagedAtomic. Result Fewer warnings Fixes #606 --- Sources/AsyncHTTPClient/HTTPClient.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 1f08fb41d..094a6d052 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Atomics import Foundation import Logging import NIOConcurrencyHelpers @@ -36,7 +37,7 @@ extension Logger { } } -let globalRequestID = NIOAtomic.makeAtomic(value: 0) +let globalRequestID = ManagedAtomic(0) /// HTTPClient class provides API for request execution. /// @@ -541,7 +542,7 @@ public class HTTPClient { logger originalLogger: Logger?, redirectState: RedirectState? ) -> Task { - let logger = (originalLogger ?? HTTPClient.loggingDisabled).attachingRequestInformation(request, requestID: globalRequestID.add(1)) + let logger = (originalLogger ?? HTTPClient.loggingDisabled).attachingRequestInformation(request, requestID: globalRequestID.wrappingIncrementThenLoad(ordering: .relaxed)) let taskEL: EventLoop switch eventLoopPreference.preference { case .indifferent: From 3960678bcd0daf1aee3e94cdcb9bc114c779b42c Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 4 Aug 2022 14:14:36 +0200 Subject: [PATCH 023/146] =?UTF-8?q?Don=E2=80=99t=20call=20`didReceiveError?= =?UTF-8?q?`=20twice=20if=20deadline=20is=20exceeded=20and=20request=20is?= =?UTF-8?q?=20canceled=20aftewards=20(#609)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../RequestBag+StateMachine.swift | 19 +++++---- .../RequestBagTests+XCTest.swift | 1 + .../RequestBagTests.swift | 42 +++++++++++++++++++ 3 files changed, 55 insertions(+), 7 deletions(-) diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index a2a90749a..9509fa2e6 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -607,13 +607,18 @@ extension RequestBag.StateMachine { // An error occurred after the request has finished. Ignore... return .none case .deadlineExceededWhileQueued: - // if we just get a `HTTPClientError.cancelled` we can use the original cancellation reason - // to give a more descriptive error to the user. - if (error as? HTTPClientError) == .cancelled { - return .failTask(HTTPClientError.deadlineExceeded, nil, nil) - } - // otherwise we already had an intermediate connection error which we should present to the user instead - return .failTask(error, nil, nil) + let realError: Error = { + if (error as? HTTPClientError) == .cancelled { + /// if we just get a `HTTPClientError.cancelled` we can use the original cancellation reason + /// to give a more descriptive error to the user. + return HTTPClientError.deadlineExceeded + } else { + /// otherwise we already had an intermediate connection error which we should present to the user instead + return error + } + }() + self.state = .finished(error: realError) + return .failTask(realError, nil, nil) case .finished(.some(_)): // this might happen, if the stream consumer has failed... let's just drop the data return .none diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index 19de474c2..72046f68c 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -29,6 +29,7 @@ extension RequestBagTests { ("testTaskIsFailedIfWritingFails", testTaskIsFailedIfWritingFails), ("testCancelFailsTaskBeforeRequestIsSent", testCancelFailsTaskBeforeRequestIsSent), ("testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart", testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart), + ("testCancelHasNoEffectAfterDeadlineExceededFailsTask", testCancelHasNoEffectAfterDeadlineExceededFailsTask), ("testCancelFailsTaskAfterRequestIsSent", testCancelFailsTaskAfterRequestIsSent), ("testCancelFailsTaskWhenTaskIsQueued", testCancelFailsTaskWhenTaskIsQueued), ("testFailsTaskWhenTaskIsWaitingForMoreFromServer", testFailsTaskWhenTaskIsWaitingForMoreFromServer), diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index b896aca0a..9e7072c19 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -266,6 +266,48 @@ final class RequestBagTests: XCTestCase { } } + func testCancelHasNoEffectAfterDeadlineExceededFailsTask() { + struct MyError: Error, Equatable {} + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org")) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + XCTAssert(bag.eventLoop === embeddedEventLoop) + + let queuer = MockTaskQueuer() + bag.requestWasQueued(queuer) + + XCTAssertEqual(queuer.hitCancelCount, 0) + bag.deadlineExceeded() + XCTAssertEqual(queuer.hitCancelCount, 1) + XCTAssertEqual(delegate.hitDidReceiveError, 0) + bag.fail(MyError()) + XCTAssertEqual(delegate.hitDidReceiveError, 1) + + bag.cancel() + XCTAssertEqual(delegate.hitDidReceiveError, 1) + + XCTAssertThrowsError(try bag.task.futureResult.wait()) { + XCTAssertEqualTypeAndValue($0, MyError()) + } + } + func testCancelFailsTaskAfterRequestIsSent() { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } From a9c3cfb38761b4b2f6f92263e25de73ae05446c5 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 4 Aug 2022 18:24:34 +0200 Subject: [PATCH 024/146] Report last connection error if request deadline is exceeded with async/await API (#608) Co-authored-by: Cory Benfield --- .../AsyncAwait/Transaction+StateMachine.swift | 46 +++++++--- .../AsyncAwait/Transaction.swift | 7 +- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 48 ++++++++++ ...Transaction+StateMachineTests+XCTest.swift | 2 + .../Transaction+StateMachineTests.swift | 89 ++++++++++++++++++- 6 files changed, 176 insertions(+), 17 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index dea1093db..53bc8c8a6 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -28,6 +28,7 @@ extension Transaction { private enum State { case initialized(CheckedContinuation) case queued(CheckedContinuation, HTTPRequestScheduler) + case deadlineExceededWhileQueued(CheckedContinuation) case executing(ExecutionContext, RequestStreamState, ResponseStreamState) case finished(error: Error?, HTTPClientResponse.Body.IteratorStream.ID?) } @@ -105,7 +106,20 @@ extension Transaction { case .queued(let continuation, let scheduler): self.state = .finished(error: error, nil) return .failResponseHead(continuation, error, scheduler, nil, bodyStreamContinuation: nil) - + case .deadlineExceededWhileQueued(let continuation): + let realError: Error = { + if (error as? HTTPClientError) == .cancelled { + /// if we just get a `HTTPClientError.cancelled` we can use the original cancellation reason + /// to give a more descriptive error to the user. + return HTTPClientError.deadlineExceeded + } else { + /// otherwise we already had an intermediate connection error which we should present to the user instead + return error + } + }() + + self.state = .finished(error: realError, nil) + return .failResponseHead(continuation, realError, nil, nil, bodyStreamContinuation: nil) case .executing(let context, let requestStreamState, .waitingForResponseHead): switch requestStreamState { case .paused(continuation: .some(let continuation)): @@ -178,6 +192,7 @@ extension Transaction { enum StartExecutionAction { case cancel(HTTPRequestExecutor) + case cancelAndFail(HTTPRequestExecutor, CheckedContinuation, with: Error) case none } @@ -191,6 +206,8 @@ extension Transaction { ) self.state = .executing(context, .requestHeadSent, .waitingForResponseHead) return .none + case .deadlineExceededWhileQueued(let continuation): + return .cancelAndFail(executor, continuation, with: HTTPClientError.deadlineExceeded) case .finished(error: .some, .none): return .cancel(executor) @@ -210,7 +227,7 @@ extension Transaction { mutating func resumeRequestBodyStream() -> ResumeProducingAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Received a resumeBodyRequest on a request, that isn't executing. Invalid state: \(self.state)") case .executing(let context, .requestHeadSent, let responseState): @@ -246,6 +263,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, .requestHeadSent, _): preconditionFailure("A request stream can only be resumed, if the request was started") @@ -271,6 +289,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, .requestHeadSent, _): preconditionFailure("A request stream can only produce, if the request was started. Invalid state: \(self.state)") @@ -301,6 +320,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, .requestHeadSent, _), .executing(_, .finished, _): preconditionFailure("A request stream can only produce, if the request was started. Invalid state: \(self.state)") @@ -334,6 +354,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, .finished, _): preconditionFailure("Invalid state: \(self.state)") @@ -372,6 +393,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseIterator), .executing(_, _, .buffering), .executing(_, _, .waitingForRemote): @@ -401,7 +423,7 @@ extension Transaction { mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponsePartAction { switch self.state { - case .initialized, .queued: + case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Received a response body part, but request hasn't started yet. Invalid state: \(self.state)") case .executing(_, _, .waitingForResponseHead): @@ -457,6 +479,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): preconditionFailure("Got notice about a deinited response, before we even received a response. Invalid state: \(self.state)") @@ -486,7 +509,7 @@ extension Transaction { mutating func responseBodyIteratorDeinited(streamID: HTTPClientResponse.Body.IteratorStream.ID) -> FailAction { switch self.state { - case .initialized, .queued, .executing(_, _, .waitingForResponseHead): + case .initialized, .queued, .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): preconditionFailure("Got notice about a deinited response body iterator, before we even received a response. Invalid state: \(self.state)") case .executing(_, _, .buffering(let registeredStreamID, _, next: _)), @@ -516,6 +539,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): preconditionFailure("If we receive a response body, we must have received a head before") @@ -635,6 +659,7 @@ extension Transaction { switch self.state { case .initialized, .queued, + .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): preconditionFailure("Received no response head, but received a response end. Invalid state: \(self.state)") @@ -677,6 +702,7 @@ extension Transaction { enum DeadlineExceededAction { case none + case cancelSchedulerOnly(scheduler: HTTPRequestScheduler) /// fail response before head received. scheduler and executor are exclusive here. case cancel( requestContinuation: CheckedContinuation, @@ -699,14 +725,12 @@ extension Transaction { ) case .queued(let continuation, let scheduler): - self.state = .finished(error: error, nil) - return .cancel( - requestContinuation: continuation, - scheduler: scheduler, - executor: nil, - bodyStreamContinuation: nil + self.state = .deadlineExceededWhileQueued(continuation) + return .cancelSchedulerOnly( + scheduler: scheduler ) - + case .deadlineExceededWhileQueued: + return .none case .executing(let context, let requestStreamState, .waitingForResponseHead): switch requestStreamState { case .paused(continuation: .some(let continuation)): diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 8830406b4..3b6db1e38 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -174,7 +174,9 @@ extension Transaction: HTTPExecutableRequest { switch action { case .cancel(let executor): executor.cancelRequest(self) - + case .cancelAndFail(let executor, let continuation, with: let error): + executor.cancelRequest(self) + continuation.resume(throwing: error) case .none: break } @@ -309,7 +311,8 @@ extension Transaction: HTTPExecutableRequest { scheduler?.cancelRequest(self) executor?.cancelRequest(self) bodyStreamContinuation?.resume(throwing: HTTPClientError.deadlineExceeded) - + case .cancelSchedulerOnly(scheduler: let scheduler): + scheduler.cancelRequest(self) case .none: break } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 6a8d923c7..fc6de5480 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -38,6 +38,7 @@ extension AsyncAwaitEndToEndTests { ("testCanceling", testCanceling), ("testDeadline", testDeadline), ("testImmediateDeadline", testImmediateDeadline), + ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), ("testInvalidURL", testInvalidURL), ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), ("testShutdown", testShutdown), diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 2cd056225..41a2d0798 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -16,6 +16,7 @@ import Logging import NIOCore import NIOPosix +import NIOSSL import XCTest private func makeDefaultHTTPClient( @@ -393,6 +394,53 @@ final class AsyncAwaitEndToEndTests: XCTestCase { #endif } + func testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded() { + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest(timeout: 5) { + /// key + cert was created with the follwing command: + /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' + let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! + let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let configuration = TLSConfiguration.makeServerConfiguration( + certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, + privateKey: .file(keyPath) + ) + let sslContext = try NIOSSLContext(configuration: configuration) + let serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { XCTAssertNoThrow(try serverGroup.syncShutdownGracefully()) } + let server = ServerBootstrap(group: serverGroup) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + } + let serverChannel = try server.bind(host: "localhost", port: 0).wait() + defer { XCTAssertNoThrow(try serverChannel.close().wait()) } + let port = serverChannel.localAddress!.port! + + var config = HTTPClient.Configuration() + config.timeout.connect = .seconds(3) + let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) + defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + let request = HTTPClientRequest(url: "https://localhost:\(port)") + await XCTAssertThrowsError(try await localClient.execute(request, deadline: .now() + .seconds(2))) { error in + #if canImport(Network) + guard let nwTLSError = error as? HTTPClient.NWTLSError else { + XCTFail("could not cast \(error) of type \(type(of: error)) to \(HTTPClient.NWTLSError.self)") + return + } + XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") + #else + guard let sslError = error as? NIOSSLError, + case .handshakeFailed(.sslError) = sslError else { + XCTFail("unexpected error \(error)") + return + } + #endif + } + } + #endif + } + func testInvalidURL() { #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift index a46c7dfc0..f86344137 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift @@ -28,7 +28,9 @@ extension Transaction_StateMachineTests { ("testRequestWasQueuedAfterWillExecuteRequestWasCalled", testRequestWasQueuedAfterWillExecuteRequestWasCalled), ("testRequestBodyStreamWasPaused", testRequestBodyStreamWasPaused), ("testQueuedRequestGetsRemovedWhenDeadlineExceeded", testQueuedRequestGetsRemovedWhenDeadlineExceeded), + ("testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect", testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect), ("testScheduledRequestGetsRemovedWhenDeadlineExceeded", testScheduledRequestGetsRemovedWhenDeadlineExceeded), + ("testDeadlineExceededRaceWithRequestWillExecute", testDeadlineExceededRaceWithRequestWillExecute), ("testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded", testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded), ] } diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift index ff1972330..8b6985386 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift @@ -73,6 +73,7 @@ final class Transaction_StateMachineTests: XCTestCase { } func testQueuedRequestGetsRemovedWhenDeadlineExceeded() { + struct MyError: Error, Equatable {} #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { @@ -82,16 +83,62 @@ final class Transaction_StateMachineTests: XCTestCase { state.requestWasQueued(queuer) - let failAction = state.deadlineExceeded() - guard case .cancel(let continuation, let scheduler, nil, nil) = failAction else { + let deadlineExceededAction = state.deadlineExceeded() + guard case .cancelSchedulerOnly(let scheduler) = deadlineExceededAction else { + return XCTFail("Unexpected fail action: \(deadlineExceededAction)") + } + XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) + + let failAction = state.fail(MyError()) + guard case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = failAction else { return XCTFail("Unexpected fail action: \(failAction)") } XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) - continuation.resume(throwing: HTTPClientError.deadlineExceeded) + continuation.resume(throwing: error) } - await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) + await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) { + XCTAssertEqualTypeAndValue($0, MyError()) + } + } + #endif + } + + func testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect() { + struct MyError: Error, Equatable {} + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + func workaround(_ continuation: CheckedContinuation) { + var state = Transaction.StateMachine(continuation) + let queuer = MockTaskQueuer() + + state.requestWasQueued(queuer) + + let deadlineExceededAction = state.deadlineExceeded() + guard case .cancelSchedulerOnly(let scheduler) = deadlineExceededAction else { + return XCTFail("Unexpected fail action: \(deadlineExceededAction)") + } + XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) + + let failAction = state.fail(MyError()) + guard case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = failAction else { + return XCTFail("Unexpected fail action: \(failAction)") + } + XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) + + let secondFailAction = state.fail(HTTPClientError.cancelled) + guard case .none = secondFailAction else { + return XCTFail("Unexpected fail action: \(secondFailAction)") + } + + continuation.resume(throwing: error) + } + + await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) { + XCTAssertEqualTypeAndValue($0, MyError()) + } } #endif } @@ -123,6 +170,40 @@ final class Transaction_StateMachineTests: XCTestCase { #endif } + func testDeadlineExceededRaceWithRequestWillExecute() { + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + let eventLoop = EmbeddedEventLoop() + XCTAsyncTest { + func workaround(_ continuation: CheckedContinuation) { + var state = Transaction.StateMachine(continuation) + let expectedExecutor = MockRequestExecutor(eventLoop: eventLoop) + let queuer = MockTaskQueuer() + + state.requestWasQueued(queuer) + + let deadlineExceededAction = state.deadlineExceeded() + guard case .cancelSchedulerOnly(let scheduler) = deadlineExceededAction else { + return XCTFail("Unexpected fail action: \(deadlineExceededAction)") + } + XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) + + let failAction = state.willExecuteRequest(expectedExecutor) + guard case .cancelAndFail(let returnedExecutor, let continuation, with: let error) = failAction else { + return XCTFail("Unexpected fail action: \(failAction)") + } + XCTAssertIdentical(returnedExecutor as? MockRequestExecutor, expectedExecutor) + + continuation.resume(throwing: error) + } + + await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) { + XCTAssertEqualTypeAndValue($0, HTTPClientError.deadlineExceeded) + } + } + #endif + } + func testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded() { #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } From 46d1c76715ee01a917e1a252b43130f1baf0c8a5 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 5 Aug 2022 17:04:57 +0200 Subject: [PATCH 025/146] Support transparent decompression with HTTP/2 (#610) --- .../HTTP2/HTTP2Connection.swift | 15 +++++-- .../HTTPClientTestUtils.swift | 3 ++ .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 43 +++++++++++++++++++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index 8eb189adc..f9854a810 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -15,6 +15,7 @@ import Logging import NIOCore import NIOHTTP2 +import NIOHTTPCompression protocol HTTP2ConnectionDelegate { func http2Connection(_: HTTP2Connection, newMaxStreamSetting: Int) @@ -79,6 +80,7 @@ final class HTTP2Connection { /// request. private var openStreams = Set() let id: HTTPConnectionPool.Connection.ID + let decompression: HTTPClient.Decompression var closeFuture: EventLoopFuture { self.channel.closeFuture @@ -86,10 +88,12 @@ final class HTTP2Connection { init(channel: Channel, connectionID: HTTPConnectionPool.Connection.ID, + decompression: HTTPClient.Decompression, delegate: HTTP2ConnectionDelegate, logger: Logger) { self.channel = channel self.id = connectionID + self.decompression = decompression self.logger = logger self.multiplexer = HTTP2StreamMultiplexer( mode: .client, @@ -118,7 +122,7 @@ final class HTTP2Connection { configuration: HTTPClient.Configuration, logger: Logger ) -> EventLoopFuture<(HTTP2Connection, Int)> { - let connection = HTTP2Connection(channel: channel, connectionID: connectionID, delegate: delegate, logger: logger) + let connection = HTTP2Connection(channel: channel, connectionID: connectionID, decompression: configuration.decompression, delegate: delegate, logger: logger) return connection.start().map { maxStreams in (connection, maxStreams) } } @@ -208,9 +212,14 @@ final class HTTP2Connection { // We only support http/2 over an https connection โ€“ using the Application-Layer // Protocol Negotiation (ALPN). For this reason it is safe to fix this to `.https`. let translate = HTTP2FramePayloadToHTTP1ClientCodec(httpProtocol: .https) - let handler = HTTP2ClientRequestHandler(eventLoop: channel.eventLoop) - try channel.pipeline.syncOperations.addHandler(translate) + + if case .enabled(let limit) = self.decompression { + let decompressHandler = NIOHTTPResponseDecompressor(limit: limit) + try channel.pipeline.syncOperations.addHandler(decompressHandler) + } + + let handler = HTTP2ClientRequestHandler(eventLoop: channel.eventLoop) try channel.pipeline.syncOperations.addHandler(handler) // We must add the new channel to the list of open channels BEFORE we write the diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 3a7f1fe90..59336d39f 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -539,6 +539,9 @@ internal final class HTTPBin where let sync = channel.pipeline.syncOperations try sync.addHandler(HTTP2FramePayloadToHTTP1ServerCodec()) + if self.mode.compress { + try sync.addHandler(HTTPResponseCompressor()) + } try sync.addHandler(self.handlerFactory(connectionID)) return channel.eventLoop.makeSucceededVoidFuture() diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index b3a13486c..421060b2e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -66,6 +66,7 @@ extension HTTPClientTests { ("testUploadStreaming", testUploadStreaming), ("testEventLoopArgument", testEventLoopArgument), ("testDecompression", testDecompression), + ("testDecompressionHTTP2", testDecompressionHTTP2), ("testDecompressionLimit", testDecompressionLimit), ("testLoopDetectionRedirectLimit", testLoopDetectionRedirectLimit), ("testCountRedirectLimit", testCountRedirectLimit), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 610df95c9..8918ea042 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -890,6 +890,49 @@ class HTTPClientTests: XCTestCase { } } + func testDecompressionHTTP2() throws { + let localHTTPBin = HTTPBin(.http2(compress: true)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init( + certificateVerification: .none, + decompression: .enabled(limit: .none) + ) + ) + + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + + var body = "" + for _ in 1...1000 { + body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." + } + + for algorithm: String? in [nil] { + var request = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/post", method: .POST) + request.body = .string(body) + if let algorithm = algorithm { + request.headers.add(name: "Accept-Encoding", value: algorithm) + } + + let response = try localClient.execute(request: request).wait() + var responseBody = try XCTUnwrap(response.body) + let data = try responseBody.readJSONDecodable(RequestInfo.self, length: responseBody.readableBytes) + + XCTAssertEqual(.ok, response.status) + let contentLength = try XCTUnwrap(response.headers["Content-Length"].first.flatMap { Int($0) }) + XCTAssertGreaterThan(body.count, contentLength) + if let algorithm = algorithm { + XCTAssertEqual(algorithm, response.headers["Content-Encoding"].first) + } else { + XCTAssertEqual("deflate", response.headers["Content-Encoding"].first) + } + XCTAssertEqual(body, data?.data) + } + } + func testDecompressionLimit() throws { let localHTTPBin = HTTPBin(.http1_1(compress: true)) let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(decompression: .enabled(limit: .ratio(1)))) From df87a860fdc41a595d5ca67f74cde9adbccc099a Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Fri, 5 Aug 2022 17:53:21 +0100 Subject: [PATCH 026/146] Limit max recursion depth delivering body parts (#611) Motivation When receiving certain patterns of response body parts, we can end up recursing almost indefinitely to deliver them to the application. This can lead to crashes, so we might politely describe it as "sub-optimal". Modifications Keep track of our stack depth and avoid creating too many stack frames. Added some unit tests. Result We no longer explode when handling bodies with lots of tiny parts. Co-authored-by: David Nadoba --- Sources/AsyncHTTPClient/RequestBag.swift | 41 +++++++++++++++++-- .../HTTP2ClientTests+XCTest.swift | 1 + .../HTTP2ClientTests.swift | 13 ++++++ .../HTTPClientTestUtils.swift | 19 +++++++++ .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 9 ++++ 6 files changed, 81 insertions(+), 3 deletions(-) diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 4ec7004c1..9c45728b7 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -19,6 +19,14 @@ import NIOHTTP1 import NIOSSL final class RequestBag { + /// Defends against the call stack getting too large when consuming body parts. + /// + /// If the response body comes in lots of tiny chunks, we'll deliver those tiny chunks to users + /// one at a time. + private static var maxConsumeBodyPartStackDepth: Int { + 50 + } + let task: HTTPClient.Task var eventLoop: EventLoop { self.task.eventLoop @@ -30,6 +38,9 @@ final class RequestBag { // the request state is synchronized on the task eventLoop private var state: StateMachine + // the consume body part stack depth is synchronized on the task event loop. + private var consumeBodyPartStackDepth: Int + // MARK: HTTPClientTask properties var logger: Logger { @@ -55,6 +66,7 @@ final class RequestBag { self.eventLoopPreference = eventLoopPreference self.task = task self.state = .init(redirectHandler: redirectHandler) + self.consumeBodyPartStackDepth = 0 self.request = request self.connectionDeadline = connectionDeadline self.requestOptions = requestOptions @@ -290,16 +302,39 @@ final class RequestBag { private func consumeMoreBodyData0(resultOfPreviousConsume result: Result) { self.task.eventLoop.assertInEventLoop() + // We get defensive here about the maximum stack depth. It's possible for the `didReceiveBodyPart` + // future to be returned to us completed. If it is, we will recurse back into this method. To + // break that recursion we have a max stack depth which we increment and decrement in this method: + // if it gets too large, instead of recurring we'll insert an `eventLoop.execute`, which will + // manually break the recursion and unwind the stack. + // + // Note that we don't bother starting this at the various other call sites that _begin_ stacks + // that risk ending up in this loop. That's because we don't need an accurate count: our limit is + // a best-effort target anyway, one stack frame here or there does not put us at risk. We're just + // trying to prevent ourselves looping out of control. + self.consumeBodyPartStackDepth += 1 + defer { + self.consumeBodyPartStackDepth -= 1 + assert(self.consumeBodyPartStackDepth >= 0) + } + let consumptionAction = self.state.consumeMoreBodyData(resultOfPreviousConsume: result) switch consumptionAction { case .consume(let byteBuffer): self.delegate.didReceiveBodyPart(task: self.task, byteBuffer) .hop(to: self.task.eventLoop) - .whenComplete { - switch $0 { + .whenComplete { result in + switch result { case .success: - self.consumeMoreBodyData0(resultOfPreviousConsume: $0) + if self.consumeBodyPartStackDepth < Self.maxConsumeBodyPartStackDepth { + self.consumeMoreBodyData0(resultOfPreviousConsume: result) + } else { + // We need to unwind the stack, let's take a break. + self.task.eventLoop.execute { + self.consumeMoreBodyData0(resultOfPreviousConsume: result) + } + } case .failure(let error): self.fail(error) } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift index e7f399658..915791cdf 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift @@ -37,6 +37,7 @@ extension HTTP2ClientTests { ("testH2CanHandleRequestsThatHaveAlreadyHitTheDeadline", testH2CanHandleRequestsThatHaveAlreadyHitTheDeadline), ("testStressCancelingRunningRequestFromDifferentThreads", testStressCancelingRunningRequestFromDifferentThreads), ("testPlatformConnectErrorIsForwardedOnTimeout", testPlatformConnectErrorIsForwardedOnTimeout), + ("testMassiveDownload", testMassiveDownload), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index eb1ac2ddc..7c0e1e56f 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -432,6 +432,19 @@ class HTTP2ClientTests: XCTestCase { ) } } + + func testMassiveDownload() { + let bin = HTTPBin(.http2(compress: false)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = self.makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + var response: HTTPClient.Response? + XCTAssertNoThrow(response = try client.get(url: "https://localhost:\(bin.port)/mega-chunked").wait()) + + XCTAssertEqual(.ok, response?.status) + XCTAssertEqual(response?.version, .http2) + XCTAssertEqual(response?.body?.readableBytes, 10_000) + } } private final class HeadReceivedCallback: HTTPClientResponseDelegate { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 59336d39f..8f7d4dfce 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -745,6 +745,22 @@ internal final class HTTPBinHandler: ChannelInboundHandler { context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) } + func writeManyChunks(context: ChannelHandlerContext) { + // This tests receiving a lot of tiny chunks: they must all be sent in a single flush or the test doesn't work. + let headers = HTTPHeaders([("Transfer-Encoding", "chunked")]) + + context.write(self.wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + let message = ByteBuffer(integer: UInt8(ascii: "a")) + + // This number (10k) is load-bearing and a bit magic: it has been experimentally verified as being sufficient to blow the stack + // in the old implementation on all testing platforms. Please don't change it without good reason. + for _ in 0..<10_000 { + context.write(wrapOutboundOut(.body(.byteBuffer(message))), promise: nil) + } + + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + } + func channelRead(context: ChannelHandlerContext, data: NIOAny) { self.isServingRequest = true switch self.unwrapInboundIn(data) { @@ -863,6 +879,9 @@ internal final class HTTPBinHandler: ChannelInboundHandler { case "/chunked": self.writeChunked(context: context) return + case "/mega-chunked": + self.writeManyChunks(context: context) + return case "/close-on-response": var headers = self.responseHeaders headers.replaceOrAdd(name: "connection", value: "close") diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 421060b2e..655e3acc5 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -142,6 +142,7 @@ extension HTTPClientTests { ("testRequestSpecificTLS", testRequestSpecificTLS), ("testConnectionPoolSizeConfigValueIsRespected", testConnectionPoolSizeConfigValueIsRespected), ("testRequestWithHeaderTransferEncodingIdentityDoesNotFail", testRequestWithHeaderTransferEncodingIdentityDoesNotFail), + ("testMassiveDownload", testMassiveDownload), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8918ea042..e2e34cf00 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3454,4 +3454,13 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try client.execute(request: request).wait()) } + + func testMassiveDownload() { + var response: HTTPClient.Response? + XCTAssertNoThrow(response = try self.defaultClient.get(url: "\(self.defaultHTTPBinURLPrefix)mega-chunked").wait()) + + XCTAssertEqual(.ok, response?.status) + XCTAssertEqual(response?.version, .http1_1) + XCTAssertEqual(response?.body?.readableBytes, 10_000) + } } From 5e3e58dafd61e3b77e6cf17a840b94addd2be7df Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 9 Aug 2022 16:23:14 +0100 Subject: [PATCH 027/146] Use Docc for documentation (#613) Motivation Documentation is nice, and we can help support users by providing useful clear docs. Modifications Add Docc to 5.6 and later builds Make sure symbol references work Add overview docs Result Nice rendering docs --- Package.swift | 3 +- Package@swift-5.4.swift | 74 ++++ Package@swift-5.5.swift | 74 ++++ .../AsyncAwait/HTTPClientRequest.swift | 83 ++++- .../AsyncAwait/HTTPClientResponse.swift | 15 + Sources/AsyncHTTPClient/Docs.docc/index.md | 349 ++++++++++++++++++ .../FileDownloadDelegate.swift | 1 + Sources/AsyncHTTPClient/HTTPClient.swift | 33 +- Sources/AsyncHTTPClient/HTTPHandler.swift | 105 ++++-- .../NIOTransportServices/NWErrorHandler.swift | 4 +- Sources/AsyncHTTPClient/Utils.swift | 4 + docker/docker-compose.1804.54.yaml | 1 + docker/docker-compose.2004.55.yaml | 1 + docker/docker-compose.yaml | 2 +- scripts/soundness.sh | 2 +- 15 files changed, 698 insertions(+), 53 deletions(-) create mode 100644 Package@swift-5.4.swift create mode 100644 Package@swift-5.5.swift create mode 100644 Sources/AsyncHTTPClient/Docs.docc/index.md diff --git a/Package.swift b/Package.swift index a9ea2ba7f..a60f012aa 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.4 +// swift-tools-version:5.6 //===----------------------------------------------------------------------===// // // This source file is part of the AsyncHTTPClient open source project @@ -28,6 +28,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), + .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), ], targets: [ .target(name: "CAsyncHTTPClient"), diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift new file mode 100644 index 000000000..a9ea2ba7f --- /dev/null +++ b/Package@swift-5.4.swift @@ -0,0 +1,74 @@ +// swift-tools-version:5.4 +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "async-http-client", + products: [ + .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-nio.git", from: "2.38.0"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.1"), + .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), + .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), + .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), + ], + targets: [ + .target(name: "CAsyncHTTPClient"), + .target( + name: "AsyncHTTPClient", + dependencies: [ + .target(name: "CAsyncHTTPClient"), + .product(name: "NIO", package: "swift-nio"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOPosix", package: "swift-nio"), + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + .product(name: "NIOHTTP2", package: "swift-nio-http2"), + .product(name: "NIOSSL", package: "swift-nio-ssl"), + .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), + .product(name: "NIOSOCKS", package: "swift-nio-extras"), + .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), + ] + ), + .testTarget( + name: "AsyncHTTPClientTests", + dependencies: [ + .target(name: "AsyncHTTPClient"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOEmbedded", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + .product(name: "NIOTestUtils", package: "swift-nio"), + .product(name: "NIOSSL", package: "swift-nio-ssl"), + .product(name: "NIOHTTP2", package: "swift-nio-http2"), + .product(name: "NIOSOCKS", package: "swift-nio-extras"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), + ], + resources: [ + .copy("Resources/self_signed_cert.pem"), + .copy("Resources/self_signed_key.pem"), + ] + ), + ] +) diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift new file mode 100644 index 000000000..a9ea2ba7f --- /dev/null +++ b/Package@swift-5.5.swift @@ -0,0 +1,74 @@ +// swift-tools-version:5.4 +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "async-http-client", + products: [ + .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), + ], + dependencies: [ + .package(url: "https://github.com/apple/swift-nio.git", from: "2.38.0"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.1"), + .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), + .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), + .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), + ], + targets: [ + .target(name: "CAsyncHTTPClient"), + .target( + name: "AsyncHTTPClient", + dependencies: [ + .target(name: "CAsyncHTTPClient"), + .product(name: "NIO", package: "swift-nio"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOPosix", package: "swift-nio"), + .product(name: "NIOHTTP1", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + .product(name: "NIOHTTP2", package: "swift-nio-http2"), + .product(name: "NIOSSL", package: "swift-nio-ssl"), + .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), + .product(name: "NIOSOCKS", package: "swift-nio-extras"), + .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), + ] + ), + .testTarget( + name: "AsyncHTTPClientTests", + dependencies: [ + .target(name: "AsyncHTTPClient"), + .product(name: "NIOCore", package: "swift-nio"), + .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), + .product(name: "NIOEmbedded", package: "swift-nio"), + .product(name: "NIOFoundationCompat", package: "swift-nio"), + .product(name: "NIOTestUtils", package: "swift-nio"), + .product(name: "NIOSSL", package: "swift-nio-ssl"), + .product(name: "NIOHTTP2", package: "swift-nio-http2"), + .product(name: "NIOSOCKS", package: "swift-nio-extras"), + .product(name: "Logging", package: "swift-log"), + .product(name: "Atomics", package: "swift-atomics"), + ], + resources: [ + .copy("Resources/self_signed_cert.pem"), + .copy("Resources/self_signed_key.pem"), + ] + ), + ] +) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index cfab828a0..a76b9239a 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -16,12 +16,21 @@ import NIOCore import NIOHTTP1 +/// A representation of an HTTP request for the Swift Concurrency HTTPClient API. +/// +/// This object is similar to ``HTTPClient/Request``, but used for the Swift Concurrency API. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) public struct HTTPClientRequest { + /// The request URL, including scheme, hostname, and optionally port. public var url: String + + /// The request method. public var method: HTTPMethod + + /// The request headers. public var headers: HTTPHeaders + /// The request body, if any. public var body: Body? public init(url: String) { @@ -34,6 +43,10 @@ public struct HTTPClientRequest { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest { + /// An HTTP request body. + /// + /// This object encapsulates the difference between streamed HTTP request bodies and those bodies that + /// are already entirely in memory. public struct Body { @usableFromInline internal enum Mode { @@ -54,10 +67,20 @@ extension HTTPClientRequest { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Body { + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `ByteBuffer`. + /// + /// - parameter byteBuffer: The bytes of the body. public static func bytes(_ byteBuffer: ByteBuffer) -> Self { self.init(.byteBuffer(byteBuffer)) } + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// - parameter bytes: The bytes of the request body. @inlinable public static func bytes( _ bytes: Bytes @@ -75,6 +98,23 @@ extension HTTPClientRequest.Body { }) } + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// Unlike ``bytes(_:)-1uns7``, this construction does not assume that the body can be replayed. As a result, + /// if a redirect is encountered that would need us to replay the request body, the redirect will instead + /// not be followed. Prefer ``bytes(_:)-1uns7`` wherever possible. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. @inlinable public static func bytes( _ bytes: Bytes, @@ -93,6 +133,19 @@ extension HTTPClientRequest.Body { }) } + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. @inlinable public static func bytes( _ bytes: Bytes, @@ -111,6 +164,17 @@ extension HTTPClientRequest.Body { }) } + /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. + /// + /// This construction will stream the upload one `ByteBuffer` at a time. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - sequenceOfBytes: The bytes of the request body. + /// - length: The length of the request body. @inlinable public static func stream( _ sequenceOfBytes: SequenceOfBytes, @@ -123,6 +187,19 @@ extension HTTPClientRequest.Body { return body } + /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. + /// + /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for + /// `AsyncSequence`s where larger chunks are buffered up and available without actually suspending, such + /// as those provided by `FileHandle`. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. @inlinable public static func stream( _ bytes: Bytes, @@ -157,10 +234,12 @@ extension Optional where Wrapped == HTTPClientRequest.Body { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Body { + /// The length of a HTTP request body. public struct Length { - /// size of the request body is not known before starting the request + /// The size of the request body is not known before starting the request public static let unknown: Self = .init(storage: .unknown) - /// size of the request body is fixed and exactly `count` bytes + + /// The size of the request body is known and exactly `count` bytes public static func known(_ count: Int) -> Self { .init(storage: .known(count)) } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index 52f03089b..7ccd74530 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -16,13 +16,28 @@ import NIOCore import NIOHTTP1 +/// A representation of an HTTP response for the Swift Concurrency HTTPClient API. +/// +/// This object is similar to ``HTTPClient/Response``, but used for the Swift Concurrency API. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) public struct HTTPClientResponse { + /// The HTTP version on which the response was received. public var version: HTTPVersion + + /// The HTTP status for this response. public var status: HTTPResponseStatus + + /// The HTTP headers of this response. public var headers: HTTPHeaders + + /// The body of this HTTP response. public var body: Body + /// A representation of the response body for an HTTP response. + /// + /// The body is streamed as an `AsyncSequence` of `ByteBuffer`, where each `ByteBuffer` contains + /// an arbitrarily large chunk of data. The boundaries between `ByteBuffer` objects in the sequence + /// are entirely synthetic and have no semantic meaning. public struct Body { private let bag: Transaction private let reference: ResponseRef diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md new file mode 100644 index 000000000..acb408684 --- /dev/null +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -0,0 +1,349 @@ +# ``AsyncHTTPClient`` + +This package provides simple HTTP Client library built on top of SwiftNIO. + +This library provides the following: +- First class support for Swift Concurrency (since version 1.9.0) +- Asynchronous and non-blocking request methods +- Simple follow-redirects (cookie headers are dropped) +- Streaming body download +- TLS support +- Automatic HTTP/2 over HTTPS (since version 1.7.0) +- Cookie parsing (but not storage) + +--- + +**NOTE**: You will need [Xcode 13.2](https://apps.apple.com/gb/app/xcode/id497799835?mt=12) or [Swift 5.5.2](https://swift.org/download/#swift-552) to try out `AsyncHTTPClient`s new async/await APIs. + +--- + +## Getting Started + +#### Adding the dependency + +Add the following entry in your Package.swift to start using HTTPClient: + +```swift +.package(url: "https://github.com/swift-server/async-http-client.git", from: "1.9.0") +``` +and `AsyncHTTPClient` dependency to your target: +```swift +.target(name: "MyApp", dependencies: [.product(name: "AsyncHTTPClient", package: "async-http-client")]), +``` + +#### Request-Response API + +The code snippet below illustrates how to make a simple GET request to a remote server. + +Please note that the example will spawn a new `EventLoopGroup` which will _create fresh threads_ which is a very costly operation. In a real-world application that uses [SwiftNIO](https://github.com/apple/swift-nio) for other parts of your application (for example a web server), please prefer `eventLoopGroupProvider: .shared(myExistingEventLoopGroup)` to share the `EventLoopGroup` used by AsyncHTTPClient with other parts of your application. + +If your application does not use SwiftNIO yet, it is acceptable to use `eventLoopGroupProvider: .createNew` but please make sure to share the returned `HTTPClient` instance throughout your whole application. Do not create a large number of `HTTPClient` instances with `eventLoopGroupProvider: .createNew`, this is very wasteful and might exhaust the resources of your program. + +```swift +import AsyncHTTPClient + +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) + +/// MARK: - Using Swift Concurrency +let request = HTTPClientRequest(url: "https://apple.com/") +let response = try await httpClient.execute(request, timeout: .seconds(30)) +print("HTTP head", response) +if response.status == .ok { + let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + // handle body +} else { + // handle remote error +} + + +/// MARK: - Using SwiftNIO EventLoopFuture +httpClient.get(url: "https://apple.com/").whenComplete { result in + switch result { + case .failure(let error): + // process error + case .success(let response): + if response.status == .ok { + // handle response + } else { + // handle remote error + } + } +} +``` + +You should always shut down ``HTTPClient`` instances you created using ``HTTPClient/syncShutdown()``. Please note that you must not call ``HTTPClient/syncShutdown()`` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. + +### async/await examples + +Examples for the async/await API can be found in the [`Examples` folder](https://github.com/swift-server/async-http-client/tree/main/Examples) in the repository. + +## Usage guide + +The default HTTP Method is `GET`. In case you need to have more control over the method, or you want to add headers or body, use the ``HTTPClientRequest`` struct: + +#### Using Swift Concurrency + +```swift +import AsyncHTTPClient + +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +do { + var request = HTTPClientRequest(url: "https://apple.com/") + request.method = .POST + request.headers.add(name: "User-Agent", value: "Swift HTTPClient") + request.body = .bytes(ByteBuffer(string: "some data")) + + let response = try await httpClient.execute(request, timeout: .seconds(30)) + if response.status == .ok { + // handle response + } else { + // handle remote error + } +} catch { + // handle error +} +// it's important to shutdown the httpClient after all requests are done, even if one failed +try await httpClient.shutdown() +``` + +#### Using SwiftNIO EventLoopFuture + +```swift +import AsyncHTTPClient + +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +defer { + try? httpClient.syncShutdown() +} + +var request = try HTTPClient.Request(url: "https://apple.com/", method: .POST) +request.headers.add(name: "User-Agent", value: "Swift HTTPClient") +request.body = .string("some-body") + +httpClient.execute(request: request).whenComplete { result in + switch result { + case .failure(let error): + // process error + case .success(let response): + if response.status == .ok { + // handle response + } else { + // handle remote error + } + } +} +``` + +### Redirects following +Enable follow-redirects behavior using the client configuration: +```swift +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, + configuration: HTTPClient.Configuration(followRedirects: true)) +``` + +### Timeouts +Timeouts (connect and read) can also be set using the client configuration: +```swift +let timeout = HTTPClient.Configuration.Timeout(connect: .seconds(1), read: .seconds(1)) +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, + configuration: HTTPClient.Configuration(timeout: timeout)) +``` +or on a per-request basis: +```swift +httpClient.execute(request: request, deadline: .now() + .milliseconds(1)) +``` + +### Streaming +When dealing with larger amount of data, it's critical to stream the response body instead of aggregating in-memory. +The following example demonstrates how to count the number of bytes in a streaming response body: + +#### Using Swift Concurrency +```swift +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +do { + let request = HTTPClientRequest(url: "https://apple.com/") + let response = try await httpClient.execute(request, timeout: .seconds(30)) + print("HTTP head", response) + + // if defined, the content-length headers announces the size of the body + let expectedBytes = response.headers.first(name: "content-length").flatMap(Int.init) + + var receivedBytes = 0 + // asynchronously iterates over all body fragments + // this loop will automatically propagate backpressure correctly + for try await buffer in response.body { + // for this example, we are just interested in the size of the fragment + receivedBytes += buffer.readableBytes + + if let expectedBytes = expectedBytes { + // if the body size is known, we calculate a progress indicator + let progress = Double(receivedBytes) / Double(expectedBytes) + print("progress: \(Int(progress * 100))%") + } + } + print("did receive \(receivedBytes) bytes") +} catch { + print("request failed:", error) +} +// it is important to shutdown the httpClient after all requests are done, even if one failed +try await httpClient.shutdown() +``` + +#### Using HTTPClientResponseDelegate and SwiftNIO EventLoopFuture + +```swift +import NIOCore +import NIOHTTP1 + +class CountingDelegate: HTTPClientResponseDelegate { + typealias Response = Int + + var count = 0 + + func didSendRequestHead(task: HTTPClient.Task, _ head: HTTPRequestHead) { + // this is executed right after request head was sent, called once + } + + func didSendRequestPart(task: HTTPClient.Task, _ part: IOData) { + // this is executed when request body part is sent, could be called zero or more times + } + + func didSendRequest(task: HTTPClient.Task) { + // this is executed when request is fully sent, called once + } + + func didReceiveHead( + task: HTTPClient.Task, + _ head: HTTPResponseHead + ) -> EventLoopFuture { + // this is executed when we receive HTTP response head part of the request + // (it contains response code and headers), called once in case backpressure + // is needed, all reads will be paused until returned future is resolved + return task.eventLoop.makeSucceededFuture(()) + } + + func didReceiveBodyPart( + task: HTTPClient.Task, + _ buffer: ByteBuffer + ) -> EventLoopFuture { + // this is executed when we receive parts of the response body, could be called zero or more times + count += buffer.readableBytes + // in case backpressure is needed, all reads will be paused until returned future is resolved + return task.eventLoop.makeSucceededFuture(()) + } + + func didFinishRequest(task: HTTPClient.Task) throws -> Int { + // this is called when the request is fully read, called once + // this is where you return a result or throw any errors you require to propagate to the client + return count + } + + func didReceiveError(task: HTTPClient.Task, _ error: Error) { + // this is called when we receive any network-related error, called once + } +} + +let request = try HTTPClient.Request(url: "https://apple.com/") +let delegate = CountingDelegate() + +httpClient.execute(request: request, delegate: delegate).futureResult.whenSuccess { count in + print(count) +} +``` + +### File downloads + +Based on the `HTTPClientResponseDelegate` example above you can build more complex delegates, +the built-in `FileDownloadDelegate` is one of them. It allows streaming the downloaded data +asynchronously, while reporting the download progress at the same time, like in the following +example: + +```swift +let client = HTTPClient(eventLoopGroupProvider: .createNew) +let request = try HTTPClient.Request( + url: "https://swift.org/builds/development/ubuntu1804/latest-build.yml" +) + +let delegate = try FileDownloadDelegate(path: "/tmp/latest-build.yml", reportProgress: { + if let totalBytes = $0.totalBytes { + print("Total bytes count: \(totalBytes)") + } + print("Downloaded \($0.receivedBytes) bytes so far") +}) + +client.execute(request: request, delegate: delegate).futureResult + .whenSuccess { progress in + if let totalBytes = progress.totalBytes { + print("Final total bytes count: \(totalBytes)") + } + print("Downloaded finished with \(progress.receivedBytes) bytes downloaded") + } +``` + +### Unix Domain Socket Paths +Connecting to servers bound to socket paths is easy: +```swift +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +httpClient.execute( + .GET, + socketPath: "/tmp/myServer.socket", + urlPath: "/path/to/resource" +).whenComplete (...) +``` + +Connecting over TLS to a unix domain socket path is possible as well: +```swift +let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +httpClient.execute( + .POST, + secureSocketPath: "/tmp/myServer.socket", + urlPath: "/path/to/resource", + body: .string("hello") +).whenComplete (...) +``` + +Direct URLs can easily be constructed to be executed in other scenarios: +```swift +let socketPathBasedURL = URL( + httpURLWithSocketPath: "/tmp/myServer.socket", + uri: "/path/to/resource" +) +let secureSocketPathBasedURL = URL( + httpsURLWithSocketPath: "/tmp/myServer.socket", + uri: "/path/to/resource" +) +``` + +### Disabling HTTP/2 +The exclusive use of HTTP/1 is possible by setting ``HTTPClient/Configuration/httpVersion-swift.property`` to ``HTTPClient/Configuration/HTTPVersion-swift.struct/http1Only`` on the ``HTTPClient/Configuration``: +```swift +var configuration = HTTPClient.Configuration() +configuration.httpVersion = .http1Only +let client = HTTPClient( + eventLoopGroupProvider: .createNew, + configuration: configuration +) +``` + +## Security + +AsyncHTTPClient's security process is documented on [GitHub](https://github.com/swift-server/async-http-client/blob/main/SECURITY.md). + +## Topics + +### HTTPClient + +- ``HTTPClient`` +- ``HTTPClientRequest`` +- ``HTTPClientResponse`` + +### HTTP Client Delegates + +- ``HTTPClientResponseDelegate`` +- ``ResponseAccumulator`` +- ``FileDownloadDelegate`` +- ``HTTPClientCopyingDelegate`` + +### Errors + +- ``HTTPClientError`` diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index 6f046dce9..75f16f52a 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -38,6 +38,7 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { private var writeFuture: EventLoopFuture? /// Initializes a new file download delegate. + /// /// - parameters: /// - path: Path to a file you'd like to write the download to. /// - pool: A thread pool to use for asynchronous file I/O. diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 094a6d052..3fbdb1366 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -65,6 +65,9 @@ let globalRequestID = ManagedAtomic(0) /// try client.syncShutdown() /// ``` public class HTTPClient { + /// The `EventLoopGroup` in use by this ``HTTPClient``. + /// + /// All HTTP transactions will occur on loops owned by this group. public let eventLoopGroup: EventLoopGroup let eventLoopGroupProvider: EventLoopGroupProvider let configuration: Configuration @@ -74,7 +77,7 @@ public class HTTPClient { internal static let loggingDisabled = Logger(label: "AHC-do-not-log", factory: { _ in SwiftLogNoOpLogHandler() }) - /// Create an `HTTPClient` with specified `EventLoopGroup` provider and configuration. + /// Create an ``HTTPClient`` with specified `EventLoopGroup` provider and configuration. /// /// - parameters: /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. @@ -86,7 +89,7 @@ public class HTTPClient { backgroundActivityLogger: HTTPClient.loggingDisabled) } - /// Create an `HTTPClient` with specified `EventLoopGroup` provider and configuration. + /// Create an ``HTTPClient`` with specified `EventLoopGroup` provider and configuration. /// /// - parameters: /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. @@ -180,7 +183,9 @@ public class HTTPClient { } } - /// Shuts down the client and event loop gracefully. This function is clearly an outlier in that it uses a completion + /// Shuts down the client and event loop gracefully. + /// + /// This function is clearly an outlier in that it uses a completion /// callback instead of an EventLoopFuture. The reason for that is that NIO's EventLoopFutures will call back on an event loop. /// The virtue of this function is to shut the event loop down. To work around that we call back on a DispatchQueue /// instead. @@ -623,11 +628,11 @@ public class HTTPClient { return task } - /// `HTTPClient` configuration. + /// ``HTTPClient`` configuration. public struct Configuration { /// TLS configuration, defaults to `TLSConfiguration.makeClientConfiguration()`. public var tlsConfiguration: Optional - /// Enables following 3xx redirects automatically, defaults to `RedirectConfiguration()`. + /// Enables following 3xx redirects automatically. /// /// Following redirects are supported: /// - `301: Moved Permanently` @@ -638,7 +643,8 @@ public class HTTPClient { /// - `307: Temporary Redirect` /// - `308: Permanent Redirect` public var redirectConfiguration: RedirectConfiguration - /// Default client timeout, defaults to no `read` timeout and 10 seconds `connect` timeout. + /// Default client timeout, defaults to no ``Timeout-swift.struct/read`` timeout + /// and 10 seconds ``Timeout-swift.struct/connect`` timeout. public var timeout: Timeout /// Connection pool configuration. public var connectionPool: ConnectionPool @@ -653,10 +659,12 @@ public class HTTPClient { set {} } - /// is set to `.automatic` by default which will use HTTP/2 if run over https and the server supports it, otherwise HTTP/1 + /// What HTTP versions to use. + /// + /// Set to ``HTTPVersion-swift.struct/automatic`` by default which will use HTTP/2 if run over https and the server supports it, otherwise HTTP/1 public var httpVersion: HTTPVersion - /// Whether `HTTPClient` will let Network.framework sit in the `.waiting` state awaiting new network changes, or fail immediately. Defaults to `true`, + /// Whether ``HTTPClient`` will let Network.framework sit in the `.waiting` state awaiting new network changes, or fail immediately. Defaults to `true`, /// which is the recommended setting. Only set this to `false` when attempting to trigger a particular error path. public var networkFrameworkWaitForConnectivity: Bool @@ -755,11 +763,11 @@ public class HTTPClient { public enum EventLoopGroupProvider { /// `EventLoopGroup` will be provided by the user. Owner of this group is responsible for its lifecycle. case shared(EventLoopGroup) - /// `EventLoopGroup` will be created by the client. When `syncShutdown` is called, created `EventLoopGroup` will be shut down as well. + /// `EventLoopGroup` will be created by the client. When ``HTTPClient/syncShutdown()`` is called, the created `EventLoopGroup` will be shut down as well. case createNew } - /// Specifies how the library will treat event loop passed by the user. + /// Specifies how the library will treat the event loop passed by the user. public struct EventLoopPreference { enum Preference { /// Event Loop will be selected by the library. @@ -830,8 +838,7 @@ extension HTTPClient.Configuration { /// Create timeout. /// /// - parameters: - /// - connect: `connect` timeout. Will default to 10 seconds, if no value is - /// provided. See `var connectionCreationTimeout` + /// - connect: `connect` timeout. Will default to 10 seconds, if no value is provided. /// - read: `read` timeout. public init(connect: TimeAmount? = nil, read: TimeAmount? = nil) { self.connect = connect @@ -897,7 +904,7 @@ extension HTTPClient.Configuration { case automatic } - /// we only use HTTP/1, even if the server would supports HTTP/2 + /// We will only use HTTP/1, even if the server would supports HTTP/2 public static let http1Only: Self = .init(configuration: .http1Only) /// HTTP/2 is used if we connect to a server with HTTPS and the server supports HTTP/2, otherwise we use HTTP/1 diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index c1ce39632..aeef71ba4 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -20,13 +20,15 @@ import NIOHTTP1 import NIOSSL extension HTTPClient { - /// Represent request body. + /// A request body. public struct Body { - /// Chunk provider. + /// A streaming uploader. + /// + /// ``StreamWriter`` abstracts public struct StreamWriter { let closure: (IOData) -> EventLoopFuture - /// Create new StreamWriter + /// Create new ``HTTPClient/Body/StreamWriter`` /// /// - parameters: /// - closure: function that will be called to write actual bytes to the channel. @@ -43,7 +45,7 @@ extension HTTPClient { } } - /// Body size. if nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` + /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `length`. public var length: Int? /// Body chunk provider. @@ -65,7 +67,7 @@ extension HTTPClient { } } - /// Create and stream body using `StreamWriter`. + /// Create and stream body using ``StreamWriter``. /// /// - parameters: /// - length: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` @@ -97,7 +99,7 @@ extension HTTPClient { } } - /// Represent HTTP request. + /// Represents an HTTP request. public struct Request { /// Request HTTP method, defaults to `GET`. public let method: HTTPMethod @@ -226,7 +228,7 @@ extension HTTPClient { } } - /// Represent HTTP response. + /// Represents an HTTP response. public struct Response { /// Remote host of the request. public var host: String @@ -272,7 +274,7 @@ extension HTTPClient { } } - /// HTTP authentication + /// HTTP authentication. public struct Authorization: Hashable { private enum Scheme: Hashable { case Basic(String) @@ -285,18 +287,24 @@ extension HTTPClient { self.scheme = scheme } + /// HTTP basic auth. public static func basic(username: String, password: String) -> HTTPClient.Authorization { return .basic(credentials: Base64.encode(bytes: "\(username):\(password)".utf8)) } + /// HTTP basic auth. + /// + /// This version uses the raw string directly. public static func basic(credentials: String) -> HTTPClient.Authorization { return .init(scheme: .Basic(credentials)) } + /// HTTP bearer auth public static func bearer(tokens: String) -> HTTPClient.Authorization { return .init(scheme: .Bearer(tokens)) } + /// The header string for this auth field. public var headerValue: String { switch self.scheme { case .Basic(let credentials): @@ -308,6 +316,10 @@ extension HTTPClient { } } +/// The default ``HTTPClientResponseDelegate``. +/// +/// This ``HTTPClientResponseDelegate`` buffers a complete HTTP response in memory. It does not stream the response body in. +/// The resulting ``Response`` type is ``HTTPClient/Response``. public class ResponseAccumulator: HTTPClientResponseDelegate { public typealias Response = HTTPClient.Response @@ -385,32 +397,34 @@ public class ResponseAccumulator: HTTPClientResponseDelegate { } } -/// `HTTPClientResponseDelegate` allows an implementation to receive notifications about request processing and to control how response parts are processed. +/// ``HTTPClientResponseDelegate`` allows an implementation to receive notifications about request processing and to control how response parts are processed. +/// /// You can implement this protocol if you need fine-grained control over an HTTP request/response, for example, if you want to inspect the response /// headers before deciding whether to accept a response body, or if you want to stream your request body. Pass an instance of your conforming -/// class to the `HTTPClient.execute()` method and this package will call each delegate method appropriately as the request takes place./ +/// class to the ``HTTPClient/execute(request:delegate:eventLoop:deadline:)`` method and this package will call each delegate method appropriately as the request takes place. /// /// ### Backpressure /// -/// A `HTTPClientResponseDelegate` can be used to exert backpressure on the server response. This is achieved by way of the futures returned from -/// `didReceiveHead` and `didReceiveBodyPart`. The following functions are part of the "backpressure system" in the delegate: +/// A ``HTTPClientResponseDelegate`` can be used to exert backpressure on the server response. This is achieved by way of the futures returned from +/// ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` and ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``. +/// The following functions are part of the "backpressure system" in the delegate: /// -/// - `didReceiveHead` -/// - `didReceiveBodyPart` -/// - `didFinishRequest` -/// - `didReceiveError` +/// - ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` +/// - ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v`` +/// - ``HTTPClientResponseDelegate/didFinishRequest(task:)`` +/// - ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg`` /// -/// The first three methods are strictly _exclusive_, with that exclusivity managed by the futures returned by `didReceiveHead` and -/// `didReceiveBodyPart`. What this means is that until the returned future is completed, none of these three methods will be called -/// again. This allows delegates to rate limit the server to a capacity it can manage. `didFinishRequest` does not return a future, +/// The first three methods are strictly _exclusive_, with that exclusivity managed by the futures returned by ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` and +/// ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``. What this means is that until the returned future is completed, none of these three methods will be called +/// again. This allows delegates to rate limit the server to a capacity it can manage. ``HTTPClientResponseDelegate/didFinishRequest(task:)`` does not return a future, /// as we are expecting no more data from the server at this time. /// -/// `didReceiveError` is somewhat special: it signals the end of this regime. `didRecieveError` is not exclusive: it may be called at -/// any time, even if a returned future is not yet completed. `didReceiveError` is terminal, meaning that once it has been called none -/// of these four methods will be called again. This can be used as a signal to abandon all outstanding work. +/// ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg`` is somewhat special: it signals the end of this regime. ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg`` +/// is not exclusive: it may be called at any time, even if a returned future is not yet completed. ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg`` is terminal, meaning +/// that once it has been called none of these four methods will be called again. This can be used as a signal to abandon all outstanding work. /// /// - note: This delegate is strongly held by the `HTTPTaskHandler` -/// for the duration of the `Request` processing and will be +/// for the duration of the ``HTTPClient/Request`` processing and will be /// released together with the `HTTPTaskHandler` when channel is closed. /// Users of the library are not required to keep a reference to the /// object that implements this protocol, but may do so if needed. @@ -428,7 +442,7 @@ public protocol HTTPClientResponseDelegate: AnyObject { /// /// - parameters: /// - task: Current request context. - /// - part: Request body `Part`. + /// - part: Request body part. func didSendRequestPart(task: HTTPClient.Task, _ part: IOData) /// Called when the request is fully sent. Will be called once. @@ -451,7 +465,7 @@ public protocol HTTPClientResponseDelegate: AnyObject { /// You must return an `EventLoopFuture` that you complete when you have finished processing the body part. /// You can create an already succeeded future by calling `task.eventLoop.makeSucceededFuture(())`. /// - /// This function will not be called until the future returned by `didReceiveHead` has completed. + /// This function will not be called until the future returned by ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` has completed. /// /// This function will not be called for subsequent body parts until the previous future returned by a /// call to this function completes. @@ -464,19 +478,22 @@ public protocol HTTPClientResponseDelegate: AnyObject { /// Called when error was thrown during request execution. Will be called zero or one time only. Request processing will be stopped after that. /// - /// This function may be called at any time: it does not respect the backpressure exerted by `didReceiveHead` and `didReceiveBodyPart`. - /// All outstanding work may be cancelled when this is received. Once called, no further calls will be made to `didReceiveHead`, `didReceiveBodyPart`, - /// or `didFinishRequest`. + /// This function may be called at any time: it does not respect the backpressure exerted by ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` + /// and ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``. + /// All outstanding work may be cancelled when this is received. Once called, no further calls will be made to + /// ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd``, ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``, + /// or ``HTTPClientResponseDelegate/didFinishRequest(task:)``. /// /// - parameters: /// - task: Current request context. /// - error: Error that occured during response processing. func didReceiveError(task: HTTPClient.Task, _ error: Error) - /// Called when the complete HTTP request is finished. You must return an instance of your `Response` associated type. Will be called once, except if an error occurred. + /// Called when the complete HTTP request is finished. You must return an instance of your ``Response`` associated type. Will be called once, except if an error occurred. /// - /// This function will not be called until all futures returned by `didReceiveHead` and `didReceiveBodyPart` have completed. Once called, - /// no further calls will be made to `didReceiveHead`, `didReceiveBodyPart`, or `didReceiveError`. + /// This function will not be called until all futures returned by ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd`` and ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v`` + /// have completed. Once called, no further calls will be made to ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd``, ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``, + /// or ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg``. /// /// - parameters: /// - task: Current request context. @@ -485,20 +502,38 @@ public protocol HTTPClientResponseDelegate: AnyObject { } extension HTTPClientResponseDelegate { + /// Default implementation of ``HTTPClientResponseDelegate/didSendRequestHead(task:_:)-6khai``. + /// + /// By default, this does nothing. public func didSendRequestHead(task: HTTPClient.Task, _ head: HTTPRequestHead) {} + /// Default implementation of ``HTTPClientResponseDelegate/didSendRequestPart(task:_:)-4qxap``. + /// + /// By default, this does nothing. public func didSendRequestPart(task: HTTPClient.Task, _ part: IOData) {} + /// Default implementation of ``HTTPClientResponseDelegate/didSendRequest(task:)-3vqgm``. + /// + /// By default, this does nothing. public func didSendRequest(task: HTTPClient.Task) {} + /// Default implementation of ``HTTPClientResponseDelegate/didReceiveHead(task:_:)-9r4xd``. + /// + /// By default, this does nothing. public func didReceiveHead(task: HTTPClient.Task, _: HTTPResponseHead) -> EventLoopFuture { - return task.eventLoop.makeSucceededFuture(()) + return task.eventLoop.makeSucceededVoidFuture() } + /// Default implementation of ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``. + /// + /// By default, this does nothing. public func didReceiveBodyPart(task: HTTPClient.Task, _: ByteBuffer) -> EventLoopFuture { - return task.eventLoop.makeSucceededFuture(()) + return task.eventLoop.makeSucceededVoidFuture() } + /// Default implementation of ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg``. + /// + /// By default, this does nothing. public func didReceiveError(task: HTTPClient.Task, _: Error) {} } @@ -560,7 +595,9 @@ protocol HTTPClientTaskDelegate { } extension HTTPClient { - /// Response execution context. Will be created by the library and could be used for obtaining + /// Response execution context. + /// + /// Will be created by the library and could be used for obtaining /// `EventLoopFuture` of the execution or cancellation of the execution. public final class Task { /// The `EventLoop` the delegate will be executed on. diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift index f732c37f4..9796bc2af 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift @@ -21,6 +21,7 @@ import NIOTransportServices extension HTTPClient { #if canImport(Network) + /// A wrapper for `POSIX` errors thrown by `Network.framework`. public struct NWPOSIXError: Error, CustomStringConvertible { /// POSIX error code (enum) public let errorCode: POSIXErrorCode @@ -40,8 +41,9 @@ extension HTTPClient { public var description: String { return self.reason } } + /// A wrapper for TLS errors thrown by `Network.framework`. public struct NWTLSError: Error, CustomStringConvertible { - /// TLS error status. List of TLS errors can be found in + /// TLS error status. List of TLS errors can be found in `` public let status: OSStatus /// actual reason, in human readable form diff --git a/Sources/AsyncHTTPClient/Utils.swift b/Sources/AsyncHTTPClient/Utils.swift index f4154df3d..ed2819fd0 100644 --- a/Sources/AsyncHTTPClient/Utils.swift +++ b/Sources/AsyncHTTPClient/Utils.swift @@ -14,6 +14,10 @@ import NIOCore +/// An ``HTTPClientResponseDelegate`` that wraps a callback. +/// +/// ``HTTPClientCopyingDelegate`` discards most parts of a HTTP response, but streams the body +/// to the `chunkHandler` provided on ``init(chunkHandler:)``. This is mostly useful for testing. public final class HTTPClientCopyingDelegate: HTTPClientResponseDelegate { public typealias Response = Void diff --git a/docker/docker-compose.1804.54.yaml b/docker/docker-compose.1804.54.yaml index 660429851..dd869549c 100644 --- a/docker/docker-compose.1804.54.yaml +++ b/docker/docker-compose.1804.54.yaml @@ -11,6 +11,7 @@ services: test: image: async-http-client:18.04-5.4 + command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" environment: [] #- SANITIZER_ARG=--sanitize=thread diff --git a/docker/docker-compose.2004.55.yaml b/docker/docker-compose.2004.55.yaml index 4d0a12ee7..71b75c0fe 100644 --- a/docker/docker-compose.2004.55.yaml +++ b/docker/docker-compose.2004.55.yaml @@ -11,6 +11,7 @@ services: test: image: async-http-client:20.04-5.5 + command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" environment: [] #- SANITIZER_ARG=--sanitize=thread diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 6269e953b..75699c60b 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -28,7 +28,7 @@ services: test: <<: *common - command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" + command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-}" # util diff --git a/scripts/soundness.sh b/scripts/soundness.sh index da9a91d24..5170ec4f1 100755 --- a/scripts/soundness.sh +++ b/scripts/soundness.sh @@ -72,7 +72,7 @@ for language in swift-or-c bash dtrace; do matching_files=( -name '*' ) case "$language" in swift-or-c) - exceptions=( -name c_nio_http_parser.c -o -name c_nio_http_parser.h -o -name cpp_magic.h -o -name Package.swift -o -name CNIOSHA1.h -o -name c_nio_sha1.c -o -name ifaddrs-android.c -o -name ifaddrs-android.h) + exceptions=( -name c_nio_http_parser.c -o -name c_nio_http_parser.h -o -name cpp_magic.h -o -name Package.swift -o -name CNIOSHA1.h -o -name c_nio_sha1.c -o -name ifaddrs-android.c -o -name ifaddrs-android.h -o -name 'Package@swift*.swift' ) matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' ) cat > "$tmp" <<"EOF" //===----------------------------------------------------------------------===// From f90cda494ce3fe2308ef2dfe82e77c3d4d0e6e20 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 16 Aug 2022 10:00:09 +0100 Subject: [PATCH 028/146] Validate missing imports in CI (#615) --- docker/docker-compose.2004.main.yaml | 3 ++- docker/docker-compose.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.2004.main.yaml b/docker/docker-compose.2004.main.yaml index 11c7517ba..d6f04b917 100644 --- a/docker/docker-compose.2004.main.yaml +++ b/docker/docker-compose.2004.main.yaml @@ -10,7 +10,8 @@ services: test: image: async-http-client:20.04-main - environment: [] + environment: + - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread shell: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 75699c60b..803fad9b1 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -28,7 +28,7 @@ services: test: <<: *common - command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-}" + command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" # util From e294c8f28fc2336a80c53fa1bfbda035d38de67f Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 16 Aug 2022 13:17:45 +0100 Subject: [PATCH 029/146] Accurately apply the connect timeout in async code (#616) Motivation We should apply the connect timeout to the complete set of connection attempts, rather than the request deadline. This allows users fine-grained control over how long we attempt to connect for. This is also the behaviour of our old-school interface. Modifications - Changed the connect deadline calculation for async/await to match that of the future-based code. - Added a connect timeout test. Result Connect timeouts are properly handled --- .../AsyncAwait/HTTPClient+execute.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 80 +++++++++++++++++++ 4 files changed, 83 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index 043ad510b..318edbff9 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -134,7 +134,7 @@ extension HTTPClient { request: request, requestOptions: .init(idleReadTimeout: nil), logger: logger, - connectionDeadline: deadline, + connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), preferredEventLoop: eventLoop, responseContinuation: continuation ) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 3fbdb1366..9cf84a2cb 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -604,7 +604,7 @@ public class HTTPClient { eventLoopPreference: eventLoopPreference, task: task, redirectHandler: redirectHandler, - connectionDeadline: .now() + (self.configuration.timeout.connect ?? .seconds(10)), + connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), requestOptions: .fromClientConfiguration(self.configuration), delegate: delegate ) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index fc6de5480..397071e08 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -38,6 +38,7 @@ extension AsyncAwaitEndToEndTests { ("testCanceling", testCanceling), ("testDeadline", testDeadline), ("testImmediateDeadline", testImmediateDeadline), + ("testConnectTimeout", testConnectTimeout), ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), ("testInvalidURL", testInvalidURL), ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 41a2d0798..68457d4ff 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -34,6 +34,27 @@ private func makeDefaultHTTPClient( } final class AsyncAwaitEndToEndTests: XCTestCase { + var clientGroup: EventLoopGroup! + var serverGroup: EventLoopGroup! + + override func setUp() { + XCTAssertNil(self.clientGroup) + XCTAssertNil(self.serverGroup) + + self.clientGroup = getDefaultEventLoopGroup(numberOfThreads: 1) + self.serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + } + + override func tearDown() { + XCTAssertNotNil(self.clientGroup) + XCTAssertNoThrow(try self.clientGroup.syncShutdownGracefully()) + self.clientGroup = nil + + XCTAssertNotNil(self.serverGroup) + XCTAssertNoThrow(try self.serverGroup.syncShutdownGracefully()) + self.serverGroup = nil + } + func testSimpleGet() { #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } @@ -394,6 +415,65 @@ final class AsyncAwaitEndToEndTests: XCTestCase { #endif } + func testConnectTimeout() { + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest(timeout: 60) { + #if os(Linux) + // 198.51.100.254 is reserved for documentation only and therefore should not accept any TCP connection + let url = "http://198.51.100.254/get" + #else + // on macOS we can use the TCP backlog behaviour when the queue is full to simulate a non reachable server. + // this makes this test a bit more stable if `198.51.100.254` actually responds to connection attempt. + // The backlog behaviour on Linux can not be used to simulate a non-reachable server. + // Linux sends a `SYN/ACK` back even if the `backlog` queue is full as it has two queues. + // The second queue is not limit by `ChannelOptions.backlog` but by `/proc/sys/net/ipv4/tcp_max_syn_backlog`. + + let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { + XCTAssertNoThrow(try group.syncShutdownGracefully()) + } + + let serverChannel = try await ServerBootstrap(group: self.serverGroup) + .serverChannelOption(ChannelOptions.backlog, value: 1) + .serverChannelOption(ChannelOptions.autoRead, value: false) + .bind(host: "127.0.0.1", port: 0) + .get() + defer { + XCTAssertNoThrow(try serverChannel.close().wait()) + } + let port = serverChannel.localAddress!.port! + let firstClientChannel = try ClientBootstrap(group: self.serverGroup) + .connect(host: "127.0.0.1", port: port) + .wait() + defer { + XCTAssertNoThrow(try firstClientChannel.close().wait()) + } + let url = "http://localhost:\(port)/get" + #endif + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150)))) + + defer { + XCTAssertNoThrow(try httpClient.syncShutdown()) + } + + let request = HTTPClientRequest(url: url) + let start = NIODeadline.now() + await XCTAssertThrowsError(try await httpClient.execute(request, deadline: .now() + .seconds(30))) { + XCTAssertEqualTypeAndValue($0, HTTPClientError.connectTimeout) + let end = NIODeadline.now() + let duration = end - start + + // We give ourselves 10x slack in order to be confident that even on slow machines this assertion passes. + // It's 30x smaller than our other timeout though. + XCTAssertLessThan(duration, .seconds(1)) + } + } + #endif + } + func testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded() { #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } From 0469acb3bda524a7a416b86f8454f51f98a705c8 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 18 Aug 2022 11:39:55 +0200 Subject: [PATCH 030/146] Tollerate more data after request body is cancelled (#617) * Tollerate more data after request body is cancelled * wait is not needed if we shutdown the server first * Remove test that depends on external resources * Remove unused conformance to Equatable * SwiftFormat * run generate_linux_tests.rb * Increase timeout for CI --- .../AsyncAwait/Transaction+StateMachine.swift | 18 ++++++++++------- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 20 +++++++++++++++++++ 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 53bc8c8a6..38709c9a7 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -429,17 +429,20 @@ extension Transaction { case .executing(_, _, .waitingForResponseHead): preconditionFailure("If we receive a response body, we must have received a head before") - case .executing(let context, let requestState, .buffering(let streamID, var currentBuffer, next: let next)): - guard case .askExecutorForMore = next else { - preconditionFailure("If we have received an error or eof before, why did we get another body part? Next: \(next)") - } + case .executing(_, _, .buffering(_, _, next: .endOfFile)): + preconditionFailure("If we have received an eof before, why did we get another body part?") + case .executing(_, _, .buffering(_, _, next: .error)): + // we might still get pending buffers if the user has canceled the request + return .none + + case .executing(let context, let requestState, .buffering(let streamID, var currentBuffer, next: .askExecutorForMore)): if currentBuffer.isEmpty { currentBuffer = buffer } else { currentBuffer.append(contentsOf: buffer) } - self.state = .executing(context, requestState, .buffering(streamID, currentBuffer, next: next)) + self.state = .executing(context, requestState, .buffering(streamID, currentBuffer, next: .askExecutorForMore)) return .none case .executing(let executor, let requestState, .waitingForResponseIterator(var currentBuffer, next: let next)): @@ -690,10 +693,11 @@ extension Transaction { case .finished: // the request failed or was cancelled before, we can ignore all events return .none - + case .executing(_, _, .buffering(_, _, next: .error)): + // we might still get pending buffers if the user has canceled the request + return .none case .executing(_, _, .waitingForResponseIterator(_, next: .error)), .executing(_, _, .waitingForResponseIterator(_, next: .endOfFile)), - .executing(_, _, .buffering(_, _, next: .error)), .executing(_, _, .buffering(_, _, next: .endOfFile)), .executing(_, _, .finished(_, _)): preconditionFailure("Already received an eof or error before. Must not receive further events. Invalid state: \(self.state)") diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 397071e08..3fb55779f 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -43,6 +43,7 @@ extension AsyncAwaitEndToEndTests { ("testInvalidURL", testInvalidURL), ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), ("testShutdown", testShutdown), + ("testCancelingBodyDoesNotCrash", testCancelingBodyDoesNotCrash), ] } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 68457d4ff..91482d08c 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -578,6 +578,26 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } #endif } + + /// Regression test for https://github.com/swift-server/async-http-client/issues/612 + func testCancelingBodyDoesNotCrash() { + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let bin = HTTPBin(.http2(compress: true)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + let request = HTTPClientRequest(url: "https://127.0.0.1:\(bin.port)/mega-chunked") + let response = try await client.execute(request, deadline: .now() + .seconds(10)) + + await XCTAssertThrowsError(try await response.body.collect(upTo: 100)) { error in + XCTAssert(error is NIOTooManyBytesError) + } + } + #endif + } } #if compiler(>=5.5.2) && canImport(_Concurrency) From fc510a39cff61b849bf5cdff17eb2bd6d0777b49 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 18 Aug 2022 11:55:55 +0200 Subject: [PATCH 031/146] Fix thread leak in `FileDownloadDelegate` (#614) * Fix thread leak in `FileDownloadDelegate` * `SwiftFormat` * Add a shared file IO thread pool per HTTPClient * User bigger thread pool and initlize lazily during first file write * thread pool is actually not used in tests * Update documentation * fix review comments * make `fileIOThreadPool` internal * Add test to verify that we actually share the same thread pool across all delegates for a given HTTPClient Co-authored-by: Cory Benfield --- .../FileDownloadDelegate.swift | 57 ++++++++++++++++--- Sources/AsyncHTTPClient/HTTPClient.swift | 42 ++++++++++++-- Sources/AsyncHTTPClient/HTTPHandler.swift | 21 +++++-- .../HTTPClientInternalTests+XCTest.swift | 1 + .../HTTPClientInternalTests.swift | 35 ++++++++++++ .../HTTPClientTestUtils.swift | 17 ++++++ .../RequestBagTests.swift | 11 ++++ 7 files changed, 166 insertions(+), 18 deletions(-) diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index 75f16f52a..6199f33ff 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -30,7 +30,7 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { public typealias Response = Progress private let filePath: String - private let io: NonBlockingFileIO + private(set) var fileIOThreadPool: NIOThreadPool? private let reportHead: ((HTTPResponseHead) -> Void)? private let reportProgress: ((Progress) -> Void)? @@ -47,14 +47,46 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { /// the total byte count and download byte count passed to it as arguments. The callbacks /// will be invoked in the same threading context that the delegate itself is invoked, /// as controlled by `EventLoopPreference`. - public init( + public convenience init( path: String, - pool: NIOThreadPool = NIOThreadPool(numberOfThreads: 1), + pool: NIOThreadPool, reportHead: ((HTTPResponseHead) -> Void)? = nil, reportProgress: ((Progress) -> Void)? = nil ) throws { - pool.start() - self.io = NonBlockingFileIO(threadPool: pool) + try self.init(path: path, pool: .some(pool), reportHead: reportHead, reportProgress: reportProgress) + } + + /// Initializes a new file download delegate and uses the shared thread pool of the ``HTTPClient`` for file I/O. + /// + /// - parameters: + /// - path: Path to a file you'd like to write the download to. + /// - reportHead: A closure called when the response head is available. + /// - reportProgress: A closure called when a body chunk has been downloaded, with + /// the total byte count and download byte count passed to it as arguments. The callbacks + /// will be invoked in the same threading context that the delegate itself is invoked, + /// as controlled by `EventLoopPreference`. + public convenience init( + path: String, + reportHead: ((HTTPResponseHead) -> Void)? = nil, + reportProgress: ((Progress) -> Void)? = nil + ) throws { + try self.init(path: path, pool: nil, reportHead: reportHead, reportProgress: reportProgress) + } + + private init( + path: String, + pool: NIOThreadPool?, + reportHead: ((HTTPResponseHead) -> Void)? = nil, + reportProgress: ((Progress) -> Void)? = nil + ) throws { + if let pool = pool { + self.fileIOThreadPool = pool + } else { + // we should use the shared thread pool from the HTTPClient which + // we will get from the `HTTPClient.Task` + self.fileIOThreadPool = nil + } + self.filePath = path self.reportHead = reportHead @@ -79,16 +111,25 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { task: HTTPClient.Task, _ buffer: ByteBuffer ) -> EventLoopFuture { + let threadPool: NIOThreadPool = { + guard let pool = self.fileIOThreadPool else { + let pool = task.fileIOThreadPool + self.fileIOThreadPool = pool + return pool + } + return pool + }() + let io = NonBlockingFileIO(threadPool: threadPool) self.progress.receivedBytes += buffer.readableBytes self.reportProgress?(self.progress) let writeFuture: EventLoopFuture if let fileHandleFuture = self.fileHandleFuture { writeFuture = fileHandleFuture.flatMap { - self.io.write(fileHandle: $0, buffer: buffer, eventLoop: task.eventLoop) + io.write(fileHandle: $0, buffer: buffer, eventLoop: task.eventLoop) } } else { - let fileHandleFuture = self.io.openFile( + let fileHandleFuture = io.openFile( path: self.filePath, mode: .write, flags: .allowFileCreation(), @@ -96,7 +137,7 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { ) self.fileHandleFuture = fileHandleFuture writeFuture = fileHandleFuture.flatMap { - self.io.write(fileHandle: $0, buffer: buffer, eventLoop: task.eventLoop) + io.write(fileHandle: $0, buffer: buffer, eventLoop: task.eventLoop) } } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 9cf84a2cb..ab4f7815e 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -72,6 +72,11 @@ public class HTTPClient { let eventLoopGroupProvider: EventLoopGroupProvider let configuration: Configuration let poolManager: HTTPConnectionPool.Manager + + /// Shared thread pool used for file IO. It is lazily created on first access of ``Task/fileIOThreadPool``. + private var fileIOThreadPool: NIOThreadPool? + private let fileIOThreadPoolLock = Lock() + private var state: State private let stateLock = Lock() @@ -213,6 +218,16 @@ public class HTTPClient { } } + private func shutdownFileIOThreadPool(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { + self.fileIOThreadPoolLock.withLockVoid { + guard let fileIOThreadPool = fileIOThreadPool else { + callback(nil) + return + } + fileIOThreadPool.shutdownGracefully(queue: queue, callback) + } + } + private func shutdown(requiresCleanClose: Bool, queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { do { try self.stateLock.withLock { @@ -241,15 +256,28 @@ public class HTTPClient { let error: Error? = (requiresClean && unclean) ? HTTPClientError.uncleanShutdown : nil return (callback, error) } - - self.shutdownEventLoop(queue: queue) { error in - let reportedError = error ?? uncleanError - callback(reportedError) + self.shutdownFileIOThreadPool(queue: queue) { ioThreadPoolError in + self.shutdownEventLoop(queue: queue) { error in + let reportedError = error ?? ioThreadPoolError ?? uncleanError + callback(reportedError) + } } } } } + private func makeOrGetFileIOThreadPool() -> NIOThreadPool { + self.fileIOThreadPoolLock.withLock { + guard let fileIOThreadPool = fileIOThreadPool else { + let fileIOThreadPool = NIOThreadPool(numberOfThreads: ProcessInfo.processInfo.processorCount) + fileIOThreadPool.start() + self.fileIOThreadPool = fileIOThreadPool + return fileIOThreadPool + } + return fileIOThreadPool + } + } + /// Execute `GET` request using specified URL. /// /// - parameters: @@ -562,6 +590,7 @@ public class HTTPClient { case .testOnly_exact(_, delegateOn: let delegateEL): taskEL = delegateEL } + logger.trace("selected EventLoop for task given the preference", metadata: ["ahc-eventloop": "\(taskEL)", "ahc-el-preference": "\(eventLoopPreference)"]) @@ -574,7 +603,8 @@ public class HTTPClient { logger.debug("client is shutting down, failing request") return Task.failedTask(eventLoop: taskEL, error: HTTPClientError.alreadyShutdown, - logger: logger) + logger: logger, + makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool) } } @@ -597,7 +627,7 @@ public class HTTPClient { } }() - let task = Task(eventLoop: taskEL, logger: logger) + let task = Task(eventLoop: taskEL, logger: logger, makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool) do { let requestBag = try RequestBag( request: request, diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index aeef71ba4..c62c2f7d1 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -17,6 +17,7 @@ import Logging import NIOConcurrencyHelpers import NIOCore import NIOHTTP1 +import NIOPosix import NIOSSL extension HTTPClient { @@ -502,7 +503,7 @@ public protocol HTTPClientResponseDelegate: AnyObject { } extension HTTPClientResponseDelegate { - /// Default implementation of ``HTTPClientResponseDelegate/didSendRequestHead(task:_:)-6khai``. + /// Default implementation of ``HTTPClientResponseDelegate/didSendRequest(task:)-9od5p``. /// /// By default, this does nothing. public func didSendRequestHead(task: HTTPClient.Task, _ head: HTTPRequestHead) {} @@ -622,15 +623,27 @@ extension HTTPClient { private var _isCancelled: Bool = false private var _taskDelegate: HTTPClientTaskDelegate? private let lock = Lock() + private let makeOrGetFileIOThreadPool: () -> NIOThreadPool - init(eventLoop: EventLoop, logger: Logger) { + /// The shared thread pool of a ``HTTPClient`` used for file IO. It is lazily created on first access. + internal var fileIOThreadPool: NIOThreadPool { + self.makeOrGetFileIOThreadPool() + } + + init(eventLoop: EventLoop, logger: Logger, makeOrGetFileIOThreadPool: @escaping () -> NIOThreadPool) { self.eventLoop = eventLoop self.promise = eventLoop.makePromise() self.logger = logger + self.makeOrGetFileIOThreadPool = makeOrGetFileIOThreadPool } - static func failedTask(eventLoop: EventLoop, error: Error, logger: Logger) -> Task { - let task = self.init(eventLoop: eventLoop, logger: logger) + static func failedTask( + eventLoop: EventLoop, + error: Error, + logger: Logger, + makeOrGetFileIOThreadPool: @escaping () -> NIOThreadPool + ) -> Task { + let task = self.init(eventLoop: eventLoop, logger: logger, makeOrGetFileIOThreadPool: makeOrGetFileIOThreadPool) task.promise.fail(error) return task } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift index 3be2c79a6..9114df259 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift @@ -36,6 +36,7 @@ extension HTTPClientInternalTests { ("testConnectErrorCalloutOnCorrectEL", testConnectErrorCalloutOnCorrectEL), ("testInternalRequestURI", testInternalRequestURI), ("testHasSuffix", testHasSuffix), + ("testSharedThreadPoolIsIdenticalForAllDelegates", testSharedThreadPoolIsIdenticalForAllDelegates), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index 492bb4c35..234185eb6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -554,4 +554,39 @@ class HTTPClientInternalTests: XCTestCase { XCTAssertFalse(elements.hasSuffix([0, 0, 0])) } } + + /// test to verify that we actually share the same thread pool across all ``FileDownloadDelegate``s for a given ``HTTPClient`` + func testSharedThreadPoolIsIdenticalForAllDelegates() throws { + let httpBin = HTTPBin() + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) + defer { + XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) + XCTAssertNoThrow(try httpBin.shutdown()) + } + var request = try Request(url: "http://localhost:\(httpBin.port)/events/10/content-length") + request.headers.add(name: "Accept", value: "text/event-stream") + + let filePaths = (0..<10).map { _ in + TemporaryFileHelpers.makeTemporaryFilePath() + } + defer { + for filePath in filePaths { + TemporaryFileHelpers.removeTemporaryFile(at: filePath) + } + } + let delegates = try filePaths.map { + try FileDownloadDelegate(path: $0) + } + + let resultFutures = delegates.map { delegate in + httpClient.execute( + request: request, + delegate: delegate + ).futureResult + } + _ = try EventLoopFuture.whenAllSucceed(resultFutures, on: self.clientGroup.next()).wait() + let threadPools = delegates.map { $0.fileIOThreadPool } + let firstThreadPool = threadPools.first ?? nil + XCTAssert(threadPools.dropFirst().allSatisfy { $0 === firstThreadPool }) + } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 8f7d4dfce..7cd9ef83d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -283,6 +283,23 @@ enum TemporaryFileHelpers { return try body(path) } + internal static func makeTemporaryFilePath( + directory: String = temporaryDirectory + ) -> String { + let (fd, path) = self.openTemporaryFile() + close(fd) + try! FileManager.default.removeItem(atPath: path) + return path + } + + internal static func removeTemporaryFile( + at path: String + ) { + if FileManager.default.fileExists(atPath: path) { + try? FileManager.default.removeItem(atPath: path) + } + } + internal static func fileSize(path: String) throws -> Int? { return try FileManager.default.attributesOfItem(atPath: path)[.size] as? Int } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 9e7072c19..6993c0df9 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -771,6 +771,17 @@ final class RequestBagTests: XCTestCase { } } +extension HTTPClient.Task { + convenience init( + eventLoop: EventLoop, + logger: Logger + ) { + self.init(eventLoop: eventLoop, logger: logger) { + preconditionFailure("thread pool not needed in tests") + } + } +} + class UploadCountingDelegate: HTTPClientResponseDelegate { typealias Response = Void From 18109d842d077e21adf60c8d545f171f495f584e Mon Sep 17 00:00:00 2001 From: Karl <5254025+karwa@users.noreply.github.com> Date: Sat, 20 Aug 2022 13:22:41 +0200 Subject: [PATCH 032/146] Use NIOCore.System.coreCount for the fileIO thread pool (#618) --- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index ab4f7815e..2e1960f09 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -269,7 +269,7 @@ public class HTTPClient { private func makeOrGetFileIOThreadPool() -> NIOThreadPool { self.fileIOThreadPoolLock.withLock { guard let fileIOThreadPool = fileIOThreadPool else { - let fileIOThreadPool = NIOThreadPool(numberOfThreads: ProcessInfo.processInfo.processorCount) + let fileIOThreadPool = NIOThreadPool(numberOfThreads: System.coreCount) fileIOThreadPool.start() self.fileIOThreadPool = fileIOThreadPool return fileIOThreadPool From d764c1ac7ca7e30ad4c233656b71d9a0dc6b2399 Mon Sep 17 00:00:00 2001 From: Karl <5254025+karwa@users.noreply.github.com> Date: Sat, 20 Aug 2022 13:26:53 +0200 Subject: [PATCH 033/146] NIOFoundationCompat does not appear to be used (#619) Co-authored-by: Cory Benfield --- Package.swift | 1 - Package@swift-5.4.swift | 1 - Package@swift-5.5.swift | 1 - 3 files changed, 3 deletions(-) diff --git a/Package.swift b/Package.swift index a60f012aa..177489899 100644 --- a/Package.swift +++ b/Package.swift @@ -41,7 +41,6 @@ let package = Package( .product(name: "NIOPosix", package: "swift-nio"), .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), .product(name: "NIOHTTP2", package: "swift-nio-http2"), .product(name: "NIOSSL", package: "swift-nio-ssl"), .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift index a9ea2ba7f..e0ca9e78a 100644 --- a/Package@swift-5.4.swift +++ b/Package@swift-5.4.swift @@ -40,7 +40,6 @@ let package = Package( .product(name: "NIOPosix", package: "swift-nio"), .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), .product(name: "NIOHTTP2", package: "swift-nio-http2"), .product(name: "NIOSSL", package: "swift-nio-ssl"), .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index a9ea2ba7f..e0ca9e78a 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -40,7 +40,6 @@ let package = Package( .product(name: "NIOPosix", package: "swift-nio"), .product(name: "NIOHTTP1", package: "swift-nio"), .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), .product(name: "NIOHTTP2", package: "swift-nio-http2"), .product(name: "NIOSSL", package: "swift-nio-ssl"), .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), From 9c7ab039fec3d224ccdbd4eb5502bdff3da25690 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 23 Aug 2022 14:46:17 +0200 Subject: [PATCH 034/146] Allow `HTTPClientRequest` to be executed multiple times if `body` is an `AsyncSequence` (#620) --- .../HTTPClientRequest+Prepared.swift | 30 ++++++++++- .../AsyncAwait/HTTPClientRequest.swift | 50 +++++++++++++------ .../AsyncAwait/Transaction.swift | 2 +- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 38 ++++++++++++++ .../HTTPClientRequestTests.swift | 4 +- 6 files changed, 107 insertions(+), 18 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index de09df5b8..b4707226d 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -14,11 +14,25 @@ #if compiler(>=5.5.2) && canImport(_Concurrency) import struct Foundation.URL +import NIOCore import NIOHTTP1 @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest { struct Prepared { + enum Body { + case asyncSequence( + length: RequestBodyLength, + nextBodyPart: (ByteBufferAllocator) async throws -> ByteBuffer? + ) + case sequence( + length: RequestBodyLength, + canBeConsumedMultipleTimes: Bool, + makeCompleteBody: (ByteBufferAllocator) -> ByteBuffer + ) + case byteBuffer(ByteBuffer) + } + var url: URL var poolKey: ConnectionPool.Key var requestFramingMetadata: RequestFramingMetadata @@ -53,11 +67,25 @@ extension HTTPClientRequest.Prepared { uri: deconstructedURL.uri, headers: headers ), - body: request.body + body: request.body.map { .init($0) } ) } } +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientRequest.Prepared.Body { + init(_ body: HTTPClientRequest.Body) { + switch body.mode { + case .asyncSequence(let length, let makeAsyncIterator): + self = .asyncSequence(length: length, nextBodyPart: makeAsyncIterator()) + case .sequence(let length, let canBeConsumedMultipleTimes, let makeCompleteBody): + self = .sequence(length: length, canBeConsumedMultipleTimes: canBeConsumedMultipleTimes, makeCompleteBody: makeCompleteBody) + case .byteBuffer(let byteBuffer): + self = .byteBuffer(byteBuffer) + } + } +} + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension RequestBodyLength { init(_ body: HTTPClientRequest.Body?) { diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index a76b9239a..0b465138f 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -50,8 +50,26 @@ extension HTTPClientRequest { public struct Body { @usableFromInline internal enum Mode { - case asyncSequence(length: RequestBodyLength, (ByteBufferAllocator) async throws -> ByteBuffer?) - case sequence(length: RequestBodyLength, canBeConsumedMultipleTimes: Bool, (ByteBufferAllocator) -> ByteBuffer) + /// - parameters: + /// - length: complete body length. + /// If `length` is `.known`, `nextBodyPart` is not allowed to produce more bytes than `length` defines. + /// - makeAsyncIterator: Creates a new async iterator under the hood and returns a function which will call `next()` on it. + /// The returned function then produce the next body buffer asynchronously. + /// We use a closure as abstraction instead of an existential to enable specialization. + case asyncSequence( + length: RequestBodyLength, + makeAsyncIterator: () -> ((ByteBufferAllocator) async throws -> ByteBuffer?) + ) + /// - parameters: + /// - length: complete body length. + /// If `length` is `.known`, `nextBodyPart` is not allowed to produce more bytes than `length` defines. + /// - canBeConsumedMultipleTimes: if `makeBody` can be called multiple times and returns the same result. + /// - makeCompleteBody: function to produce the complete body. + case sequence( + length: RequestBodyLength, + canBeConsumedMultipleTimes: Bool, + makeCompleteBody: (ByteBufferAllocator) -> ByteBuffer + ) case byteBuffer(ByteBuffer) } @@ -180,9 +198,11 @@ extension HTTPClientRequest.Body { _ sequenceOfBytes: SequenceOfBytes, length: Length ) -> Self where SequenceOfBytes.Element == ByteBuffer { - var iterator = sequenceOfBytes.makeAsyncIterator() - let body = self.init(.asyncSequence(length: length.storage) { _ -> ByteBuffer? in - try await iterator.next() + let body = self.init(.asyncSequence(length: length.storage) { + var iterator = sequenceOfBytes.makeAsyncIterator() + return { _ -> ByteBuffer? in + try await iterator.next() + } }) return body } @@ -205,16 +225,18 @@ extension HTTPClientRequest.Body { _ bytes: Bytes, length: Length ) -> Self where Bytes.Element == UInt8 { - var iterator = bytes.makeAsyncIterator() - let body = self.init(.asyncSequence(length: length.storage) { allocator -> ByteBuffer? in - var buffer = allocator.buffer(capacity: 1024) // TODO: Magic number - while buffer.writableBytes > 0, let byte = try await iterator.next() { - buffer.writeInteger(byte) - } - if buffer.readableBytes > 0 { - return buffer + let body = self.init(.asyncSequence(length: length.storage) { + var iterator = bytes.makeAsyncIterator() + return { allocator -> ByteBuffer? in + var buffer = allocator.buffer(capacity: 1024) // TODO: Magic number + while buffer.writableBytes > 0, let byte = try await iterator.next() { + buffer.writeInteger(byte) + } + if buffer.readableBytes > 0 { + return buffer + } + return nil } - return nil }) return body } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 3b6db1e38..66050bcde 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -194,7 +194,7 @@ extension Transaction: HTTPExecutableRequest { break case .startStream(let allocator): - switch self.request.body?.mode { + switch self.request.body { case .asyncSequence(_, let next): // it is safe to call this async here. it dispatches... self.continueRequestBodyStream(allocator, next: next) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 3fb55779f..b8431757c 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -44,6 +44,7 @@ extension AsyncAwaitEndToEndTests { ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), ("testShutdown", testShutdown), ("testCancelingBodyDoesNotCrash", testCancelingBodyDoesNotCrash), + ("testAsyncSequenceReuse", testAsyncSequenceReuse), ] } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 91482d08c..a9728e1df 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -598,6 +598,44 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } #endif } + + func testAsyncSequenceReuse() { + #if compiler(>=5.5.2) && canImport(_Concurrency) + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") + request.method = .POST + request.body = .stream([ + ByteBuffer(string: "1"), + ByteBuffer(string: "2"), + ByteBuffer(string: "34"), + ].asAsyncSequence(), length: .unknown) + + guard let response1 = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { return } + XCTAssertEqual(response1.headers["content-length"], []) + guard let body = await XCTAssertNoThrowWithResult( + try await response1.body.collect() + ) else { return } + XCTAssertEqual(body, ByteBuffer(string: "1234")) + + guard let response2 = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { return } + XCTAssertEqual(response2.headers["content-length"], []) + guard let body = await XCTAssertNoThrowWithResult( + try await response2.body.collect() + ) else { return } + XCTAssertEqual(body, ByteBuffer(string: "1234")) + } + #endif + } } #if compiler(>=5.5.2) && canImport(_Concurrency) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 1ebe7e939..99b9097ef 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -489,11 +489,11 @@ private struct LengthMismatch: Error { } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension Optional where Wrapped == HTTPClientRequest.Body { +extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { /// Accumulates all data from `self` into a single `ByteBuffer` and checks that the user specified length matches /// the length of the accumulated data. fileprivate func read() async throws -> ByteBuffer { - switch self?.mode { + switch self { case .none: return ByteBuffer() case .byteBuffer(let buffer): From c3c90aab58840d5f8285c7110e8346d327c5459f Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 25 Aug 2022 11:45:13 +0200 Subject: [PATCH 035/146] Adopt `Sendable` (#621) --- Package.swift | 8 +- Package@swift-5.4.swift | 8 +- Package@swift-5.5.swift | 8 +- .../AsyncAwait/HTTPClientRequest.swift | 168 +++++++++++++++++- .../AsyncAwait/HTTPClientResponse.swift | 6 +- .../HTTPConnectionPool+Manager.swift | 2 +- .../ConnectionPool/RequestBodyLength.swift | 4 +- .../HTTPClient+HTTPCookie.swift | 3 +- .../AsyncHTTPClient/HTTPClient+Proxy.swift | 4 +- Sources/AsyncHTTPClient/HTTPClient.swift | 61 +++++-- Sources/AsyncHTTPClient/HTTPHandler.swift | 43 ++++- Sources/AsyncHTTPClient/SSLContextCache.swift | 6 +- Sources/AsyncHTTPClient/UnsafeTransfer.swift | 31 ++++ Sources/AsyncHTTPClient/Utils.swift | 7 + .../AsyncAwaitEndToEndTests.swift | 47 ++++- .../AsyncTestHelpers.swift | 3 +- .../HTTPClientRequestTests.swift | 10 +- .../HTTPClientTests.swift | 2 +- .../TransactionTests.swift | 32 ++-- 19 files changed, 392 insertions(+), 61 deletions(-) create mode 100644 Sources/AsyncHTTPClient/UnsafeTransfer.swift diff --git a/Package.swift b/Package.swift index 177489899..23fde6647 100644 --- a/Package.swift +++ b/Package.swift @@ -21,12 +21,12 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.38.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), - .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), + .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), ], diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift index e0ca9e78a..a69d9bc8a 100644 --- a/Package@swift-5.4.swift +++ b/Package@swift-5.4.swift @@ -21,12 +21,12 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.38.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), - .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), + .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), ], targets: [ diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index e0ca9e78a..a69d9bc8a 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -21,12 +21,12 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.38.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.14.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), - .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.10.0"), + .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.4.0"), + .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), ], targets: [ diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index 0b465138f..2e5bcfe88 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -20,7 +20,7 @@ import NIOHTTP1 /// /// This object is similar to ``HTTPClient/Request``, but used for the Swift Concurrency API. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -public struct HTTPClientRequest { +public struct HTTPClientRequest: Sendable { /// The request URL, including scheme, hostname, and optionally port. public var url: String @@ -47,18 +47,18 @@ extension HTTPClientRequest { /// /// This object encapsulates the difference between streamed HTTP request bodies and those bodies that /// are already entirely in memory. - public struct Body { + public struct Body: Sendable { @usableFromInline - internal enum Mode { + internal enum Mode: Sendable { /// - parameters: /// - length: complete body length. /// If `length` is `.known`, `nextBodyPart` is not allowed to produce more bytes than `length` defines. /// - makeAsyncIterator: Creates a new async iterator under the hood and returns a function which will call `next()` on it. /// The returned function then produce the next body buffer asynchronously. - /// We use a closure as abstraction instead of an existential to enable specialization. + /// We use a closure as an abstraction instead of an existential to enable specialization. case asyncSequence( length: RequestBodyLength, - makeAsyncIterator: () -> ((ByteBufferAllocator) async throws -> ByteBuffer?) + makeAsyncIterator: @Sendable () -> ((ByteBufferAllocator) async throws -> ByteBuffer?) ) /// - parameters: /// - length: complete body length. @@ -68,7 +68,7 @@ extension HTTPClientRequest { case sequence( length: RequestBodyLength, canBeConsumedMultipleTimes: Bool, - makeCompleteBody: (ByteBufferAllocator) -> ByteBuffer + makeCompleteBody: @Sendable (ByteBufferAllocator) -> ByteBuffer ) case byteBuffer(ByteBuffer) } @@ -92,6 +92,22 @@ extension HTTPClientRequest.Body { self.init(.byteBuffer(byteBuffer)) } + #if swift(>=5.6) + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// - parameter bytes: The bytes of the request body. + @inlinable + @preconcurrency + public static func bytes( + _ bytes: Bytes + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes) + } + #else /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -102,6 +118,14 @@ extension HTTPClientRequest.Body { @inlinable public static func bytes( _ bytes: Bytes + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes) + } + #endif + + @inlinable + static func _bytes( + _ bytes: Bytes ) -> Self where Bytes.Element == UInt8 { self.init(.sequence( length: .known(bytes.count), @@ -116,6 +140,33 @@ extension HTTPClientRequest.Body { }) } + #if swift(>=5.6) + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// Unlike ``bytes(_:)-1uns7``, this construction does not assume that the body can be replayed. As a result, + /// if a redirect is encountered that would need us to replay the request body, the redirect will instead + /// not be followed. Prefer ``bytes(_:)-1uns7`` wherever possible. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. + @inlinable + @preconcurrency + public static func bytes( + _ bytes: Bytes, + length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes, length: length) + } + #else /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -137,6 +188,15 @@ extension HTTPClientRequest.Body { public static func bytes( _ bytes: Bytes, length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes, length: length) + } + #endif + + @inlinable + static func _bytes( + _ bytes: Bytes, + length: Length ) -> Self where Bytes.Element == UInt8 { self.init(.sequence( length: length.storage, @@ -151,6 +211,29 @@ extension HTTPClientRequest.Body { }) } + #if swift(>=5.6) + /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. + /// + /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory + /// usage of this construction will be double the size of the original collection. The construction + /// of the `ByteBuffer` will be delayed until it's needed. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. + @inlinable + @preconcurrency + public static func bytes( + _ bytes: Bytes, + length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes, length: length) + } + #else /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -168,6 +251,15 @@ extension HTTPClientRequest.Body { public static func bytes( _ bytes: Bytes, length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._bytes(bytes, length: length) + } + #endif + + @inlinable + static func _bytes( + _ bytes: Bytes, + length: Length ) -> Self where Bytes.Element == UInt8 { self.init(.sequence( length: length.storage, @@ -182,6 +274,27 @@ extension HTTPClientRequest.Body { }) } + #if swift(>=5.6) + /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. + /// + /// This construction will stream the upload one `ByteBuffer` at a time. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - sequenceOfBytes: The bytes of the request body. + /// - length: The length of the request body. + @inlinable + @preconcurrency + public static func stream( + _ sequenceOfBytes: SequenceOfBytes, + length: Length + ) -> Self where SequenceOfBytes.Element == ByteBuffer { + Self._stream(sequenceOfBytes, length: length) + } + #else /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. /// /// This construction will stream the upload one `ByteBuffer` at a time. @@ -197,6 +310,15 @@ extension HTTPClientRequest.Body { public static func stream( _ sequenceOfBytes: SequenceOfBytes, length: Length + ) -> Self where SequenceOfBytes.Element == ByteBuffer { + Self._stream(sequenceOfBytes, length: length) + } + #endif + + @inlinable + static func _stream( + _ sequenceOfBytes: SequenceOfBytes, + length: Length ) -> Self where SequenceOfBytes.Element == ByteBuffer { let body = self.init(.asyncSequence(length: length.storage) { var iterator = sequenceOfBytes.makeAsyncIterator() @@ -207,6 +329,29 @@ extension HTTPClientRequest.Body { return body } + #if swift(>=5.6) + /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. + /// + /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for + /// `AsyncSequence`s where larger chunks are buffered up and available without actually suspending, such + /// as those provided by `FileHandle`. + /// + /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths + /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// + /// - parameters: + /// - bytes: The bytes of the request body. + /// - length: The length of the request body. + @inlinable + @preconcurrency + public static func stream( + _ bytes: Bytes, + length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._stream(bytes, length: length) + } + #else /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. /// /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for @@ -224,6 +369,15 @@ extension HTTPClientRequest.Body { public static func stream( _ bytes: Bytes, length: Length + ) -> Self where Bytes.Element == UInt8 { + Self._stream(bytes, length: length) + } + #endif + + @inlinable + static func _stream( + _ bytes: Bytes, + length: Length ) -> Self where Bytes.Element == UInt8 { let body = self.init(.asyncSequence(length: length.storage) { var iterator = bytes.makeAsyncIterator() @@ -257,7 +411,7 @@ extension Optional where Wrapped == HTTPClientRequest.Body { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Body { /// The length of a HTTP request body. - public struct Length { + public struct Length: Sendable { /// The size of the request body is not known before starting the request public static let unknown: Self = .init(storage: .unknown) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index 7ccd74530..e6cc47210 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -20,7 +20,7 @@ import NIOHTTP1 /// /// This object is similar to ``HTTPClient/Response``, but used for the Swift Concurrency API. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -public struct HTTPClientResponse { +public struct HTTPClientResponse: Sendable { /// The HTTP version on which the response was received. public var version: HTTPVersion @@ -38,7 +38,7 @@ public struct HTTPClientResponse { /// The body is streamed as an `AsyncSequence` of `ByteBuffer`, where each `ByteBuffer` contains /// an arbitrarily large chunk of data. The boundaries between `ByteBuffer` objects in the sequence /// are entirely synthetic and have no semantic meaning. - public struct Body { + public struct Body: Sendable { private let bag: Transaction private let reference: ResponseRef @@ -87,7 +87,7 @@ extension HTTPClientResponse.Body { /// The purpose of this object is to inform the transaction about the response body being deinitialized. /// If the users has not called `makeAsyncIterator` on the body, before it is deinited, the http /// request needs to be cancelled. - fileprivate class ResponseRef { + fileprivate final class ResponseRef: Sendable { private let transaction: Transaction init(transaction: Transaction) { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift index 8500c59da..08d1e9431 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift @@ -163,7 +163,7 @@ extension HTTPConnectionPool.Manager: HTTPConnectionPoolDelegate { } extension HTTPConnectionPool.Connection.ID { - static var globalGenerator = Generator() + static let globalGenerator = Generator() struct Generator { private let atomic: ManagedAtomic diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift index 38d90e057..fdbd8b2e7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift @@ -12,9 +12,11 @@ // //===----------------------------------------------------------------------===// +import NIOCore + /// - Note: use `HTTPClientRequest.Body.Length` if you want to expose `RequestBodyLength` publicly @usableFromInline -internal enum RequestBodyLength: Hashable { +internal enum RequestBodyLength: Hashable, NIOSendable { /// size of the request body is not known before starting the request case unknown /// size of the request body is fixed and exactly `count` bytes diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index 75fc28de4..cb67099ec 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -19,10 +19,11 @@ import Darwin import Glibc #endif import CAsyncHTTPClient +import NIOCore extension HTTPClient { /// A representation of an HTTP cookie. - public struct Cookie { + public struct Cookie: NIOSendable { /// The name of the cookie. public var name: String /// The cookie's string value. diff --git a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift index 4d2b9388f..16be51ca5 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift @@ -12,6 +12,8 @@ // //===----------------------------------------------------------------------===// +import NIOCore + extension HTTPClient.Configuration { /// Proxy server configuration /// Specifies the remote address of an HTTP proxy. @@ -23,7 +25,7 @@ extension HTTPClient.Configuration { /// If a `TLSConfiguration` is used in conjunction with `HTTPClient.Configuration.Proxy`, /// TLS will be established _after_ successful proxy, between your client /// and the destination server. - public struct Proxy { + public struct Proxy: NIOSendable { enum ProxyType: Hashable { case http(HTTPClient.Authorization?) case socks diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 2e1960f09..55d2573ba 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -170,24 +170,38 @@ public class HTTPClient { """) } let errorStorageLock = Lock() - var errorStorage: Error? + let errorStorage: UnsafeMutableTransferBox = .init(nil) let continuation = DispatchWorkItem {} self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown")) { error in if let error = error { errorStorageLock.withLock { - errorStorage = error + errorStorage.wrappedValue = error } } continuation.perform() } continuation.wait() try errorStorageLock.withLock { - if let error = errorStorage { + if let error = errorStorage.wrappedValue { throw error } } } + #if swift(>=5.6) + /// Shuts down the client and event loop gracefully. + /// + /// This function is clearly an outlier in that it uses a completion + /// callback instead of an EventLoopFuture. The reason for that is that NIO's EventLoopFutures will call back on an event loop. + /// The virtue of this function is to shut the event loop down. To work around that we call back on a DispatchQueue + /// instead. + @preconcurrency public func shutdown( + queue: DispatchQueue = .global(), + _ callback: @Sendable @escaping (Error?) -> Void + ) { + self.shutdown(requiresCleanClose: false, queue: queue, callback) + } + #else /// Shuts down the client and event loop gracefully. /// /// This function is clearly an outlier in that it uses a completion @@ -197,8 +211,9 @@ public class HTTPClient { public func shutdown(queue: DispatchQueue = .global(), _ callback: @escaping (Error?) -> Void) { self.shutdown(requiresCleanClose: false, queue: queue, callback) } + #endif - private func shutdownEventLoop(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { + private func shutdownEventLoop(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { self.stateLock.withLock { switch self.eventLoopGroupProvider { case .shared: @@ -218,7 +233,7 @@ public class HTTPClient { } } - private func shutdownFileIOThreadPool(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { + private func shutdownFileIOThreadPool(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { self.fileIOThreadPoolLock.withLockVoid { guard let fileIOThreadPool = fileIOThreadPool else { callback(nil) @@ -228,7 +243,7 @@ public class HTTPClient { } } - private func shutdown(requiresCleanClose: Bool, queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { + private func shutdown(requiresCleanClose: Bool, queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { do { try self.stateLock.withLock { guard case .upAndRunning = self.state else { @@ -248,7 +263,7 @@ public class HTTPClient { case .failure: preconditionFailure("Shutting down the connection pool must not fail, ever.") case .success(let unclean): - let (callback, uncleanError) = self.stateLock.withLock { () -> ((Error?) -> Void, Error?) in + let (callback, uncleanError) = self.stateLock.withLock { () -> (ShutdownCallback, Error?) in guard case .shuttingDown(let requiresClean, callback: let callback) = self.state else { preconditionFailure("Why did the pool manager shut down, if it was not instructed to") } @@ -838,23 +853,43 @@ public class HTTPClient { } /// Specifies decompression settings. - public enum Decompression { + public enum Decompression: NIOSendable { /// Decompression is disabled. case disabled /// Decompression is enabled. case enabled(limit: NIOHTTPDecompression.DecompressionLimit) } + #if swift(>=5.6) + typealias ShutdownCallback = @Sendable (Error?) -> Void + #else + typealias ShutdownCallback = (Error?) -> Void + #endif + enum State { case upAndRunning - case shuttingDown(requiresCleanClose: Bool, callback: (Error?) -> Void) + case shuttingDown(requiresCleanClose: Bool, callback: ShutdownCallback) case shutDown } } +#if swift(>=5.7) +extension HTTPClient.Configuration: Sendable {} +#endif + +#if swift(>=5.6) +extension HTTPClient.EventLoopGroupProvider: Sendable {} +extension HTTPClient.EventLoopPreference: Sendable {} +#endif + +#if swift(>=5.5) && canImport(_Concurrency) +// HTTPClient is thread-safe because its shared mutable state is protected through a lock +extension HTTPClient: @unchecked Sendable {} +#endif + extension HTTPClient.Configuration { /// Timeout configuration. - public struct Timeout { + public struct Timeout: NIOSendable { /// Specifies connect timeout. If no connect timeout is given, a default 30 seconds timeout will applied. public var connect: TimeAmount? /// Specifies read timeout. @@ -877,7 +912,7 @@ extension HTTPClient.Configuration { } /// Specifies redirect processing settings. - public struct RedirectConfiguration { + public struct RedirectConfiguration: NIOSendable { enum Mode { /// Redirects are not followed. case disallow @@ -909,7 +944,7 @@ extension HTTPClient.Configuration { } /// Connection pool configuration. - public struct ConnectionPool: Hashable { + public struct ConnectionPool: Hashable, NIOSendable { /// Specifies amount of time connections are kept idle in the pool. After this time has passed without a new /// request the connections are closed. public var idleTimeout: TimeAmount @@ -928,7 +963,7 @@ extension HTTPClient.Configuration { } } - public struct HTTPVersion { + public struct HTTPVersion: NIOSendable { internal enum Configuration { case http1Only case automatic diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index c62c2f7d1..3ecb68446 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -49,11 +49,19 @@ extension HTTPClient { /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `length`. public var length: Int? + #if swift(>=5.6) /// Body chunk provider. + public var stream: @Sendable (StreamWriter) -> EventLoopFuture + + @usableFromInline typealias StreamCallback = @Sendable (StreamWriter) -> EventLoopFuture + #else public var stream: (StreamWriter) -> EventLoopFuture + @usableFromInline typealias StreamCallback = (StreamWriter) -> EventLoopFuture + #endif + @inlinable - init(length: Int?, stream: @escaping (StreamWriter) -> EventLoopFuture) { + init(length: Int?, stream: @escaping StreamCallback) { self.length = length self.stream = stream } @@ -68,6 +76,18 @@ extension HTTPClient { } } + #if swift(>=5.6) + /// Create and stream body using ``StreamWriter``. + /// + /// - parameters: + /// - length: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` + /// header is set with the given `length`. + /// - stream: Body chunk provider. + @preconcurrency + public static func stream(length: Int? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { + return Body(length: length, stream: stream) + } + #else /// Create and stream body using ``StreamWriter``. /// /// - parameters: @@ -77,7 +97,21 @@ extension HTTPClient { public static func stream(length: Int? = nil, _ stream: @escaping (StreamWriter) -> EventLoopFuture) -> Body { return Body(length: length, stream: stream) } + #endif + #if swift(>=5.6) + /// Create and stream body using a collection of bytes. + /// + /// - parameters: + /// - data: Body binary representation. + @preconcurrency + @inlinable + public static func bytes(_ bytes: Bytes) -> Body where Bytes: RandomAccessCollection, Bytes: Sendable, Bytes.Element == UInt8 { + return Body(length: bytes.count) { writer in + writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) + } + } + #else /// Create and stream body using a collection of bytes. /// /// - parameters: @@ -88,6 +122,7 @@ extension HTTPClient { writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) } } + #endif /// Create and stream body using `String`. /// @@ -276,7 +311,7 @@ extension HTTPClient { } /// HTTP authentication. - public struct Authorization: Hashable { + public struct Authorization: Hashable, NIOSendable { private enum Scheme: Hashable { case Basic(String) case Bearer(String) @@ -685,6 +720,10 @@ extension HTTPClient { } } +#if swift(>=5.5) && canImport(_Concurrency) +extension HTTPClient.Task: @unchecked Sendable {} +#endif + internal struct TaskCancelEvent {} // MARK: - RedirectHandler diff --git a/Sources/AsyncHTTPClient/SSLContextCache.swift b/Sources/AsyncHTTPClient/SSLContextCache.swift index 31ed106a0..16916929a 100644 --- a/Sources/AsyncHTTPClient/SSLContextCache.swift +++ b/Sources/AsyncHTTPClient/SSLContextCache.swift @@ -18,7 +18,7 @@ import NIOConcurrencyHelpers import NIOCore import NIOSSL -class SSLContextCache { +final class SSLContextCache { private let lock = Lock() private var sslContextCache = LRUCache() private let offloadQueue = DispatchQueue(label: "io.github.swift-server.AsyncHTTPClient.SSLContextCache") @@ -55,3 +55,7 @@ extension SSLContextCache { return newSSLContext } } + +#if swift(>=5.5) && canImport(_Concurrency) +extension SSLContextCache: @unchecked Sendable {} +#endif diff --git a/Sources/AsyncHTTPClient/UnsafeTransfer.swift b/Sources/AsyncHTTPClient/UnsafeTransfer.swift new file mode 100644 index 000000000..2df9d1238 --- /dev/null +++ b/Sources/AsyncHTTPClient/UnsafeTransfer.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +/// ``UnsafeMutableTransferBox`` can be used to make non-`Sendable` values `Sendable` and mutable. +/// It can be used to capture local mutable values in a `@Sendable` closure and mutate them from within the closure. +/// As the name implies, the usage of this is unsafe because it disables the sendable checking of the compiler and does not add any synchronisation. +@usableFromInline +final class UnsafeMutableTransferBox { + @usableFromInline + var wrappedValue: Wrapped + + @inlinable + init(_ wrappedValue: Wrapped) { + self.wrappedValue = wrappedValue + } +} + +#if swift(>=5.5) && canImport(_Concurrency) +extension UnsafeMutableTransferBox: @unchecked Sendable {} +#endif diff --git a/Sources/AsyncHTTPClient/Utils.swift b/Sources/AsyncHTTPClient/Utils.swift index ed2819fd0..3bbb97904 100644 --- a/Sources/AsyncHTTPClient/Utils.swift +++ b/Sources/AsyncHTTPClient/Utils.swift @@ -23,9 +23,16 @@ public final class HTTPClientCopyingDelegate: HTTPClientResponseDelegate { let chunkHandler: (ByteBuffer) -> EventLoopFuture + #if swift(>=5.6) + @preconcurrency + public init(chunkHandler: @Sendable @escaping (ByteBuffer) -> EventLoopFuture) { + self.chunkHandler = chunkHandler + } + #else public init(chunkHandler: @escaping (ByteBuffer) -> EventLoopFuture) { self.chunkHandler = chunkHandler } + #endif public func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { return self.chunkHandler(buffer) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index a9728e1df..1420f187e 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -137,7 +137,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") request.method = .POST - request.body = .bytes(AnySequence("1234".utf8), length: .unknown) + request.body = .bytes(AnySendableSequence("1234".utf8), length: .unknown) guard let response = await XCTAssertNoThrowWithResult( try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) @@ -162,7 +162,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") request.method = .POST - request.body = .bytes(AnyCollection("1234".utf8), length: .unknown) + request.body = .bytes(AnySendableCollection("1234".utf8), length: .unknown) guard let response = await XCTAssertNoThrowWithResult( try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) @@ -647,4 +647,47 @@ extension AsyncSequence where Element == ByteBuffer { } } } + +struct AnySendableSequence: @unchecked Sendable { + private let wrapped: AnySequence + init( + _ sequence: WrappedSequence + ) where WrappedSequence.Element == Element { + self.wrapped = .init(sequence) + } +} + +extension AnySendableSequence: Sequence { + func makeIterator() -> AnySequence.Iterator { + self.wrapped.makeIterator() + } +} + +struct AnySendableCollection: @unchecked Sendable { + private let wrapped: AnyCollection + init( + _ collection: WrappedCollection + ) where WrappedCollection.Element == Element { + self.wrapped = .init(collection) + } +} + +extension AnySendableCollection: Collection { + var startIndex: AnyCollection.Index { + self.wrapped.startIndex + } + + var endIndex: AnyCollection.Index { + self.wrapped.endIndex + } + + func index(after i: AnyIndex) -> AnyIndex { + self.wrapped.index(after: i) + } + + subscript(position: AnyCollection.Index) -> Element { + self.wrapped[position] + } +} + #endif diff --git a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift index 312008959..f0376382f 100644 --- a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift +++ b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift @@ -16,8 +16,9 @@ import NIOConcurrencyHelpers import NIOCore +/// ``AsyncSequenceWriter`` is `Sendable` because its state is protected by a Lock @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -class AsyncSequenceWriter: AsyncSequence { +final class AsyncSequenceWriter: AsyncSequence, @unchecked Sendable { typealias AsyncIterator = Iterator struct Iterator: AsyncIteratorProtocol { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 99b9097ef..42ce4d537 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -296,7 +296,7 @@ class HTTPClientRequestTests: XCTestCase { XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST - let sequence = AnySequence(ByteBuffer(string: "post body").readableBytesView) + let sequence = AnySendableSequence(ByteBuffer(string: "post body").readableBytesView) request.body = .bytes(sequence, length: .unknown) var preparedRequest: PreparedRequest? XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) @@ -333,7 +333,7 @@ class HTTPClientRequestTests: XCTestCase { var request = Request(url: "http://example.com/post") request.method = .POST - let sequence = AnySequence(ByteBuffer(string: "post body").readableBytesView) + let sequence = AnySendableSequence(ByteBuffer(string: "post body").readableBytesView) request.body = .bytes(sequence, length: .known(9)) var preparedRequest: PreparedRequest? XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) @@ -541,6 +541,8 @@ struct ChunkedSequence: Sequence { } } +extension ChunkedSequence: Sendable where Wrapped: Sendable {} + extension Collection { /// Lazily splits `self` into `SubSequence`s with `maxChunkSize` elements. /// - Parameter maxChunkSize: size of each chunk except the last one which can be smaller if not enough elements are remaining. @@ -550,7 +552,7 @@ extension Collection { } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -struct AsyncSequenceFromSyncSequence: AsyncSequence { +struct AsyncSequenceFromSyncSequence: AsyncSequence, Sendable { typealias Element = Wrapped.Element struct AsyncIterator: AsyncIteratorProtocol { fileprivate var iterator: Wrapped.Iterator @@ -567,7 +569,7 @@ struct AsyncSequenceFromSyncSequence: AsyncSequence { } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension Sequence { +extension Sequence where Self: Sendable { /// Turns `self` into an `AsyncSequence` by wending each element of `self` asynchronously. func asAsyncSequence() -> AsyncSequenceFromSyncSequence { .init(wrapped: self) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index e2e34cf00..fd324c68d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1050,11 +1050,11 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try server?.close().wait()) } + let url = "http://127.0.0.1:\(server?.localAddress?.port ?? -1)/hello" let g = DispatchGroup() for workerID in 0.. 0 }.makeAsyncIterator()) + let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }) for i in 0..<100 { XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") @@ -237,7 +237,7 @@ final class TransactionTests: XCTestCase { XCTAssertEqual(response.headers, responseHead.headers) XCTAssertEqual(response.version, responseHead.version) - let iterator = SharedIterator(response.body.makeAsyncIterator()) + let iterator = SharedIterator(response.body) XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") async let part = iterator.next() @@ -433,7 +433,7 @@ final class TransactionTests: XCTestCase { XCTAssertEqual(response.version, responseHead.version) XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") - let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }.makeAsyncIterator()) + let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }) async let part1 = iterator.next() XCTAssertNoThrow(try executor.receiveResponseDemand()) @@ -510,7 +510,7 @@ final class TransactionTests: XCTestCase { XCTAssertEqual(response.version, .http2) XCTAssertEqual(delegate.hitStreamClosed, 0) - let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }.makeAsyncIterator()) + let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }) // at this point we can start to write to the stream and wait for the results @@ -541,16 +541,26 @@ final class TransactionTests: XCTestCase { // tasks. Since we want to wait for things to happen in tests, we need to `async let`, which creates // implicit tasks. Therefore we need to wrap our iterator struct. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -actor SharedIterator { - private var iterator: Iterator +actor SharedIterator where Wrapped.Element: Sendable { + private var wrappedIterator: Wrapped.AsyncIterator + private var nextCallInProgress: Bool = false - init(_ iterator: Iterator) { - self.iterator = iterator + init(_ sequence: Wrapped) { + self.wrappedIterator = sequence.makeAsyncIterator() } - func next() async throws -> Iterator.Element? { - var iter = self.iterator - defer { self.iterator = iter } + func next() async throws -> Wrapped.Element? { + precondition(self.nextCallInProgress == false) + self.nextCallInProgress = true + var iter = self.wrappedIterator + defer { + // auto-closure of `precondition(_:)` messes with actor isolation analyses in Swift 5.5 + // we therefore need to move the access to `self.nextCallInProgress` out of the auto-closure + let nextCallInProgress = nextCallInProgress + precondition(nextCallInProgress == true) + self.nextCallInProgress = false + self.wrappedIterator = iter + } return try await iter.next() } } From 8994e1f379c44182f559ebe4fbc0c333d524cc08 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Mon, 5 Sep 2022 09:55:44 +0100 Subject: [PATCH 036/146] not that simple anymore ;) (#624) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6dad76de3..611f5ce2c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # AsyncHTTPClient -This package provides simple HTTP Client library built on top of SwiftNIO. +This package provides an HTTP Client library built on top of SwiftNIO. This library provides the following: - First class support for Swift Concurrency (since version 1.9.0) From 7f998f5118de081fa2034833ce95bb5925dc6459 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Wed, 7 Sep 2022 14:12:05 +0100 Subject: [PATCH 037/146] add a future-returning shutdown method (#626) Co-authored-by: Johannes Weiss --- .../AsyncAwait/HTTPClient+shutdown.swift | 2 ++ Sources/AsyncHTTPClient/HTTPClient.swift | 22 +++++++++++++++++++ .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 5 +++++ 4 files changed, 30 insertions(+) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift index 4e7090dbf..36dd3588f 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift @@ -12,6 +12,8 @@ // //===----------------------------------------------------------------------===// +import NIOCore + #if compiler(>=5.5.2) && canImport(_Concurrency) extension HTTPClient { diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 55d2573ba..fa9ca0f83 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -213,6 +213,28 @@ public class HTTPClient { } #endif + /// Shuts down the ``HTTPClient`` and releases its resources. + /// + /// - note: You cannot use this method if you sharted the ``HTTPClient`` with + /// ``init(eventLoopGroupProvider: .createNew)`` because that will shut down the ``EventLoopGroup`` the + /// returned future would run in. + public func shutdown() -> EventLoopFuture { + switch self.eventLoopGroupProvider { + case .shared(let group): + let promise = group.any().makePromise(of: Void.self) + self.shutdown(queue: .global()) { error in + if let error = error { + promise.fail(error) + } else { + promise.succeed(()) + } + } + return promise.futureResult + case .createNew: + preconditionFailure("Cannot use the shutdown() method which returns a future when owning the EventLoopGroup. Please use the one of the other shutdown methods.") + } + } + private func shutdownEventLoop(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { self.stateLock.withLock { switch self.eventLoopGroupProvider { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 655e3acc5..2427d6cbf 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -143,6 +143,7 @@ extension HTTPClientTests { ("testConnectionPoolSizeConfigValueIsRespected", testConnectionPoolSizeConfigValueIsRespected), ("testRequestWithHeaderTransferEncodingIdentityDoesNotFail", testRequestWithHeaderTransferEncodingIdentityDoesNotFail), ("testMassiveDownload", testMassiveDownload), + ("testShutdownWithFutures", testShutdownWithFutures), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index fd324c68d..3c275c5eb 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3463,4 +3463,9 @@ class HTTPClientTests: XCTestCase { XCTAssertEqual(response?.version, .http1_1) XCTAssertEqual(response?.body?.readableBytes, 10_000) } + + func testShutdownWithFutures() { + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) + XCTAssertNoThrow(try httpClient.shutdown().wait()) + } } From 897d49aa1b33d1fa9754c7d15d8cc665003f0bdc Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 27 Sep 2022 15:42:47 +0200 Subject: [PATCH 038/146] Replace Lock with NIOLock (#628) SwiftNIO 2.42.0 has deprecated Lock and replaced it with a new NIOLock. This commit removes all uses of Lock and replaces them with NIOLock. Further, now, we must require SwiftNIO 2.42.0 --- Package.swift | 2 +- .../AsyncHTTPClient/AsyncAwait/Transaction.swift | 2 +- .../HTTPConnectionPool+Manager.swift | 2 +- .../ConnectionPool/HTTPConnectionPool.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 8 ++++---- Sources/AsyncHTTPClient/HTTPHandler.swift | 2 +- Sources/AsyncHTTPClient/SSLContextCache.swift | 2 +- Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift | 2 +- .../HTTP1ConnectionTests.swift | 14 +++++++------- .../HTTP2ConnectionTests.swift | 10 +++++----- .../AsyncHTTPClientTests/HTTPClientTestUtils.swift | 2 +- 11 files changed, 24 insertions(+), 24 deletions(-) diff --git a/Package.swift b/Package.swift index 23fde6647..f72de842e 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 66050bcde..1974778e9 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -29,7 +29,7 @@ final class Transaction: @unchecked Sendable { let preferredEventLoop: EventLoop let requestOptions: RequestOptions - private let stateLock = Lock() + private let stateLock = NIOLock() private var state: StateMachine init( diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift index 08d1e9431..f5a0540cf 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift @@ -35,7 +35,7 @@ extension HTTPConnectionPool { private var state: State = .active private var _pools: [Key: HTTPConnectionPool] = [:] - private let lock = Lock() + private let lock = NIOLock() private let sslContextCache = SSLContextCache() diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 5a0b2708e..74b7c044c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -22,7 +22,7 @@ protocol HTTPConnectionPoolDelegate { } final class HTTPConnectionPool { - private let stateLock = Lock() + private let stateLock = NIOLock() private var _state: StateMachine /// The connection idle timeout timers. Protected by the stateLock private var _idleTimer = [Connection.ID: Scheduled]() diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index fa9ca0f83..fba012b09 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -75,10 +75,10 @@ public class HTTPClient { /// Shared thread pool used for file IO. It is lazily created on first access of ``Task/fileIOThreadPool``. private var fileIOThreadPool: NIOThreadPool? - private let fileIOThreadPoolLock = Lock() + private let fileIOThreadPoolLock = NIOLock() private var state: State - private let stateLock = Lock() + private let stateLock = NIOLock() internal static let loggingDisabled = Logger(label: "AHC-do-not-log", factory: { _ in SwiftLogNoOpLogHandler() }) @@ -169,7 +169,7 @@ public class HTTPClient { Current eventLoop: \(eventLoop) """) } - let errorStorageLock = Lock() + let errorStorageLock = NIOLock() let errorStorage: UnsafeMutableTransferBox = .init(nil) let continuation = DispatchWorkItem {} self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown")) { error in @@ -256,7 +256,7 @@ public class HTTPClient { } private func shutdownFileIOThreadPool(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { - self.fileIOThreadPoolLock.withLockVoid { + self.fileIOThreadPoolLock.withLock { guard let fileIOThreadPool = fileIOThreadPool else { callback(nil) return diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 3ecb68446..744139f68 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -657,7 +657,7 @@ extension HTTPClient { private var _isCancelled: Bool = false private var _taskDelegate: HTTPClientTaskDelegate? - private let lock = Lock() + private let lock = NIOLock() private let makeOrGetFileIOThreadPool: () -> NIOThreadPool /// The shared thread pool of a ``HTTPClient`` used for file IO. It is lazily created on first access. diff --git a/Sources/AsyncHTTPClient/SSLContextCache.swift b/Sources/AsyncHTTPClient/SSLContextCache.swift index 16916929a..f1e8623a6 100644 --- a/Sources/AsyncHTTPClient/SSLContextCache.swift +++ b/Sources/AsyncHTTPClient/SSLContextCache.swift @@ -19,7 +19,7 @@ import NIOCore import NIOSSL final class SSLContextCache { - private let lock = Lock() + private let lock = NIOLock() private var sslContextCache = LRUCache() private let offloadQueue = DispatchQueue(label: "io.github.swift-server.AsyncHTTPClient.SSLContextCache") } diff --git a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift index f0376382f..332cb2227 100644 --- a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift +++ b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift @@ -45,7 +45,7 @@ final class AsyncSequenceWriter: AsyncSequence, @unchecked Sendable { } private var _state = State.buffering(.init(), nil) - private let lock = Lock() + private let lock = NIOLock() public var hasDemand: Bool { self.lock.withLock { diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift index 3575a6080..d240d2686 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift @@ -595,12 +595,12 @@ class HTTP1ConnectionTests: XCTestCase { var _reads = 0 var _channel: Channel? - let lock: Lock + let lock: NIOLock let backpressurePromise: EventLoopPromise let messageReceived: EventLoopPromise init(eventLoop: EventLoop) { - self.lock = Lock() + self.lock = NIOLock() self.backpressurePromise = eventLoop.makePromise() self.messageReceived = eventLoop.makePromise() } @@ -612,7 +612,7 @@ class HTTP1ConnectionTests: XCTestCase { } func willExecuteOnChannel(_ channel: Channel) { - self.lock.withLockVoid { + self.lock.withLock { self._channel = channel } } @@ -623,7 +623,7 @@ class HTTP1ConnectionTests: XCTestCase { func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { // We count a number of reads received. - self.lock.withLockVoid { + self.lock.withLock { self._reads += 1 } // We need to notify the test when first byte of the message is arrived. @@ -805,7 +805,7 @@ class AfterRequestCloseConnectionChannelHandler: ChannelInboundHandler { } class MockConnectionDelegate: HTTP1ConnectionDelegate { - private var lock = Lock() + private var lock = NIOLock() private var _hitConnectionReleased = 0 private var _hitConnectionClosed = 0 @@ -821,13 +821,13 @@ class MockConnectionDelegate: HTTP1ConnectionDelegate { init() {} func http1ConnectionReleased(_: HTTP1Connection) { - self.lock.withLockVoid { + self.lock.withLock { self._hitConnectionReleased += 1 } } func http1ConnectionClosed(_: HTTP1Connection) { - self.lock.withLockVoid { + self.lock.withLock { self._hitConnectionClosed += 1 } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index bcdaf1af2..9d0517657 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -235,7 +235,7 @@ class TestConnectionCreator { } private var state: State = .idle - private let lock = Lock() + private let lock = NIOLock() init() {} @@ -428,7 +428,7 @@ class TestHTTP2ConnectionDelegate: HTTP2ConnectionDelegate { self.lock.withLock { self._maxStreamSetting } } - private let lock = Lock() + private let lock = NIOLock() private var _hitStreamClosed: Int = 0 private var _hitGoAwayReceived: Int = 0 private var _hitConnectionClosed: Int = 0 @@ -439,19 +439,19 @@ class TestHTTP2ConnectionDelegate: HTTP2ConnectionDelegate { func http2Connection(_: HTTP2Connection, newMaxStreamSetting: Int) {} func http2ConnectionStreamClosed(_: HTTP2Connection, availableStreams: Int) { - self.lock.withLockVoid { + self.lock.withLock { self._hitStreamClosed += 1 } } func http2ConnectionGoAwayReceived(_: HTTP2Connection) { - self.lock.withLockVoid { + self.lock.withLock { self._hitGoAwayReceived += 1 } } func http2ConnectionClosed(_: HTTP2Connection) { - self.lock.withLockVoid { + self.lock.withLock { self._hitConnectionClosed += 1 } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 7cd9ef83d..91358a361 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -1118,7 +1118,7 @@ struct CollectEverythingLogHandler: LogHandler { var metadata: [String: String] } - var lock = Lock() + var lock = NIOLock() var logs: [Entry] = [] var allEntries: [Entry] { From 03b3e7b34153299e9b4c4b5c2a6ac790a582a3ac Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 27 Sep 2022 15:50:07 +0100 Subject: [PATCH 039/146] Update files missed by #628 (#629) --- Package@swift-5.4.swift | 2 +- Package@swift-5.5.swift | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift index a69d9bc8a..2718138eb 100644 --- a/Package@swift-5.4.swift +++ b/Package@swift-5.4.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index a69d9bc8a..2718138eb 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.41.1"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), From b57bcb90a81432f83b19d21a67780de837d4270a Mon Sep 17 00:00:00 2001 From: George Barnett Date: Wed, 28 Sep 2022 17:16:26 +0100 Subject: [PATCH 040/146] Raise minimum supported Swift version from 5.4 to 5.5 (#630) Motivation: SwiftNIO periodically drops support for older Swift versions. Now that 5.7 has been released, 5.4 will be dropped. Modifications: - Remove 5.4 specific Package.swift and docker-compose - Update the 5.7 docker-compose to use the released 5.7 and move from focal (2004) to jammy (2204) - Update tools version in Package@swift-5.5.swift to 5.5 (from 5.4) - Add supported versions section to README Results: Minimum Swift version is 5.5 --- Package@swift-5.4.swift | 73 ---------------------------- Package@swift-5.5.swift | 2 +- README.md | 41 ++++++++++------ Tests/LinuxMain.swift | 77 ++++++++++++++++-------------- docker/Dockerfile | 4 +- docker/docker-compose.1804.54.yaml | 19 -------- docker/docker-compose.2004.57.yaml | 17 ------- docker/docker-compose.2204.57.yaml | 18 +++++++ scripts/generate_linux_tests.rb | 11 +++-- 9 files changed, 96 insertions(+), 166 deletions(-) delete mode 100644 Package@swift-5.4.swift delete mode 100644 docker/docker-compose.1804.54.yaml delete mode 100644 docker/docker-compose.2004.57.yaml create mode 100644 docker/docker-compose.2204.57.yaml diff --git a/Package@swift-5.4.swift b/Package@swift-5.4.swift deleted file mode 100644 index 2718138eb..000000000 --- a/Package@swift-5.4.swift +++ /dev/null @@ -1,73 +0,0 @@ -// swift-tools-version:5.4 -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import PackageDescription - -let package = Package( - name: "async-http-client", - products: [ - .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), - ], - dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), - .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), - .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), - .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), - .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), - ], - targets: [ - .target(name: "CAsyncHTTPClient"), - .target( - name: "AsyncHTTPClient", - dependencies: [ - .target(name: "CAsyncHTTPClient"), - .product(name: "NIO", package: "swift-nio"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOPosix", package: "swift-nio"), - .product(name: "NIOHTTP1", package: "swift-nio"), - .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOHTTP2", package: "swift-nio-http2"), - .product(name: "NIOSSL", package: "swift-nio-ssl"), - .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), - .product(name: "NIOSOCKS", package: "swift-nio-extras"), - .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), - .product(name: "Logging", package: "swift-log"), - .product(name: "Atomics", package: "swift-atomics"), - ] - ), - .testTarget( - name: "AsyncHTTPClientTests", - dependencies: [ - .target(name: "AsyncHTTPClient"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOEmbedded", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), - .product(name: "NIOTestUtils", package: "swift-nio"), - .product(name: "NIOSSL", package: "swift-nio-ssl"), - .product(name: "NIOHTTP2", package: "swift-nio-http2"), - .product(name: "NIOSOCKS", package: "swift-nio-extras"), - .product(name: "Logging", package: "swift-log"), - .product(name: "Atomics", package: "swift-atomics"), - ], - resources: [ - .copy("Resources/self_signed_cert.pem"), - .copy("Resources/self_signed_key.pem"), - ] - ), - ] -) diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index 2718138eb..8ae20ed9c 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.4 +// swift-tools-version:5.5 //===----------------------------------------------------------------------===// // // This source file is part of the AsyncHTTPClient open source project diff --git a/README.md b/README.md index 611f5ce2c..261fcab56 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ httpClient.execute(request: request, deadline: .now() + .milliseconds(1)) ``` ### Streaming -When dealing with larger amount of data, it's critical to stream the response body instead of aggregating in-memory. +When dealing with larger amount of data, it's critical to stream the response body instead of aggregating in-memory. The following example demonstrates how to count the number of bytes in a streaming response body: #### Using Swift Concurrency @@ -172,7 +172,7 @@ do { for try await buffer in response.body { // for this example, we are just interested in the size of the fragment receivedBytes += buffer.readableBytes - + if let expectedBytes = expectedBytes { // if the body size is known, we calculate a progress indicator let progress = Double(receivedBytes) / Double(expectedBytes) @@ -181,9 +181,9 @@ do { } print("did receive \(receivedBytes) bytes") } catch { - print("request failed:", error) + print("request failed:", error) } -// it is important to shutdown the httpClient after all requests are done, even if one failed +// it is important to shutdown the httpClient after all requests are done, even if one failed try await httpClient.shutdown() ``` @@ -211,17 +211,17 @@ class CountingDelegate: HTTPClientResponseDelegate { } func didReceiveHead( - task: HTTPClient.Task, + task: HTTPClient.Task, _ head: HTTPResponseHead ) -> EventLoopFuture { - // this is executed when we receive HTTP response head part of the request - // (it contains response code and headers), called once in case backpressure + // this is executed when we receive HTTP response head part of the request + // (it contains response code and headers), called once in case backpressure // is needed, all reads will be paused until returned future is resolved return task.eventLoop.makeSucceededFuture(()) } func didReceiveBodyPart( - task: HTTPClient.Task, + task: HTTPClient.Task, _ buffer: ByteBuffer ) -> EventLoopFuture { // this is executed when we receive parts of the response body, could be called zero or more times @@ -283,8 +283,8 @@ Connecting to servers bound to socket paths is easy: ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) httpClient.execute( - .GET, - socketPath: "/tmp/myServer.socket", + .GET, + socketPath: "/tmp/myServer.socket", urlPath: "/path/to/resource" ).whenComplete (...) ``` @@ -293,9 +293,9 @@ Connecting over TLS to a unix domain socket path is possible as well: ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) httpClient.execute( - .POST, - secureSocketPath: "/tmp/myServer.socket", - urlPath: "/path/to/resource", + .POST, + secureSocketPath: "/tmp/myServer.socket", + urlPath: "/path/to/resource", body: .string("hello") ).whenComplete (...) ``` @@ -303,11 +303,11 @@ httpClient.execute( Direct URLs can easily be constructed to be executed in other scenarios: ```swift let socketPathBasedURL = URL( - httpURLWithSocketPath: "/tmp/myServer.socket", + httpURLWithSocketPath: "/tmp/myServer.socket", uri: "/path/to/resource" ) let secureSocketPathBasedURL = URL( - httpsURLWithSocketPath: "/tmp/myServer.socket", + httpsURLWithSocketPath: "/tmp/myServer.socket", uri: "/path/to/resource" ) ``` @@ -326,3 +326,14 @@ let client = HTTPClient( ## Security Please have a look at [SECURITY.md](SECURITY.md) for AsyncHTTPClient's security process. + +## Supported Versions + +The most recent versions of AsyncHTTPClient support Swift 5.5 and newer. The minimum Swift version supported by AsyncHTTPClient releases are detailed below: + +AsyncHTTPClient | Minimum Swift Version +--------------------|---------------------- +`1.0.0 ..< 1.5.0` | 5.0 +`1.5.0 ..< 1.10.0` | 5.2 +`1.10.0 ..< 1.13.0` | 5.4 +`1.13.0 ...` | 5.5 diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index cebced614..2d7e744af 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -25,40 +25,45 @@ import XCTest #if os(Linux) || os(FreeBSD) @testable import AsyncHTTPClientTests -XCTMain([ - testCase(AsyncAwaitEndToEndTests.allTests), - testCase(HTTP1ClientChannelHandlerTests.allTests), - testCase(HTTP1ConnectionStateMachineTests.allTests), - testCase(HTTP1ConnectionTests.allTests), - testCase(HTTP1ProxyConnectHandlerTests.allTests), - testCase(HTTP2ClientRequestHandlerTests.allTests), - testCase(HTTP2ClientTests.allTests), - testCase(HTTP2ConnectionTests.allTests), - testCase(HTTP2IdleHandlerTests.allTests), - testCase(HTTPClientCookieTests.allTests), - testCase(HTTPClientInternalTests.allTests), - testCase(HTTPClientNIOTSTests.allTests), - testCase(HTTPClientReproTests.allTests), - testCase(HTTPClientRequestTests.allTests), - testCase(HTTPClientSOCKSTests.allTests), - testCase(HTTPClientTests.allTests), - testCase(HTTPClientUncleanSSLConnectionShutdownTests.allTests), - testCase(HTTPConnectionPoolTests.allTests), - testCase(HTTPConnectionPool_FactoryTests.allTests), - testCase(HTTPConnectionPool_HTTP1ConnectionsTests.allTests), - testCase(HTTPConnectionPool_HTTP1StateMachineTests.allTests), - testCase(HTTPConnectionPool_HTTP2ConnectionsTests.allTests), - testCase(HTTPConnectionPool_HTTP2StateMachineTests.allTests), - testCase(HTTPConnectionPool_ManagerTests.allTests), - testCase(HTTPConnectionPool_RequestQueueTests.allTests), - testCase(HTTPRequestStateMachineTests.allTests), - testCase(LRUCacheTests.allTests), - testCase(RequestBagTests.allTests), - testCase(RequestValidationTests.allTests), - testCase(SOCKSEventsHandlerTests.allTests), - testCase(SSLContextCacheTests.allTests), - testCase(TLSEventsHandlerTests.allTests), - testCase(TransactionTests.allTests), - testCase(Transaction_StateMachineTests.allTests), -]) +@main +struct LinuxMain { + static func main() { + XCTMain([ + testCase(AsyncAwaitEndToEndTests.allTests), + testCase(HTTP1ClientChannelHandlerTests.allTests), + testCase(HTTP1ConnectionStateMachineTests.allTests), + testCase(HTTP1ConnectionTests.allTests), + testCase(HTTP1ProxyConnectHandlerTests.allTests), + testCase(HTTP2ClientRequestHandlerTests.allTests), + testCase(HTTP2ClientTests.allTests), + testCase(HTTP2ConnectionTests.allTests), + testCase(HTTP2IdleHandlerTests.allTests), + testCase(HTTPClientCookieTests.allTests), + testCase(HTTPClientInternalTests.allTests), + testCase(HTTPClientNIOTSTests.allTests), + testCase(HTTPClientReproTests.allTests), + testCase(HTTPClientRequestTests.allTests), + testCase(HTTPClientSOCKSTests.allTests), + testCase(HTTPClientTests.allTests), + testCase(HTTPClientUncleanSSLConnectionShutdownTests.allTests), + testCase(HTTPConnectionPoolTests.allTests), + testCase(HTTPConnectionPool_FactoryTests.allTests), + testCase(HTTPConnectionPool_HTTP1ConnectionsTests.allTests), + testCase(HTTPConnectionPool_HTTP1StateMachineTests.allTests), + testCase(HTTPConnectionPool_HTTP2ConnectionsTests.allTests), + testCase(HTTPConnectionPool_HTTP2StateMachineTests.allTests), + testCase(HTTPConnectionPool_ManagerTests.allTests), + testCase(HTTPConnectionPool_RequestQueueTests.allTests), + testCase(HTTPRequestStateMachineTests.allTests), + testCase(LRUCacheTests.allTests), + testCase(RequestBagTests.allTests), + testCase(RequestValidationTests.allTests), + testCase(SOCKSEventsHandlerTests.allTests), + testCase(SSLContextCacheTests.allTests), + testCase(TLSEventsHandlerTests.allTests), + testCase(TransactionTests.allTests), + testCase(Transaction_StateMachineTests.allTests), + ]) + } +} #endif diff --git a/docker/Dockerfile b/docker/Dockerfile index 1cd4f2140..2d1e57def 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,5 +1,5 @@ -ARG swift_version=5.4 -ARG ubuntu_version=bionic +ARG swift_version=5.7 +ARG ubuntu_version=jammy ARG base_image=swift:$swift_version-$ubuntu_version FROM $base_image # needed to do again after FROM due to docker limitation diff --git a/docker/docker-compose.1804.54.yaml b/docker/docker-compose.1804.54.yaml deleted file mode 100644 index dd869549c..000000000 --- a/docker/docker-compose.1804.54.yaml +++ /dev/null @@ -1,19 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:18.04-5.4 - build: - args: - ubuntu_version: "bionic" - swift_version: "5.4" - - test: - image: async-http-client:18.04-5.4 - command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:18.04-5.4 diff --git a/docker/docker-compose.2004.57.yaml b/docker/docker-compose.2004.57.yaml deleted file mode 100644 index bf6dd6e97..000000000 --- a/docker/docker-compose.2004.57.yaml +++ /dev/null @@ -1,17 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:20.04-5.7 - build: - args: - base_image: "swiftlang/swift:nightly-5.7-focal" - - test: - image: async-http-client:20.04-5.7 - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:20.04-5.7 diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml new file mode 100644 index 000000000..bea713bad --- /dev/null +++ b/docker/docker-compose.2204.57.yaml @@ -0,0 +1,18 @@ +version: "3" + +services: + + runtime-setup: + image: async-http-client:22.04-5.7 + build: + args: + ubuntu_version: "jammy" + swift_version: "5.7" + + test: + image: async-http-client:22.04-5.7 + environment: [] + #- SANITIZER_ARG=--sanitize=thread + + shell: + image: async-http-client:22.04-5.7 diff --git a/scripts/generate_linux_tests.rb b/scripts/generate_linux_tests.rb index ed887f83c..fe5726f6c 100755 --- a/scripts/generate_linux_tests.rb +++ b/scripts/generate_linux_tests.rb @@ -99,7 +99,10 @@ def createLinuxMain(testsDirectory, allTestSubDirectories, files) file.write '@testable import ' + testSubDirectory + "\n" end file.write "\n" - file.write "XCTMain([\n" + file.write "@main\n" + file.write "struct LinuxMain {\n" + file.write " static func main() {\n" + file.write " XCTMain([\n" testCases = [] for classes in files @@ -109,9 +112,11 @@ def createLinuxMain(testsDirectory, allTestSubDirectories, files) end for testCase in testCases.sort { |x, y| x <=> y } - file.write ' testCase(' + testCase + ".allTests),\n" + file.write ' testCase(' + testCase + ".allTests),\n" end - file.write "])\n" + file.write " ])\n" + file.write " }\n" + file.write "}\n" file.write "#endif\n" end end From 64ff430582b22218866a4f7f2964312a71d9bb2d Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Fri, 7 Oct 2022 16:49:29 +0200 Subject: [PATCH 041/146] Add Hashable conformace to `HTTPClient.Configuration.Proxy` (#634) --- Sources/AsyncHTTPClient/HTTPClient+Proxy.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift index 16be51ca5..148a4e888 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift @@ -25,7 +25,7 @@ extension HTTPClient.Configuration { /// If a `TLSConfiguration` is used in conjunction with `HTTPClient.Configuration.Proxy`, /// TLS will be established _after_ successful proxy, between your client /// and the destination server. - public struct Proxy: NIOSendable { + public struct Proxy: NIOSendable, Hashable { enum ProxyType: Hashable { case http(HTTPClient.Authorization?) case socks From af5966f1d1baccf210b5aa9e56b6b247b22f7982 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Fri, 7 Oct 2022 17:05:28 +0200 Subject: [PATCH 042/146] Reduce use of `HTTPClient.Configuration` in the Connection objects (#635) --- .../HTTP1/HTTP1Connection.swift | 8 ++-- .../HTTP2/HTTP2Connection.swift | 10 ++++- .../HTTPConnectionPool+Factory.swift | 39 +------------------ .../EmbeddedChannel+HTTPConvenience.swift | 2 +- .../HTTP1ConnectionTests.swift | 24 ++++++------ .../HTTP2ConnectionTests.swift | 2 +- 6 files changed, 28 insertions(+), 57 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift index 173ac79e4..3485ada6c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift @@ -57,11 +57,11 @@ final class HTTP1Connection { channel: Channel, connectionID: HTTPConnectionPool.Connection.ID, delegate: HTTP1ConnectionDelegate, - configuration: HTTPClient.Configuration, + decompression: HTTPClient.Decompression, logger: Logger ) throws -> HTTP1Connection { let connection = HTTP1Connection(channel: channel, connectionID: connectionID, delegate: delegate) - try connection.start(configuration: configuration, logger: logger) + try connection.start(decompression: decompression, logger: logger) return connection } @@ -101,7 +101,7 @@ final class HTTP1Connection { self.channel.write(request, promise: nil) } - private func start(configuration: HTTPClient.Configuration, logger: Logger) throws { + private func start(decompression: HTTPClient.Decompression, logger: Logger) throws { self.channel.eventLoop.assertInEventLoop() guard case .initialized = self.state else { @@ -127,7 +127,7 @@ final class HTTP1Connection { try sync.addHandler(requestEncoder) try sync.addHandler(ByteToMessageHandler(responseDecoder)) - if case .enabled(let limit) = configuration.decompression { + if case .enabled(let limit) = decompression { let decompressHandler = NIOHTTPResponseDecompressor(limit: limit) try sync.addHandler(decompressHandler) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index f9854a810..701e630c2 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -119,10 +119,16 @@ final class HTTP2Connection { channel: Channel, connectionID: HTTPConnectionPool.Connection.ID, delegate: HTTP2ConnectionDelegate, - configuration: HTTPClient.Configuration, + decompression: HTTPClient.Decompression, logger: Logger ) -> EventLoopFuture<(HTTP2Connection, Int)> { - let connection = HTTP2Connection(channel: channel, connectionID: connectionID, decompression: configuration.decompression, delegate: delegate, logger: logger) + let connection = HTTP2Connection( + channel: channel, + connectionID: connectionID, + decompression: decompression, + delegate: delegate, + logger: logger + ) return connection.start().map { maxStreams in (connection, maxStreams) } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 1444df9bb..e94e967a6 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -71,7 +71,7 @@ extension HTTPConnectionPool.ConnectionFactory { channel: channel, connectionID: connectionID, delegate: http1ConnectionDelegate, - configuration: self.clientConfiguration, + decompression: self.clientConfiguration.decompression, logger: logger ) requester.http1ConnectionCreated(connection) @@ -83,7 +83,7 @@ extension HTTPConnectionPool.ConnectionFactory { channel: channel, connectionID: connectionID, delegate: http2ConnectionDelegate, - configuration: self.clientConfiguration, + decompression: self.clientConfiguration.decompression, logger: logger ).whenComplete { result in switch result { @@ -105,41 +105,6 @@ extension HTTPConnectionPool.ConnectionFactory { case http2(Channel) } - func makeHTTP1Channel( - requester: Requester, - connectionID: HTTPConnectionPool.Connection.ID, - deadline: NIODeadline, - eventLoop: EventLoop, - logger: Logger - ) -> EventLoopFuture { - self.makeChannel( - requester: requester, - connectionID: connectionID, - deadline: deadline, - eventLoop: eventLoop, - logger: logger - ).flatMapThrowing { negotiated -> Channel in - - guard case .http1_1(let channel) = negotiated else { - preconditionFailure("Expected to create http/1.1 connections only for now") - } - - // add the http1.1 channel handlers - let syncOperations = channel.pipeline.syncOperations - try syncOperations.addHTTPClientHandlers(leftOverBytesStrategy: .forwardBytes) - - switch self.clientConfiguration.decompression { - case .disabled: - () - case .enabled(let limit): - let decompressHandler = NIOHTTPResponseDecompressor(limit: limit) - try syncOperations.addHandler(decompressHandler) - } - - return channel - } - } - func makeChannel( requester: Requester, connectionID: HTTPConnectionPool.Connection.ID, diff --git a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift index f46c6fbf9..5e7a1a9bc 100644 --- a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift +++ b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift @@ -77,7 +77,7 @@ extension EmbeddedChannel { channel: self, connectionID: 1, delegate: connectionDelegate, - configuration: .init(), + decompression: .disabled, logger: logger ) diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift index d240d2686..3ff73de06 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift @@ -35,7 +35,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: MockHTTP1ConnectionDelegate(), - configuration: .init(decompression: .enabled(limit: .ratio(4))), + decompression: .enabled(limit: .ratio(4)), logger: logger )) @@ -58,7 +58,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: MockHTTP1ConnectionDelegate(), - configuration: .init(decompression: .disabled), + decompression: .disabled, logger: logger )) @@ -82,7 +82,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: MockHTTP1ConnectionDelegate(), - configuration: .init(), + decompression: .disabled, logger: logger )) } @@ -106,7 +106,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: $0, connectionID: 0, delegate: delegate, - configuration: .init(decompression: .disabled), + decompression: .disabled, logger: logger ) } @@ -206,7 +206,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: XCTUnwrap(maybeChannel), connectionID: 0, delegate: connectionDelegate, - configuration: .init(), + decompression: .disabled, logger: logger ) }.wait()) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } @@ -260,7 +260,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: XCTUnwrap(maybeChannel), connectionID: 0, delegate: connectionDelegate, - configuration: .init(), + decompression: .disabled, logger: logger ) }.wait()) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } @@ -332,7 +332,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: XCTUnwrap(maybeChannel), connectionID: 0, delegate: connectionDelegate, - configuration: .init(), + decompression: .disabled, logger: logger ) }.wait()) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } @@ -377,7 +377,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: connectionDelegate, - configuration: .init(decompression: .enabled(limit: .ratio(4))), + decompression: .enabled(limit: .ratio(4)), logger: logger )) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } @@ -442,7 +442,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: connectionDelegate, - configuration: .init(decompression: .enabled(limit: .ratio(4))), + decompression: .enabled(limit: .ratio(4)), logger: logger )) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } @@ -504,7 +504,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: connectionDelegate, - configuration: .init(decompression: .enabled(limit: .ratio(4))), + decompression: .enabled(limit: .ratio(4)), logger: logger )) @@ -539,7 +539,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: connectionDelegate, - configuration: .init(decompression: .enabled(limit: .ratio(4))), + decompression: .enabled(limit: .ratio(4)), logger: logger )) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } @@ -691,7 +691,7 @@ class HTTP1ConnectionTests: XCTestCase { channel: channel, connectionID: 0, delegate: connectionDelegate, - configuration: .init(), + decompression: .disabled, logger: logger ) }.wait()) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point") } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index 9d0517657..a9ff14f49 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -37,7 +37,7 @@ class HTTP2ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, delegate: TestHTTP2ConnectionDelegate(), - configuration: .init(), + decompression: .disabled, logger: logger ).wait()) } From 4d69c84617e2313bc23e7f9f31fb9b153879f40b Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Fri, 7 Oct 2022 17:15:14 +0200 Subject: [PATCH 043/146] Add `Hashable` conformance to `HTTPClient.Configuration.HTTPVersion` (#636) --- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index fba012b09..0f5026fce 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -985,7 +985,7 @@ extension HTTPClient.Configuration { } } - public struct HTTPVersion: NIOSendable { + public struct HTTPVersion: NIOSendable, Hashable { internal enum Configuration { case http1Only case automatic From f7a84af5d670c559c31cf53aca69225da77b516a Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 7 Oct 2022 17:22:56 +0200 Subject: [PATCH 044/146] Fix request hang if delegate fails promise returned by `didReceiveBodyPart` (#633) * Add test which currently hangs indefinitely * Fix request hang if delegate fails promise * Run `generate_linux_tests.rb` and `SwiftFormat` --- Sources/AsyncHTTPClient/RequestBag.swift | 24 ++----- .../RequestBagTests+XCTest.swift | 1 + .../RequestBagTests.swift | 67 +++++++++++++++++++ 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 9c45728b7..50c0057ba 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -276,14 +276,7 @@ final class RequestBag { self.delegate.didReceiveBodyPart(task: self.task, buffer) .hop(to: self.task.eventLoop) .whenComplete { - switch $0 { - case .success: - self.consumeMoreBodyData0(resultOfPreviousConsume: $0) - case .failure(let error): - // if in the response stream consumption an error has occurred, we need to - // cancel the running request and fail the task. - self.fail(error) - } + self.consumeMoreBodyData0(resultOfPreviousConsume: $0) } case .succeedRequest: @@ -325,18 +318,13 @@ final class RequestBag { self.delegate.didReceiveBodyPart(task: self.task, byteBuffer) .hop(to: self.task.eventLoop) .whenComplete { result in - switch result { - case .success: - if self.consumeBodyPartStackDepth < Self.maxConsumeBodyPartStackDepth { + if self.consumeBodyPartStackDepth < Self.maxConsumeBodyPartStackDepth { + self.consumeMoreBodyData0(resultOfPreviousConsume: result) + } else { + // We need to unwind the stack, let's take a break. + self.task.eventLoop.execute { self.consumeMoreBodyData0(resultOfPreviousConsume: result) - } else { - // We need to unwind the stack, let's take a break. - self.task.eventLoop.execute { - self.consumeMoreBodyData0(resultOfPreviousConsume: result) - } } - case .failure(let error): - self.fail(error) } } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index 72046f68c..b6a05733c 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -34,6 +34,7 @@ extension RequestBagTests { ("testCancelFailsTaskWhenTaskIsQueued", testCancelFailsTaskWhenTaskIsQueued), ("testFailsTaskWhenTaskIsWaitingForMoreFromServer", testFailsTaskWhenTaskIsWaitingForMoreFromServer), ("testChannelBecomingWritableDoesntCrashCancelledTask", testChannelBecomingWritableDoesntCrashCancelledTask), + ("testDidReceiveBodyPartFailedPromise", testDidReceiveBodyPartFailedPromise), ("testHTTPUploadIsCancelledEvenThoughRequestSucceeds", testHTTPUploadIsCancelledEvenThoughRequestSucceeds), ("testRaceBetweenConnectionCloseAndDemandMoreData", testRaceBetweenConnectionCloseAndDemandMoreData), ("testRedirectWith3KBBody", testRedirectWith3KBBody), diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 6993c0df9..43062405c 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -455,6 +455,73 @@ final class RequestBagTests: XCTestCase { } } + func testDidReceiveBodyPartFailedPromise() { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let logger = Logger(label: "test") + + var maybeRequest: HTTPClient.Request? + + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( + url: "https://swift.org", + method: .POST, + body: .byteBuffer(.init(bytes: [1])) + )) + guard let request = maybeRequest else { return XCTFail("Expected to have a request") } + + struct MyError: Error, Equatable {} + final class Delegate: HTTPClientResponseDelegate { + typealias Response = Void + let didFinishPromise: EventLoopPromise + init(didFinishPromise: EventLoopPromise) { + self.didFinishPromise = didFinishPromise + } + + func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { + task.eventLoop.makeFailedFuture(MyError()) + } + + func didReceiveError(task: HTTPClient.Task, _ error: Error) { + self.didFinishPromise.fail(error) + } + + func didFinishRequest(task: AsyncHTTPClient.HTTPClient.Task) throws { + XCTFail("\(#function) should not be called") + self.didFinishPromise.succeed(()) + } + } + let delegate = Delegate(didFinishPromise: embeddedEventLoop.makePromise()) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + )) + guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } + + let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + + executor.runRequest(bag) + + bag.resumeRequestBodyStream() + XCTAssertNoThrow(try executor.receiveRequestBody { XCTAssertEqual($0, ByteBuffer(bytes: [1])) }) + + bag.receiveResponseHead(.init(version: .http1_1, status: .ok)) + + bag.succeedRequest([ByteBuffer([1])]) + + XCTAssertThrowsError(try delegate.didFinishPromise.futureResult.wait()) { error in + XCTAssertEqualTypeAndValue(error, MyError()) + } + XCTAssertThrowsError(try bag.task.futureResult.wait()) { error in + XCTAssertEqualTypeAndValue(error, MyError()) + } + } + func testHTTPUploadIsCancelledEvenThoughRequestSucceeds() { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } From 9937d8751a83fde631d26929f56de99abef0d957 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Mon, 10 Oct 2022 11:18:58 +0100 Subject: [PATCH 045/146] Handle ResponseAccumulator not being able to buffer large response in memory (#637) * Handle ResponseAccumulator not being able to buffer large response in memory Check content length header for early exit * Add test which currently hangs indefinitely * Run `generate_linux_tests.rb` and `SwiftFormat` * Print type and value if assert fails * Run `generate_linux_tests.rb` and `SwiftFormat` * Remove duplicate test due too merge conflict * Validate that maxBodySize is positive * Address review comments --- Sources/AsyncHTTPClient/HTTPHandler.swift | 63 ++++++++++++- .../HTTPClientTestUtils.swift | 31 +++++++ .../HTTPClientTests+XCTest.swift | 5 ++ .../HTTPClientTests.swift | 88 ++++++++++++++++++- 4 files changed, 183 insertions(+), 4 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 744139f68..26880ef8b 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -356,7 +356,7 @@ extension HTTPClient { /// /// This ``HTTPClientResponseDelegate`` buffers a complete HTTP response in memory. It does not stream the response body in. /// The resulting ``Response`` type is ``HTTPClient/Response``. -public class ResponseAccumulator: HTTPClientResponseDelegate { +public final class ResponseAccumulator: HTTPClientResponseDelegate { public typealias Response = HTTPClient.Response enum State { @@ -367,16 +367,63 @@ public class ResponseAccumulator: HTTPClientResponseDelegate { case error(Error) } + public struct ResponseTooBigError: Error, CustomStringConvertible { + public var maxBodySize: Int + public init(maxBodySize: Int) { + self.maxBodySize = maxBodySize + } + + public var description: String { + return "ResponseTooBigError: received response body exceeds maximum accepted size of \(self.maxBodySize) bytes" + } + } + var state = State.idle let request: HTTPClient.Request - public init(request: HTTPClient.Request) { + static let maxByteBufferSize = Int(UInt32.max) + + /// Maximum size in bytes of the HTTP response body that ``ResponseAccumulator`` will accept + /// until it will abort the request and throw an ``ResponseTooBigError``. + /// + /// Default is 2^32. + /// - precondition: not allowed to exceed 2^32 because ``ByteBuffer`` can not store more bytes + public let maxBodySize: Int + + public convenience init(request: HTTPClient.Request) { + self.init(request: request, maxBodySize: Self.maxByteBufferSize) + } + + /// - Parameters: + /// - request: The corresponding request of the response this delegate will be accumulating. + /// - maxBodySize: Maximum size in bytes of the HTTP response body that ``ResponseAccumulator`` will accept + /// until it will abort the request and throw an ``ResponseTooBigError``. + /// Default is 2^32. + /// - precondition: maxBodySize is not allowed to exceed 2^32 because ``ByteBuffer`` can not store more bytes + /// - warning: You can use ``ResponseAccumulator`` for just one request. + /// If you start another request, you need to initiate another ``ResponseAccumulator``. + public init(request: HTTPClient.Request, maxBodySize: Int) { + precondition(maxBodySize >= 0, "maxBodyLength is not allowed to be negative") + precondition( + maxBodySize <= Self.maxByteBufferSize, + "maxBodyLength is not allowed to exceed 2^32 because ByteBuffer can not store more bytes" + ) self.request = request + self.maxBodySize = maxBodySize } public func didReceiveHead(task: HTTPClient.Task, _ head: HTTPResponseHead) -> EventLoopFuture { switch self.state { case .idle: + if self.request.method != .HEAD, + let contentLength = head.headers.first(name: "Content-Length"), + let announcedBodySize = Int(contentLength), + announcedBodySize > self.maxBodySize { + let error = ResponseTooBigError(maxBodySize: maxBodySize) + self.state = .error(error) + return task.eventLoop.makeFailedFuture(error) + } + self.state = .head(head) case .head: preconditionFailure("head already set") @@ -395,8 +442,20 @@ public class ResponseAccumulator: HTTPClientResponseDelegate { case .idle: preconditionFailure("no head received before body") case .head(let head): + guard part.readableBytes <= self.maxBodySize else { + let error = ResponseTooBigError(maxBodySize: self.maxBodySize) + self.state = .error(error) + return task.eventLoop.makeFailedFuture(error) + } self.state = .body(head, part) case .body(let head, var body): + let newBufferSize = body.writerIndex + part.readableBytes + guard newBufferSize <= self.maxBodySize else { + let error = ResponseTooBigError(maxBodySize: self.maxBodySize) + self.state = .error(error) + return task.eventLoop.makeFailedFuture(error) + } + // The compiler can't prove that `self.state` is dead here (and it kinda isn't, there's // a cross-module call in the way) so we need to drop the original reference to `body` in // `self.state` or we'll get a CoW. To fix that we temporarily set the state to `.end` (which diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 91358a361..884681123 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -366,6 +366,18 @@ internal final class HTTPBin where return self.serverChannel.localAddress! } + var baseURL: String { + let scheme: String = { + switch mode { + case .http1_1, .refuse: + return "http" + case .http2: + return "https" + } + }() + return "\(scheme)://localhost:\(self.port)/" + } + private let mode: Mode private let sslContext: NIOSSLContext? private var serverChannel: Channel! @@ -1319,6 +1331,25 @@ class HTTPEchoHandler: ChannelInboundHandler { } } +final class HTTPEchoHeaders: ChannelInboundHandler { + typealias InboundIn = HTTPServerRequestPart + typealias OutboundOut = HTTPServerResponsePart + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let request = self.unwrapInboundIn(data) + switch request { + case .head(let requestHead): + context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: requestHead.headers))), promise: nil) + case .body: + break + case .end: + context.writeAndFlush(self.wrapOutboundOut(.end(nil))).whenSuccess { + context.close(promise: nil) + } + } + } +} + final class HTTP200DelayedHandler: ChannelInboundHandler { typealias InboundIn = HTTPServerRequestPart typealias OutboundOut = HTTPServerResponsePart diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 2427d6cbf..ef6690c00 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -132,6 +132,11 @@ extension HTTPClientTests { ("testSSLHandshakeErrorPropagationDelayedClose", testSSLHandshakeErrorPropagationDelayedClose), ("testWeCloseConnectionsWhenConnectionCloseSetByServer", testWeCloseConnectionsWhenConnectionCloseSetByServer), ("testBiDirectionalStreaming", testBiDirectionalStreaming), + ("testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLength", testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLength), + ("testResponseAccumulatorMaxBodySizeLimitNotExceedingWithContentLength", testResponseAccumulatorMaxBodySizeLimitNotExceedingWithContentLength), + ("testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLengthButMethodIsHead", testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLengthButMethodIsHead), + ("testResponseAccumulatorMaxBodySizeLimitExceedingWithTransferEncodingChuncked", testResponseAccumulatorMaxBodySizeLimitExceedingWithTransferEncodingChuncked), + ("testResponseAccumulatorMaxBodySizeLimitNotExceedingWithTransferEncodingChuncked", testResponseAccumulatorMaxBodySizeLimitNotExceedingWithTransferEncodingChuncked), ("testBiDirectionalStreamingEarly200", testBiDirectionalStreamingEarly200), ("testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests", testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests), ("testCloseConnectionAfterEarly2XXWhenStreaming", testCloseConnectionAfterEarly2XXWhenStreaming), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 3c275c5eb..d5108bb27 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -2677,8 +2677,8 @@ class HTTPClientTests: XCTestCase { let delegate = TestDelegate() XCTAssertThrowsError(try httpClient.execute(request: request, delegate: delegate).wait()) { - XCTAssertEqual(.connectTimeout, $0 as? HTTPClientError) - XCTAssertEqual(.connectTimeout, delegate.error as? HTTPClientError) + XCTAssertEqualTypeAndValue($0, HTTPClientError.connectTimeout) + XCTAssertEqualTypeAndValue(delegate.error, HTTPClientError.connectTimeout) } } @@ -3092,6 +3092,90 @@ class HTTPClientTests: XCTestCase { XCTAssertNil(try delegate.next().wait()) } + func testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLength() throws { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let body = ByteBuffer(bytes: 0..<11) + + var request = try Request(url: httpBin.baseURL) + request.body = .byteBuffer(body) + XCTAssertThrowsError(try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait()) { error in + XCTAssertTrue(error is ResponseAccumulator.ResponseTooBigError, "unexpected error \(error)") + } + } + + func testResponseAccumulatorMaxBodySizeLimitNotExceedingWithContentLength() throws { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let body = ByteBuffer(bytes: 0..<10) + + var request = try Request(url: httpBin.baseURL) + request.body = .byteBuffer(body) + let response = try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait() + + XCTAssertEqual(response.body, body) + } + + func testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLengthButMethodIsHead() throws { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTPEchoHeaders() } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let body = ByteBuffer(bytes: 0..<11) + + var request = try Request(url: httpBin.baseURL, method: .HEAD) + request.body = .byteBuffer(body) + let response = try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait() + + XCTAssertEqual(response.body ?? ByteBuffer(), ByteBuffer()) + } + + func testResponseAccumulatorMaxBodySizeLimitExceedingWithTransferEncodingChuncked() throws { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let body = ByteBuffer(bytes: 0..<11) + + var request = try Request(url: httpBin.baseURL) + request.body = .stream { writer in + writer.write(.byteBuffer(body)) + } + XCTAssertThrowsError(try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait()) { error in + XCTAssertTrue(error is ResponseAccumulator.ResponseTooBigError, "unexpected error \(error)") + } + } + + func testResponseAccumulatorMaxBodySizeLimitNotExceedingWithTransferEncodingChuncked() throws { + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let body = ByteBuffer(bytes: 0..<10) + + var request = try Request(url: httpBin.baseURL) + request.body = .stream { writer in + writer.write(.byteBuffer(body)) + } + let response = try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait() + + XCTAssertEqual(response.body, body) + } + // In this test, we test that a request can continue to stream its body after the response head and end // was received where the end is a 200. func testBiDirectionalStreamingEarly200() { From f17a47e91684541c84a0cdd8864af68e7e30c463 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Mon, 10 Oct 2022 13:34:42 +0100 Subject: [PATCH 046/146] Allow immediate request failure on connection error (#625) --- .../ConnectionPool/HTTPConnectionPool.swift | 3 +- ...HTTPConnectionPool+HTTP1StateMachine.swift | 25 ++++ ...HTTPConnectionPool+HTTP2StateMachine.swift | 28 ++++ .../HTTPConnectionPool+StateMachine.swift | 15 ++- Sources/AsyncHTTPClient/HTTPClient.swift | 10 ++ .../AsyncAwaitEndToEndTests.swift | 2 +- .../HTTPClientNIOTSTests.swift | 3 + ...onnectionPool+HTTP1StateTests+XCTest.swift | 1 + .../HTTPConnectionPool+HTTP1StateTests.swift | 92 +++++++++++-- ...onPool+HTTP2StateMachineTests+XCTest.swift | 1 + ...onnectionPool+HTTP2StateMachineTests.swift | 126 +++++++++++++++--- .../Mocks/MockConnectionPool.swift | 6 +- 12 files changed, 283 insertions(+), 29 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 74b7c044c..593802a58 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -70,7 +70,8 @@ final class HTTPConnectionPool { self._state = StateMachine( idGenerator: idGenerator, - maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool.concurrentHTTP1ConnectionsPerHostSoftLimit + maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool.concurrentHTTP1ConnectionsPerHostSoftLimit, + retryConnectionEstablishment: clientConfiguration.connectionPool.retryConnectionEstablishment ) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift index 2cd667bb3..669e43f13 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift @@ -17,6 +17,7 @@ import NIOCore extension HTTPConnectionPool { struct HTTP1StateMachine { typealias Action = HTTPConnectionPool.StateMachine.Action + typealias RequestAction = HTTPConnectionPool.StateMachine.RequestAction typealias ConnectionMigrationAction = HTTPConnectionPool.StateMachine.ConnectionMigrationAction typealias EstablishedAction = HTTPConnectionPool.StateMachine.EstablishedAction typealias EstablishedConnectionAction = HTTPConnectionPool.StateMachine.EstablishedConnectionAction @@ -29,16 +30,21 @@ extension HTTPConnectionPool { private(set) var requests: RequestQueue private(set) var lifecycleState: StateMachine.LifecycleState + /// The property was introduced to fail fast during testing. + /// Otherwise this should always be true and not turned off. + private let retryConnectionEstablishment: Bool init( idGenerator: Connection.ID.Generator, maximumConcurrentConnections: Int, + retryConnectionEstablishment: Bool, lifecycleState: StateMachine.LifecycleState ) { self.connections = HTTP1Connections( maximumConcurrentConnections: maximumConcurrentConnections, generator: idGenerator ) + self.retryConnectionEstablishment = retryConnectionEstablishment self.requests = RequestQueue() self.lifecycleState = lifecycleState @@ -219,6 +225,17 @@ extension HTTPConnectionPool { switch self.lifecycleState { case .running: + guard self.retryConnectionEstablishment else { + guard let (index, _) = self.connections.failConnection(connectionID) else { + preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + } + self.connections.removeConnection(at: index) + + return .init( + request: self.failAllRequests(reason: error), + connection: .none + ) + } // We don't care how many waiting requests we have at this point, we will schedule a // retry. More tasks, may appear until the backoff has completed. The final // decision about the retry will be made in `connectionCreationBackoffDone(_:)` @@ -523,6 +540,14 @@ extension HTTPConnectionPool { return .none } + private mutating func failAllRequests(reason error: Error) -> RequestAction { + let allRequests = self.requests.removeAll() + guard !allRequests.isEmpty else { + return .none + } + return .failRequestsAndCancelTimeouts(allRequests, error) + } + // MARK: HTTP2 mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index d517d82e6..003de4223 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -18,6 +18,7 @@ import NIOHTTP2 extension HTTPConnectionPool { struct HTTP2StateMachine { typealias Action = HTTPConnectionPool.StateMachine.Action + typealias RequestAction = HTTPConnectionPool.StateMachine.RequestAction typealias ConnectionMigrationAction = HTTPConnectionPool.StateMachine.ConnectionMigrationAction typealias EstablishedAction = HTTPConnectionPool.StateMachine.EstablishedAction typealias EstablishedConnectionAction = HTTPConnectionPool.StateMachine.EstablishedConnectionAction @@ -33,9 +34,13 @@ extension HTTPConnectionPool { private let idGenerator: Connection.ID.Generator private(set) var lifecycleState: StateMachine.LifecycleState + /// The property was introduced to fail fast during testing. + /// Otherwise this should always be true and not turned off. + private let retryConnectionEstablishment: Bool init( idGenerator: Connection.ID.Generator, + retryConnectionEstablishment: Bool, lifecycleState: StateMachine.LifecycleState ) { self.idGenerator = idGenerator @@ -43,6 +48,7 @@ extension HTTPConnectionPool { self.connections = HTTP2Connections(generator: idGenerator) self.lifecycleState = lifecycleState + self.retryConnectionEstablishment = retryConnectionEstablishment } mutating func migrateFromHTTP1( @@ -398,9 +404,22 @@ extension HTTPConnectionPool { } mutating func failedToCreateNewConnection(_ error: Error, connectionID: Connection.ID) -> Action { + // TODO: switch over state https://github.com/swift-server/async-http-client/issues/638 self.failedConsecutiveConnectionAttempts += 1 self.lastConnectFailure = error + guard self.retryConnectionEstablishment else { + guard let (index, _) = self.connections.failConnection(connectionID) else { + preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + } + self.connections.removeConnection(at: index) + + return .init( + request: self.failAllRequests(reason: error), + connection: .none + ) + } + let eventLoop = self.connections.backoffNextConnectionAttempt(connectionID) let backoff = calculateBackoff(failedAttempt: self.failedConsecutiveConnectionAttempts) return .init(request: .none, connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop)) @@ -408,6 +427,7 @@ extension HTTPConnectionPool { mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { self.lastConnectFailure = error + return .init(request: .none, connection: .none) } @@ -421,6 +441,14 @@ extension HTTPConnectionPool { return self.nextActionForFailedConnection(at: index, on: context.eventLoop) } + private mutating func failAllRequests(reason error: Error) -> RequestAction { + let allRequests = self.requests.removeAll() + guard !allRequests.isEmpty else { + return .none + } + return .failRequestsAndCancelTimeouts(allRequests, error) + } + mutating func timeoutRequest(_ requestID: Request.ID) -> Action { // 1. check requests in queue if let request = self.requests.remove(requestID) { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift index 63f3e5a9a..0460849cc 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift @@ -96,13 +96,22 @@ extension HTTPConnectionPool { let idGenerator: Connection.ID.Generator let maximumConcurrentHTTP1Connections: Int - - init(idGenerator: Connection.ID.Generator, maximumConcurrentHTTP1Connections: Int) { + /// The property was introduced to fail fast during testing. + /// Otherwise this should always be true and not turned off. + private let retryConnectionEstablishment: Bool + + init( + idGenerator: Connection.ID.Generator, + maximumConcurrentHTTP1Connections: Int, + retryConnectionEstablishment: Bool + ) { self.maximumConcurrentHTTP1Connections = maximumConcurrentHTTP1Connections + self.retryConnectionEstablishment = retryConnectionEstablishment self.idGenerator = idGenerator let http1State = HTTP1StateMachine( idGenerator: idGenerator, maximumConcurrentConnections: maximumConcurrentHTTP1Connections, + retryConnectionEstablishment: retryConnectionEstablishment, lifecycleState: .running ) self.state = .http1(http1State) @@ -127,6 +136,7 @@ extension HTTPConnectionPool { var http1StateMachine = HTTP1StateMachine( idGenerator: self.idGenerator, maximumConcurrentConnections: self.maximumConcurrentHTTP1Connections, + retryConnectionEstablishment: self.retryConnectionEstablishment, lifecycleState: http2StateMachine.lifecycleState ) @@ -147,6 +157,7 @@ extension HTTPConnectionPool { var http2StateMachine = HTTP2StateMachine( idGenerator: self.idGenerator, + retryConnectionEstablishment: self.retryConnectionEstablishment, lifecycleState: http1StateMachine.lifecycleState ) let migrationAction = http2StateMachine.migrateFromHTTP1( diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 0f5026fce..1580bbe9c 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -975,6 +975,15 @@ extension HTTPClient.Configuration { /// an explicit eventLoopRequirement are sent, this number might be exceeded due to overflow connections. public var concurrentHTTP1ConnectionsPerHostSoftLimit: Int + /// If true, ``HTTPClient`` will try to create new connections on connection failure with an exponential backoff. + /// Requests will only fail after the ``HTTPClient/Configuration/Timeout-swift.struct/connect`` timeout exceeded. + /// If false, all requests that have no assigned connection will fail immediately after a connection could not be established. + /// Defaults to `true`. + /// - warning: We highly recommend leaving this on. + /// It is very common that connections establishment is flaky at scale. + /// ``HTTPClient`` will automatically mitigate these kind of issues if this flag is turned on. + var retryConnectionEstablishment: Bool + public init(idleTimeout: TimeAmount = .seconds(60)) { self.init(idleTimeout: idleTimeout, concurrentHTTP1ConnectionsPerHostSoftLimit: 8) } @@ -982,6 +991,7 @@ extension HTTPClient.Configuration { public init(idleTimeout: TimeAmount, concurrentHTTP1ConnectionsPerHostSoftLimit: Int) { self.idleTimeout = idleTimeout self.concurrentHTTP1ConnectionsPerHostSoftLimit = concurrentHTTP1ConnectionsPerHostSoftLimit + self.retryConnectionEstablishment = true } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 1420f187e..92b01dd02 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -409,7 +409,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { return XCTFail("unexpected error \(error)") } // a race between deadline and connect timer can result in either error - XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error)) + XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error), "unexpected error \(error)") } } #endif diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 3b659a14a..9727746cc 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -57,6 +57,7 @@ class HTTPClientNIOTSTests: XCTestCase { let httpBin = HTTPBin(.http1_1(ssl: true)) var config = HTTPClient.Configuration() config.networkFrameworkWaitForConnectivity = false + config.connectionPool.retryConnectionEstablishment = false let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) defer { @@ -85,6 +86,7 @@ class HTTPClientNIOTSTests: XCTestCase { let httpBin = HTTPBin(.http1_1(ssl: false)) var config = HTTPClient.Configuration() config.networkFrameworkWaitForConnectivity = false + config.connectionPool.retryConnectionEstablishment = false let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) @@ -140,6 +142,7 @@ class HTTPClientNIOTSTests: XCTestCase { var clientConfig = HTTPClient.Configuration(tlsConfiguration: tlsConfig) clientConfig.networkFrameworkWaitForConnectivity = false + clientConfig.connectionPool.retryConnectionEstablishment = false let httpClient = HTTPClient( eventLoopGroupProvider: .shared(self.clientGroup), configuration: clientConfig diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift index 16377d07f..d50ab9893 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift @@ -26,6 +26,7 @@ extension HTTPConnectionPool_HTTP1StateMachineTests { static var allTests: [(String, (HTTPConnectionPool_HTTP1StateMachineTests) -> () throws -> Void)] { return [ ("testCreatingAndFailingConnections", testCreatingAndFailingConnections), + ("testCreatingAndFailingConnectionsWithoutRetry", testCreatingAndFailingConnectionsWithoutRetry), ("testConnectionFailureBackoff", testConnectionFailureBackoff), ("testCancelRequestWorks", testCancelRequestWorks), ("testExecuteOnShuttingDownPool", testExecuteOnShuttingDownPool), diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift index 7f59fd4e1..125ba1a74 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift @@ -27,7 +27,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 8 + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true ) var connections = MockConnectionPool() @@ -102,13 +103,82 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssert(connections.isEmpty) } + func testCreatingAndFailingConnectionsWithoutRetry() { + struct SomeError: Error, Equatable {} + let elg = EmbeddedEventLoopGroup(loops: 4) + defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } + + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: false + ) + + var connections = MockConnectionPool() + var queuer = MockRequestQueuer() + + // for the first eight requests, the pool should try to create new connections. + + for _ in 0..<8 { + let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let request = HTTPConnectionPool.Request(mockRequest) + let action = state.executeRequest(request) + guard case .createConnection(let connectionID, let connectionEL) = action.connection else { + return XCTFail("Unexpected connection action") + } + XCTAssertEqual(.scheduleRequestTimeout(for: request, on: mockRequest.eventLoop), action.request) + XCTAssert(connectionEL === mockRequest.eventLoop) + + XCTAssertNoThrow(try connections.createConnection(connectionID, on: connectionEL)) + XCTAssertNoThrow(try queuer.queue(mockRequest, id: request.id)) + } + + // the next eight requests should only be queued. + + for _ in 0..<8 { + let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let request = HTTPConnectionPool.Request(mockRequest) + let action = state.executeRequest(request) + guard case .none = action.connection else { + return XCTFail("Unexpected connection action") + } + XCTAssertEqual(.scheduleRequestTimeout(for: request, on: mockRequest.eventLoop), action.request) + XCTAssertNoThrow(try queuer.queue(mockRequest, id: request.id)) + } + + // the first failure should cancel all requests because we have disabled connection establishtment retry + let randomConnectionID = connections.randomStartingConnection()! + XCTAssertNoThrow(try connections.failConnectionCreation(randomConnectionID)) + let action = state.failedToCreateNewConnection(SomeError(), connectionID: randomConnectionID) + XCTAssertEqual(action.connection, .none) + guard case .failRequestsAndCancelTimeouts(let requestsToFail, let requestError) = action.request else { + return XCTFail("Unexpected request action: \(action.request)") + } + XCTAssertEqualTypeAndValue(requestError, SomeError()) + for requestToFail in requestsToFail { + XCTAssertNoThrow(try queuer.fail(requestToFail.id, request: requestToFail.__testOnly_wrapped_request())) + } + + // all requests have been canceled and therefore nothing should happen if a connection fails + while let randomConnectionID = connections.randomStartingConnection() { + XCTAssertNoThrow(try connections.failConnectionCreation(randomConnectionID)) + let action = state.failedToCreateNewConnection(SomeError(), connectionID: randomConnectionID) + + XCTAssertEqual(action, .none) + } + + XCTAssert(queuer.isEmpty) + XCTAssert(connections.isEmpty) + } + func testConnectionFailureBackoff() { let elg = EmbeddedEventLoopGroup(loops: 4) defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 2 + maximumConcurrentHTTP1Connections: 2, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: elg.next()) @@ -165,7 +235,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 2 + maximumConcurrentHTTP1Connections: 2, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: elg.next()) @@ -201,7 +272,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 2 + maximumConcurrentHTTP1Connections: 2, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: elg.next()) @@ -591,7 +663,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 6 + maximumConcurrentHTTP1Connections: 6, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) @@ -629,7 +702,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 6 + maximumConcurrentHTTP1Connections: 6, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) @@ -660,7 +734,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 6 + maximumConcurrentHTTP1Connections: 6, + retryConnectionEstablishment: true ) let mockRequest = MockHTTPRequest(eventLoop: eventLoop.next(), requiresEventLoopForChannel: false) @@ -683,7 +758,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 6 + maximumConcurrentHTTP1Connections: 6, + retryConnectionEstablishment: true ) let mockRequest1 = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift index 9dca0c934..95ea8c580 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift @@ -27,6 +27,7 @@ extension HTTPConnectionPool_HTTP2StateMachineTests { return [ ("testCreatingOfConnection", testCreatingOfConnection), ("testConnectionFailureBackoff", testConnectionFailureBackoff), + ("testConnectionFailureWithoutRetry", testConnectionFailureWithoutRetry), ("testCancelRequestWorks", testCancelRequestWorks), ("testExecuteOnShuttingDownPool", testExecuteOnShuttingDownPool), ("testHTTP1ToHTTP2MigrationAndShutdownIfFirstConnectionIsHTTP1", testHTTP1ToHTTP2MigrationAndShutdownIfFirstConnectionIsHTTP1), diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index e42a98ac7..699909c09 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -29,7 +29,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el1 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: .init(), lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: .init(), + retryConnectionEstablishment: true, + lifecycleState: .running + ) /// first request should create a new connection let mockRequest = MockHTTPRequest(eventLoop: el1) @@ -138,6 +142,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: .init(), + retryConnectionEstablishment: true, lifecycleState: .running ) @@ -189,12 +194,45 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertEqual(state.connectionCreationBackoffDone(newConnectionID), .none) } + func testConnectionFailureWithoutRetry() { + struct SomeError: Error, Equatable {} + let elg = EmbeddedEventLoopGroup(loops: 4) + defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } + + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: .init(), + retryConnectionEstablishment: false, + lifecycleState: .running + ) + + let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let request = HTTPConnectionPool.Request(mockRequest) + + let action = state.executeRequest(request) + XCTAssertEqual(.scheduleRequestTimeout(for: request, on: mockRequest.eventLoop), action.request) + + // 1. connection attempt + guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { + return XCTFail("Unexpected connection action: \(action.connection)") + } + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + + let failedConnectAction = state.failedToCreateNewConnection(SomeError(), connectionID: connectionID) + XCTAssertEqual(failedConnectAction.connection, .none) + guard case .failRequestsAndCancelTimeouts(let requestsToFail, let requestError) = failedConnectAction.request else { + return XCTFail("Unexpected request action: \(action.request)") + } + XCTAssertEqualTypeAndValue(requestError, SomeError()) + XCTAssertEqualTypeAndValue(requestsToFail, [request]) + } + func testCancelRequestWorks() { let elg = EmbeddedEventLoopGroup(loops: 4) defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: .init(), + retryConnectionEstablishment: true, lifecycleState: .running ) @@ -233,6 +271,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: .init(), + retryConnectionEstablishment: true, lifecycleState: .running ) @@ -287,7 +326,12 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el1 = elg.next() let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1State = HTTPConnectionPool.HTTP1StateMachine(idGenerator: idGenerator, maximumConcurrentConnections: 8, lifecycleState: .running) + var http1State = HTTPConnectionPool.HTTP1StateMachine( + idGenerator: idGenerator, + maximumConcurrentConnections: 8, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let mockRequest1 = MockHTTPRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest1) @@ -313,7 +357,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // second connection is a HTTP2 connection and we need to migrate let conn2: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn2ID, eventLoop: el1) - var http2State = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var http2State = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let http2ConnectAction = http2State.migrateFromHTTP1( http1Connections: http1State.connections, @@ -353,7 +401,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) @@ -398,7 +450,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) @@ -426,7 +482,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) XCTAssertEqual(connectAction.request, .none) @@ -461,7 +521,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) @@ -491,7 +555,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) let connectAction = state.migrateFromHTTP1( @@ -532,7 +600,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let idGenerator = HTTPConnectionPool.Connection.ID.Generator() var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator) let conn1ID = http1Conns.createNewConnection(on: el1) - var state = HTTPConnectionPool.HTTP2StateMachine(idGenerator: idGenerator, lifecycleState: .running) + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: idGenerator, + retryConnectionEstablishment: true, + lifecycleState: .running + ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) let connectAction1 = state.migrateFromHTTP1( @@ -592,7 +664,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el1 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) /// first 8 request should create a new connection var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] @@ -678,7 +754,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el1 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) /// create a new connection let mockRequest = MockHTTPRequest(eventLoop: el1) @@ -720,7 +800,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el1 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) /// first 8 request should create a new connection var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] @@ -855,7 +939,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el2 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) // create http2 connection let mockRequest = MockHTTPRequest(eventLoop: el1) @@ -921,7 +1009,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el2 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) // create http2 connection let mockRequest = MockHTTPRequest(eventLoop: el1) @@ -993,7 +1085,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let el2 = elg.next() var connections = MockConnectionPool() var queuer = MockRequestQueuer() - var state = HTTPConnectionPool.StateMachine(idGenerator: .init(), maximumConcurrentHTTP1Connections: 8) + var state = HTTPConnectionPool.StateMachine( + idGenerator: .init(), + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true + ) var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] for el in [el1, el2, el2] { diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift index eedc499ad..1b2c27b68 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift @@ -541,7 +541,8 @@ extension MockConnectionPool { ) throws -> (Self, HTTPConnectionPool.StateMachine) { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: maxNumberOfConnections + maximumConcurrentHTTP1Connections: maxNumberOfConnections, + retryConnectionEstablishment: true ) var connections = MockConnectionPool() var queuer = MockRequestQueuer() @@ -604,7 +605,8 @@ extension MockConnectionPool { ) throws -> (Self, HTTPConnectionPool.StateMachine) { var state = HTTPConnectionPool.StateMachine( idGenerator: .init(), - maximumConcurrentHTTP1Connections: 8 + maximumConcurrentHTTP1Connections: 8, + retryConnectionEstablishment: true ) var connections = MockConnectionPool() var queuer = MockRequestQueuer() From d7b69d9d560f14e4f19fbb8660f8bfa3f1da11ca Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 11 Oct 2022 10:36:21 +0100 Subject: [PATCH 047/146] Make `HTTPClientResponse.init` public (#632) --- .../AsyncAwait/AnyAsyncSequence.swift | 48 ++++++ .../AsyncAwait/AsyncLazySequence.swift | 52 +++++++ .../AsyncAwait/HTTPClientResponse.swift | 139 +++++++++++------- .../SingleIteratorPrecondition.swift | 45 ++++++ .../AsyncAwait/Transaction+StateMachine.swift | 16 +- .../AsyncAwait/Transaction.swift | 6 +- .../AsyncAwait/TransactionBody.swift | 66 +++++++++ .../AsyncAwaitEndToEndTests.swift | 6 +- .../HTTPClientRequestTests.swift | 29 +--- 9 files changed, 310 insertions(+), 97 deletions(-) create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequence.swift create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/AsyncLazySequence.swift create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/SingleIteratorPrecondition.swift create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift diff --git a/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequence.swift b/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequence.swift new file mode 100644 index 000000000..8f6b32bd2 --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequence.swift @@ -0,0 +1,48 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@usableFromInline +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +struct AnyAsyncSequence: Sendable, AsyncSequence { + @usableFromInline typealias AsyncIteratorNextCallback = () async throws -> Element? + + @usableFromInline struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline let nextCallback: AsyncIteratorNextCallback + + @inlinable init(nextCallback: @escaping AsyncIteratorNextCallback) { + self.nextCallback = nextCallback + } + + @inlinable mutating func next() async throws -> Element? { + try await self.nextCallback() + } + } + + @usableFromInline var makeAsyncIteratorCallback: @Sendable () -> AsyncIteratorNextCallback + + @inlinable init( + _ asyncSequence: SequenceOfBytes + ) where SequenceOfBytes: AsyncSequence & Sendable, SequenceOfBytes.Element == Element { + self.makeAsyncIteratorCallback = { + var iterator = asyncSequence.makeAsyncIterator() + return { + try await iterator.next() + } + } + } + + @inlinable func makeAsyncIterator() -> AsyncIterator { + .init(nextCallback: self.makeAsyncIteratorCallback()) + } +} diff --git a/Sources/AsyncHTTPClient/AsyncAwait/AsyncLazySequence.swift b/Sources/AsyncHTTPClient/AsyncAwait/AsyncLazySequence.swift new file mode 100644 index 000000000..fe37dd5e7 --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/AsyncLazySequence.swift @@ -0,0 +1,52 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +@usableFromInline +struct AsyncLazySequence: AsyncSequence { + @usableFromInline typealias Element = Base.Element + @usableFromInline struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline var iterator: Base.Iterator + @inlinable init(iterator: Base.Iterator) { + self.iterator = iterator + } + + @inlinable mutating func next() async throws -> Base.Element? { + self.iterator.next() + } + } + + @usableFromInline var base: Base + + @inlinable init(base: Base) { + self.base = base + } + + @inlinable func makeAsyncIterator() -> AsyncIterator { + .init(iterator: self.base.makeIterator()) + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension AsyncLazySequence: Sendable where Base: Sendable {} +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension AsyncLazySequence.AsyncIterator: Sendable where Base.Iterator: Sendable {} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension Sequence { + /// Turns `self` into an `AsyncSequence` by vending each element of `self` asynchronously. + @inlinable var async: AsyncLazySequence { + .init(base: self) + } +} diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index e6cc47210..786b49eaf 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -33,98 +33,125 @@ public struct HTTPClientResponse: Sendable { /// The body of this HTTP response. public var body: Body - /// A representation of the response body for an HTTP response. - /// - /// The body is streamed as an `AsyncSequence` of `ByteBuffer`, where each `ByteBuffer` contains - /// an arbitrarily large chunk of data. The boundaries between `ByteBuffer` objects in the sequence - /// are entirely synthetic and have no semantic meaning. - public struct Body: Sendable { - private let bag: Transaction - private let reference: ResponseRef - - fileprivate init(_ transaction: Transaction) { - self.bag = transaction - self.reference = ResponseRef(transaction: transaction) - } - } - init( bag: Transaction, version: HTTPVersion, status: HTTPResponseStatus, headers: HTTPHeaders ) { - self.body = Body(bag) self.version = version self.status = status self.headers = headers + self.body = Body(TransactionBody(bag)) + } + + @inlinable public init( + version: HTTPVersion = .http1_1, + status: HTTPResponseStatus = .ok, + headers: HTTPHeaders = [:], + body: Body = Body() + ) { + self.version = version + self.status = status + self.headers = headers + self.body = body } } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension HTTPClientResponse.Body: AsyncSequence { - public typealias Element = AsyncIterator.Element +extension HTTPClientResponse { + /// A representation of the response body for an HTTP response. + /// + /// The body is streamed as an `AsyncSequence` of `ByteBuffer`, where each `ByteBuffer` contains + /// an arbitrarily large chunk of data. The boundaries between `ByteBuffer` objects in the sequence + /// are entirely synthetic and have no semantic meaning. + public struct Body: AsyncSequence, Sendable { + public typealias Element = ByteBuffer + public struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline var storage: Storage.AsyncIterator - public struct AsyncIterator: AsyncIteratorProtocol { - private let stream: IteratorStream + @inlinable init(storage: Storage.AsyncIterator) { + self.storage = storage + } - fileprivate init(stream: IteratorStream) { - self.stream = stream + @inlinable public mutating func next() async throws -> ByteBuffer? { + try await self.storage.next() + } } - public mutating func next() async throws -> ByteBuffer? { - try await self.stream.next() + @usableFromInline var storage: Storage + + @inlinable public func makeAsyncIterator() -> AsyncIterator { + .init(storage: self.storage.makeAsyncIterator()) } } +} - public func makeAsyncIterator() -> AsyncIterator { - AsyncIterator(stream: IteratorStream(bag: self.bag)) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientResponse.Body { + @usableFromInline enum Storage: Sendable { + case transaction(TransactionBody) + case anyAsyncSequence(AnyAsyncSequence) } } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension HTTPClientResponse.Body { - /// The purpose of this object is to inform the transaction about the response body being deinitialized. - /// If the users has not called `makeAsyncIterator` on the body, before it is deinited, the http - /// request needs to be cancelled. - fileprivate final class ResponseRef: Sendable { - private let transaction: Transaction - - init(transaction: Transaction) { - self.transaction = transaction +extension HTTPClientResponse.Body.Storage: AsyncSequence { + @usableFromInline typealias Element = ByteBuffer + + @inlinable func makeAsyncIterator() -> AsyncIterator { + switch self { + case .transaction(let transaction): + return .transaction(transaction.makeAsyncIterator()) + case .anyAsyncSequence(let anyAsyncSequence): + return .anyAsyncSequence(anyAsyncSequence.makeAsyncIterator()) } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientResponse.Body.Storage { + @usableFromInline enum AsyncIterator { + case transaction(TransactionBody.AsyncIterator) + case anyAsyncSequence(AnyAsyncSequence.AsyncIterator) + } +} - deinit { - self.transaction.responseBodyDeinited() +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientResponse.Body.Storage.AsyncIterator: AsyncIteratorProtocol { + @inlinable mutating func next() async throws -> ByteBuffer? { + switch self { + case .transaction(let iterator): + return try await iterator.next() + case .anyAsyncSequence(var iterator): + defer { self = .anyAsyncSequence(iterator) } + return try await iterator.next() } } } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientResponse.Body { - internal class IteratorStream { - struct ID: Hashable { - private let objectID: ObjectIdentifier - - init(_ object: IteratorStream) { - self.objectID = ObjectIdentifier(object) - } - } + init(_ body: TransactionBody) { + self.init(.transaction(body)) + } - private var id: ID { ID(self) } - private let bag: Transaction + @usableFromInline init(_ storage: Storage) { + self.storage = storage + } - init(bag: Transaction) { - self.bag = bag - } + public init() { + self = .stream(EmptyCollection().async) + } - deinit { - self.bag.responseBodyIteratorDeinited(streamID: self.id) - } + @inlinable public static func stream( + _ sequenceOfBytes: SequenceOfBytes + ) -> Self where SequenceOfBytes: AsyncSequence & Sendable, SequenceOfBytes.Element == ByteBuffer { + self.init(.anyAsyncSequence(AnyAsyncSequence(sequenceOfBytes.singleIteratorPrecondition))) + } - func next() async throws -> ByteBuffer? { - try await self.bag.nextResponsePart(streamID: self.id) - } + public static func bytes(_ byteBuffer: ByteBuffer) -> Self { + .stream(CollectionOfOne(byteBuffer).async) } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/SingleIteratorPrecondition.swift b/Sources/AsyncHTTPClient/AsyncAwait/SingleIteratorPrecondition.swift new file mode 100644 index 000000000..04034db2d --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/SingleIteratorPrecondition.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Atomics + +/// Makes sure that a consumer of this `AsyncSequence` only calls `makeAsyncIterator()` at most once. +/// If `makeAsyncIterator()` is called multiple times, the program crashes. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +@usableFromInline struct SingleIteratorPrecondition: AsyncSequence { + @usableFromInline let base: Base + @usableFromInline let didCreateIterator: ManagedAtomic = .init(false) + @usableFromInline typealias Element = Base.Element + @inlinable init(base: Base) { + self.base = base + } + + @inlinable func makeAsyncIterator() -> Base.AsyncIterator { + precondition( + self.didCreateIterator.exchange(true, ordering: .relaxed) == false, + "makeAsyncIterator() is only allowed to be called at most once." + ) + return self.base.makeAsyncIterator() + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension SingleIteratorPrecondition: @unchecked Sendable where Base: Sendable {} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension AsyncSequence { + @inlinable var singleIteratorPrecondition: SingleIteratorPrecondition { + .init(base: self) + } +} diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 38709c9a7..3ec6b4eb7 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -30,7 +30,7 @@ extension Transaction { case queued(CheckedContinuation, HTTPRequestScheduler) case deadlineExceededWhileQueued(CheckedContinuation) case executing(ExecutionContext, RequestStreamState, ResponseStreamState) - case finished(error: Error?, HTTPClientResponse.Body.IteratorStream.ID?) + case finished(error: Error?, TransactionBody.AsyncIterator.ID?) } fileprivate enum RequestStreamState { @@ -52,9 +52,9 @@ extension Transaction { // We are waiting for the user to create a response body iterator and to call next on // it for the first time. case waitingForResponseIterator(CircularBuffer, next: Next) - case buffering(HTTPClientResponse.Body.IteratorStream.ID, CircularBuffer, next: Next) - case waitingForRemote(HTTPClientResponse.Body.IteratorStream.ID, CheckedContinuation) - case finished(HTTPClientResponse.Body.IteratorStream.ID, CheckedContinuation) + case buffering(TransactionBody.AsyncIterator.ID, CircularBuffer, next: Next) + case waitingForRemote(TransactionBody.AsyncIterator.ID, CheckedContinuation) + case finished(TransactionBody.AsyncIterator.ID, CheckedContinuation) } private var state: State @@ -510,7 +510,7 @@ extension Transaction { } } - mutating func responseBodyIteratorDeinited(streamID: HTTPClientResponse.Body.IteratorStream.ID) -> FailAction { + mutating func responseBodyIteratorDeinited(streamID: TransactionBody.AsyncIterator.ID) -> FailAction { switch self.state { case .initialized, .queued, .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): preconditionFailure("Got notice about a deinited response body iterator, before we even received a response. Invalid state: \(self.state)") @@ -536,7 +536,7 @@ extension Transaction { } mutating func consumeNextResponsePart( - streamID: HTTPClientResponse.Body.IteratorStream.ID, + streamID: TransactionBody.AsyncIterator.ID, continuation: CheckedContinuation ) -> ConsumeAction { switch self.state { @@ -639,8 +639,8 @@ extension Transaction { } private func verifyStreamIDIsEqual( - registered: HTTPClientResponse.Body.IteratorStream.ID, - this: HTTPClientResponse.Body.IteratorStream.ID, + registered: TransactionBody.AsyncIterator.ID, + this: TransactionBody.AsyncIterator.ID, file: StaticString = #file, line: UInt = #line ) { diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 1974778e9..f5c90557a 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -20,7 +20,7 @@ import NIOHTTP1 import NIOSSL @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -final class Transaction: @unchecked Sendable { +@usableFromInline final class Transaction: @unchecked Sendable { let logger: Logger let request: HTTPClientRequest.Prepared @@ -334,7 +334,7 @@ extension Transaction { } } - func nextResponsePart(streamID: HTTPClientResponse.Body.IteratorStream.ID) async throws -> ByteBuffer? { + func nextResponsePart(streamID: TransactionBody.AsyncIterator.ID) async throws -> ByteBuffer? { try await withCheckedThrowingContinuation { continuation in let action = self.stateLock.withLock { self.state.consumeNextResponsePart(streamID: streamID, continuation: continuation) @@ -355,7 +355,7 @@ extension Transaction { } } - func responseBodyIteratorDeinited(streamID: HTTPClientResponse.Body.IteratorStream.ID) { + func responseBodyIteratorDeinited(streamID: TransactionBody.AsyncIterator.ID) { let action = self.stateLock.withLock { self.state.responseBodyIteratorDeinited(streamID: streamID) } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift b/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift new file mode 100644 index 000000000..497a3cc72 --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift @@ -0,0 +1,66 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2021-2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +/// This is a class because we need to inform the transaction about the response body being deinitialized. +/// If the users has not called `makeAsyncIterator` on the body, before it is deinited, the http +/// request needs to be cancelled. +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +@usableFromInline final class TransactionBody: Sendable { + @usableFromInline let transaction: Transaction + + init(_ transaction: Transaction) { + self.transaction = transaction + } + + deinit { + self.transaction.responseBodyDeinited() + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension TransactionBody: AsyncSequence { + @usableFromInline typealias Element = AsyncIterator.Element + + @usableFromInline final class AsyncIterator: AsyncIteratorProtocol { + @usableFromInline struct ID: Hashable { + private let objectID: ObjectIdentifier + + init(_ object: AsyncIterator) { + self.objectID = ObjectIdentifier(object) + } + } + + @usableFromInline var id: ID { ID(self) } + @usableFromInline let transaction: Transaction + + @inlinable init(transaction: Transaction) { + self.transaction = transaction + } + + deinit { + self.transaction.responseBodyIteratorDeinited(streamID: self.id) + } + + // TODO: this should be @inlinable + @usableFromInline func next() async throws -> ByteBuffer? { + try await self.transaction.nextResponsePart(streamID: self.id) + } + } + + @inlinable func makeAsyncIterator() -> AsyncIterator { + AsyncIterator(transaction: self.transaction) + } +} diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 92b01dd02..e87d3d392 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -216,7 +216,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ByteBuffer(string: "1"), ByteBuffer(string: "2"), ByteBuffer(string: "34"), - ].asAsyncSequence(), length: .unknown) + ].async, length: .unknown) guard let response = await XCTAssertNoThrowWithResult( try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) @@ -241,7 +241,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") request.method = .POST - request.body = .stream("1234".utf8.asAsyncSequence(), length: .unknown) + request.body = .stream("1234".utf8.async, length: .unknown) guard let response = await XCTAssertNoThrowWithResult( try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) @@ -614,7 +614,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ByteBuffer(string: "1"), ByteBuffer(string: "2"), ByteBuffer(string: "34"), - ].asAsyncSequence(), length: .unknown) + ].async, length: .unknown) guard let response1 = await XCTAssertNoThrowWithResult( try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 42ce4d537..271144cc4 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -408,7 +408,7 @@ class HTTPClientRequestTests: XCTestCase { let asyncSequence = ByteBuffer(string: "post body") .readableBytesView .chunked(maxChunkSize: 2) - .asAsyncSequence() + .async .map { ByteBuffer($0) } request.body = .stream(asyncSequence, length: .unknown) @@ -449,7 +449,7 @@ class HTTPClientRequestTests: XCTestCase { let asyncSequence = ByteBuffer(string: "post body") .readableBytesView .chunked(maxChunkSize: 2) - .asAsyncSequence() + .async .map { ByteBuffer($0) } request.body = .stream(asyncSequence, length: .known(9)) @@ -551,29 +551,4 @@ extension Collection { } } -@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -struct AsyncSequenceFromSyncSequence: AsyncSequence, Sendable { - typealias Element = Wrapped.Element - struct AsyncIterator: AsyncIteratorProtocol { - fileprivate var iterator: Wrapped.Iterator - mutating func next() async throws -> Wrapped.Element? { - self.iterator.next() - } - } - - fileprivate let wrapped: Wrapped - - func makeAsyncIterator() -> AsyncIterator { - .init(iterator: self.wrapped.makeIterator()) - } -} - -@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension Sequence where Self: Sendable { - /// Turns `self` into an `AsyncSequence` by wending each element of `self` asynchronously. - func asAsyncSequence() -> AsyncSequenceFromSyncSequence { - .init(wrapped: self) - } -} - #endif From 9195d3bcb4b581893d07f21e4ddb6d124f96c568 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 11 Oct 2022 16:40:26 +0100 Subject: [PATCH 048/146] Speedup tests (#639) * Enable fast failure mode for testing * Split-up HTTPClientTests into multiple subclasses * Move test subclasses into separate files --- .../AsyncAwaitEndToEndTests.swift | 5 +- ...zeConfigValueIsRespectedTests+XCTest.swift | 31 +++ ...nPoolSizeConfigValueIsRespectedTests.swift | 75 ++++++ .../HTTP2ClientTests.swift | 2 +- .../HTTPClient+SOCKSTests.swift | 20 +- .../AsyncHTTPClientTests/HTTPClientBase.swift | 86 +++++++ .../HTTPClientInternalTests.swift | 13 +- .../HTTPClientTests+XCTest.swift | 6 - .../HTTPClientTests.swift | 227 ++---------------- .../HTTPConnectionPool+FactoryTests.swift | 9 +- .../IdleTimeoutNoReuseTests+XCTest.swift | 31 +++ .../IdleTimeoutNoReuseTests.swift | 40 +++ ...NoBytesSentOverBodyLimitTests+XCTest.swift | 31 +++ .../NoBytesSentOverBodyLimitTests.swift | 80 ++++++ ...oolIdleConnectionsAndGetTests+XCTest.swift | 31 +++ .../RacePoolIdleConnectionsAndGetTests.swift | 44 ++++ .../ResponseDelayGetTests+XCTest.swift | 31 +++ .../ResponseDelayGetTests.swift | 43 ++++ .../StressGetHttpsTests+XCTest.swift | 31 +++ .../StressGetHttpsTests.swift | 51 ++++ Tests/LinuxMain.swift | 6 + 21 files changed, 662 insertions(+), 231 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift create mode 100644 Tests/AsyncHTTPClientTests/HTTPClientBase.swift create mode 100644 Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift create mode 100644 Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift create mode 100644 Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift create mode 100644 Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift create mode 100644 Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index e87d3d392..8995acfb1 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -497,8 +497,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { defer { XCTAssertNoThrow(try serverChannel.close().wait()) } let port = serverChannel.localAddress!.port! - var config = HTTPClient.Configuration() - config.timeout.connect = .seconds(3) + let config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() + let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://localhost:\(port)") diff --git a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift new file mode 100644 index 000000000..f76fea3c4 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension ConnectionPoolSizeConfigValueIsRespectedTests { + static var allTests: [(String, (ConnectionPoolSizeConfigValueIsRespectedTests) -> () throws -> Void)] { + return [ + ("testConnectionPoolSizeConfigValueIsRespected", testConnectionPoolSizeConfigValueIsRespected), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift new file mode 100644 index 000000000..46cddeead --- /dev/null +++ b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift @@ -0,0 +1,75 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class ConnectionPoolSizeConfigValueIsRespectedTests: XCTestCaseHTTPClientTestsBaseClass { + func testConnectionPoolSizeConfigValueIsRespected() { + let numberOfRequestsPerThread = 1000 + let numberOfParallelWorkers = 16 + let poolSize = 12 + + let httpBin = HTTPBin() + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let group = MultiThreadedEventLoopGroup(numberOfThreads: 4) + defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } + + let configuration = HTTPClient.Configuration( + connectionPool: .init( + idleTimeout: .seconds(30), + concurrentHTTP1ConnectionsPerHostSoftLimit: poolSize + ) + ) + let client = HTTPClient(eventLoopGroupProvider: .shared(group), configuration: configuration) + defer { XCTAssertNoThrow(try client.syncShutdown()) } + + let g = DispatchGroup() + for workerID in 0..! + var defaultClient: HTTPClient! + var backgroundLogStore: CollectEverythingLogHandler.LogStore! + + var defaultHTTPBinURLPrefix: String { + return "http://localhost:\(self.defaultHTTPBin.port)/" + } + + override func setUp() { + XCTAssertNil(self.clientGroup) + XCTAssertNil(self.serverGroup) + XCTAssertNil(self.defaultHTTPBin) + XCTAssertNil(self.defaultClient) + XCTAssertNil(self.backgroundLogStore) + + self.clientGroup = getDefaultEventLoopGroup(numberOfThreads: 1) + self.serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + self.defaultHTTPBin = HTTPBin() + self.backgroundLogStore = CollectEverythingLogHandler.LogStore() + var backgroundLogger = Logger(label: "\(#function)", factory: { _ in + CollectEverythingLogHandler(logStore: self.backgroundLogStore!) + }) + backgroundLogger.logLevel = .trace + self.defaultClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration().enableFastFailureModeForTesting(), + backgroundActivityLogger: backgroundLogger) + } + + override func tearDown() { + if let defaultClient = self.defaultClient { + XCTAssertNoThrow(try defaultClient.syncShutdown()) + self.defaultClient = nil + } + + XCTAssertNotNil(self.defaultHTTPBin) + XCTAssertNoThrow(try self.defaultHTTPBin.shutdown()) + self.defaultHTTPBin = nil + + XCTAssertNotNil(self.clientGroup) + XCTAssertNoThrow(try self.clientGroup.syncShutdownGracefully()) + self.clientGroup = nil + + XCTAssertNotNil(self.serverGroup) + XCTAssertNoThrow(try self.serverGroup.syncShutdownGracefully()) + self.serverGroup = nil + + XCTAssertNotNil(self.backgroundLogStore) + self.backgroundLogStore = nil + } +} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index 234185eb6..6f412a30d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -429,8 +429,8 @@ class HTTPClientInternalTests: XCTestCase { let el2 = elg.next() let httpBin = HTTPBin(.refuse) - var config = HTTPClient.Configuration() - config.networkFrameworkWaitForConnectivity = false + let config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() let client = HTTPClient(eventLoopGroupProvider: .shared(elg), configuration: config) defer { @@ -590,3 +590,12 @@ class HTTPClientInternalTests: XCTestCase { XCTAssert(threadPools.dropFirst().allSatisfy { $0 === firstThreadPool }) } } + +extension HTTPClient.Configuration { + func enableFastFailureModeForTesting() -> Self { + var copy = self + copy.networkFrameworkWaitForConnectivity = false + copy.connectionPool.retryConnectionEstablishment = false + return copy + } +} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index ef6690c00..ef81b1dde 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -77,13 +77,10 @@ extension HTTPClientTests { ("testWorksWhenServerClosesConnectionAfterReceivingRequest", testWorksWhenServerClosesConnectionAfterReceivingRequest), ("testSubsequentRequestsWorkWithServerSendingConnectionClose", testSubsequentRequestsWorkWithServerSendingConnectionClose), ("testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose", testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose), - ("testStressGetHttps", testStressGetHttps), ("testStressGetHttpsSSLError", testStressGetHttpsSSLError), ("testSelfSignedCertificateIsRejectedWithCorrectError", testSelfSignedCertificateIsRejectedWithCorrectError), ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), ("testFailingConnectionIsReleased", testFailingConnectionIsReleased), - ("testResponseDelayGet", testResponseDelayGet), - ("testIdleTimeoutNoReuse", testIdleTimeoutNoReuse), ("testStressGetClose", testStressGetClose), ("testManyConcurrentRequestsWork", testManyConcurrentRequestsWork), ("testRepeatedRequestsWorkWhenServerAlwaysCloses", testRepeatedRequestsWorkWhenServerAlwaysCloses), @@ -104,7 +101,6 @@ extension HTTPClientTests { ("testUseExistingConnectionOnDifferentEL", testUseExistingConnectionOnDifferentEL), ("testWeRecoverFromServerThatClosesTheConnectionOnUs", testWeRecoverFromServerThatClosesTheConnectionOnUs), ("testPoolClosesIdleConnections", testPoolClosesIdleConnections), - ("testRacePoolIdleConnectionsAndGet", testRacePoolIdleConnectionsAndGet), ("testAvoidLeakingTLSHandshakeCompletionPromise", testAvoidLeakingTLSHandshakeCompletionPromise), ("testAsyncShutdown", testAsyncShutdown), ("testAsyncShutdownDefaultQueue", testAsyncShutdownDefaultQueue), @@ -126,7 +122,6 @@ extension HTTPClientTests { ("testContentLengthTooLongFails", testContentLengthTooLongFails), ("testContentLengthTooShortFails", testContentLengthTooShortFails), ("testBodyUploadAfterEndFails", testBodyUploadAfterEndFails), - ("testNoBytesSentOverBodyLimit", testNoBytesSentOverBodyLimit), ("testDoubleError", testDoubleError), ("testSSLHandshakeErrorPropagation", testSSLHandshakeErrorPropagation), ("testSSLHandshakeErrorPropagationDelayedClose", testSSLHandshakeErrorPropagationDelayedClose), @@ -145,7 +140,6 @@ extension HTTPClientTests { ("testCloseWhileBackpressureIsExertedIsFine", testCloseWhileBackpressureIsExertedIsFine), ("testErrorAfterCloseWhileBackpressureExerted", testErrorAfterCloseWhileBackpressureExerted), ("testRequestSpecificTLS", testRequestSpecificTLS), - ("testConnectionPoolSizeConfigValueIsRespected", testConnectionPoolSizeConfigValueIsRespected), ("testRequestWithHeaderTransferEncodingIdentityDoesNotFail", testRequestWithHeaderTransferEncodingIdentityDoesNotFail), ("testMassiveDownload", testMassiveDownload), ("testShutdownWithFutures", testShutdownWithFutures), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index d5108bb27..a8036ab81 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -29,60 +29,7 @@ import NIOTestUtils import NIOTransportServices import XCTest -class HTTPClientTests: XCTestCase { - typealias Request = HTTPClient.Request - - var clientGroup: EventLoopGroup! - var serverGroup: EventLoopGroup! - var defaultHTTPBin: HTTPBin! - var defaultClient: HTTPClient! - var backgroundLogStore: CollectEverythingLogHandler.LogStore! - - var defaultHTTPBinURLPrefix: String { - return "http://localhost:\(self.defaultHTTPBin.port)/" - } - - override func setUp() { - XCTAssertNil(self.clientGroup) - XCTAssertNil(self.serverGroup) - XCTAssertNil(self.defaultHTTPBin) - XCTAssertNil(self.defaultClient) - XCTAssertNil(self.backgroundLogStore) - - self.clientGroup = getDefaultEventLoopGroup(numberOfThreads: 1) - self.serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) - self.defaultHTTPBin = HTTPBin() - self.backgroundLogStore = CollectEverythingLogHandler.LogStore() - var backgroundLogger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: self.backgroundLogStore!) - }) - backgroundLogger.logLevel = .trace - self.defaultClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - backgroundActivityLogger: backgroundLogger) - } - - override func tearDown() { - if let defaultClient = self.defaultClient { - XCTAssertNoThrow(try defaultClient.syncShutdown()) - self.defaultClient = nil - } - - XCTAssertNotNil(self.defaultHTTPBin) - XCTAssertNoThrow(try self.defaultHTTPBin.shutdown()) - self.defaultHTTPBin = nil - - XCTAssertNotNil(self.clientGroup) - XCTAssertNoThrow(try self.clientGroup.syncShutdownGracefully()) - self.clientGroup = nil - - XCTAssertNotNil(self.serverGroup) - XCTAssertNoThrow(try self.serverGroup.syncShutdownGracefully()) - self.serverGroup = nil - - XCTAssertNotNil(self.backgroundLogStore) - self.backgroundLogStore = nil - } - +final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testRequestURI() throws { let request1 = try Request(url: "https://someserver.com:8888/some/path?foo=bar") XCTAssertEqual(request1.url.host, "someserver.com") @@ -781,11 +728,19 @@ class HTTPClientTests: XCTestCase { func testProxyPlaintextWithIncorrectlyAuthorization() throws { let localHTTPBin = HTTPBin(proxy: .simulate(authorization: "Basic YWxhZGRpbjpvcGVuc2VzYW1l")) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .server(host: "localhost", - port: localHTTPBin.port, - authorization: .basic(username: "aladdin", - password: "opensesamefoo")))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init( + proxy: .server( + host: "localhost", + port: localHTTPBin.port, + authorization: .basic( + username: "aladdin", + password: "opensesamefoo" + ) + ) + ).enableFastFailureModeForTesting() + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -1208,31 +1163,9 @@ class HTTPClientTests: XCTestCase { } } - func testStressGetHttps() throws { - let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localHTTPBin.shutdown()) - } - - let eventLoop = localClient.eventLoopGroup.next() - let requestCount = 200 - var futureResults = [EventLoopFuture]() - for _ in 1...requestCount { - let req = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, headers: ["X-internal-delay": "100"]) - futureResults.append(localClient.execute(request: req)) - } - XCTAssertNoThrow(try EventLoopFuture.andAllSucceed(futureResults, on: eventLoop).wait()) - } - func testStressGetHttpsSSLError() throws { - var config = HTTPClient.Configuration() - config.networkFrameworkWaitForConnectivity = false - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: config) + configuration: HTTPClient.Configuration().enableFastFailureModeForTesting()) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -1292,8 +1225,8 @@ class HTTPClientTests: XCTestCase { defer { XCTAssertNoThrow(try serverChannel.close().wait()) } let port = serverChannel.localAddress!.port! - var config = HTTPClient.Configuration() - config.timeout.connect = .seconds(2) + let config = HTTPClient.Configuration().enableFastFailureModeForTesting() + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(port)").wait()) { error in @@ -1332,8 +1265,8 @@ class HTTPClientTests: XCTestCase { defer { XCTAssertNoThrow(try serverChannel.close().wait()) } let port = serverChannel.localAddress!.port! - var config = HTTPClient.Configuration() - config.timeout.connect = .seconds(3) + let config = HTTPClient.Configuration().enableFastFailureModeForTesting() + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -1372,25 +1305,6 @@ class HTTPClientTests: XCTestCase { } } - func testResponseDelayGet() throws { - let req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", - method: .GET, - headers: ["X-internal-delay": "2000"], - body: nil) - let start = NIODeadline.now() - let response = try self.defaultClient.execute(request: req).wait() - XCTAssertGreaterThanOrEqual(.now() - start, .milliseconds(1_900 /* 1.9 seconds */ )) - XCTAssertEqual(response.status, .ok) - } - - func testIdleTimeoutNoReuse() throws { - var req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .GET) - XCTAssertNoThrow(try self.defaultClient.execute(request: req, deadline: .now() + .seconds(2)).wait()) - req.headers.add(name: "X-internal-delay", value: "2500") - try self.defaultClient.eventLoopGroup.next().scheduleTask(in: .milliseconds(250)) {}.futureResult.wait() - XCTAssertNoThrow(try self.defaultClient.execute(request: req).timeout(after: .seconds(10)).wait()) - } - func testStressGetClose() throws { let eventLoop = self.defaultClient.eventLoopGroup.next() let requestCount = 200 @@ -1924,18 +1838,6 @@ class HTTPClientTests: XCTestCase { XCTAssertEqual(self.defaultHTTPBin.activeConnections, 0) } - func testRacePoolIdleConnectionsAndGet() { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(connectionPool: .init(idleTimeout: .milliseconds(10)))) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - } - for _ in 1...500 { - XCTAssertNoThrow(try localClient.get(url: self.defaultHTTPBinURLPrefix + "get").wait()) - Thread.sleep(forTimeInterval: 0.01 + .random(in: -0.05...0.05)) - } - } - func testAvoidLeakingTLSHandshakeCompletionPromise() { let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(timeout: .init(connect: .milliseconds(100)))) let localHTTPBin = HTTPBin() @@ -2818,54 +2720,6 @@ class HTTPClientTests: XCTestCase { XCTAssertNoThrow(try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "get").wait()) } - func testNoBytesSentOverBodyLimit() throws { - let server = NIOHTTP1TestServer(group: self.serverGroup) - defer { - XCTAssertNoThrow(try server.stop()) - } - - let tooLong = "XBAD BAD BAD NOT HTTP/1.1\r\n\r\n" - - let request = try Request( - url: "http://localhost:\(server.serverPort)", - body: .stream(length: 1) { streamWriter in - streamWriter.write(.byteBuffer(ByteBuffer(string: tooLong))) - } - ) - - let future = self.defaultClient.execute(request: request) - - // Okay, what happens here needs an explanation: - // - // In the request state machine, we should start the request, which will lead to an - // invocation of `context.write(HTTPRequestHead)`. Since we will receive a streamed request - // body a `context.flush()` will be issued. Further the request stream will be started. - // Since the request stream immediately produces to much data, the request will be failed - // and the connection will be closed. - // - // Even though a flush was issued after the request head, there is no guarantee that the - // request head was written to the network. For this reason we must accept not receiving a - // request and receiving a request head. - - do { - _ = try server.receiveHead() - - // A request head was sent. We expect the request now to fail with a parsing error, - // since the client ended the connection to early (from the server's point of view.) - XCTAssertThrowsError(try server.readInbound()) { - XCTAssertEqual($0 as? HTTPParserError, HTTPParserError.invalidEOFState) - } - } catch { - // TBD: We sadly can't verify the error type, since it is private in `NIOTestUtils`: - // NIOTestUtils.BlockingQueue.TimeoutError - } - - // request must always be failed with this error - XCTAssertThrowsError(try future.wait()) { - XCTAssertEqual($0 as? HTTPClientError, .bodyLengthMismatch) - } - } - func testDoubleError() throws { // This is needed to that connection pool will not get into closed state when we release // second connection. @@ -3476,49 +3330,6 @@ class HTTPClientTests: XCTestCase { XCTAssertNotEqual(thirdConnectionNumber, firstConnectionNumber, "Different TLS configurations did not use different connections.") } - func testConnectionPoolSizeConfigValueIsRespected() { - let numberOfRequestsPerThread = 1000 - let numberOfParallelWorkers = 16 - let poolSize = 12 - - let httpBin = HTTPBin() - defer { XCTAssertNoThrow(try httpBin.shutdown()) } - - let group = MultiThreadedEventLoopGroup(numberOfThreads: 4) - defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } - - let configuration = HTTPClient.Configuration( - connectionPool: .init( - idleTimeout: .seconds(30), - concurrentHTTP1ConnectionsPerHostSoftLimit: poolSize - ) - ) - let client = HTTPClient(eventLoopGroupProvider: .shared(group), configuration: configuration) - defer { XCTAssertNoThrow(try client.syncShutdown()) } - - let g = DispatchGroup() - for workerID in 0.. () throws -> Void)] { + return [ + ("testIdleTimeoutNoReuse", testIdleTimeoutNoReuse), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift b/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift new file mode 100644 index 000000000..e7cfed4d0 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift @@ -0,0 +1,40 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class TestIdleTimeoutNoReuse: XCTestCaseHTTPClientTestsBaseClass { + func testIdleTimeoutNoReuse() throws { + var req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .GET) + XCTAssertNoThrow(try self.defaultClient.execute(request: req, deadline: .now() + .seconds(2)).wait()) + req.headers.add(name: "X-internal-delay", value: "2500") + try self.defaultClient.eventLoopGroup.next().scheduleTask(in: .milliseconds(250)) {}.futureResult.wait() + XCTAssertNoThrow(try self.defaultClient.execute(request: req).timeout(after: .seconds(10)).wait()) + } +} diff --git a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift new file mode 100644 index 000000000..9ed1ca2a8 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// NoBytesSentOverBodyLimitTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension NoBytesSentOverBodyLimitTests { + static var allTests: [(String, (NoBytesSentOverBodyLimitTests) -> () throws -> Void)] { + return [ + ("testNoBytesSentOverBodyLimit", testNoBytesSentOverBodyLimit), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift new file mode 100644 index 000000000..41285d5c5 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift @@ -0,0 +1,80 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class NoBytesSentOverBodyLimitTests: XCTestCaseHTTPClientTestsBaseClass { + func testNoBytesSentOverBodyLimit() throws { + let server = NIOHTTP1TestServer(group: self.serverGroup) + defer { + XCTAssertNoThrow(try server.stop()) + } + + let tooLong = "XBAD BAD BAD NOT HTTP/1.1\r\n\r\n" + + let request = try Request( + url: "http://localhost:\(server.serverPort)", + body: .stream(length: 1) { streamWriter in + streamWriter.write(.byteBuffer(ByteBuffer(string: tooLong))) + } + ) + + let future = self.defaultClient.execute(request: request) + + // Okay, what happens here needs an explanation: + // + // In the request state machine, we should start the request, which will lead to an + // invocation of `context.write(HTTPRequestHead)`. Since we will receive a streamed request + // body a `context.flush()` will be issued. Further the request stream will be started. + // Since the request stream immediately produces to much data, the request will be failed + // and the connection will be closed. + // + // Even though a flush was issued after the request head, there is no guarantee that the + // request head was written to the network. For this reason we must accept not receiving a + // request and receiving a request head. + + do { + _ = try server.receiveHead() + + // A request head was sent. We expect the request now to fail with a parsing error, + // since the client ended the connection to early (from the server's point of view.) + XCTAssertThrowsError(try server.readInbound()) { + XCTAssertEqual($0 as? HTTPParserError, HTTPParserError.invalidEOFState) + } + } catch { + // TBD: We sadly can't verify the error type, since it is private in `NIOTestUtils`: + // NIOTestUtils.BlockingQueue.TimeoutError + } + + // request must always be failed with this error + XCTAssertThrowsError(try future.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .bodyLengthMismatch) + } + } +} diff --git a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift new file mode 100644 index 000000000..b4aa20dad --- /dev/null +++ b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// RacePoolIdleConnectionsAndGetTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension RacePoolIdleConnectionsAndGetTests { + static var allTests: [(String, (RacePoolIdleConnectionsAndGetTests) -> () throws -> Void)] { + return [ + ("testRacePoolIdleConnectionsAndGet", testRacePoolIdleConnectionsAndGet), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift new file mode 100644 index 000000000..fd8e45273 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift @@ -0,0 +1,44 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class RacePoolIdleConnectionsAndGetTests: XCTestCaseHTTPClientTestsBaseClass { + func testRacePoolIdleConnectionsAndGet() { + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(connectionPool: .init(idleTimeout: .milliseconds(10)))) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + } + for _ in 1...200 { + XCTAssertNoThrow(try localClient.get(url: self.defaultHTTPBinURLPrefix + "get").wait()) + Thread.sleep(forTimeInterval: 0.01 + .random(in: -0.01...0.01)) + } + } +} diff --git a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift new file mode 100644 index 000000000..5d3c8bfb1 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// ResponseDelayGetTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension ResponseDelayGetTests { + static var allTests: [(String, (ResponseDelayGetTests) -> () throws -> Void)] { + return [ + ("testResponseDelayGet", testResponseDelayGet), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift new file mode 100644 index 000000000..0af5c7243 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift @@ -0,0 +1,43 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class ResponseDelayGetTests: XCTestCaseHTTPClientTestsBaseClass { + func testResponseDelayGet() throws { + let req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", + method: .GET, + headers: ["X-internal-delay": "2000"], + body: nil) + let start = NIODeadline.now() + let response = try self.defaultClient.execute(request: req).wait() + XCTAssertGreaterThanOrEqual(.now() - start, .milliseconds(1_900 /* 1.9 seconds */ )) + XCTAssertEqual(response.status, .ok) + } +} diff --git a/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift b/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift new file mode 100644 index 000000000..faf64cb19 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift @@ -0,0 +1,31 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// StressGetHttpsTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension StressGetHttpsTests { + static var allTests: [(String, (StressGetHttpsTests) -> () throws -> Void)] { + return [ + ("testStressGetHttps", testStressGetHttps), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift b/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift new file mode 100644 index 000000000..4c5cd1816 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift @@ -0,0 +1,51 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import Atomics +#if canImport(Network) +import Network +#endif +import Logging +import NIOConcurrencyHelpers +import NIOCore +import NIOFoundationCompat +import NIOHTTP1 +import NIOHTTPCompression +import NIOPosix +import NIOSSL +import NIOTestUtils +import NIOTransportServices +import XCTest + +final class StressGetHttpsTests: XCTestCaseHTTPClientTestsBaseClass { + func testStressGetHttps() throws { + let localHTTPBin = HTTPBin(.http1_1(ssl: true)) + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none)) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + + let eventLoop = localClient.eventLoopGroup.next() + let requestCount = 200 + var futureResults = [EventLoopFuture]() + for _ in 1...requestCount { + let req = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, headers: ["X-internal-delay": "100"]) + futureResults.append(localClient.execute(request: req)) + } + XCTAssertNoThrow(try EventLoopFuture.andAllSucceed(futureResults, on: eventLoop).wait()) + } +} diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index 2d7e744af..ca8478326 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -30,6 +30,7 @@ struct LinuxMain { static func main() { XCTMain([ testCase(AsyncAwaitEndToEndTests.allTests), + testCase(ConnectionPoolSizeConfigValueIsRespectedTests.allTests), testCase(HTTP1ClientChannelHandlerTests.allTests), testCase(HTTP1ConnectionStateMachineTests.allTests), testCase(HTTP1ConnectionTests.allTests), @@ -56,11 +57,16 @@ struct LinuxMain { testCase(HTTPConnectionPool_RequestQueueTests.allTests), testCase(HTTPRequestStateMachineTests.allTests), testCase(LRUCacheTests.allTests), + testCase(NoBytesSentOverBodyLimitTests.allTests), + testCase(RacePoolIdleConnectionsAndGetTests.allTests), testCase(RequestBagTests.allTests), testCase(RequestValidationTests.allTests), + testCase(ResponseDelayGetTests.allTests), testCase(SOCKSEventsHandlerTests.allTests), testCase(SSLContextCacheTests.allTests), + testCase(StressGetHttpsTests.allTests), testCase(TLSEventsHandlerTests.allTests), + testCase(TestIdleTimeoutNoReuse.allTests), testCase(TransactionTests.allTests), testCase(Transaction_StateMachineTests.allTests), ]) From 0b5bec741bfcf941e208d937de2ec29affe750a7 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 12 Oct 2022 08:50:28 +0100 Subject: [PATCH 049/146] Replace `NIOSendable` with `Sendable` (#640) --- .../ConnectionPool/RequestBodyLength.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient+Proxy.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 10 +++++----- Sources/AsyncHTTPClient/HTTPHandler.swift | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift index fdbd8b2e7..83f0e6edf 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift @@ -16,7 +16,7 @@ import NIOCore /// - Note: use `HTTPClientRequest.Body.Length` if you want to expose `RequestBodyLength` publicly @usableFromInline -internal enum RequestBodyLength: Hashable, NIOSendable { +internal enum RequestBodyLength: Hashable, Sendable { /// size of the request body is not known before starting the request case unknown /// size of the request body is fixed and exactly `count` bytes diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index cb67099ec..c35540114 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -23,7 +23,7 @@ import NIOCore extension HTTPClient { /// A representation of an HTTP cookie. - public struct Cookie: NIOSendable { + public struct Cookie: Sendable { /// The name of the cookie. public var name: String /// The cookie's string value. diff --git a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift index 148a4e888..25b4b4555 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift @@ -25,7 +25,7 @@ extension HTTPClient.Configuration { /// If a `TLSConfiguration` is used in conjunction with `HTTPClient.Configuration.Proxy`, /// TLS will be established _after_ successful proxy, between your client /// and the destination server. - public struct Proxy: NIOSendable, Hashable { + public struct Proxy: Sendable, Hashable { enum ProxyType: Hashable { case http(HTTPClient.Authorization?) case socks diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 1580bbe9c..6403c38f4 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -875,7 +875,7 @@ public class HTTPClient { } /// Specifies decompression settings. - public enum Decompression: NIOSendable { + public enum Decompression: Sendable { /// Decompression is disabled. case disabled /// Decompression is enabled. @@ -911,7 +911,7 @@ extension HTTPClient: @unchecked Sendable {} extension HTTPClient.Configuration { /// Timeout configuration. - public struct Timeout: NIOSendable { + public struct Timeout: Sendable { /// Specifies connect timeout. If no connect timeout is given, a default 30 seconds timeout will applied. public var connect: TimeAmount? /// Specifies read timeout. @@ -934,7 +934,7 @@ extension HTTPClient.Configuration { } /// Specifies redirect processing settings. - public struct RedirectConfiguration: NIOSendable { + public struct RedirectConfiguration: Sendable { enum Mode { /// Redirects are not followed. case disallow @@ -966,7 +966,7 @@ extension HTTPClient.Configuration { } /// Connection pool configuration. - public struct ConnectionPool: Hashable, NIOSendable { + public struct ConnectionPool: Hashable, Sendable { /// Specifies amount of time connections are kept idle in the pool. After this time has passed without a new /// request the connections are closed. public var idleTimeout: TimeAmount @@ -995,7 +995,7 @@ extension HTTPClient.Configuration { } } - public struct HTTPVersion: NIOSendable, Hashable { + public struct HTTPVersion: Sendable, Hashable { internal enum Configuration { case http1Only case automatic diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 26880ef8b..9cb9102f1 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -311,7 +311,7 @@ extension HTTPClient { } /// HTTP authentication. - public struct Authorization: Hashable, NIOSendable { + public struct Authorization: Hashable, Sendable { private enum Scheme: Hashable { case Basic(String) case Bearer(String) From 0bdc425a84cc447b8d83f5385c5a25d4da2c2026 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 12 Oct 2022 16:18:47 +0100 Subject: [PATCH 050/146] Remove `#if compiler(>=5.5)` (#641) * Remove `#if compiler(>=5.5)` * Run SwiftFormat --- README.md | 4 +- .../AsyncAwait/HTTPClient+execute.swift | 3 -- .../AsyncAwait/HTTPClient+shutdown.swift | 4 -- .../HTTPClientRequest+Prepared.swift | 3 -- .../AsyncAwait/HTTPClientRequest.swift | 3 -- .../AsyncAwait/HTTPClientResponse.swift | 3 -- .../AsyncAwait/Transaction+StateMachine.swift | 4 +- .../AsyncAwait/Transaction.swift | 2 - Sources/AsyncHTTPClient/HTTPClient.swift | 2 - Sources/AsyncHTTPClient/HTTPHandler.swift | 2 - Sources/AsyncHTTPClient/SSLContextCache.swift | 2 - Sources/AsyncHTTPClient/UnsafeTransfer.swift | 2 - .../AsyncAwaitEndToEndTests.swift | 43 ------------------- .../AsyncTestHelpers.swift | 2 - .../HTTPClientRequestTests.swift | 31 ------------- .../Transaction+StateMachineTests.swift | 16 ------- .../TransactionTests.swift | 25 +---------- .../XCTest+AsyncAwait.swift | 4 +- 18 files changed, 5 insertions(+), 150 deletions(-) diff --git a/README.md b/README.md index 261fcab56..b334db8da 100644 --- a/README.md +++ b/README.md @@ -329,11 +329,11 @@ Please have a look at [SECURITY.md](SECURITY.md) for AsyncHTTPClient's security ## Supported Versions -The most recent versions of AsyncHTTPClient support Swift 5.5 and newer. The minimum Swift version supported by AsyncHTTPClient releases are detailed below: +The most recent versions of AsyncHTTPClient support Swift 5.5.2 and newer. The minimum Swift version supported by AsyncHTTPClient releases are detailed below: AsyncHTTPClient | Minimum Swift Version --------------------|---------------------- `1.0.0 ..< 1.5.0` | 5.0 `1.5.0 ..< 1.10.0` | 5.2 `1.10.0 ..< 1.13.0` | 5.4 -`1.13.0 ...` | 5.5 +`1.13.0 ...` | 5.5.2 diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index 318edbff9..5328b7688 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import struct Foundation.URL import Logging import NIOCore @@ -215,5 +214,3 @@ private actor TransactionCancelHandler { } } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift index 36dd3588f..43020c3e5 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+shutdown.swift @@ -14,8 +14,6 @@ import NIOCore -#if compiler(>=5.5.2) && canImport(_Concurrency) - extension HTTPClient { /// Shuts down the client and `EventLoopGroup` if it was created by the client. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -32,5 +30,3 @@ extension HTTPClient { } } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index b4707226d..bd7417725 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import struct Foundation.URL import NIOCore import NIOHTTP1 @@ -122,5 +121,3 @@ extension HTTPClientRequest { return newRequest } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index 2e5bcfe88..6f3637f77 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import NIOCore import NIOHTTP1 @@ -424,5 +423,3 @@ extension HTTPClientRequest.Body { internal var storage: RequestBodyLength } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index 786b49eaf..b68e8db8b 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import NIOCore import NIOHTTP1 @@ -154,5 +153,3 @@ extension HTTPClientResponse.Body { .stream(CollectionOfOne(byteBuffer).async) } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 3ec6b4eb7..32f93582b 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -11,7 +11,7 @@ // SPDX-License-Identifier: Apache-2.0 // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) + import Logging import NIOCore import NIOHTTP1 @@ -765,5 +765,3 @@ extension Transaction { } } } - -#endif diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index f5c90557a..d81fbfd28 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import Logging import NIOConcurrencyHelpers import NIOCore @@ -362,4 +361,3 @@ extension Transaction { self.performFailAction(action) } } -#endif diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 6403c38f4..0ca48b0c4 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -904,10 +904,8 @@ extension HTTPClient.EventLoopGroupProvider: Sendable {} extension HTTPClient.EventLoopPreference: Sendable {} #endif -#if swift(>=5.5) && canImport(_Concurrency) // HTTPClient is thread-safe because its shared mutable state is protected through a lock extension HTTPClient: @unchecked Sendable {} -#endif extension HTTPClient.Configuration { /// Timeout configuration. diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 9cb9102f1..ab82d4f91 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -779,9 +779,7 @@ extension HTTPClient { } } -#if swift(>=5.5) && canImport(_Concurrency) extension HTTPClient.Task: @unchecked Sendable {} -#endif internal struct TaskCancelEvent {} diff --git a/Sources/AsyncHTTPClient/SSLContextCache.swift b/Sources/AsyncHTTPClient/SSLContextCache.swift index f1e8623a6..660a04942 100644 --- a/Sources/AsyncHTTPClient/SSLContextCache.swift +++ b/Sources/AsyncHTTPClient/SSLContextCache.swift @@ -56,6 +56,4 @@ extension SSLContextCache { } } -#if swift(>=5.5) && canImport(_Concurrency) extension SSLContextCache: @unchecked Sendable {} -#endif diff --git a/Sources/AsyncHTTPClient/UnsafeTransfer.swift b/Sources/AsyncHTTPClient/UnsafeTransfer.swift index 2df9d1238..ea5af56da 100644 --- a/Sources/AsyncHTTPClient/UnsafeTransfer.swift +++ b/Sources/AsyncHTTPClient/UnsafeTransfer.swift @@ -26,6 +26,4 @@ final class UnsafeMutableTransferBox { } } -#if swift(>=5.5) && canImport(_Concurrency) extension UnsafeMutableTransferBox: @unchecked Sendable {} -#endif diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 8995acfb1..bd6720220 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -56,7 +56,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testSimpleGet() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) @@ -75,11 +74,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(response.status, .ok) XCTAssertEqual(response.version, .http2) } - #endif } func testSimplePost() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) @@ -98,11 +95,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(response.status, .ok) XCTAssertEqual(response.version, .http2) } - #endif } func testPostWithByteBuffer() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -123,11 +118,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithSequenceOfUInt8() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -148,11 +141,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithCollectionOfUInt8() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -173,11 +164,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithRandomAccessCollectionOfUInt8() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -198,11 +187,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithAsyncSequenceOfByteBuffers() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -227,11 +214,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithAsyncSequenceOfUInt8() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -252,11 +237,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } func testPostWithFragmentedAsyncSequenceOfByteBuffers() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -294,11 +277,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(lastResult, nil) } - #endif } func testPostWithFragmentedAsyncSequenceOfLargeByteBuffers() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -337,11 +318,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(lastResult, nil) } - #endif } func testCanceling() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) @@ -362,11 +341,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(error as? HTTPClientError, .cancelled) } } - #endif } func testDeadline() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) @@ -387,11 +364,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error)) } } - #endif } func testImmediateDeadline() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) @@ -412,11 +387,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error), "unexpected error \(error)") } } - #endif } func testConnectTimeout() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 60) { #if os(Linux) @@ -471,11 +444,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertLessThan(duration, .seconds(1)) } } - #endif } func testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { /// key + cert was created with the follwing command: @@ -519,11 +490,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { #endif } } - #endif } func testInvalidURL() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let client = makeDefaultHTTPClient() @@ -535,11 +504,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual($0 as? HTTPClientError, .invalidURL) } } - #endif } func testRedirectChangesHostHeader() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) @@ -564,11 +531,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(response.version, .http2) XCTAssertEqual(requestInfo.data, "localhost:\(bin.port)") } - #endif } func testShutdown() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let client = makeDefaultHTTPClient() @@ -577,12 +542,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqualTypeAndValue(error, HTTPClientError.alreadyShutdown) } } - #endif } /// Regression test for https://github.com/swift-server/async-http-client/issues/612 func testCancelingBodyDoesNotCrash() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let client = makeDefaultHTTPClient() @@ -597,11 +560,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssert(error is NIOTooManyBytesError) } } - #endif } func testAsyncSequenceReuse() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } @@ -635,11 +596,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } - #endif } } -#if compiler(>=5.5.2) && canImport(_Concurrency) extension AsyncSequence where Element == ByteBuffer { func collect() async rethrows -> ByteBuffer { try await self.reduce(into: ByteBuffer()) { accumulatingBuffer, nextBuffer in @@ -690,5 +649,3 @@ extension AnySendableCollection: Collection { self.wrapped[position] } } - -#endif diff --git a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift index 332cb2227..147b24dca 100644 --- a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift +++ b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -#if compiler(>=5.5.2) && canImport(_Concurrency) import NIOConcurrencyHelpers import NIOCore @@ -192,4 +191,3 @@ final class AsyncSequenceWriter: AsyncSequence, @unchecked Sendable { } } } -#endif diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 271144cc4..fa424b042 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -17,16 +17,13 @@ import NIOCore import XCTest class HTTPClientRequestTests: XCTestCase { - #if compiler(>=5.5.2) && canImport(_Concurrency) @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) private typealias Request = HTTPClientRequest @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) private typealias PreparedRequest = HTTPClientRequest.Prepared - #endif func testCustomHeadersAreRespected() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "https://example.com/get") @@ -58,11 +55,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testUnixScheme() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "unix://%2Fexample%2Ffolder.sock/some_path") @@ -89,11 +84,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testHTTPUnixScheme() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http+unix://%2Fexample%2Ffolder.sock/some_path") @@ -120,11 +113,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testHTTPSUnixScheme() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "https+unix://%2Fexample%2Ffolder.sock/some_path") @@ -151,11 +142,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testGetWithoutBody() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let request = Request(url: "https://example.com/get") @@ -181,11 +170,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testPostWithoutBody() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -216,11 +203,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testPostWithEmptyByteBuffer() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -252,11 +237,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } - #endif } func testPostWithByteBuffer() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -287,11 +270,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } func testPostWithSequenceOfUnknownLength() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -323,11 +304,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } func testPostWithSequenceWithFixedLength() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -360,11 +339,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } func testPostWithRandomAccessCollection() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -396,11 +373,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } func testPostWithAsyncSequenceOfUnknownLength() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -437,11 +412,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } func testPostWithAsyncSequenceWithKnownLength() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") @@ -478,11 +451,9 @@ class HTTPClientRequestTests: XCTestCase { guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } - #endif } } -#if compiler(>=5.5.2) && canImport(_Concurrency) private struct LengthMismatch: Error { var announcedLength: Int var actualLength: Int @@ -550,5 +521,3 @@ extension Collection { .init(wrapped: self, maxChunkSize: maxChunkSize) } } - -#endif diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift index 8b6985386..bb3a2f03e 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift @@ -20,7 +20,6 @@ import XCTest final class Transaction_StateMachineTests: XCTestCase { func testRequestWasQueuedAfterWillExecuteRequestWasCalled() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { @@ -46,11 +45,9 @@ final class Transaction_StateMachineTests: XCTestCase { await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) } - #endif } func testRequestBodyStreamWasPaused() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { @@ -69,12 +66,10 @@ final class Transaction_StateMachineTests: XCTestCase { await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) } - #endif } func testQueuedRequestGetsRemovedWhenDeadlineExceeded() { struct MyError: Error, Equatable {} - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -102,12 +97,10 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertEqualTypeAndValue($0, MyError()) } } - #endif } func testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect() { struct MyError: Error, Equatable {} - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -140,11 +133,9 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertEqualTypeAndValue($0, MyError()) } } - #endif } func testScheduledRequestGetsRemovedWhenDeadlineExceeded() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { @@ -167,11 +158,9 @@ final class Transaction_StateMachineTests: XCTestCase { await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) } - #endif } func testDeadlineExceededRaceWithRequestWillExecute() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { @@ -201,11 +190,9 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertEqualTypeAndValue($0, HTTPClientError.deadlineExceeded) } } - #endif } func testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { @@ -231,11 +218,9 @@ final class Transaction_StateMachineTests: XCTestCase { await XCTAssertThrowsError(try await withCheckedThrowingContinuation(workaround)) } - #endif } } -#if compiler(>=5.5.2) && canImport(_Concurrency) @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction.StateMachine.StartExecutionAction: Equatable { public static func == (lhs: Self, rhs: Self) -> Bool { @@ -286,4 +271,3 @@ extension Transaction.StateMachine.NextWriteAction: Equatable { } } } -#endif diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index c1257c11d..7aee8c642 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -21,14 +21,11 @@ import NIOHTTP1 import NIOPosix import XCTest -#if compiler(>=5.5.2) && canImport(_Concurrency) @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) typealias PreparedRequest = HTTPClientRequest.Prepared -#endif final class TransactionTests: XCTestCase { func testCancelAsyncRequest() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -60,11 +57,9 @@ final class TransactionTests: XCTestCase { } XCTAssertEqual(queuer.hitCancelCount, 1) } - #endif } func testResponseStreamingWorks() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -123,11 +118,9 @@ final class TransactionTests: XCTestCase { let result = try await part XCTAssertNil(result) } - #endif } func testIgnoringResponseBodyWorks() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -174,11 +167,9 @@ final class TransactionTests: XCTestCase { transaction.receiveResponseBodyParts([ByteBuffer(string: "foo bar")]) transaction.succeedRequest(nil) } - #endif } func testWriteBackpressureWorks() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -248,11 +239,9 @@ final class TransactionTests: XCTestCase { let result = try await part XCTAssertNil(result) } - #endif } func testSimpleGetRequest() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) @@ -306,11 +295,9 @@ final class TransactionTests: XCTestCase { RequestInfo(data: "", requestNumber: 1, connectionNumber: 0) ) } - #endif } func testSimplePostRequest() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -346,11 +333,9 @@ final class TransactionTests: XCTestCase { XCTAssertEqual(response.version, .http1_1) XCTAssertEqual(response.headers, ["foo": "bar"]) } - #endif } func testPostStreamFails() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() @@ -391,11 +376,9 @@ final class TransactionTests: XCTestCase { } XCTAssertNoThrow(try executor.receiveCancellation()) } - #endif } func testResponseStreamFails() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 30) { let embeddedEventLoop = EmbeddedEventLoop() @@ -456,11 +439,9 @@ final class TransactionTests: XCTestCase { XCTAssertEqual($0 as? HTTPClientError, .readTimeout) } } - #endif } func testBiDirectionalStreamingHTTP2() { - #if compiler(>=5.5.2) && canImport(_Concurrency) guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) @@ -531,12 +512,9 @@ final class TransactionTests: XCTestCase { XCTAssertNil(final) XCTAssertEqual(delegate.hitStreamClosed, 1) } - #endif } } -#if compiler(>=5.5.2) && canImport(_Concurrency) - // This needs a small explanation. If an iterator is a struct, it can't be used across multiple // tasks. Since we want to wait for things to happen in tests, we need to `async let`, which creates // implicit tasks. Therefore we need to wrap our iterator struct. @@ -567,7 +545,7 @@ actor SharedIterator where Wrapped.Element: Sendable { /// non fail-able promise that only supports one observer @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -fileprivate actor Promise { +private actor Promise { private enum State { case initialised case fulfilled(Value) @@ -633,4 +611,3 @@ extension Transaction { return (await transactionPromise.value, task) } } -#endif diff --git a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift index fbc429b10..3f2d1185e 100644 --- a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift +++ b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift @@ -26,7 +26,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -#if compiler(>=5.5.2) && canImport(_Concurrency) + import XCTest extension XCTestCase { @@ -89,5 +89,3 @@ internal func XCTAssertNoThrowWithResult( } return nil } - -#endif From 10f42e647a15d6e5c0e9de2e081761d23730a249 Mon Sep 17 00:00:00 2001 From: Max Desiatov Date: Wed, 26 Oct 2022 10:50:36 +0100 Subject: [PATCH 051/146] FileDownloadDelegate: mark `Progress` as `Sendable` (#643) This is a trivial struct that should be `Sendable`, as it's a common use case to pass `Progress` values to `@Sendable` closures. --- Sources/AsyncHTTPClient/FileDownloadDelegate.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index 6199f33ff..c328c7211 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -20,7 +20,7 @@ import NIOPosix public final class FileDownloadDelegate: HTTPClientResponseDelegate { /// The response type for this delegate: the total count of bytes as reported by the response /// "Content-Length" header (if available) and the count of bytes downloaded. - public struct Progress { + public struct Progress: Sendable { public var totalBytes: Int? public var receivedBytes: Int } From 8b84142a7079b7a3707d493ecb221cb4674ef60e Mon Sep 17 00:00:00 2001 From: carolinacass <67160898+carolinacass@users.noreply.github.com> Date: Fri, 4 Nov 2022 15:20:19 +0000 Subject: [PATCH 052/146] Use #fileID/#filePath instead of #file (#644) --- .../AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift | 2 +- .../ConnectionPoolSizeConfigValueIsRespectedTests.swift | 2 +- Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 2 +- .../HTTPConnectionPool+HTTP2StateMachineTests.swift | 2 +- Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift | 2 +- Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift | 2 +- Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift | 2 +- Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift | 4 ++-- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 32f93582b..8700d6b32 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -641,7 +641,7 @@ extension Transaction { private func verifyStreamIDIsEqual( registered: TransactionBody.AsyncIterator.ID, this: TransactionBody.AsyncIterator.ID, - file: StaticString = #file, + file: StaticString = #fileID, line: UInt = #line ) { if registered != this { diff --git a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift index 46cddeead..79c304fc2 100644 --- a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift +++ b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift @@ -52,7 +52,7 @@ final class ConnectionPoolSizeConfigValueIsRespectedTests: XCTestCaseHTTPClientT let g = DispatchGroup() for workerID in 0..( _ lhs: @autoclosure () throws -> Left, _ rhs: @autoclosure () throws -> Right, - file: StaticString = #file, + file: StaticString = #filePath, line: UInt = #line ) { XCTAssertNoThrow(try { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift index 60e5077ee..e85571ad1 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift @@ -419,7 +419,7 @@ class HTTPConnectionPoolTests: XCTestCase { let dispatchGroup = DispatchGroup() for workerID in 0..( _ expectedError: Error, _ expectedFinalStreamAction: HTTPRequestStateMachine.Action.FinalFailedRequestAction, - file: StaticString = #file, + file: StaticString = #filePath, line: UInt = #line ) where Error: Swift.Error & Equatable { guard case .failRequest(let actualError, let actualFinalStreamAction) = self else { diff --git a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift index d7c97e6fe..d888769b4 100644 --- a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift @@ -40,7 +40,7 @@ class MockSOCKSServer { self.channel.localAddress!.port! } - init(expectedURL: String, expectedResponse: String, misbehave: Bool = false, file: String = #file, line: UInt = #line) throws { + init(expectedURL: String, expectedResponse: String, misbehave: Bool = false, file: String = #filePath, line: UInt = #line) throws { let elg = MultiThreadedEventLoopGroup(numberOfThreads: 1) let bootstrap: ServerBootstrap if misbehave { diff --git a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift index 3f2d1185e..bf297413c 100644 --- a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift +++ b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift @@ -65,7 +65,7 @@ extension XCTestCase { internal func XCTAssertThrowsError( _ expression: @autoclosure () async throws -> T, verify: (Error) -> Void = { _ in }, - file: StaticString = #file, + file: StaticString = #filePath, line: UInt = #line ) async { do { @@ -79,7 +79,7 @@ internal func XCTAssertThrowsError( @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) internal func XCTAssertNoThrowWithResult( _ expression: @autoclosure () async throws -> Result, - file: StaticString = #file, + file: StaticString = #filePath, line: UInt = #line ) async -> Result? { do { From 708ead8e60ee18880c5ea7994ecd9e3d1664cd50 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 8 Nov 2022 10:44:59 +0100 Subject: [PATCH 053/146] Remove obsolete notes in README.md (#645) AsyncHTTPClient now requires at least Swift 5.5.2. The async/await APIs are therefore available in all supported Swift compiler versions. Swift Concurrency and HTTP/2 are now also supported for some time now. No need to call out the specific version in the readme. This information can still be found in the release notes if needed. --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index b334db8da..3e705d50c 100644 --- a/README.md +++ b/README.md @@ -2,20 +2,14 @@ This package provides an HTTP Client library built on top of SwiftNIO. This library provides the following: -- First class support for Swift Concurrency (since version 1.9.0) +- First class support for Swift Concurrency - Asynchronous and non-blocking request methods - Simple follow-redirects (cookie headers are dropped) - Streaming body download - TLS support -- Automatic HTTP/2 over HTTPS (since version 1.7.0) +- Automatic HTTP/2 over HTTPS - Cookie parsing (but not storage) ---- - -**NOTE**: You will need [Xcode 13.2](https://apps.apple.com/gb/app/xcode/id497799835?mt=12) or [Swift 5.5.2](https://swift.org/download/#swift-552) to try out `AsyncHTTPClient`s new async/await APIs. - ---- - ## Getting Started #### Adding the dependency From fd03ed01c770649922e870b4029e94867aeebcfc Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 8 Nov 2022 17:48:56 +0100 Subject: [PATCH 054/146] Tolerate shutdown message after channel is closed (#646) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Tolerate shutdown message after channel is closed ### Motivation A channel can close unexpectedly if something goes wrong. We may in the meantime have scheduled the connection for graceful shutdown but the connection has not yet seen the message. We need to still accept the shutdown message and just ignore it if we are already closed. ### Modification - ignore calls to shutdown if the channel is already closed - add a test which would previously crash because we have transition from the closed state to the closing state and we hit the deinit precondition - include the current state in preconditions if we are in the wrong state ### Result We donโ€™t hit the precondition in the deinit in the scenario described above and have more descriptive crashes if something still goes wrong. --- .../HTTP2/HTTP2Connection.swift | 44 +++++++++++++------ .../HTTP2ConnectionTests+XCTest.swift | 1 + .../HTTP2ConnectionTests.swift | 24 ++++++++++ 3 files changed, 56 insertions(+), 13 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index 701e630c2..5859e619a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -111,7 +111,7 @@ final class HTTP2Connection { deinit { guard case .closed = self.state else { - preconditionFailure("Connection must be closed, before we can deinit it") + preconditionFailure("Connection must be closed, before we can deinit it. Current state: \(self.state)") } } @@ -129,7 +129,7 @@ final class HTTP2Connection { delegate: delegate, logger: logger ) - return connection.start().map { maxStreams in (connection, maxStreams) } + return connection._start0().map { maxStreams in (connection, maxStreams) } } func executeRequest(_ request: HTTPExecutableRequest) { @@ -164,15 +164,23 @@ final class HTTP2Connection { return promise.futureResult } - private func start() -> EventLoopFuture { + func _start0() -> EventLoopFuture { self.channel.eventLoop.assertInEventLoop() let readyToAcceptConnectionsPromise = self.channel.eventLoop.makePromise(of: Int.self) self.state = .starting(readyToAcceptConnectionsPromise) self.channel.closeFuture.whenComplete { _ in - self.state = .closed - self.delegate.http2ConnectionClosed(self) + switch self.state { + case .initialized, .closed: + preconditionFailure("invalid state \(self.state)") + case .starting(let readyToAcceptConnectionsPromise): + self.state = .closed + readyToAcceptConnectionsPromise.fail(HTTPClientError.remoteConnectionClosed) + case .active, .closing: + self.state = .closed + self.delegate.http2ConnectionClosed(self) + } } do { @@ -258,16 +266,26 @@ final class HTTP2Connection { private func shutdown0() { self.channel.eventLoop.assertInEventLoop() - self.state = .closing + switch self.state { + case .active: + self.state = .closing - // inform all open streams, that the currently running request should be cancelled. - self.openStreams.forEach { box in - box.channel.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) - } + // inform all open streams, that the currently running request should be cancelled. + self.openStreams.forEach { box in + box.channel.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) + } - // inform the idle connection handler, that connection should be closed, once all streams - // are closed. - self.channel.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) + // inform the idle connection handler, that connection should be closed, once all streams + // are closed. + self.channel.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) + + case .closed, .closing: + // we are already closing/closed and we need to tolerate this + break + + case .initialized, .starting: + preconditionFailure("invalid state \(self.state)") + } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift index 9f9582d9f..06b60f757 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift @@ -26,6 +26,7 @@ extension HTTP2ConnectionTests { static var allTests: [(String, (HTTP2ConnectionTests) -> () throws -> Void)] { return [ ("testCreateNewConnectionFailureClosedIO", testCreateNewConnectionFailureClosedIO), + ("testConnectionToleratesShutdownEventsAfterAlreadyClosed", testConnectionToleratesShutdownEventsAfterAlreadyClosed), ("testSimpleGetRequest", testSimpleGetRequest), ("testEveryDoneRequestLeadsToAStreamAvailableCall", testEveryDoneRequestLeadsToAStreamAvailableCall), ("testCancelAllRunningRequests", testCancelAllRunningRequests), diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index a9ff14f49..652884a84 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -42,6 +42,30 @@ class HTTP2ConnectionTests: XCTestCase { ).wait()) } + func testConnectionToleratesShutdownEventsAfterAlreadyClosed() { + let embedded = EmbeddedChannel() + XCTAssertNoThrow(try embedded.connect(to: SocketAddress(ipAddress: "127.0.0.1", port: 3000)).wait()) + + let logger = Logger(label: "test.http2.connection") + let connection = HTTP2Connection( + channel: embedded, + connectionID: 0, + decompression: .disabled, + delegate: TestHTTP2ConnectionDelegate(), + logger: logger + ) + let startFuture = connection._start0() + + XCTAssertNoThrow(try embedded.close().wait()) + // to really destroy the channel we need to tick once + embedded.embeddedEventLoop.run() + + XCTAssertThrowsError(try startFuture.wait()) + + // should not crash + connection.shutdown() + } + func testSimpleGetRequest() { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) let eventLoop = eventLoopGroup.next() From 5bee16a79922e3efcb5cea06ecd27e6f8048b56b Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 9 Nov 2022 15:30:16 +0100 Subject: [PATCH 055/146] Switch over state in `HTTPConnectionPool.HTTP2StateMachine.failedToCreateNewConnection` (#647) --- ...HTTPConnectionPool+HTTP2StateMachine.swift | 35 ++++++++++------- ...onPool+HTTP2StateMachineTests+XCTest.swift | 1 + ...onnectionPool+HTTP2StateMachineTests.swift | 38 +++++++++++++++++++ 3 files changed, 61 insertions(+), 13 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index 003de4223..9964ccd05 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -404,25 +404,34 @@ extension HTTPConnectionPool { } mutating func failedToCreateNewConnection(_ error: Error, connectionID: Connection.ID) -> Action { - // TODO: switch over state https://github.com/swift-server/async-http-client/issues/638 self.failedConsecutiveConnectionAttempts += 1 self.lastConnectFailure = error - guard self.retryConnectionEstablishment else { - guard let (index, _) = self.connections.failConnection(connectionID) else { - preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + switch self.lifecycleState { + case .running: + guard self.retryConnectionEstablishment else { + guard let (index, _) = self.connections.failConnection(connectionID) else { + preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + } + self.connections.removeConnection(at: index) + + return .init( + request: self.failAllRequests(reason: error), + connection: .none + ) } - self.connections.removeConnection(at: index) - return .init( - request: self.failAllRequests(reason: error), - connection: .none - ) + let eventLoop = self.connections.backoffNextConnectionAttempt(connectionID) + let backoff = calculateBackoff(failedAttempt: self.failedConsecutiveConnectionAttempts) + return .init(request: .none, connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop)) + case .shuttingDown: + guard let (index, context) = self.connections.failConnection(connectionID) else { + preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + } + return self.nextActionForFailedConnection(at: index, on: context.eventLoop) + case .shutDown: + preconditionFailure("If the pool is already shutdown, all connections must have been torn down.") } - - let eventLoop = self.connections.backoffNextConnectionAttempt(connectionID) - let backoff = calculateBackoff(failedAttempt: self.failedConsecutiveConnectionAttempts) - return .init(request: .none, connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop)) } mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift index 95ea8c580..12b031cc0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift @@ -27,6 +27,7 @@ extension HTTPConnectionPool_HTTP2StateMachineTests { return [ ("testCreatingOfConnection", testCreatingOfConnection), ("testConnectionFailureBackoff", testConnectionFailureBackoff), + ("testConnectionFailureWhileShuttingDown", testConnectionFailureWhileShuttingDown), ("testConnectionFailureWithoutRetry", testConnectionFailureWithoutRetry), ("testCancelRequestWorks", testCancelRequestWorks), ("testExecuteOnShuttingDownPool", testExecuteOnShuttingDownPool), diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index bf1ef4a98..10fad7bd6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -194,6 +194,44 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertEqual(state.connectionCreationBackoffDone(newConnectionID), .none) } + func testConnectionFailureWhileShuttingDown() { + struct SomeError: Error, Equatable {} + let elg = EmbeddedEventLoopGroup(loops: 4) + defer { XCTAssertNoThrow(try elg.syncShutdownGracefully()) } + + var state = HTTPConnectionPool.HTTP2StateMachine( + idGenerator: .init(), + retryConnectionEstablishment: false, + lifecycleState: .running + ) + + let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let request = HTTPConnectionPool.Request(mockRequest) + + let action = state.executeRequest(request) + XCTAssertEqual(.scheduleRequestTimeout(for: request, on: mockRequest.eventLoop), action.request) + + // 1. connection attempt + guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { + return XCTFail("Unexpected connection action: \(action.connection)") + } + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + + // 2. initialise shutdown + let shutdownAction = state.shutdown() + XCTAssertEqual(shutdownAction.connection, .cleanupConnections(.init(), isShutdown: .no)) + guard case .failRequestsAndCancelTimeouts(let requestsToFail, let requestError) = shutdownAction.request else { + return XCTFail("Unexpected request action: \(action.request)") + } + XCTAssertEqualTypeAndValue(requestError, HTTPClientError.cancelled) + XCTAssertEqualTypeAndValue(requestsToFail, [request]) + + // 3. connection attempt fails + let failedConnectAction = state.failedToCreateNewConnection(SomeError(), connectionID: connectionID) + XCTAssertEqual(failedConnectAction.request, .none) + XCTAssertEqual(failedConnectAction.connection, .cleanupConnections(.init(), isShutdown: .yes(unclean: true))) + } + func testConnectionFailureWithoutRetry() { struct SomeError: Error, Equatable {} let elg = EmbeddedEventLoopGroup(loops: 4) From 2859c10ce412978a4ad00c1ada4e9b79c4c5aa2a Mon Sep 17 00:00:00 2001 From: Yim Lee Date: Fri, 2 Dec 2022 13:16:44 -0800 Subject: [PATCH 056/146] Add .spi.yml for Swift Package Index DocC support (#648) --- .spi.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .spi.yml diff --git a/.spi.yml b/.spi.yml new file mode 100644 index 000000000..795484b35 --- /dev/null +++ b/.spi.yml @@ -0,0 +1,4 @@ +version: 1 +builder: + configs: + - documentation_targets: [AsyncHTTPClient] From 49abfc30ae80951dbd9e8d12bef6d7722a4837dd Mon Sep 17 00:00:00 2001 From: iMike Date: Tue, 6 Dec 2022 14:13:10 +0300 Subject: [PATCH 057/146] Add `Host` header (#650) (#651) --- .../ChannelHandler/HTTP1ProxyConnectHandler.swift | 1 + .../AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift index 7340a59ea..fbcd4f9c0 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift @@ -155,6 +155,7 @@ final class HTTP1ProxyConnectHandler: ChannelDuplexHandler, RemovableChannelHand method: .CONNECT, uri: "\(self.targetHost):\(self.targetPort)" ) + head.headers.replaceOrAdd(name: "host", value: "\(self.targetHost)") if let authorization = self.proxyAuthorization { head.headers.replaceOrAdd(name: "proxy-authorization", value: authorization.headerValue) } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift index bbe6fab1f..b3917173f 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift @@ -43,6 +43,7 @@ class HTTP1ProxyConnectHandlerTests: XCTestCase { XCTAssertEqual(head.method, .CONNECT) XCTAssertEqual(head.uri, "swift.org:443") + XCTAssertEqual(head.headers["host"].first, "swift.org") XCTAssertNil(head.headers["proxy-authorization"].first) XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) @@ -76,6 +77,7 @@ class HTTP1ProxyConnectHandlerTests: XCTestCase { XCTAssertEqual(head.method, .CONNECT) XCTAssertEqual(head.uri, "swift.org:443") + XCTAssertEqual(head.headers["host"].first, "swift.org") XCTAssertEqual(head.headers["proxy-authorization"].first, "Basic abc123") XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) @@ -109,6 +111,7 @@ class HTTP1ProxyConnectHandlerTests: XCTestCase { XCTAssertEqual(head.method, .CONNECT) XCTAssertEqual(head.uri, "swift.org:443") + XCTAssertEqual(head.headers["host"].first, "swift.org") XCTAssertNil(head.headers["proxy-authorization"].first) XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) @@ -148,6 +151,7 @@ class HTTP1ProxyConnectHandlerTests: XCTestCase { XCTAssertEqual(head.method, .CONNECT) XCTAssertEqual(head.uri, "swift.org:443") + XCTAssertEqual(head.headers["host"].first, "swift.org") XCTAssertNil(head.headers["proxy-authorization"].first) XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) @@ -187,6 +191,7 @@ class HTTP1ProxyConnectHandlerTests: XCTestCase { XCTAssertEqual(head.method, .CONNECT) XCTAssertEqual(head.uri, "swift.org:443") + XCTAssertEqual(head.headers["host"].first, "swift.org") XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) From 7f05a8da46cc2a4ab43218722298b81ac7a08031 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 17 Jan 2023 11:06:08 +0000 Subject: [PATCH 058/146] Merge pull request from GHSA-v3r5-pjpm-mwgq Motivation Allowing arbitrary data in outbound header field values allows for the possibility that users of AHC will accidentally pass untrusted data into those values. That untrusted data can substantially alter the parsing and content of the HTTP requests, which is extremely dangerous. The result of this is vulnerability to CRLF injection. Modifications Add validation of outbound header field values. Result No longer vulnerable to CRLF injection --- Sources/AsyncHTTPClient/HTTPClient.swift | 5 + .../AsyncHTTPClient/RequestValidation.swift | 51 +++++++ .../AsyncAwaitEndToEndTests+XCTest.swift | 4 + .../AsyncAwaitEndToEndTests.swift | 128 ++++++++++++++++++ 4 files changed, 188 insertions(+) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 0ca48b0c4..1089db86c 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -1032,6 +1032,7 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { case uncleanShutdown case traceRequestWithBody case invalidHeaderFieldNames([String]) + case invalidHeaderFieldValues([String]) case bodyLengthMismatch case writeAfterRequestSent @available(*, deprecated, message: "AsyncHTTPClient now silently corrects invalid headers.") @@ -1100,6 +1101,8 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { return "Trace request with body" case .invalidHeaderFieldNames: return "Invalid header field names" + case .invalidHeaderFieldValues: + return "Invalid header field values" case .bodyLengthMismatch: return "Body length mismatch" case .writeAfterRequestSent: @@ -1166,6 +1169,8 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { public static let traceRequestWithBody = HTTPClientError(code: .traceRequestWithBody) /// Header field names contain invalid characters. public static func invalidHeaderFieldNames(_ names: [String]) -> HTTPClientError { return HTTPClientError(code: .invalidHeaderFieldNames(names)) } + /// Header field values contain invalid characters. + public static func invalidHeaderFieldValues(_ values: [String]) -> HTTPClientError { return HTTPClientError(code: .invalidHeaderFieldValues(values)) } /// Body length is not equal to `Content-Length`. public static let bodyLengthMismatch = HTTPClientError(code: .bodyLengthMismatch) /// Body part was written after request was fully sent. diff --git a/Sources/AsyncHTTPClient/RequestValidation.swift b/Sources/AsyncHTTPClient/RequestValidation.swift index e23c35423..87224a3b2 100644 --- a/Sources/AsyncHTTPClient/RequestValidation.swift +++ b/Sources/AsyncHTTPClient/RequestValidation.swift @@ -21,6 +21,7 @@ extension HTTPHeaders { bodyLength: RequestBodyLength ) throws -> RequestFramingMetadata { try self.validateFieldNames() + try self.validateFieldValues() if case .TRACE = method { switch bodyLength { @@ -80,6 +81,56 @@ extension HTTPHeaders { } } + private func validateFieldValues() throws { + let invalidValues = self.compactMap { _, value -> String? in + let satisfy = value.utf8.allSatisfy { char -> Bool in + /// Validates a byte of a given header field value against the definition in RFC 9110. + /// + /// The spec in [RFC 9110](https://httpwg.org/specs/rfc9110.html#fields.values) defines the valid + /// characters as the following: + /// + /// ``` + /// field-value = *field-content + /// field-content = field-vchar + /// [ 1*( SP / HTAB / field-vchar ) field-vchar ] + /// field-vchar = VCHAR / obs-text + /// obs-text = %x80-FF + /// ``` + /// + /// Additionally, it makes the following note: + /// + /// "Field values containing CR, LF, or NUL characters are invalid and dangerous, due to the + /// varying ways that implementations might parse and interpret those characters; a recipient + /// of CR, LF, or NUL within a field value MUST either reject the message or replace each of + /// those characters with SP before further processing or forwarding of that message. Field + /// values containing other CTL characters are also invalid; however, recipients MAY retain + /// such characters for the sake of robustness when they appear within a safe context (e.g., + /// an application-specific quoted string that will not be processed by any downstream HTTP + /// parser)." + /// + /// As we cannot guarantee the context is safe, this code will reject all ASCII control characters + /// directly _except_ for HTAB, which is explicitly allowed. + switch char { + case UInt8(ascii: "\t"): + // HTAB, explicitly allowed. + return true + case 0...0x1f, 0x7F: + // ASCII control character, forbidden. + return false + default: + // Printable or non-ASCII, allowed. + return true + } + } + + return satisfy ? nil : value + } + + guard invalidValues.count == 0 else { + throw HTTPClientError.invalidHeaderFieldValues(invalidValues) + } + } + private mutating func setTransportFraming( method: HTTPMethod, bodyLength: RequestBodyLength diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index b8431757c..c0b028905 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -45,6 +45,10 @@ extension AsyncAwaitEndToEndTests { ("testShutdown", testShutdown), ("testCancelingBodyDoesNotCrash", testCancelingBodyDoesNotCrash), ("testAsyncSequenceReuse", testAsyncSequenceReuse), + ("testRejectsInvalidCharactersInHeaderFieldNames_http1", testRejectsInvalidCharactersInHeaderFieldNames_http1), + ("testRejectsInvalidCharactersInHeaderFieldNames_http2", testRejectsInvalidCharactersInHeaderFieldNames_http2), + ("testRejectsInvalidCharactersInHeaderFieldValues_http1", testRejectsInvalidCharactersInHeaderFieldValues_http1), + ("testRejectsInvalidCharactersInHeaderFieldValues_http2", testRejectsInvalidCharactersInHeaderFieldValues_http2), ] } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index bd6720220..9a77ee9fb 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -597,6 +597,134 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(body, ByteBuffer(string: "1234")) } } + + func testRejectsInvalidCharactersInHeaderFieldNames_http1() { + self._rejectsInvalidCharactersInHeaderFieldNames(mode: .http1_1(ssl: true)) + } + + func testRejectsInvalidCharactersInHeaderFieldNames_http2() { + self._rejectsInvalidCharactersInHeaderFieldNames(mode: .http2(compress: false)) + } + + private func _rejectsInvalidCharactersInHeaderFieldNames(mode: HTTPBin.Mode) { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let bin = HTTPBin(mode) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + + // The spec in [RFC 9110](https://httpwg.org/specs/rfc9110.html#fields.values) defines the valid + // characters as the following: + // + // ``` + // field-name = token + // + // token = 1*tchar + // + // tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" + // / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" + // / DIGIT / ALPHA + // ; any VCHAR, except delimiters + let weirdAllowedFieldName = "!#$%&'*+-.^_`|~0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") + request.headers.add(name: weirdAllowedFieldName, value: "present") + + // This should work fine. + guard let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { + return + } + + XCTAssertEqual(response.status, .ok) + + // Now, let's confirm all other bytes are rejected. We want to stay within the ASCII space as the HTTPHeaders type will forbid anything else. + for byte in UInt8(0)...UInt8(127) { + // Skip bytes that we already believe are allowed. + if weirdAllowedFieldName.utf8.contains(byte) { + continue + } + let forbiddenFieldName = weirdAllowedFieldName + String(decoding: [byte], as: UTF8.self) + + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") + request.headers.add(name: forbiddenFieldName, value: "present") + + await XCTAssertThrowsError(try await client.execute(request, deadline: .now() + .seconds(10), logger: logger)) { error in + XCTAssertEqual(error as? HTTPClientError, .invalidHeaderFieldNames([forbiddenFieldName])) + } + } + } + } + + func testRejectsInvalidCharactersInHeaderFieldValues_http1() { + self._rejectsInvalidCharactersInHeaderFieldValues(mode: .http1_1(ssl: true)) + } + + func testRejectsInvalidCharactersInHeaderFieldValues_http2() { + self._rejectsInvalidCharactersInHeaderFieldValues(mode: .http2(compress: false)) + } + + private func _rejectsInvalidCharactersInHeaderFieldValues(mode: HTTPBin.Mode) { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let bin = HTTPBin(mode) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + + // We reject all ASCII control characters except HTAB and tolerate everything else. + let weirdAllowedFieldValue = "!\" \t#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") + request.headers.add(name: "Weird-Value", value: weirdAllowedFieldValue) + + // This should work fine. + guard let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { + return + } + + XCTAssertEqual(response.status, .ok) + + // Now, let's confirm all other bytes in the ASCII range ar rejected + for byte in UInt8(0)...UInt8(127) { + // Skip bytes that we already believe are allowed. + if weirdAllowedFieldValue.utf8.contains(byte) { + continue + } + let forbiddenFieldValue = weirdAllowedFieldValue + String(decoding: [byte], as: UTF8.self) + + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") + request.headers.add(name: "Weird-Value", value: forbiddenFieldValue) + + await XCTAssertThrowsError(try await client.execute(request, deadline: .now() + .seconds(10), logger: logger)) { error in + XCTAssertEqual(error as? HTTPClientError, .invalidHeaderFieldValues([forbiddenFieldValue])) + } + } + + // All the bytes outside the ASCII range are fine though. + for byte in UInt8(128)...UInt8(255) { + let evenWeirderAllowedValue = weirdAllowedFieldValue + String(decoding: [byte], as: UTF8.self) + + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") + request.headers.add(name: "Weird-Value", value: evenWeirderAllowedValue) + + // This should work fine. + guard let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { + return + } + + XCTAssertEqual(response.status, .ok) + } + } + } } extension AsyncSequence where Element == ByteBuffer { From 5daaa38ba13e7d02e17f65310b85331b5b7165cb Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 18 Jan 2023 11:29:51 +0100 Subject: [PATCH 059/146] Add Swift 5.8 CI and update nightly CI to Ubuntu 22.04 (#655) * Add Swift 5.8 CI * Update nightly CI to Ubuntu 22.04 * Fix warnings in test target with latest NIO release Replace calls to `.wait()` with calls to `.get()` --- .../AsyncAwaitEndToEndTests.swift | 6 +++--- ...4.main.yaml => docker-compose.2204.58.yaml} | 8 ++++---- docker/docker-compose.2204.main.yaml | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 7 deletions(-) rename docker/{docker-compose.2004.main.yaml => docker-compose.2204.58.yaml} (54%) create mode 100644 docker/docker-compose.2204.main.yaml diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 9a77ee9fb..97c802319 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -416,9 +416,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertNoThrow(try serverChannel.close().wait()) } let port = serverChannel.localAddress!.port! - let firstClientChannel = try ClientBootstrap(group: self.serverGroup) + let firstClientChannel = try await ClientBootstrap(group: self.serverGroup) .connect(host: "127.0.0.1", port: port) - .wait() + .get() defer { XCTAssertNoThrow(try firstClientChannel.close().wait()) } @@ -464,7 +464,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { .childChannelInitializer { channel in channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) } - let serverChannel = try server.bind(host: "localhost", port: 0).wait() + let serverChannel = try await server.bind(host: "localhost", port: 0).get() defer { XCTAssertNoThrow(try serverChannel.close().wait()) } let port = serverChannel.localAddress!.port! diff --git a/docker/docker-compose.2004.main.yaml b/docker/docker-compose.2204.58.yaml similarity index 54% rename from docker/docker-compose.2004.main.yaml rename to docker/docker-compose.2204.58.yaml index d6f04b917..d5dc8432e 100644 --- a/docker/docker-compose.2004.main.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -3,16 +3,16 @@ version: "3" services: runtime-setup: - image: async-http-client:20.04-main + image: async-http-client:22.04-5.8 build: args: - base_image: "swiftlang/swift:nightly-main-focal" + base_image: "swiftlang/swift:nightly-5.8-jammy" test: - image: async-http-client:20.04-main + image: async-http-client:22.04-5.8 environment: - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread shell: - image: async-http-client:20.04-main + image: async-http-client:22.04-5.8 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml new file mode 100644 index 000000000..9132ed322 --- /dev/null +++ b/docker/docker-compose.2204.main.yaml @@ -0,0 +1,18 @@ +version: "3" + +services: + + runtime-setup: + image: async-http-client:22.04-main + build: + args: + base_image: "swiftlang/swift:nightly-main-jammy" + + test: + image: async-http-client:22.04-main + environment: + - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error + #- SANITIZER_ARG=--sanitize=thread + + shell: + image: async-http-client:22.04-main From 817d9aa99fe4c71e7bea18a9a74bc7a5fc166692 Mon Sep 17 00:00:00 2001 From: Felix Schlegel Date: Sun, 22 Jan 2023 17:32:01 +0000 Subject: [PATCH 060/146] Make Task.logger accessible to delegate implementations outside of Package (#587) Motivation: This change was proposed in issue [#389](https://github.com/swift-server/async-http-client/issues/389). Users writing their own `HttpClientResponseDelegate` implementation might want to emit log messages to the `task`'s `logger`. Modifications: Changed the access level of `Task`'s `logger` property from `internal` to `public`. Result: Users can access the `logger` property of a `Task` outside of the `async-http-client`. --- Sources/AsyncHTTPClient/HTTPHandler.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index ab82d4f91..73f467d09 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -697,9 +697,10 @@ extension HTTPClient { public final class Task { /// The `EventLoop` the delegate will be executed on. public let eventLoop: EventLoop + /// The `Logger` used by the `Task` for logging. + public let logger: Logger // We are okay to store the logger here because a Task is for only one request. let promise: EventLoopPromise - let logger: Logger // We are okay to store the logger here because a Task is for only one request. var isCancelled: Bool { self.lock.withLock { self._isCancelled } From 67f99d1798518c7cb78887545398b6cabb9b765d Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 26 Jan 2023 12:11:30 +0100 Subject: [PATCH 061/146] Add test for HTTP1 request with large header (#658) * Reproducer * Refactor test case * Refactor tests * Remove debugging artefacts * Fix typo * Fix formatting * Remove `promise?.succeed(())` * Rename `onRequestCompleted` to `onConnectionIdle` --- .../HTTP1/HTTP1ClientChannelHandler.swift | 31 ++-- .../HTTP1/HTTP1Connection.swift | 7 +- ...TTP1ClientChannelHandlerTests+XCTest.swift | 1 + .../HTTP1ClientChannelHandlerTests.swift | 31 ++++ .../AsyncHTTPClientTests/HTTPClientBase.swift | 2 +- .../HTTPClientTestUtils.swift | 13 +- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 15 ++ .../HTTPConnectionPool+HTTP1StateTests.swift | 38 ++-- ...onnectionPool+HTTP2StateMachineTests.swift | 56 +++--- .../Mocks/MockConnectionPool.swift | 6 +- .../Mocks/MockHTTPExecutableRequest.swift | 163 ++++++++++++++++++ 12 files changed, 298 insertions(+), 66 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/Mocks/MockHTTPExecutableRequest.swift diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index ac92e4bc8..626a6fc23 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -35,8 +35,8 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { didSet { if let newRequest = self.request { var requestLogger = newRequest.logger - requestLogger[metadataKey: "ahc-connection-id"] = "\(self.connection.id)" - requestLogger[metadataKey: "ahc-el"] = "\(self.connection.channel.eventLoop)" + requestLogger[metadataKey: "ahc-connection-id"] = self.connectionIdLoggerMetadata + requestLogger[metadataKey: "ahc-el"] = "\(self.eventLoop)" self.logger = requestLogger if let idleReadTimeout = newRequest.requestOptions.idleReadTimeout { @@ -59,15 +59,15 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { private let backgroundLogger: Logger private var logger: Logger + private let eventLoop: EventLoop + private let connectionIdLoggerMetadata: Logger.MetadataValue - let connection: HTTP1Connection - let eventLoop: EventLoop - - init(connection: HTTP1Connection, eventLoop: EventLoop, logger: Logger) { - self.connection = connection + var onConnectionIdle: () -> Void = {} + init(eventLoop: EventLoop, backgroundLogger: Logger, connectionIdLoggerMetadata: Logger.MetadataValue) { self.eventLoop = eventLoop - self.backgroundLogger = logger - self.logger = self.backgroundLogger + self.backgroundLogger = backgroundLogger + self.logger = backgroundLogger + self.connectionIdLoggerMetadata = connectionIdLoggerMetadata } func handlerAdded(context: ChannelHandlerContext) { @@ -108,6 +108,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { let action = self.state.writabilityChanged(writable: context.channel.isWritable) self.run(action, context: context) + context.fireChannelWritabilityChanged() } func channelRead(context: ChannelHandlerContext, data: NIOAny) { @@ -274,7 +275,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { if shouldClose { context.close(promise: nil) } else { - self.connection.taskCompleted() + self.onConnectionIdle() } oldRequest.succeedRequest(buffer) @@ -286,7 +287,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) case .informConnectionIsIdle: - self.connection.taskCompleted() + self.onConnectionIdle() oldRequest.succeedRequest(buffer) } @@ -303,7 +304,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { oldRequest.fail(error) case .informConnectionIsIdle: - self.connection.taskCompleted() + self.onConnectionIdle() oldRequest.fail(error) case .failWritePromise(let writePromise): @@ -328,6 +329,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { // we must check if the request is still present here. guard let request = self.request else { return } request.requestHeadSent() + request.resumeRequestBodyStream() } else { context.write(self.wrapOutboundOut(.head(head)), promise: nil) @@ -434,6 +436,11 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } } +#if swift(>=5.6) +@available(*, unavailable) +extension HTTP1ClientChannelHandler: Sendable {} +#endif + extension HTTP1ClientChannelHandler: HTTPRequestExecutor { func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift index 3485ada6c..ee0a78498 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift @@ -133,10 +133,13 @@ final class HTTP1Connection { } let channelHandler = HTTP1ClientChannelHandler( - connection: self, eventLoop: channel.eventLoop, - logger: logger + backgroundLogger: logger, + connectionIdLoggerMetadata: "\(self.id)" ) + channelHandler.onConnectionIdle = { + self.taskCompleted() + } try sync.addHandler(channelHandler) } catch { diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift index 66c1a48d1..2502e6fb7 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift @@ -33,6 +33,7 @@ extension HTTP1ClientChannelHandlerTests { ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand), ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), ("testHandlerClosesChannelIfLastActionIsSendEndAndItFails", testHandlerClosesChannelIfLastActionIsSendEndAndItFails), + ("testChannelBecomesNonWritableDuringHeaderWrite", testChannelBecomesNonWritableDuringHeaderWrite), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index f97580372..820e6cf10 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -526,6 +526,37 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { XCTAssertTrue(error is FailEndHandler.Error) } } + + func testChannelBecomesNonWritableDuringHeaderWrite() throws { + try XCTSkipIf(true, "this currently fails and will be fixed in follow up PR") + final class ChangeWritabilityOnFlush: ChannelOutboundHandler { + typealias OutboundIn = Any + func flush(context: ChannelHandlerContext) { + context.flush() + (context.channel as! EmbeddedChannel).isWritable = false + context.fireChannelWritabilityChanged() + } + } + let eventLoopGroup = EmbeddedEventLoopGroup(loops: 1) + let eventLoop = eventLoopGroup.next() as! EmbeddedEventLoop + let handler = HTTP1ClientChannelHandler( + eventLoop: eventLoop, + backgroundLogger: Logger(label: "no-op", factory: SwiftLogNoOpLogHandler.init), + connectionIdLoggerMetadata: "test connection" + ) + let channel = EmbeddedChannel(handlers: [ + ChangeWritabilityOnFlush(), + handler, + ], loop: eventLoop) + try channel.connect(to: .init(ipAddress: "127.0.0.1", port: 80)).wait() + + let request = MockHTTPExecutableRequest() + // non empty body is important to trigger this bug as we otherwise finish the request in a single flush + request.requestFramingMetadata.body = .fixedSize(1) + request.raiseErrorIfUnimplementedMethodIsCalled = false + channel.writeAndFlush(request, promise: nil) + XCTAssertEqual(request.events.map(\.kind), [.willExecuteRequest, .requestHeadSent]) + } } class TestBackpressureWriter { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientBase.swift b/Tests/AsyncHTTPClientTests/HTTPClientBase.swift index af310953a..188a6959f 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientBase.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientBase.swift @@ -39,7 +39,7 @@ class XCTestCaseHTTPClientTestsBaseClass: XCTestCase { var backgroundLogStore: CollectEverythingLogHandler.LogStore! var defaultHTTPBinURLPrefix: String { - return "http://localhost:\(self.defaultHTTPBin.port)/" + self.defaultHTTPBin.baseURL } override func setUp() { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 884681123..e50dab3b6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -375,7 +375,18 @@ internal final class HTTPBin where return "https" } }() - return "\(scheme)://localhost:\(self.port)/" + let host: String = { + switch self.socketAddress { + case .v4: + return self.socketAddress.ipAddress! + case .v6: + return "[\(self.socketAddress.ipAddress!)]" + case .unixDomainSocket: + return self.socketAddress.pathname! + } + }() + + return "\(scheme)://\(host):\(self.port)/" } private let mode: Mode diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index ef81b1dde..f9ddb1c8b 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -143,6 +143,7 @@ extension HTTPClientTests { ("testRequestWithHeaderTransferEncodingIdentityDoesNotFail", testRequestWithHeaderTransferEncodingIdentityDoesNotFail), ("testMassiveDownload", testMassiveDownload), ("testShutdownWithFutures", testShutdownWithFutures), + ("testMassiveHeaderHTTP1", testMassiveHeaderHTTP1), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8f4126c43..8e5d5fbfa 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3363,4 +3363,19 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup)) XCTAssertNoThrow(try httpClient.shutdown().wait()) } + + func testMassiveHeaderHTTP1() throws { + try XCTSkipIf(true, "this currently crashes and will be fixed in follow up PR") + var request = try HTTPClient.Request(url: defaultHTTPBin.baseURL, method: .POST) + // add ~64 KB header + let headerValue = String(repeating: "0", count: 1024) + for headerID in 0..<64 { + request.headers.replaceOrAdd(name: "larg-header-\(headerID)", value: headerValue) + } + + // non empty body is important to trigger this bug as we otherwise finish the request in a single flush + request.body = .byteBuffer(ByteBuffer(bytes: [0])) + + XCTAssertNoThrow(try defaultClient.execute(request: request).wait()) + } } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift index 125ba1a74..6cb097b04 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift @@ -37,7 +37,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // for the first eight requests, the pool should try to create new connections. for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let connectionID, let connectionEL) = action.connection else { @@ -53,7 +53,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // the next eight requests should only be queued. for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .none = action.connection else { @@ -120,7 +120,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // for the first eight requests, the pool should try to create new connections. for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let connectionID, let connectionEL) = action.connection else { @@ -136,7 +136,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // the next eight requests should only be queued. for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .none = action.connection else { @@ -181,7 +181,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -239,7 +239,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -276,7 +276,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -310,7 +310,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(cleanupContext.connectBackoff, []) // 4. execute another request - let finalMockRequest = MockHTTPRequest(eventLoop: elg.next()) + let finalMockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let finalRequest = HTTPConnectionPool.Request(finalMockRequest) let failAction = state.executeRequest(finalRequest) XCTAssertEqual(failAction.connection, .none) @@ -339,7 +339,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { return XCTFail("Expected to still have connections available") } - let mockRequest = MockHTTPRequest(eventLoop: eventLoop) + let mockRequest = MockHTTPScheduableRequest(eventLoop: eventLoop) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -359,7 +359,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { var queuer = MockRequestQueuer() for _ in 0..<100 { let eventLoop = elg.next() - let mockRequest = MockHTTPRequest(eventLoop: eventLoop, requiresEventLoopForChannel: false) + let mockRequest = MockHTTPScheduableRequest(eventLoop: eventLoop, requiresEventLoopForChannel: false) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -418,7 +418,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // 10% of the cases enforce the eventLoop let elRequired = (0..<10).randomElement().flatMap { $0 == 0 ? true : false }! - let mockRequest = MockHTTPRequest(eventLoop: reqEventLoop, requiresEventLoopForChannel: elRequired) + let mockRequest = MockHTTPScheduableRequest(eventLoop: reqEventLoop, requiresEventLoopForChannel: elRequired) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -482,7 +482,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(connections.parked, 8) // close a leased connection == abort - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) guard let connectionToAbort = connections.newestParkedConnection else { return XCTFail("Expected to have a parked connection") @@ -536,7 +536,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { return XCTFail("Expected to still have connections available") } - let mockRequest = MockHTTPRequest(eventLoop: eventLoop) + let mockRequest = MockHTTPScheduableRequest(eventLoop: eventLoop) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -553,7 +553,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { for _ in 0..<100 { let eventLoop = elg.next() - let mockRequest = MockHTTPRequest(eventLoop: eventLoop, requiresEventLoopForChannel: false) + let mockRequest = MockHTTPScheduableRequest(eventLoop: eventLoop, requiresEventLoopForChannel: false) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -667,7 +667,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -706,7 +706,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -738,7 +738,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest = MockHTTPRequest(eventLoop: eventLoop.next(), requiresEventLoopForChannel: false) + let mockRequest = MockHTTPScheduableRequest(eventLoop: eventLoop.next(), requiresEventLoopForChannel: false) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -762,7 +762,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { retryConnectionEstablishment: true ) - let mockRequest1 = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) let request1 = HTTPConnectionPool.Request(mockRequest1) let executeAction1 = state.executeRequest(request1) @@ -773,7 +773,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(executeAction1.request, .scheduleRequestTimeout(for: request1, on: mockRequest1.eventLoop)) - let mockRequest2 = MockHTTPRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) + let mockRequest2 = MockHTTPScheduableRequest(eventLoop: elg.next(), requiresEventLoopForChannel: false) let request2 = HTTPConnectionPool.Request(mockRequest2) let executeAction2 = state.executeRequest(request2) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index 10fad7bd6..2fefa697b 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -36,7 +36,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) /// first request should create a new connection - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -52,7 +52,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// subsequent requests should not create a connection for _ in 0..<9 { - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -103,7 +103,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// 4 streams are available and therefore request should be executed immediately for _ in 0..<4 { - let mockRequest = MockHTTPRequest(eventLoop: el1, requiresEventLoopForChannel: true) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1, requiresEventLoopForChannel: true) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -146,7 +146,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -205,7 +205,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -243,7 +243,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -274,7 +274,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -313,7 +313,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest = MockHTTPRequest(eventLoop: elg.next()) + let mockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) @@ -347,7 +347,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertEqual(cleanupContext.connectBackoff, []) // 4. execute another request - let finalMockRequest = MockHTTPRequest(eventLoop: elg.next()) + let finalMockRequest = MockHTTPScheduableRequest(eventLoop: elg.next()) let finalRequest = HTTPConnectionPool.Request(finalMockRequest) let failAction = state.executeRequest(finalRequest) XCTAssertEqual(failAction.connection, .none) @@ -371,9 +371,9 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { lifecycleState: .running ) - let mockRequest1 = MockHTTPRequest(eventLoop: el1) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest1) - let mockRequest2 = MockHTTPRequest(eventLoop: el1) + let mockRequest2 = MockHTTPScheduableRequest(eventLoop: el1) let request2 = HTTPConnectionPool.Request(mockRequest2) let executeAction1 = http1State.executeRequest(request1) @@ -456,7 +456,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { )) // execute request on idle connection - let mockRequest1 = MockHTTPRequest(eventLoop: el1) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest1) let request1Action = state.executeRequest(request1) XCTAssertEqual(request1Action.request, .executeRequest(request1, conn1, cancelTimeout: false)) @@ -468,7 +468,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertEqual(closeStream1Action.connection, .scheduleTimeoutTimer(conn1ID, on: el1)) // execute request on idle connection with required event loop - let mockRequest2 = MockHTTPRequest(eventLoop: el1, requiresEventLoopForChannel: true) + let mockRequest2 = MockHTTPScheduableRequest(eventLoop: el1, requiresEventLoopForChannel: true) let request2 = HTTPConnectionPool.Request(mockRequest2) let request2Action = state.executeRequest(request2) XCTAssertEqual(request2Action.request, .executeRequest(request2, conn1, cancelTimeout: false)) @@ -535,7 +535,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { )) // create new http2 connection - let mockRequest1 = MockHTTPRequest(eventLoop: el2, requiresEventLoopForChannel: true) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) let request1 = HTTPConnectionPool.Request(mockRequest1) let executeAction = state.executeRequest(request1) XCTAssertEqual(executeAction.request, .scheduleRequestTimeout(for: request1, on: el2)) @@ -614,7 +614,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { )) // execute request on idle connection - let mockRequest1 = MockHTTPRequest(eventLoop: el1) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest1) let request1Action = state.executeRequest(request1) XCTAssertEqual(request1Action.request, .executeRequest(request1, conn1, cancelTimeout: false)) @@ -659,14 +659,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { )) // execute request - let mockRequest1 = MockHTTPRequest(eventLoop: el1) + let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest1) let request1Action = state.executeRequest(request1) XCTAssertEqual(request1Action.request, .executeRequest(request1, conn1, cancelTimeout: false)) XCTAssertEqual(request1Action.connection, .cancelTimeoutTimer(conn1ID)) // queue request - let mockRequest2 = MockHTTPRequest(eventLoop: el1) + let mockRequest2 = MockHTTPScheduableRequest(eventLoop: el1) let request2 = HTTPConnectionPool.Request(mockRequest2) let request2Action = state.executeRequest(request2) XCTAssertEqual(request2Action.request, .scheduleRequestTimeout(for: request2, on: el1)) @@ -711,7 +711,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// first 8 request should create a new connection var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let connID, let eventLoop) = action.connection else { @@ -730,7 +730,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// after we reached the `maximumConcurrentHTTP1Connections`, we will not create new connections for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) XCTAssertEqual(action.connection, .none) @@ -799,7 +799,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) /// create a new connection - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let conn1ID, let eventLoop) = action.connection else { @@ -847,7 +847,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// first 8 request should create a new connection var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let connID, let eventLoop) = action.connection else { @@ -862,7 +862,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// after we reached the `maximumConcurrentHTTP1Connections`, we will not create new connections for _ in 0..<8 { - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) XCTAssertEqual(action.connection, .none) @@ -984,7 +984,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) // create http2 connection - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest) let action1 = state.executeRequest(request1) guard case .createConnection(let http2ConnID, let http2EventLoop) = action1.connection else { @@ -1008,7 +1008,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } // a request with new required event loop should create a new connection - let mockRequestWithRequiredEventLoop = MockHTTPRequest(eventLoop: el2, requiresEventLoopForChannel: true) + let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) let requestWithRequiredEventLoop = HTTPConnectionPool.Request(mockRequestWithRequiredEventLoop) let action2 = state.executeRequest(requestWithRequiredEventLoop) guard case .createConnection(let http1ConnId, let http1EventLoop) = action2.connection else { @@ -1054,7 +1054,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) // create http2 connection - let mockRequest = MockHTTPRequest(eventLoop: el1) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el1) let request1 = HTTPConnectionPool.Request(mockRequest) let action1 = state.executeRequest(request1) guard case .createConnection(let http2ConnID, let http2EventLoop) = action1.connection else { @@ -1078,7 +1078,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } // a request with new required event loop should create a new connection - let mockRequestWithRequiredEventLoop = MockHTTPRequest(eventLoop: el2, requiresEventLoopForChannel: true) + let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) let requestWithRequiredEventLoop = HTTPConnectionPool.Request(mockRequestWithRequiredEventLoop) let action2 = state.executeRequest(requestWithRequiredEventLoop) guard case .createConnection(let http1ConnId, let http1EventLoop) = action2.connection else { @@ -1131,7 +1131,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] for el in [el1, el2, el2] { - let mockRequest = MockHTTPRequest(eventLoop: el, requiresEventLoopForChannel: true) + let mockRequest = MockHTTPScheduableRequest(eventLoop: el, requiresEventLoopForChannel: true) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) guard case .createConnection(let connID, let eventLoop) = action.connection else { @@ -1210,7 +1210,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // shall be queued. for i in 0..<1000 { let requestEL = elg.next() - let mockRequest = MockHTTPRequest(eventLoop: requestEL) + let mockRequest = MockHTTPScheduableRequest(eventLoop: requestEL) let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift index 1b2c27b68..4374c713d 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift @@ -548,7 +548,7 @@ extension MockConnectionPool { var queuer = MockRequestQueuer() for _ in 0..) + case succeedRequest(CircularBuffer?) + case fail(Error) + + var kind: Kind { + switch self { + case .willExecuteRequest: return .willExecuteRequest + case .requestHeadSent: return .requestHeadSent + case .resumeRequestBodyStream: return .resumeRequestBodyStream + case .pauseRequestBodyStream: return .pauseRequestBodyStream + case .receiveResponseHead: return .receiveResponseHead + case .receiveResponseBodyParts: return .receiveResponseBodyParts + case .succeedRequest: return .succeedRequest + case .fail: return .fail + } + } + } + + var logger: Logging.Logger = Logger(label: "request") + var requestHead: NIOHTTP1.HTTPRequestHead + var requestFramingMetadata: RequestFramingMetadata + var requestOptions: RequestOptions = .forTests() + + /// if true and ``HTTPExecutableRequest`` method is called without setting a corresponding callback on `self` e.g. + /// If ``HTTPExecutableRequest\.willExecuteRequest(_:)`` is called but ``willExecuteRequestCallback`` is not set, + /// ``XCTestFail(_:)`` will be called to fail the current test. + var raiseErrorIfUnimplementedMethodIsCalled: Bool = true + private var file: StaticString + private var line: UInt + + var willExecuteRequestCallback: ((HTTPRequestExecutor) -> Void)? + var requestHeadSentCallback: (() -> Void)? + var resumeRequestBodyStreamCallback: (() -> Void)? + var pauseRequestBodyStreamCallback: (() -> Void)? + var receiveResponseHeadCallback: ((HTTPResponseHead) -> Void)? + var receiveResponseBodyPartsCallback: ((CircularBuffer) -> Void)? + var succeedRequestCallback: ((CircularBuffer?) -> Void)? + var failCallback: ((Error) -> Void)? + + /// captures all ``HTTPExecutableRequest`` method calls in the order of occurrence, including arguments. + /// If you are not interested in the arguments you can use `events.map(\.kind)` to get all events without arguments. + private(set) var events: [Event] = [] + + init( + head: NIOHTTP1.HTTPRequestHead = .init(version: .http1_1, method: .GET, uri: "http://localhost/"), + framingMetadata: RequestFramingMetadata = .init(connectionClose: false, body: .fixedSize(0)), + file: StaticString = #file, + line: UInt = #line + ) { + self.requestHead = head + self.requestFramingMetadata = framingMetadata + self.file = file + self.line = line + } + + private func calledUnimplementedMethod(_ name: String) { + guard self.raiseErrorIfUnimplementedMethodIsCalled else { return } + XCTFail("\(name) invoked but it is not implemented", file: self.file, line: self.line) + } + + func willExecuteRequest(_ executor: HTTPRequestExecutor) { + self.events.append(.willExecuteRequest(executor)) + guard let willExecuteRequestCallback = willExecuteRequestCallback else { + return self.calledUnimplementedMethod(#function) + } + willExecuteRequestCallback(executor) + } + + func requestHeadSent() { + self.events.append(.requestHeadSent) + guard let requestHeadSentCallback = requestHeadSentCallback else { + return self.calledUnimplementedMethod(#function) + } + requestHeadSentCallback() + } + + func resumeRequestBodyStream() { + self.events.append(.resumeRequestBodyStream) + guard let resumeRequestBodyStreamCallback = resumeRequestBodyStreamCallback else { + return self.calledUnimplementedMethod(#function) + } + resumeRequestBodyStreamCallback() + } + + func pauseRequestBodyStream() { + self.events.append(.pauseRequestBodyStream) + guard let pauseRequestBodyStreamCallback = pauseRequestBodyStreamCallback else { + return self.calledUnimplementedMethod(#function) + } + pauseRequestBodyStreamCallback() + } + + func receiveResponseHead(_ head: HTTPResponseHead) { + self.events.append(.receiveResponseHead(head)) + guard let receiveResponseHeadCallback = receiveResponseHeadCallback else { + return self.calledUnimplementedMethod(#function) + } + receiveResponseHeadCallback(head) + } + + func receiveResponseBodyParts(_ buffer: CircularBuffer) { + self.events.append(.receiveResponseBodyParts(buffer)) + guard let receiveResponseBodyPartsCallback = receiveResponseBodyPartsCallback else { + return self.calledUnimplementedMethod(#function) + } + receiveResponseBodyPartsCallback(buffer) + } + + func succeedRequest(_ buffer: CircularBuffer?) { + self.events.append(.succeedRequest(buffer)) + guard let succeedRequestCallback = succeedRequestCallback else { + return self.calledUnimplementedMethod(#function) + } + succeedRequestCallback(buffer) + } + + func fail(_ error: Error) { + self.events.append(.fail(error)) + guard let failCallback = failCallback else { + return self.calledUnimplementedMethod(#function) + } + failCallback(error) + } +} From 59bfb96afb2a45feb79ed8a3f51d97ad4da879cb Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 26 Jan 2023 14:26:58 +0100 Subject: [PATCH 062/146] Add test for HTTP2 request with large header (#659) Motivation We currently don't handle large headers well which trigger a channel writability change event. Modification Add failing (but currently skipped) tests which reproduces the issue Result We can reliably reproduce the large request header issue in an integration and unit test. Note that the actual fix is not included to make reviewing easier and will come in a follow up PR. --- ...TTP2ClientRequestHandlerTests+XCTest.swift | 1 + .../HTTP2ClientRequestHandlerTests.swift | 29 +++++++++++++++++++ .../HTTPClientTestUtils.swift | 25 +++++++++++----- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 29 +++++++++++++++++++ 5 files changed, 78 insertions(+), 7 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift index 8fa219838..221a63211 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift @@ -30,6 +30,7 @@ extension HTTP2ClientRequestHandlerTests { ("testIdleReadTimeout", testIdleReadTimeout), ("testIdleReadTimeoutIsCanceledIfRequestIsCanceled", testIdleReadTimeoutIsCanceledIfRequestIsCanceled), ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), + ("testChannelBecomesNonWritableDuringHeaderWrite", testChannelBecomesNonWritableDuringHeaderWrite), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index e67529ad8..5dfce3f9d 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -345,4 +345,33 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { XCTAssertEqual(embedded.isActive, false) } } + + func testChannelBecomesNonWritableDuringHeaderWrite() throws { + try XCTSkipIf(true, "this currently fails and will be fixed in follow up PR") + final class ChangeWritabilityOnFlush: ChannelOutboundHandler { + typealias OutboundIn = Any + func flush(context: ChannelHandlerContext) { + context.flush() + (context.channel as! EmbeddedChannel).isWritable = false + context.fireChannelWritabilityChanged() + } + } + let eventLoopGroup = EmbeddedEventLoopGroup(loops: 1) + let eventLoop = eventLoopGroup.next() as! EmbeddedEventLoop + let handler = HTTP2ClientRequestHandler( + eventLoop: eventLoop + ) + let channel = EmbeddedChannel(handlers: [ + ChangeWritabilityOnFlush(), + handler, + ], loop: eventLoop) + try channel.connect(to: .init(ipAddress: "127.0.0.1", port: 80)).wait() + + let request = MockHTTPExecutableRequest() + // non empty body is important to trigger this bug as we otherwise finish the request in a single flush + request.requestFramingMetadata.body = .fixedSize(1) + request.raiseErrorIfUnimplementedMethodIsCalled = false + channel.writeAndFlush(request, promise: nil) + XCTAssertEqual(request.events.map(\.kind), [.willExecuteRequest, .requestHeadSent]) + } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index e50dab3b6..ca24cba1c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -329,17 +329,32 @@ internal final class HTTPBin where // supports http1.1 connections only, which can be either plain text or encrypted case http1_1(ssl: Bool = false, compress: Bool = false) // supports http1.1 and http2 connections which must be always encrypted - case http2(compress: Bool) + case http2( + compress: Bool = false, + settings: HTTP2Settings? = nil + ) // supports request decompression and http response compression var compress: Bool { switch self { case .refuse: return false - case .http1_1(ssl: _, compress: let compress), .http2(compress: let compress): + case .http1_1(ssl: _, compress: let compress), .http2(compress: let compress, _): return compress } } + + var httpSettings: HTTP2Settings { + switch self { + case .http1_1, .http2(_, nil), .refuse: + return [ + HTTP2Setting(parameter: .maxConcurrentStreams, value: 10), + HTTP2Setting(parameter: .maxHeaderListSize, value: HPACKDecoder.defaultMaxHeaderListSize), + ] + case .http2(_, .some(let customSettings)): + return customSettings + } + } } enum Proxy { @@ -565,11 +580,7 @@ internal final class HTTPBin where // Successful upgrade to HTTP/2. Let the user configure the pipeline. let http2Handler = NIOHTTP2Handler( mode: .server, - initialSettings: [ - // TODO: make max concurrent streams configurable - HTTP2Setting(parameter: .maxConcurrentStreams, value: 10), - HTTP2Setting(parameter: .maxHeaderListSize, value: HPACKDecoder.defaultMaxHeaderListSize), - ] + initialSettings: self.mode.httpSettings ) let multiplexer = HTTP2StreamMultiplexer( mode: .server, diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index f9ddb1c8b..6e84f9d29 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -144,6 +144,7 @@ extension HTTPClientTests { ("testMassiveDownload", testMassiveDownload), ("testShutdownWithFutures", testShutdownWithFutures), ("testMassiveHeaderHTTP1", testMassiveHeaderHTTP1), + ("testMassiveHeaderHTTP2", testMassiveHeaderHTTP2), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8e5d5fbfa..54d854bf0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3378,4 +3378,33 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try defaultClient.execute(request: request).wait()) } + + func testMassiveHeaderHTTP2() throws { + try XCTSkipIf(true, "this currently crashes and will be fixed in follow up PR") + let bin = HTTPBin(.http2(settings: [ + .init(parameter: .maxConcurrentStreams, value: 100), + .init(parameter: .maxHeaderListSize, value: 1024 * 256), + .init(parameter: .maxFrameSize, value: 1024 * 256), + ])) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + let client = HTTPClient( + eventLoopGroupProvider: .shared(clientGroup), + configuration: .init(certificateVerification: .none) + ) + + defer { XCTAssertNoThrow(try client.syncShutdown()) } + + var request = try HTTPClient.Request(url: bin.baseURL, method: .POST) + // add ~200 KB header + let headerValue = String(repeating: "0", count: 1024) + for headerID in 0..<200 { + request.headers.replaceOrAdd(name: "larg-header-\(headerID)", value: headerValue) + } + + // non empty body is important to trigger this bug as we otherwise finish the request in a single flush + request.body = .byteBuffer(ByteBuffer(bytes: [0])) + + XCTAssertNoThrow(try client.execute(request: request).wait()) + } } From f65f45b72743a3246f5bd43cfe2ba27e02045f6b Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 7 Feb 2023 17:18:56 +0100 Subject: [PATCH 063/146] Fix Request streaming memory leak (#665) --- Sources/AsyncHTTPClient/HTTPHandler.swift | 12 +++-- .../RequestBag+StateMachine.swift | 47 +++++++++--------- Sources/AsyncHTTPClient/RequestBag.swift | 3 +- .../RequestBagTests+XCTest.swift | 1 + .../RequestBagTests.swift | 49 +++++++++++++++++++ 5 files changed, 82 insertions(+), 30 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 73f467d09..7b2c5c6ff 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -379,7 +379,8 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { } var state = State.idle - let request: HTTPClient.Request + let requestMethod: HTTPMethod + let requestHost: String static let maxByteBufferSize = Int(UInt32.max) @@ -408,14 +409,15 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { maxBodySize <= Self.maxByteBufferSize, "maxBodyLength is not allowed to exceed 2^32 because ByteBuffer can not store more bytes" ) - self.request = request + self.requestMethod = request.method + self.requestHost = request.host self.maxBodySize = maxBodySize } public func didReceiveHead(task: HTTPClient.Task, _ head: HTTPResponseHead) -> EventLoopFuture { switch self.state { case .idle: - if self.request.method != .HEAD, + if self.requestMethod != .HEAD, let contentLength = head.headers.first(name: "Content-Length"), let announcedBodySize = Int(contentLength), announcedBodySize > self.maxBodySize { @@ -481,9 +483,9 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { case .idle: preconditionFailure("no head received before end") case .head(let head): - return Response(host: self.request.host, status: head.status, version: head.version, headers: head.headers, body: nil) + return Response(host: self.requestHost, status: head.status, version: head.version, headers: head.headers, body: nil) case .body(let head, let body): - return Response(host: self.request.host, status: head.status, version: head.version, headers: head.headers, body: body) + return Response(host: self.requestHost, status: head.status, version: head.version, headers: head.headers, body: body) case .end: preconditionFailure("request already processed") case .error(let error): diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index 9509fa2e6..e7fad6850 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -29,15 +29,15 @@ extension HTTPClient { extension RequestBag { struct StateMachine { fileprivate enum State { - case initialized - case queued(HTTPRequestScheduler) + case initialized(RedirectHandler?) + case queued(HTTPRequestScheduler, RedirectHandler?) /// if the deadline was exceeded while in the `.queued(_:)` state, /// we wait until the request pool fails the request with a potential more descriptive error message, /// if a connection failure has occured while the request was queued. case deadlineExceededWhileQueued case executing(HTTPRequestExecutor, RequestStreamState, ResponseStreamState) case finished(error: Error?) - case redirected(HTTPRequestExecutor, Int, HTTPResponseHead, URL) + case redirected(HTTPRequestExecutor, RedirectHandler, Int, HTTPResponseHead, URL) case modifying } @@ -55,23 +55,22 @@ extension RequestBag { case eof } - case initialized + case initialized(RedirectHandler?) case buffering(CircularBuffer, next: Next) case waitingForRemote } - private var state: State = .initialized - private let redirectHandler: RedirectHandler? + private var state: State init(redirectHandler: RedirectHandler?) { - self.redirectHandler = redirectHandler + self.state = .initialized(redirectHandler) } } } extension RequestBag.StateMachine { mutating func requestWasQueued(_ scheduler: HTTPRequestScheduler) { - guard case .initialized = self.state else { + guard case .initialized(let redirectHandler) = self.state else { // There might be a race between `requestWasQueued` and `willExecuteRequest`: // // If the request is created and passed to the HTTPClient on thread A, it will move into @@ -91,7 +90,7 @@ extension RequestBag.StateMachine { return } - self.state = .queued(scheduler) + self.state = .queued(scheduler, redirectHandler) } enum WillExecuteRequestAction { @@ -102,8 +101,8 @@ extension RequestBag.StateMachine { mutating func willExecuteRequest(_ executor: HTTPRequestExecutor) -> WillExecuteRequestAction { switch self.state { - case .initialized, .queued: - self.state = .executing(executor, .initialized, .initialized) + case .initialized(let redirectHandler), .queued(_, let redirectHandler): + self.state = .executing(executor, .initialized, .initialized(redirectHandler)) return .none case .deadlineExceededWhileQueued: let error: Error = HTTPClientError.deadlineExceeded @@ -127,8 +126,8 @@ extension RequestBag.StateMachine { case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("A request stream can only be resumed, if the request was started") - case .executing(let executor, .initialized, .initialized): - self.state = .executing(executor, .producing, .initialized) + case .executing(let executor, .initialized, .initialized(let redirectHandler)): + self.state = .executing(executor, .producing, .initialized(redirectHandler)) return .startWriter case .executing(_, .producing, _): @@ -299,11 +298,11 @@ extension RequestBag.StateMachine { case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("How can we receive a response, if the request hasn't started yet.") case .executing(let executor, let requestState, let responseState): - guard case .initialized = responseState else { + guard case .initialized(let redirectHandler) = responseState else { preconditionFailure("If we receive a response, we must not have received something else before") } - if let redirectURL = self.redirectHandler?.redirectTarget( + if let redirectHandler = redirectHandler, let redirectURL = redirectHandler.redirectTarget( status: head.status, responseHeaders: head.headers ) { @@ -312,11 +311,11 @@ extension RequestBag.StateMachine { // smaller than 3kb. switch head.contentLength { case .some(0...(HTTPClient.maxBodySizeRedirectResponse)), .none: - self.state = .redirected(executor, 0, head, redirectURL) + self.state = .redirected(executor, redirectHandler, 0, head, redirectURL) return .signalBodyDemand(executor) case .some: self.state = .finished(error: HTTPClientError.cancelled) - return .redirect(executor, self.redirectHandler!, head, redirectURL) + return .redirect(executor, redirectHandler, head, redirectURL) } } else { self.state = .executing(executor, requestState, .buffering(.init(), next: .askExecutorForMore)) @@ -369,15 +368,15 @@ extension RequestBag.StateMachine { } else { return .none } - case .redirected(let executor, var receivedBytes, let head, let redirectURL): + case .redirected(let executor, let redirectHandler, var receivedBytes, let head, let redirectURL): let partsLength = buffer.reduce(into: 0) { $0 += $1.readableBytes } receivedBytes += partsLength if receivedBytes > HTTPClient.maxBodySizeRedirectResponse { self.state = .finished(error: HTTPClientError.cancelled) - return .redirect(executor, self.redirectHandler!, head, redirectURL) + return .redirect(executor, redirectHandler, head, redirectURL) } else { - self.state = .redirected(executor, receivedBytes, head, redirectURL) + self.state = .redirected(executor, redirectHandler, receivedBytes, head, redirectURL) return .signalBodyDemand(executor) } @@ -428,9 +427,9 @@ extension RequestBag.StateMachine { self.state = .executing(executor, requestState, .buffering(newChunks, next: .eof)) return .consume(first) - case .redirected(_, _, let head, let redirectURL): + case .redirected(_, let redirectHandler, _, let head, let redirectURL): self.state = .finished(error: nil) - return .redirect(self.redirectHandler!, head, redirectURL) + return .redirect(redirectHandler, head, redirectURL) case .finished(error: .some): return .none @@ -553,7 +552,7 @@ extension RequestBag.StateMachine { mutating func deadlineExceeded() -> DeadlineExceededAction { switch self.state { - case .queued(let queuer): + case .queued(let queuer, _): /// We do not fail the request immediately because we want to give the scheduler a chance of throwing a better error message /// We therefore depend on the scheduler failing the request after we cancel the request. self.state = .deadlineExceededWhileQueued @@ -582,7 +581,7 @@ extension RequestBag.StateMachine { case .initialized: self.state = .finished(error: error) return .failTask(error, nil, nil) - case .queued(let queuer): + case .queued(let queuer, _): self.state = .finished(error: error) return .failTask(error, queuer, nil) case .executing(let executor, let requestState, .buffering(_, next: .eof)): diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 50c0057ba..1119236fb 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -33,7 +33,7 @@ final class RequestBag { } private let delegate: Delegate - private let request: HTTPClient.Request + private var request: HTTPClient.Request // the request state is synchronized on the task eventLoop private var state: StateMachine @@ -126,6 +126,7 @@ final class RequestBag { guard let body = self.request.body else { preconditionFailure("Expected to have a body, if the `HTTPRequestStateMachine` resume a request stream") } + self.request.body = nil let writer = HTTPClient.Body.StreamWriter { self.writeNextRequestPart($0) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift index b6a05733c..53c152c06 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift @@ -40,6 +40,7 @@ extension RequestBagTests { ("testRedirectWith3KBBody", testRedirectWith3KBBody), ("testRedirectWith4KBBodyAnnouncedInResponseHead", testRedirectWith4KBBodyAnnouncedInResponseHead), ("testRedirectWith4KBBodyNotAnnouncedInResponseHead", testRedirectWith4KBBodyNotAnnouncedInResponseHead), + ("testWeDontLeakTheRequestIfTheRequestWriterWasCapturedByAPromise", testWeDontLeakTheRequestIfTheRequestWriterWasCapturedByAPromise), ] } } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 43062405c..43134d453 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -17,6 +17,7 @@ import Logging import NIOCore import NIOEmbedded import NIOHTTP1 +import NIOPosix import XCTest final class RequestBagTests: XCTestCase { @@ -836,6 +837,54 @@ final class RequestBagTests: XCTestCase { XCTAssertTrue(redirectTriggered) } + + func testWeDontLeakTheRequestIfTheRequestWriterWasCapturedByAPromise() { + final class LeakDetector {} + + let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(group)) + defer { XCTAssertNoThrow(try httpClient.shutdown().wait()) } + + let httpBin = HTTPBin() + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + var leakDetector = LeakDetector() + + do { + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/", method: .POST)) + guard var request = maybeRequest else { return XCTFail("Expected to have a request here") } + + let writerPromise = group.any().makePromise(of: HTTPClient.Body.StreamWriter.self) + let donePromise = group.any().makePromise(of: Void.self) + request.body = .stream { [leakDetector] writer in + _ = leakDetector + writerPromise.succeed(writer) + return donePromise.futureResult + } + + let resultFuture = httpClient.execute(request: request) + request.body = nil + writerPromise.futureResult.whenSuccess { writer in + writer.write(.byteBuffer(ByteBuffer(string: "hello"))).map { + print("written") + }.cascade(to: donePromise) + } + XCTAssertNoThrow(try donePromise.futureResult.wait()) + print("HTTP sent") + + var result: HTTPClient.Response? + XCTAssertNoThrow(result = try resultFuture.wait()) + + XCTAssertEqual(.ok, result?.status) + let body = result?.body.map { String(buffer: $0) } + XCTAssertNotNil(body) + print("HTTP done") + } + XCTAssertTrue(isKnownUniquelyReferenced(&leakDetector)) + } } extension HTTPClient.Task { From aa66da80fa1ea29d728ee093757f4fe5db89dca7 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 10 Feb 2023 15:13:21 +0100 Subject: [PATCH 064/146] Fix request head continuation misuse (#666) * Fix request head continuation misuse Fixes #664 * remove unused function * format & generate linux tests --- .../AsyncAwait/Transaction+StateMachine.swift | 4 +- .../TransactionTests+XCTest.swift | 1 + .../TransactionTests.swift | 49 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 8700d6b32..008f1f823 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -207,7 +207,9 @@ extension Transaction { self.state = .executing(context, .requestHeadSent, .waitingForResponseHead) return .none case .deadlineExceededWhileQueued(let continuation): - return .cancelAndFail(executor, continuation, with: HTTPClientError.deadlineExceeded) + let error = HTTPClientError.deadlineExceeded + self.state = .finished(error: error, nil) + return .cancelAndFail(executor, continuation, with: error) case .finished(error: .some, .none): return .cancel(executor) diff --git a/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift index 190260647..de63914a9 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift @@ -26,6 +26,7 @@ extension TransactionTests { static var allTests: [(String, (TransactionTests) -> () throws -> Void)] { return [ ("testCancelAsyncRequest", testCancelAsyncRequest), + ("testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask", testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask), ("testResponseStreamingWorks", testResponseStreamingWorks), ("testIgnoringResponseBodyWorks", testIgnoringResponseBodyWorks), ("testWriteBackpressureWorks", testWriteBackpressureWorks), diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 7aee8c642..1ef0bfed1 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -59,6 +59,55 @@ final class TransactionTests: XCTestCase { } } + func testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let embeddedEventLoop = EmbeddedEventLoop() + defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + + var request = HTTPClientRequest(url: "https://localhost/") + request.method = .GET + var maybePreparedRequest: PreparedRequest? + XCTAssertNoThrow(maybePreparedRequest = try PreparedRequest(request)) + guard let preparedRequest = maybePreparedRequest else { + return XCTFail("Expected to have a request here.") + } + let (transaction, responseTask) = await Transaction.makeWithResultTask( + request: preparedRequest, + preferredEventLoop: embeddedEventLoop + ) + + let queuer = MockTaskQueuer() + transaction.requestWasQueued(queuer) + + transaction.deadlineExceeded() + + struct Executor: HTTPRequestExecutor { + func writeRequestBodyPart(_: NIOCore.IOData, request: AsyncHTTPClient.HTTPExecutableRequest, promise: NIOCore.EventLoopPromise?) { + XCTFail() + } + + func finishRequestBodyStream(_ task: AsyncHTTPClient.HTTPExecutableRequest, promise: NIOCore.EventLoopPromise?) { + XCTFail() + } + + func demandResponseBodyStream(_: AsyncHTTPClient.HTTPExecutableRequest) { + XCTFail() + } + + func cancelRequest(_ task: AsyncHTTPClient.HTTPExecutableRequest) { + task.fail(HTTPClientError.cancelled) + } + } + + transaction.willExecuteRequest(Executor()) + + await XCTAssertThrowsError(try await responseTask.value) { error in + XCTAssertEqualTypeAndValue(error, HTTPClientError.deadlineExceeded) + } + } + } + func testResponseStreamingWorks() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { From 1d24271feee99403c80e5387a1775299c84f902f Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 10 Feb 2023 15:41:26 +0100 Subject: [PATCH 065/146] Fix crash for large HTTP request headers (#661) * Reproducer * Refactor test case * Refactor tests * Remove debugging artefacts * Fix typo * Fix formatting * Remove `promise?.succeed(())` * Add test for HTTP2 request with large header Motivation We currently don't handle large headers well which trigger a channel writability change event. Modification Add failing (but currently skipped) tests which reproduces the issue Result We can reliably reproduce the large request header issue in an integration and unit test. Note that the actual fix is not included to make reviewing easier and will come in a follow up PR. * Remove logging * Fix crash for large HTTP request headers Fix crash for when sending HTTP request headers result in a channel writability change event * Formatting and linux tests * Formatting and linux tests * Generate linux tests * Use previous default max concurrent streams value of 10 * Fix crash if request is canceled after request header is send * generate linux tests and run swift format --------- Co-authored-by: Cory Benfield --- .../HTTP1/HTTP1ClientChannelHandler.swift | 47 +++++------ .../HTTP1/HTTP1ConnectionStateMachine.swift | 23 +++++- .../HTTP2/HTTP2ClientRequestHandler.swift | 46 +++++------ .../HTTPRequestStateMachine.swift | 81 ++++++++++++++----- .../HTTP1ClientChannelHandlerTests.swift | 1 - .../HTTP1ConnectionStateMachineTests.swift | 40 ++++++--- .../HTTP2ClientRequestHandlerTests.swift | 1 - .../HTTPClientTests+XCTest.swift | 2 + .../HTTPClientTests.swift | 34 +++++++- .../HTTPRequestStateMachineTests.swift | 75 +++++++++-------- 10 files changed, 230 insertions(+), 120 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 626a6fc23..8af70ac23 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -183,9 +183,23 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { private func run(_ action: HTTP1ConnectionStateMachine.Action, context: ChannelHandlerContext) { switch action { - case .sendRequestHead(let head, startBody: let startBody): - self.sendRequestHead(head, startBody: startBody, context: context) - + case .sendRequestHead(let head, let sendEnd): + self.sendRequestHead(head, sendEnd: sendEnd, context: context) + case .notifyRequestHeadSendSuccessfully(let resumeRequestBodyStream, let startIdleTimer): + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet + self.request!.requestHeadSent() + if resumeRequestBodyStream, let request = self.request { + // The above request head send notification might lead the request to mark itself as + // cancelled, which in turn might pop the request of the handler. For this reason we + // must check if the request is still present here. + request.resumeRequestBodyStream() + } + if startIdleTimer { + if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(timeoutAction, context: context) + } + } case .sendBodyPart(let part, let writePromise): context.writeAndFlush(self.wrapOutboundOut(.body(part)), promise: writePromise) @@ -320,32 +334,15 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } } - private func sendRequestHead(_ head: HTTPRequestHead, startBody: Bool, context: ChannelHandlerContext) { - if startBody { - context.writeAndFlush(self.wrapOutboundOut(.head(head)), promise: nil) - - // The above write might trigger an error, which may lead to a call to `errorCaught`, - // which in turn, may fail the request and pop it from the handler. For this reason - // we must check if the request is still present here. - guard let request = self.request else { return } - request.requestHeadSent() - - request.resumeRequestBodyStream() - } else { + private func sendRequestHead(_ head: HTTPRequestHead, sendEnd: Bool, context: ChannelHandlerContext) { + if sendEnd { context.write(self.wrapOutboundOut(.head(head)), promise: nil) context.write(self.wrapOutboundOut(.end(nil)), promise: nil) context.flush() - - // The above write might trigger an error, which may lead to a call to `errorCaught`, - // which in turn, may fail the request and pop it from the handler. For this reason - // we must check if the request is still present here. - guard let request = self.request else { return } - request.requestHeadSent() - - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) - } + } else { + context.writeAndFlush(self.wrapOutboundOut(.head(head)), promise: nil) } + self.run(self.state.headSent(), context: context) } private func runTimeoutAction(_ action: IdleReadStateMachine.Action, context: ChannelHandlerContext) { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index e7258611c..a908ded9a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -57,7 +57,11 @@ struct HTTP1ConnectionStateMachine { case none } - case sendRequestHead(HTTPRequestHead, startBody: Bool) + case sendRequestHead(HTTPRequestHead, sendEnd: Bool) + case notifyRequestHeadSendSuccessfully( + resumeRequestBodyStream: Bool, + startIdleTimer: Bool + ) case sendBodyPart(IOData, EventLoopPromise?) case sendRequestEnd(EventLoopPromise?) case failSendBodyPart(Error, EventLoopPromise?) @@ -350,6 +354,17 @@ struct HTTP1ConnectionStateMachine { return state.modify(with: action) } } + + mutating func headSent() -> Action { + guard case .inRequest(var requestStateMachine, let close) = self.state else { + return .wait + } + return self.avoidingStateMachineCoW { state in + let action = requestStateMachine.headSent() + state = .inRequest(requestStateMachine, close: close) + return state.modify(with: action) + } + } } extension HTTP1ConnectionStateMachine { @@ -390,8 +405,10 @@ extension HTTP1ConnectionStateMachine { extension HTTP1ConnectionStateMachine.State { fileprivate mutating func modify(with action: HTTPRequestStateMachine.Action) -> HTTP1ConnectionStateMachine.Action { switch action { - case .sendRequestHead(let head, let startBody): - return .sendRequestHead(head, startBody: startBody) + case .sendRequestHead(let head, let sendEnd): + return .sendRequestHead(head, sendEnd: sendEnd) + case .notifyRequestHeadSendSuccessfully(let resumeRequestBodyStream, let startIdleTimer): + return .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: resumeRequestBodyStream, startIdleTimer: startIdleTimer) case .pauseRequestBodyStream: return .pauseRequestBodyStream case .resumeRequestBodyStream: diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 578b83029..e7412f5c2 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -140,9 +140,23 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { private func run(_ action: HTTPRequestStateMachine.Action, context: ChannelHandlerContext) { switch action { - case .sendRequestHead(let head, let startBody): - self.sendRequestHead(head, startBody: startBody, context: context) - + case .sendRequestHead(let head, let sendEnd): + self.sendRequestHead(head, sendEnd: sendEnd, context: context) + case .notifyRequestHeadSendSuccessfully(let resumeRequestBodyStream, let startIdleTimer): + // We can force unwrap the request here, as we have just validated in the state machine, + // that the request is neither failed nor finished yet + self.request!.requestHeadSent() + if resumeRequestBodyStream, let request = self.request { + // The above request head send notification might lead the request to mark itself as + // cancelled, which in turn might pop the request of the handler. For this reason we + // must check if the request is still present here. + request.resumeRequestBodyStream() + } + if startIdleTimer { + if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(timeoutAction, context: context) + } + } case .pauseRequestBodyStream: // We can force unwrap the request here, as we have just validated in the state machine, // that the request is neither failed nor finished yet @@ -210,31 +224,15 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } } - private func sendRequestHead(_ head: HTTPRequestHead, startBody: Bool, context: ChannelHandlerContext) { - if startBody { - context.writeAndFlush(self.wrapOutboundOut(.head(head)), promise: nil) - - // The above write might trigger an error, which may lead to a call to `errorCaught`, - // which in turn, may fail the request and pop it from the handler. For this reason - // we must check if the request is still present here. - guard let request = self.request else { return } - request.requestHeadSent() - request.resumeRequestBodyStream() - } else { + private func sendRequestHead(_ head: HTTPRequestHead, sendEnd: Bool, context: ChannelHandlerContext) { + if sendEnd { context.write(self.wrapOutboundOut(.head(head)), promise: nil) context.write(self.wrapOutboundOut(.end(nil)), promise: nil) context.flush() - - // The above write might trigger an error, which may lead to a call to `errorCaught`, - // which in turn, may fail the request and pop it from the handler. For this reason - // we must check if the request is still present here. - guard let request = self.request else { return } - request.requestHeadSent() - - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) - } + } else { + context.writeAndFlush(self.wrapOutboundOut(.head(head)), promise: nil) } + self.run(self.state.headSent(), context: context) } private func runSuccessfulFinalAction(_ action: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, context: ChannelHandlerContext) { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift index aafa3d28b..4835feac3 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift @@ -20,21 +20,24 @@ struct HTTPRequestStateMachine { fileprivate enum State { /// The initial state machine state. The only valid mutation is `start()`. The state will /// transitions to: - /// - `.waitForChannelToBecomeWritable` - /// - `.running(.streaming, .initialized)` (if the Channel is writable and if a request body is expected) - /// - `.running(.endSent, .initialized)` (if the Channel is writable and no request body is expected) + /// - `.waitForChannelToBecomeWritable` (if the channel becomes non writable while sending the header) + /// - `.sendingHead` if the channel is writable case initialized + /// Waiting for the channel to be writable. Valid transitions are: - /// - `.running(.streaming, .initialized)` (once the Channel is writable again and if a request body is expected) - /// - `.running(.endSent, .initialized)` (once the Channel is writable again and no request body is expected) + /// - `.running(.streaming, .waitingForHead)` (once the Channel is writable again and if a request body is expected) + /// - `.running(.endSent, .waitingForHead)` (once the Channel is writable again and no request body is expected) /// - `.failed` (if a connection error occurred) case waitForChannelToBecomeWritable(HTTPRequestHead, RequestFramingMetadata) + /// A request is on the wire. Valid transitions are: /// - `.finished` /// - `.failed` case running(RequestState, ResponseState) + /// The request has completed successfully case finished + /// The request has failed case failed(Error) @@ -93,7 +96,11 @@ struct HTTPRequestStateMachine { case none } - case sendRequestHead(HTTPRequestHead, startBody: Bool) + case sendRequestHead(HTTPRequestHead, sendEnd: Bool) + case notifyRequestHeadSendSuccessfully( + resumeRequestBodyStream: Bool, + startIdleTimer: Bool + ) case sendBodyPart(IOData, EventLoopPromise?) case sendRequestEnd(EventLoopPromise?) case failSendBodyPart(Error, EventLoopPromise?) @@ -223,6 +230,7 @@ struct HTTPRequestStateMachine { // the request failed, before it was sent onto the wire. self.state = .failed(error) return .failRequest(error, .none) + case .running: self.state = .failed(error) return .failRequest(error, .close(nil)) @@ -520,7 +528,7 @@ struct HTTPRequestStateMachine { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response head before sending a request head ourselves") + preconditionFailure("How can we receive a response head before sending a request head ourselves \(self.state)") case .running(.streaming(let expectedBodyLength, let sentBodyBytes, producer: .paused), .waitingForHead): self.state = .running( @@ -561,7 +569,7 @@ struct HTTPRequestStateMachine { mutating func receivedHTTPResponseBodyPart(_ body: ByteBuffer) -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response head before sending a request head ourselves. Invalid state: \(self.state)") + preconditionFailure("How can we receive a response head before completely sending a request head ourselves. Invalid state: \(self.state)") case .running(_, .waitingForHead): preconditionFailure("How can we receive a response body, if we haven't received a head. Invalid state: \(self.state)") @@ -587,7 +595,7 @@ struct HTTPRequestStateMachine { private mutating func receivedHTTPResponseEnd() -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response head before sending a request head ourselves. Invalid state: \(self.state)") + preconditionFailure("How can we receive a response end before completely sending a request head ourselves. Invalid state: \(self.state)") case .running(_, .waitingForHead): preconditionFailure("How can we receive a response end, if we haven't a received a head. Invalid state: \(self.state)") @@ -654,7 +662,7 @@ struct HTTPRequestStateMachine { case .initialized, .running(_, .waitingForHead), .waitForChannelToBecomeWritable: - preconditionFailure("The response is expected to only ask for more data after the response head was forwarded") + preconditionFailure("The response is expected to only ask for more data after the response head was forwarded \(self.state)") case .running(let requestState, .receivingBody(let head, var responseStreamState)): return self.avoidingStateMachineCoW { state -> Action in @@ -697,18 +705,51 @@ struct HTTPRequestStateMachine { } private mutating func startSendingRequest(head: HTTPRequestHead, metadata: RequestFramingMetadata) -> Action { - switch metadata.body { - case .stream: - self.state = .running(.streaming(expectedBodyLength: nil, sentBodyBytes: 0, producer: .producing), .waitingForHead) - return .sendRequestHead(head, startBody: true) - case .fixedSize(0): + let length = metadata.body.expectedLength + if length == 0 { // no body self.state = .running(.endSent, .waitingForHead) - return .sendRequestHead(head, startBody: false) - case .fixedSize(let length): - // length is greater than zero and we therefore have a body to send - self.state = .running(.streaming(expectedBodyLength: length, sentBodyBytes: 0, producer: .producing), .waitingForHead) - return .sendRequestHead(head, startBody: true) + return .sendRequestHead(head, sendEnd: true) + } else { + self.state = .running(.streaming(expectedBodyLength: length, sentBodyBytes: 0, producer: .paused), .waitingForHead) + return .sendRequestHead(head, sendEnd: false) + } + } + + mutating func headSent() -> Action { + switch self.state { + case .initialized, .waitForChannelToBecomeWritable, .finished: + preconditionFailure("Not a valid transition after `.sendingHeader`: \(self.state)") + + case .running(.streaming(let expectedBodyLength, let sentBodyBytes, producer: .paused), let responseState): + let startProducing = self.isChannelWritable && expectedBodyLength != sentBodyBytes + self.state = .running(.streaming( + expectedBodyLength: expectedBodyLength, + sentBodyBytes: sentBodyBytes, + producer: startProducing ? .producing : .paused + ), responseState) + return .notifyRequestHeadSendSuccessfully( + resumeRequestBodyStream: startProducing, + startIdleTimer: false + ) + case .running(.endSent, _): + return .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + case .running(.streaming(_, _, producer: .producing), _): + preconditionFailure("request body producing can not start before we have successfully send the header \(self.state)") + case .failed: + return .wait + + case .modifying: + preconditionFailure("Invalid state: \(self.state)") + } + } +} + +extension RequestFramingMetadata.Body { + var expectedLength: Int? { + switch self { + case .fixedSize(let length): return length + case .stream: return nil } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index 820e6cf10..bdf897b3d 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -528,7 +528,6 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { } func testChannelBecomesNonWritableDuringHeaderWrite() throws { - try XCTSkipIf(true, "this currently fails and will be fixed in follow up PR") final class ChangeWritabilityOnFlush: ChannelOutboundHandler { typealias OutboundIn = Any func flush(context: ChannelHandlerContext) { diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index fd771aca0..ce8e6ed17 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -26,7 +26,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: ["content-length": "4"]) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.runNewRequest(head: requestHead, metadata: metadata), .wait) - XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) @@ -64,7 +65,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "12"]) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -92,7 +94,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/", headers: ["connection": "close"]) let metadata = RequestFramingMetadata(connectionClose: true, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -108,7 +111,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_0, status: .ok, headers: ["content-length": "4"]) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -124,7 +128,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_0, status: .ok, headers: ["content-length": "4", "connection": "keep-alive"]) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -141,7 +146,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["connection": "close"]) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -170,9 +176,11 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) XCTAssertEqual(state.channelInactive(), .failRequest(HTTPClientError.remoteConnectionClosed, .none)) + + XCTAssertEqual(state.headSent(), .wait) } func testRequestWasCancelledWhileUploadingData() { @@ -182,7 +190,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: ["content-length": "4"]) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.runNewRequest(head: requestHead, metadata: metadata), .wait) - XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) @@ -235,7 +244,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) XCTAssertEqual(state.channelRead(.body(ByteBuffer(string: "Hello world!\n"))), .wait) @@ -250,7 +260,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .switchingProtocols) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, [])) @@ -262,7 +273,8 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) - XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .init(statusCode: 103, reasonPhrase: "Early Hints")) XCTAssertEqual(state.channelRead(.head(responseHead)), .wait) XCTAssertEqual(state.channelInactive(), .failRequest(HTTPClientError.remoteConnectionClosed, .none)) @@ -295,6 +307,12 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { case (.sendRequestHead(let lhsHead, let lhsStartBody), .sendRequestHead(let rhsHead, let rhsStartBody)): return lhsHead == rhsHead && lhsStartBody == rhsStartBody + case ( + .notifyRequestHeadSendSuccessfully(let lhsResumeRequestBodyStream, let lhsStartIdleTimer), + .notifyRequestHeadSendSuccessfully(let rhsResumeRequestBodyStream, let rhsStartIdleTimer) + ): + return lhsResumeRequestBodyStream == rhsResumeRequestBodyStream && lhsStartIdleTimer == rhsStartIdleTimer + case (.sendBodyPart(let lhsData, let lhsPromise), .sendBodyPart(let rhsData, let rhsPromise)): return lhsData == rhsData && lhsPromise?.futureResult == rhsPromise?.futureResult diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index 5dfce3f9d..4873bc169 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -347,7 +347,6 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { } func testChannelBecomesNonWritableDuringHeaderWrite() throws { - try XCTSkipIf(true, "this currently fails and will be fixed in follow up PR") final class ChangeWritabilityOnFlush: ChannelOutboundHandler { typealias OutboundIn = Any func flush(context: ChannelHandlerContext) { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 6e84f9d29..d5a8160b6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -145,6 +145,8 @@ extension HTTPClientTests { ("testShutdownWithFutures", testShutdownWithFutures), ("testMassiveHeaderHTTP1", testMassiveHeaderHTTP1), ("testMassiveHeaderHTTP2", testMassiveHeaderHTTP2), + ("testCancelingHTTP1RequestAfterHeaderSend", testCancelingHTTP1RequestAfterHeaderSend), + ("testCancelingHTTP2RequestAfterHeaderSend", testCancelingHTTP2RequestAfterHeaderSend), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 54d854bf0..49f94a7d4 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3365,7 +3365,6 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testMassiveHeaderHTTP1() throws { - try XCTSkipIf(true, "this currently crashes and will be fixed in follow up PR") var request = try HTTPClient.Request(url: defaultHTTPBin.baseURL, method: .POST) // add ~64 KB header let headerValue = String(repeating: "0", count: 1024) @@ -3380,7 +3379,6 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testMassiveHeaderHTTP2() throws { - try XCTSkipIf(true, "this currently crashes and will be fixed in follow up PR") let bin = HTTPBin(.http2(settings: [ .init(parameter: .maxConcurrentStreams, value: 100), .init(parameter: .maxHeaderListSize, value: 1024 * 256), @@ -3407,4 +3405,36 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try client.execute(request: request).wait()) } + + func testCancelingHTTP1RequestAfterHeaderSend() throws { + var request = try HTTPClient.Request(url: self.defaultHTTPBin.baseURL + "/wait", method: .POST) + // non-empty body is important + request.body = .byteBuffer(ByteBuffer([1])) + + class CancelAfterHeadSend: HTTPClientResponseDelegate { + init() {} + func didFinishRequest(task: AsyncHTTPClient.HTTPClient.Task) throws {} + func didSendRequestHead(task: HTTPClient.Task, _ head: HTTPRequestHead) { + task.cancel() + } + } + XCTAssertThrowsError(try defaultClient.execute(request: request, delegate: CancelAfterHeadSend()).wait()) + } + + func testCancelingHTTP2RequestAfterHeaderSend() throws { + let bin = HTTPBin(.http2()) + defer { XCTAssertNoThrow(try bin.shutdown()) } + var request = try HTTPClient.Request(url: bin.baseURL + "/wait", method: .POST) + // non-empty body is important + request.body = .byteBuffer(ByteBuffer([1])) + + class CancelAfterHeadSend: HTTPClientResponseDelegate { + init() {} + func didFinishRequest(task: AsyncHTTPClient.HTTPClient.Task) throws {} + func didSendRequestHead(task: HTTPClient.Task, _ head: HTTPRequestHead) { + task.cancel() + } + } + XCTAssertThrowsError(try defaultClient.execute(request: request, delegate: CancelAfterHeadSend()).wait()) + } } diff --git a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift index 61ea4702b..92bf42b1d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift @@ -24,7 +24,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -38,7 +38,8 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "4")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) let part2 = IOData.byteBuffer(ByteBuffer(bytes: [2])) @@ -72,7 +73,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "4")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -87,7 +88,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "8")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(8)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -98,7 +99,8 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) @@ -132,7 +134,8 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) XCTAssertEqual(state.writabilityChanged(writable: false), .pauseRequestBodyStream) @@ -157,7 +160,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -179,7 +182,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -200,7 +203,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -219,7 +222,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -239,7 +242,7 @@ class HTTPRequestStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .wait) XCTAssertEqual(state.read(), .read) - XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -261,7 +264,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -288,7 +291,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -315,7 +318,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -362,7 +365,7 @@ class HTTPRequestStateMachineTests: XCTestCase { // --- sending request let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) // --- receiving response let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "4"]) @@ -377,7 +380,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) } @@ -385,7 +388,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/", headers: .init([("content-length", "4")])) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(.init(bytes: 1...3)), promise: nil), .failSendBodyPart(HTTPClientError.cancelled, nil)) } @@ -394,7 +397,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -411,7 +414,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let continueHead = HTTPResponseHead(version: .http1_1, status: .continue) XCTAssertEqual(state.channelRead(.head(continueHead)), .wait) @@ -427,7 +430,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -439,7 +442,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -451,7 +454,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) state.errorHappened(HTTPParserError.invalidChunkSize).assertFailRequest(HTTPParserError.invalidChunkSize, .close(nil)) XCTAssertEqual(state.requestCancelled(), .wait, "A cancellation that happens to late is ignored") @@ -461,7 +464,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_0, status: .internalServerError) XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) @@ -477,7 +480,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_0, status: .internalServerError) let body = ByteBuffer(string: "foo bar") @@ -495,7 +498,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .stream) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: true)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) let part1: ByteBuffer = .init(string: "foo") XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(part1), promise: nil), .sendBodyPart(.byteBuffer(part1), nil)) @@ -515,7 +518,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) let body = ByteBuffer(string: "foo bar") @@ -531,7 +534,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) XCTAssertEqual(state.errorHappened(NIOSSLError.uncleanShutdown), .wait) state.channelInactive().assertFailRequest(HTTPClientError.remoteConnectionClosed, .none) @@ -542,7 +545,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) state.errorHappened(ArbitraryError()).assertFailRequest(ArbitraryError(), .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) @@ -552,7 +555,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "30"]) let body = ByteBuffer(string: "foo bar") @@ -570,7 +573,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") @@ -591,7 +594,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") @@ -612,7 +615,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") @@ -632,7 +635,7 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, startBody: false)) + XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") @@ -668,6 +671,12 @@ extension HTTPRequestStateMachine.Action: Equatable { case (.sendRequestHead(let lhsHead, let lhsStartBody), .sendRequestHead(let rhsHead, let rhsStartBody)): return lhsHead == rhsHead && lhsStartBody == rhsStartBody + case ( + .notifyRequestHeadSendSuccessfully(let lhsResumeRequestBodyStream, let lhsStartIdleTimer), + .notifyRequestHeadSendSuccessfully(let rhsResumeRequestBodyStream, let rhsStartIdleTimer) + ): + return lhsResumeRequestBodyStream == rhsResumeRequestBodyStream && lhsStartIdleTimer == rhsStartIdleTimer + case (.sendBodyPart(let lhsData, let lhsPromise), .sendBodyPart(let rhsData, let rhsPromise)): return lhsData == rhsData && lhsPromise?.futureResult == rhsPromise?.futureResult From 9401037091ae0b90d48029a9a59e835061a2b848 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 13 Feb 2023 17:44:22 +0000 Subject: [PATCH 066/146] Make syncShutdown unavailable from async (#667) * Make syncShutdown unavailable from async Motivation syncShutdown can cause unbounded thread blocking, we shouldn't allow it in concurrent code. Modification Mark syncShutdown unavailable from async. Result Users are warned if they try to syncShutdown in an async context * Only noasync on 5.7 --- README.md | 2 +- Sources/AsyncHTTPClient/Docs.docc/index.md | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 14 +++++++++++++- 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3e705d50c..27354d8da 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ httpClient.get(url: "https://apple.com/").whenComplete { result in } ``` -You should always shut down `HTTPClient` instances you created using `try httpClient.syncShutdown()`. Please note that you must not call `httpClient.syncShutdown` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. +You should always shut down `HTTPClient` instances you created using `try httpClient.shutdown()`. Please note that you must not call `httpClient.shutdown` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. ### async/await examples diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md index acb408684..60e928e7d 100644 --- a/Sources/AsyncHTTPClient/Docs.docc/index.md +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -71,7 +71,7 @@ httpClient.get(url: "https://apple.com/").whenComplete { result in } ``` -You should always shut down ``HTTPClient`` instances you created using ``HTTPClient/syncShutdown()``. Please note that you must not call ``HTTPClient/syncShutdown()`` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. +You should always shut down ``HTTPClient`` instances you created using ``HTTPClient/shutdown()-96ayw()``. Please note that you must not call ``HTTPClient/shutdown()-96ayw()`` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. ### async/await examples diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 1089db86c..2f4368402 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -139,17 +139,29 @@ public class HTTPClient { """) case .upAndRunning: preconditionFailure(""" - Client not shut down before the deinit. Please call client.syncShutdown() when no \ + Client not shut down before the deinit. Please call client.shutdown() when no \ longer needed. Otherwise memory will leak. """) } } } + #if swift(>=5.7) /// Shuts down the client and `EventLoopGroup` if it was created by the client. + /// + /// This method blocks the thread indefinitely, prefer using ``shutdown()-96ayw``. + @available(*, noasync, message: "syncShutdown() can block indefinitely, prefer shutdown()", renamed: "shutdown()") + public func syncShutdown() throws { + try self.syncShutdown(requiresCleanClose: false) + } + #else + /// Shuts down the client and `EventLoopGroup` if it was created by the client. + /// + /// This method blocks the thread indefinitely, prefer using ``shutdown()-96ayw``. public func syncShutdown() throws { try self.syncShutdown(requiresCleanClose: false) } + #endif /// Shuts down the client and `EventLoopGroup` if it was created by the client. /// From e26459902c2f859a6dbc2155c9e58be424caafe2 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 14 Feb 2023 10:04:27 +0100 Subject: [PATCH 067/146] Fix HTTP2StreamChannel leak (#657) * Fix HTTP2StreamChannel leak * Update code comments. --- .../HTTP2/HTTP2ClientRequestHandler.swift | 16 ++-- .../HTTP2/HTTP2Connection.swift | 9 +- .../HTTP2ClientRequestHandlerTests.swift | 3 +- .../HTTP2ConnectionTests+XCTest.swift | 1 + .../HTTP2ConnectionTests.swift | 93 +++++++++++++++++++ 5 files changed, 113 insertions(+), 9 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index e7412f5c2..0e8e819e8 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -237,21 +237,25 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { private func runSuccessfulFinalAction(_ action: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, context: ChannelHandlerContext) { switch action { - case .close: - context.close(promise: nil) + case .close, .none: + // The actions returned here come from an `HTTPRequestStateMachine` that assumes http/1.1 + // semantics. For this reason we can ignore the close here, since an h2 stream is closed + // after every request anyway. + break case .sendRequestEnd(let writePromise): context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) - - case .none: - break } } private func runFailedFinalAction(_ action: HTTPRequestStateMachine.Action.FinalFailedRequestAction, context: ChannelHandlerContext, error: Error) { + // We must close the http2 stream after the request has finished. Since the request failed, + // we have no idea what the h2 streams state was. To be on the save side, we explicitly close + // the h2 stream. This will break a reference cycle in HTTP2Connection. + context.close(promise: nil) + switch action { case .close(let writePromise): - context.close(promise: nil) writePromise?.fail(error) case .none: diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index 5859e619a..0cad92cfe 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -77,7 +77,7 @@ final class HTTP2Connection { /// We use this channel set to remember, which open streams we need to inform that /// we want to close the connection. The channels shall than cancel their currently running - /// request. + /// request. This property must only be accessed from the connections `EventLoop`. private var openStreams = Set() let id: HTTPConnectionPool.Connection.ID let decompression: HTTPClient.Decompression @@ -241,7 +241,7 @@ final class HTTP2Connection { // before. let box = ChannelBox(channel) self.openStreams.insert(box) - self.channel.closeFuture.whenComplete { _ in + channel.closeFuture.whenComplete { _ in self.openStreams.remove(box) } @@ -287,6 +287,11 @@ final class HTTP2Connection { preconditionFailure("invalid state \(self.state)") } } + + func __forTesting_getStreamChannels() -> [Channel] { + self.channel.eventLoop.preconditionInEventLoop() + return self.openStreams.map { $0.channel } + } } extension HTTP2Connection: HTTP2IdleHandlerDelegate { diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index 4873bc169..c0e5b6054 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -335,6 +335,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { // the handler only writes once the channel is writable XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .none) + XCTAssertTrue(embedded.isActive) embedded.isWritable = true embedded.pipeline.fireChannelWritabilityChanged() @@ -342,7 +343,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { XCTAssertEqual($0 as? WriteError, WriteError()) } - XCTAssertEqual(embedded.isActive, false) + XCTAssertFalse(embedded.isActive) } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift index 06b60f757..f26ca7d38 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift @@ -30,6 +30,7 @@ extension HTTP2ConnectionTests { ("testSimpleGetRequest", testSimpleGetRequest), ("testEveryDoneRequestLeadsToAStreamAvailableCall", testEveryDoneRequestLeadsToAStreamAvailableCall), ("testCancelAllRunningRequests", testCancelAllRunningRequests), + ("testChildStreamsAreRemovedFromTheOpenChannelListOnceTheRequestIsDone", testChildStreamsAreRemovedFromTheOpenChannelListOnceTheRequestIsDone), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index 652884a84..951a64494 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -243,6 +243,99 @@ class HTTP2ConnectionTests: XCTestCase { XCTAssertNoThrow(try http2Connection.closeFuture.wait()) } + + func testChildStreamsAreRemovedFromTheOpenChannelListOnceTheRequestIsDone() { + class SucceedPromiseOnRequestHandler: ChannelInboundHandler { + typealias InboundIn = HTTPServerRequestPart + typealias OutboundOut = HTTPServerResponsePart + + let dataArrivedPromise: EventLoopPromise + let triggerResponseFuture: EventLoopFuture + + init(dataArrivedPromise: EventLoopPromise, triggerResponseFuture: EventLoopFuture) { + self.dataArrivedPromise = dataArrivedPromise + self.triggerResponseFuture = triggerResponseFuture + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + self.dataArrivedPromise.succeed(()) + + self.triggerResponseFuture.hop(to: context.eventLoop).whenSuccess { + switch self.unwrapInboundIn(data) { + case .head: + context.write(self.wrapOutboundOut(.head(.init(version: .http2, status: .ok))), promise: nil) + context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) + case .body, .end: + break + } + } + } + } + + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + let eventLoop = eventLoopGroup.next() + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + + let serverReceivedRequestPromise = eventLoop.makePromise(of: Void.self) + let triggerResponsePromise = eventLoop.makePromise(of: Void.self) + let httpBin = HTTPBin(.http2(compress: false)) { _ in + SucceedPromiseOnRequestHandler( + dataArrivedPromise: serverReceivedRequestPromise, + triggerResponseFuture: triggerResponsePromise.futureResult + ) + } + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let connectionCreator = TestConnectionCreator() + let delegate = TestHTTP2ConnectionDelegate() + var maybeHTTP2Connection: HTTP2Connection? + XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + )) + guard let http2Connection = maybeHTTP2Connection else { + return XCTFail("Expected to have an HTTP2 connection here.") + } + + var maybeRequest: HTTPClient.Request? + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + )) + guard let requestBag = maybeRequestBag else { + return XCTFail("Expected to have a request bag at this point") + } + + http2Connection.executeRequest(requestBag) + + XCTAssertNoThrow(try serverReceivedRequestPromise.futureResult.wait()) + var channelCount: Int? + XCTAssertNoThrow(channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait()) + XCTAssertEqual(channelCount, 1) + triggerResponsePromise.succeed(()) + + XCTAssertNoThrow(try requestBag.task.futureResult.wait()) + + // this is racy. for this reason we allow a couple of tries + var retryCount = 0 + let maxRetries = 1000 + while retryCount < maxRetries { + XCTAssertNoThrow(channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait()) + if channelCount == 0 { + break + } + retryCount += 1 + } + XCTAssertLessThan(retryCount, maxRetries) + } } class TestConnectionCreator { From 864c8d9e0ead5de7ba70b61c8982f89126710863 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 14 Feb 2023 09:48:22 +0000 Subject: [PATCH 068/146] Mark Task.wait() noasync and provide Task.get() (#668) Motivation Task.wait() is a convenience method to avoid needing to wait for the response future. This has had the effect of "laundering" a noasync warning from EventLoopFuture.wait(), hiding it within a purely sync call that may itself be used in an async context. We should discourage using these and prefer using .get() instead. Modifications Mark Task.wait() noasync. Add Task.get() for backward compatibility. Result Safer migration to Swift concurrency --- Sources/AsyncHTTPClient/HTTPHandler.swift | 24 +++++++++++++++++-- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 20 ++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 7b2c5c6ff..278bf18a8 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -750,13 +750,33 @@ extension HTTPClient { return self.promise.futureResult } + #if swift(>=5.7) /// Waits for execution of this request to complete. /// - /// - returns: The value of the `EventLoopFuture` when it completes. - /// - throws: The error value of the `EventLoopFuture` if it errors. + /// - returns: The value of ``futureResult`` when it completes. + /// - throws: The error value of ``futureResult`` if it errors. + @available(*, noasync, message: "wait() can block indefinitely, prefer get()", renamed: "get()") public func wait() throws -> Response { return try self.promise.futureResult.wait() } + #else + /// Waits for execution of this request to complete. + /// + /// - returns: The value of ``futureResult`` when it completes. + /// - throws: The error value of ``futureResult`` if it errors. + public func wait() throws -> Response { + return try self.promise.futureResult.wait() + } + #endif + + /// Provides the result of this request. + /// + /// - returns: The value of ``futureResult`` when it completes. + /// - throws: The error value of ``futureResult`` if it errors. + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + public func get() async throws -> Response { + return try await self.promise.futureResult.get() + } /// Cancels the request execution. public func cancel() { diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index c0b028905..20538c43a 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -49,6 +49,7 @@ extension AsyncAwaitEndToEndTests { ("testRejectsInvalidCharactersInHeaderFieldNames_http2", testRejectsInvalidCharactersInHeaderFieldNames_http2), ("testRejectsInvalidCharactersInHeaderFieldValues_http1", testRejectsInvalidCharactersInHeaderFieldValues_http1), ("testRejectsInvalidCharactersInHeaderFieldValues_http2", testRejectsInvalidCharactersInHeaderFieldValues_http2), + ("testUsingGetMethodInsteadOfWait", testUsingGetMethodInsteadOfWait), ] } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 97c802319..3d0e709d6 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -725,6 +725,26 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } } + + func testUsingGetMethodInsteadOfWait() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let bin = HTTPBin(.http2(compress: false)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let request = try HTTPClient.Request(url: "https://localhost:\(bin.port)/get") + + guard let response = await XCTAssertNoThrowWithResult( + try await client.execute(request: request).get() + ) else { + return + } + + XCTAssertEqual(response.status, .ok) + XCTAssertEqual(response.version, .http2) + } + } } extension AsyncSequence where Element == ByteBuffer { From 423fd0bd6bef482d3f43a2d8cbe6d548034bcd45 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Tue, 14 Mar 2023 10:35:06 +0000 Subject: [PATCH 069/146] Fix crash if connection is closed very early (#671) * Fix crash if connection is closed very early If the channel is closed before flatMap is executed, all ChannelHandler are removed and `TLSEventsHandler` is therefore not present either. We need to tolerate this even though it is very rare. Testing ideas welcome. Fixes #670 * drop precondition to assert --- .../HTTPConnectionPool+Factory.swift | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index e94e967a6..60338f615 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -361,14 +361,19 @@ extension HTTPConnectionPool.ConnectionFactory { var channelFuture = bootstrapFuture.flatMap { bootstrap -> EventLoopFuture in return bootstrap.connect(target: self.key.connectionTarget) }.flatMap { channel -> EventLoopFuture<(Channel, String?)> in - // It is save to use `try!` here, since we are sure, that a `TLSEventsHandler` exists - // within the pipeline. It is added in `makeTLSBootstrap`. - let tlsEventHandler = try! channel.pipeline.syncOperations.handler(type: TLSEventsHandler.self) - - // The tlsEstablishedFuture is set as soon as the TLSEventsHandler is in a - // pipeline. It is created in TLSEventsHandler's handlerAdded method. - return tlsEventHandler.tlsEstablishedFuture!.flatMap { negotiated in - channel.pipeline.removeHandler(tlsEventHandler).map { (channel, negotiated) } + do { + // if the channel is closed before flatMap is executed, all ChannelHandler are removed + // and TLSEventsHandler is therefore not present either + let tlsEventHandler = try channel.pipeline.syncOperations.handler(type: TLSEventsHandler.self) + + // The tlsEstablishedFuture is set as soon as the TLSEventsHandler is in a + // pipeline. It is created in TLSEventsHandler's handlerAdded method. + return tlsEventHandler.tlsEstablishedFuture!.flatMap { negotiated in + channel.pipeline.removeHandler(tlsEventHandler).map { (channel, negotiated) } + } + } catch { + assert(channel.isActive == false, "if the channel is still active then TLSEventsHandler must be present but got error \(error)") + return channel.eventLoop.makeFailedFuture(HTTPClientError.remoteConnectionClosed) } } From 98b45ed1cd6a138f998383918a59c837ca7f51d3 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 30 Mar 2023 08:21:41 +0100 Subject: [PATCH 070/146] Allow DNS override (#675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sometimes it can be useful to connect to one host e.g. `x.example.com` but request and validate the certificate chain as if we would connect to `y.example.com`. This is what this PR adds support for by adding a `dnsOverride` configuration to `HTTPClient.Configuration`. This is similar to curls `โ€”resolve-to` option but only allows overriding host and not ports for now. --- Package.swift | 2 + Package@swift-5.5.swift | 2 + .../AsyncAwait/HTTPClient+execute.swift | 4 +- .../HTTPClientRequest+Prepared.swift | 4 +- Sources/AsyncHTTPClient/ConnectionPool.swift | 54 ++++++++++++++--- .../HTTPConnectionPool+Factory.swift | 13 +++-- .../ConnectionPool/RequestOptions.swift | 8 ++- Sources/AsyncHTTPClient/HTTPClient.swift | 11 ++++ .../TLSConfiguration.swift | 12 +++- Sources/AsyncHTTPClient/RequestBag.swift | 7 +-- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 58 +++++++++++++++++++ .../HTTPClientNIOTSTests.swift | 2 +- .../HTTPClientRequestTests.swift | 39 ++++++++----- .../HTTPClientTestUtils.swift | 56 +++++++++--------- .../RequestBagTests.swift | 8 ++- .../Resources/example.com.cert.pem | 12 ++++ .../Resources/example.com.private-key.pem | 6 ++ 18 files changed, 232 insertions(+), 67 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem create mode 100644 Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem diff --git a/Package.swift b/Package.swift index f72de842e..83d5c65bc 100644 --- a/Package.swift +++ b/Package.swift @@ -68,6 +68,8 @@ let package = Package( resources: [ .copy("Resources/self_signed_cert.pem"), .copy("Resources/self_signed_key.pem"), + .copy("Resources/example.com.cert.pem"), + .copy("Resources/example.com.private-key.pem"), ] ), ] diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index 8ae20ed9c..02c91c7ef 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -67,6 +67,8 @@ let package = Package( resources: [ .copy("Resources/self_signed_cert.pem"), .copy("Resources/self_signed_key.pem"), + .copy("Resources/example.com.cert.pem"), + .copy("Resources/example.com.private-key.pem"), ] ), ] diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index 5328b7688..5f0a5f7c5 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -77,7 +77,7 @@ extension HTTPClient { // this loop is there to follow potential redirects while true { - let preparedRequest = try HTTPClientRequest.Prepared(currentRequest) + let preparedRequest = try HTTPClientRequest.Prepared(currentRequest, dnsOverride: configuration.dnsOverride) let response = try await executeCancellable(preparedRequest, deadline: deadline, logger: logger) guard var redirectState = currentRedirectState else { @@ -131,7 +131,7 @@ extension HTTPClient { return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) -> Void in let transaction = Transaction( request: request, - requestOptions: .init(idleReadTimeout: nil), + requestOptions: .fromClientConfiguration(self.configuration), logger: logger, connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), preferredEventLoop: eventLoop, diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index bd7417725..489ba5626 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -42,7 +42,7 @@ extension HTTPClientRequest { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Prepared { - init(_ request: HTTPClientRequest) throws { + init(_ request: HTTPClientRequest, dnsOverride: [String: String] = [:]) throws { guard let url = URL(string: request.url) else { throw HTTPClientError.invalidURL } @@ -58,7 +58,7 @@ extension HTTPClientRequest.Prepared { self.init( url: url, - poolKey: .init(url: deconstructedURL, tlsConfiguration: nil), + poolKey: .init(url: deconstructedURL, tlsConfiguration: nil, dnsOverride: dnsOverride), requestFramingMetadata: metadata, head: .init( version: .http1_1, diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index 0dac50e5f..b27e3fb97 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -14,6 +14,25 @@ import NIOSSL +#if canImport(Darwin) +import Darwin.C +#elseif os(Linux) || os(FreeBSD) || os(Android) +import Glibc +#else +#error("unsupported target operating system") +#endif + +extension String { + var isIPAddress: Bool { + var ipv4Address = in_addr() + var ipv6Address = in6_addr() + return self.withCString { host in + inet_pton(AF_INET, host, &ipv4Address) == 1 || + inet_pton(AF_INET6, host, &ipv6Address) == 1 + } + } +} + enum ConnectionPool { /// Used by the `ConnectionPool` to index its `HTTP1ConnectionProvider`s /// @@ -24,15 +43,18 @@ enum ConnectionPool { var scheme: Scheme var connectionTarget: ConnectionTarget private var tlsConfiguration: BestEffortHashableTLSConfiguration? + var serverNameIndicatorOverride: String? init( scheme: Scheme, connectionTarget: ConnectionTarget, - tlsConfiguration: BestEffortHashableTLSConfiguration? = nil + tlsConfiguration: BestEffortHashableTLSConfiguration? = nil, + serverNameIndicatorOverride: String? ) { self.scheme = scheme self.connectionTarget = connectionTarget self.tlsConfiguration = tlsConfiguration + self.serverNameIndicatorOverride = serverNameIndicatorOverride } var description: String { @@ -48,26 +70,44 @@ enum ConnectionPool { case .unixSocket(let socketPath): hostDescription = socketPath } - return "\(self.scheme)://\(hostDescription) TLS-hash: \(hash)" + return "\(self.scheme)://\(hostDescription)\(self.serverNameIndicatorOverride.map { " SNI: \($0)" } ?? "") TLS-hash: \(hash) " } } } +extension DeconstructedURL { + func applyDNSOverride(_ dnsOverride: [String: String]) -> (ConnectionTarget, serverNameIndicatorOverride: String?) { + guard + let originalHost = self.connectionTarget.host, + let hostOverride = dnsOverride[originalHost] + else { + return (self.connectionTarget, nil) + } + return ( + .init(remoteHost: hostOverride, port: self.connectionTarget.port ?? self.scheme.defaultPort), + serverNameIndicatorOverride: originalHost.isIPAddress ? nil : originalHost + ) + } +} + extension ConnectionPool.Key { - init(url: DeconstructedURL, tlsConfiguration: TLSConfiguration?) { + init(url: DeconstructedURL, tlsConfiguration: TLSConfiguration?, dnsOverride: [String: String]) { + let (connectionTarget, serverNameIndicatorOverride) = url.applyDNSOverride(dnsOverride) self.init( scheme: url.scheme, - connectionTarget: url.connectionTarget, + connectionTarget: connectionTarget, tlsConfiguration: tlsConfiguration.map { BestEffortHashableTLSConfiguration(wrapping: $0) - } + }, + serverNameIndicatorOverride: serverNameIndicatorOverride ) } - init(_ request: HTTPClient.Request) { + init(_ request: HTTPClient.Request, dnsOverride: [String: String] = [:]) { self.init( url: request.deconstructedURL, - tlsConfiguration: request.tlsConfiguration + tlsConfiguration: request.tlsConfiguration, + dnsOverride: dnsOverride ) } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 60338f615..48aedfd8e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -281,7 +281,7 @@ extension HTTPConnectionPool.ConnectionFactory { } let tlsEventHandler = TLSEventsHandler(deadline: deadline) - let sslServerHostname = self.key.connectionTarget.sslServerHostname + let sslServerHostname = self.key.serverNameIndicator let sslContextFuture = self.sslContextCache.sslContext( tlsConfiguration: tlsConfig, eventLoop: channel.eventLoop, @@ -409,7 +409,7 @@ extension HTTPConnectionPool.ConnectionFactory { #if canImport(Network) if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { // create NIOClientTCPBootstrap with NIOTS TLS provider - let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions(on: eventLoop).map { + let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions(on: eventLoop, serverNameIndicatorOverride: key.serverNameIndicatorOverride).map { options -> NIOClientTCPBootstrapProtocol in tsBootstrap @@ -434,7 +434,6 @@ extension HTTPConnectionPool.ConnectionFactory { } #endif - let sslServerHostname = self.key.connectionTarget.sslServerHostname let sslContextFuture = sslContextCache.sslContext( tlsConfiguration: tlsConfig, eventLoop: eventLoop, @@ -449,7 +448,7 @@ extension HTTPConnectionPool.ConnectionFactory { let sync = channel.pipeline.syncOperations let sslHandler = try NIOSSLClientHandler( context: sslContext, - serverHostname: sslServerHostname + serverHostname: self.key.serverNameIndicator ) let tlsEventHandler = TLSEventsHandler(deadline: deadline) @@ -488,6 +487,12 @@ extension Scheme { } } +extension ConnectionPool.Key { + var serverNameIndicator: String? { + serverNameIndicatorOverride ?? connectionTarget.sslServerHostname + } +} + extension ConnectionTarget { fileprivate var sslServerHostname: String? { switch self { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift index 2092498d8..c46f1289c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift @@ -18,15 +18,19 @@ struct RequestOptions { /// The maximal `TimeAmount` that is allowed to pass between `channelRead`s from the Channel. var idleReadTimeout: TimeAmount? - init(idleReadTimeout: TimeAmount?) { + var dnsOverride: [String: String] + + init(idleReadTimeout: TimeAmount?, dnsOverride: [String: String]) { self.idleReadTimeout = idleReadTimeout + self.dnsOverride = dnsOverride } } extension RequestOptions { static func fromClientConfiguration(_ configuration: HTTPClient.Configuration) -> Self { RequestOptions( - idleReadTimeout: configuration.timeout.read + idleReadTimeout: configuration.timeout.read, + dnsOverride: configuration.dnsOverride ) } } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 2f4368402..beb2ea458 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -711,6 +711,17 @@ public class HTTPClient { public struct Configuration { /// TLS configuration, defaults to `TLSConfiguration.makeClientConfiguration()`. public var tlsConfiguration: Optional + + /// Sometimes it can be useful to connect to one host e.g. `x.example.com` but + /// request and validate the certificate chain as if we would connect to `y.example.com`. + /// ``dnsOverride`` allows to do just that by mapping host names which we will request and validate the certificate chain, to a different + /// host name which will be used to actually connect to. + /// + /// **Example:** if ``dnsOverride`` is set to `["example.com": "localhost"]` and we execute a request with a + /// `url` of `https://example.com/`, the ``HTTPClient`` will actually open a connection to `localhost` instead of `example.com`. + /// ``HTTPClient`` will still request certificates from the server for `example.com` and validate them as if we would connect to `example.com`. + public var dnsOverride: [String: String] = [:] + /// Enables following 3xx redirects automatically. /// /// Following redirects are supported: diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index e20f52634..f79954da7 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -66,11 +66,11 @@ extension TLSConfiguration { /// /// - Parameter eventLoop: EventLoop to wait for creation of options on /// - Returns: Future holding NWProtocolTLS Options - func getNWProtocolTLSOptions(on eventLoop: EventLoop) -> EventLoopFuture { + func getNWProtocolTLSOptions(on eventLoop: EventLoop, serverNameIndicatorOverride: String?) -> EventLoopFuture { let promise = eventLoop.makePromise(of: NWProtocolTLS.Options.self) Self.tlsDispatchQueue.async { do { - let options = try self.getNWProtocolTLSOptions() + let options = try self.getNWProtocolTLSOptions(serverNameIndicatorOverride: serverNameIndicatorOverride) promise.succeed(options) } catch { promise.fail(error) @@ -82,7 +82,7 @@ extension TLSConfiguration { /// create NWProtocolTLS.Options for use with NIOTransportServices from the NIOSSL TLSConfiguration /// /// - Returns: Equivalent NWProtocolTLS Options - func getNWProtocolTLSOptions() throws -> NWProtocolTLS.Options { + func getNWProtocolTLSOptions(serverNameIndicatorOverride: String?) throws -> NWProtocolTLS.Options { let options = NWProtocolTLS.Options() let useMTELGExplainer = """ @@ -92,6 +92,12 @@ extension TLSConfiguration { platform networking stack). """ + if let serverNameIndicatorOverride = serverNameIndicatorOverride { + serverNameIndicatorOverride.withCString { serverNameIndicatorOverride in + sec_protocol_options_set_tls_server_name(options.securityProtocolOptions, serverNameIndicatorOverride) + } + } + // minimum TLS protocol if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) { sec_protocol_options_set_min_tls_protocol_version(options.securityProtocolOptions, self.minimumTLSVersion.nwTLSProtocolVersion) diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 1119236fb..2b20193b4 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -27,6 +27,8 @@ final class RequestBag { 50 } + let poolKey: ConnectionPool.Key + let task: HTTPClient.Task var eventLoop: EventLoop { self.task.eventLoop @@ -63,6 +65,7 @@ final class RequestBag { connectionDeadline: NIODeadline, requestOptions: RequestOptions, delegate: Delegate) throws { + self.poolKey = .init(request, dnsOverride: requestOptions.dnsOverride) self.eventLoopPreference = eventLoopPreference self.task = task self.state = .init(redirectHandler: redirectHandler) @@ -392,10 +395,6 @@ final class RequestBag { } extension RequestBag: HTTPSchedulableRequest { - var poolKey: ConnectionPool.Key { - ConnectionPool.Key(self.request) - } - var tlsConfiguration: TLSConfiguration? { self.request.tlsConfiguration } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 20538c43a..ce0e2846d 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -40,6 +40,7 @@ extension AsyncAwaitEndToEndTests { ("testImmediateDeadline", testImmediateDeadline), ("testConnectTimeout", testConnectTimeout), ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), + ("testDnsOverride", testDnsOverride), ("testInvalidURL", testInvalidURL), ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), ("testShutdown", testShutdown), diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 3d0e709d6..e80957079 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -492,6 +492,64 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } + func testDnsOverride() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest(timeout: 5) { + /// key + cert was created with the following code (depends on swift-certificates) + /// ``` + /// let privateKey = P384.Signing.PrivateKey() + /// let name = try DistinguishedName { + /// OrganizationName("Self Signed") + /// CommonName("localhost") + /// } + /// let certificate = try Certificate( + /// version: .v3, + /// serialNumber: .init(), + /// publicKey: .init(privateKey.publicKey), + /// notValidBefore: Date(), + /// notValidAfter: Date() + .days(365), + /// issuer: name, + /// subject: name, + /// signatureAlgorithm: .ecdsaWithSHA384, + /// extensions: try .init { + /// SubjectAlternativeNames([.dnsName("example.com")]) + /// ExtendedKeyUsage([.serverAuth]) + /// }, + /// issuerPrivateKey: .init(privateKey) + /// ) + /// ``` + let certPath = Bundle.module.path(forResource: "example.com.cert", ofType: "pem")! + let keyPath = Bundle.module.path(forResource: "example.com.private-key", ofType: "pem")! + let localhostCert = try NIOSSLCertificate.fromPEMFile(certPath) + let configuration = TLSConfiguration.makeServerConfiguration( + certificateChain: localhostCert.map { .certificate($0) }, + privateKey: .file(keyPath) + ) + let bin = HTTPBin(.http2(tlsConfiguration: configuration)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + var config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() + var tlsConfig = TLSConfiguration.makeClientConfiguration() + + tlsConfig.trustRoots = .certificates(localhostCert) + config.tlsConfiguration = tlsConfig + // this is the actual configuration under test + config.dnsOverride = ["example.com": "localhost"] + + let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) + defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echohostheader") + let response = await XCTAssertNoThrowWithResult(try await localClient.execute(request, deadline: .now() + .seconds(2))) + XCTAssertEqual(response?.status, .ok) + XCTAssertEqual(response?.version, .http2) + var body = try await response?.body.collect(upTo: 1024) + let readableBytes = body?.readableBytes ?? 0 + let responseInfo = try body?.readJSONDecodable(RequestInfo.self, length: readableBytes) + XCTAssertEqual(responseInfo?.data, "example.com\(bin.port == 443 ? "" : ":\(bin.port)")") + } + } + func testInvalidURL() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 9727746cc..3db4385cd 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -165,7 +165,7 @@ class HTTPClientNIOTSTests: XCTestCase { var tlsConfig = TLSConfiguration.makeClientConfiguration() tlsConfig.trustRoots = .file("not/a/certificate") - XCTAssertThrowsError(try tlsConfig.getNWProtocolTLSOptions()) { error in + XCTAssertThrowsError(try tlsConfig.getNWProtocolTLSOptions(serverNameIndicatorOverride: nil)) { error in switch error { case let error as NIOSSL.NIOSSLError where error == .failedToLoadCertificate: break diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index fa424b042..aa1071de6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -37,7 +37,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .https, connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -69,7 +70,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .unix, connectionTarget: .unixSocket(path: "/some_path"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -98,7 +100,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .httpUnix, connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -127,7 +130,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .httpsUnix, connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -155,7 +159,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .https, connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -184,7 +189,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -218,7 +224,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -252,7 +259,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -286,7 +294,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -321,7 +330,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -355,7 +365,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -394,7 +405,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, @@ -433,7 +445,8 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(preparedRequest.poolKey, .init( scheme: .http, connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil + tlsConfiguration: nil, + serverNameIndicatorOverride: nil )) XCTAssertEqual(preparedRequest.head, .init( version: .http1_1, diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index ca24cba1c..c617555c6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -312,6 +312,10 @@ enum TemporaryFileHelpers { enum TestTLS { static let certificate = try! NIOSSLCertificate(bytes: Array(cert.utf8), format: .pem) static let privateKey = try! NIOSSLPrivateKey(bytes: Array(key.utf8), format: .pem) + static let serverConfiguration: TLSConfiguration = .makeServerConfiguration( + certificateChain: [.certificate(TestTLS.certificate)], + privateKey: .privateKey(TestTLS.privateKey) + ) } internal final class HTTPBin where @@ -327,34 +331,54 @@ internal final class HTTPBin where // refuses all connections case refuse // supports http1.1 connections only, which can be either plain text or encrypted - case http1_1(ssl: Bool = false, compress: Bool = false) + case http1_1( + tlsConfiguration: TLSConfiguration? = nil, + compress: Bool = false + ) // supports http1.1 and http2 connections which must be always encrypted case http2( + tlsConfiguration: TLSConfiguration = TestTLS.serverConfiguration, compress: Bool = false, settings: HTTP2Settings? = nil ) + static func http1_1(ssl: Bool, compress: Bool = false) -> Self { + .http1_1(tlsConfiguration: ssl ? TestTLS.serverConfiguration : nil, compress: compress) + } + // supports request decompression and http response compression var compress: Bool { switch self { case .refuse: return false - case .http1_1(ssl: _, compress: let compress), .http2(compress: let compress, _): + case .http1_1(_, let compress), .http2(_, let compress, _): return compress } } var httpSettings: HTTP2Settings { switch self { - case .http1_1, .http2(_, nil), .refuse: + case .http1_1, .http2(_, _, nil), .refuse: return [ HTTP2Setting(parameter: .maxConcurrentStreams, value: 10), HTTP2Setting(parameter: .maxHeaderListSize, value: HPACKDecoder.defaultMaxHeaderListSize), ] - case .http2(_, .some(let customSettings)): + case .http2(_, _, .some(let customSettings)): return customSettings } } + + var tlsConfiguration: TLSConfiguration? { + switch self { + case .refuse: + return nil + case .http1_1(let tlsConfiguration, _): + return tlsConfiguration + case .http2(var tlsConfiguration, _, _): + tlsConfiguration.applicationProtocols = NIOHTTP2SupportedALPNProtocols + return tlsConfiguration + } + } } enum Proxy { @@ -540,30 +564,8 @@ internal final class HTTPBin where } } - private static func tlsConfiguration(for mode: Mode) -> TLSConfiguration? { - var configuration: TLSConfiguration? - - switch mode { - case .refuse, .http1_1(ssl: false, compress: _): - break - case .http2: - configuration = .makeServerConfiguration( - certificateChain: [.certificate(TestTLS.certificate)], - privateKey: .privateKey(TestTLS.privateKey) - ) - configuration!.applicationProtocols = NIOHTTP2SupportedALPNProtocols - case .http1_1(ssl: true, compress: _): - configuration = .makeServerConfiguration( - certificateChain: [.certificate(TestTLS.certificate)], - privateKey: .privateKey(TestTLS.privateKey) - ) - } - - return configuration - } - private static func sslContext(for mode: Mode) -> NIOSSLContext? { - if let tlsConfiguration = self.tlsConfiguration(for: mode) { + if let tlsConfiguration = mode.tlsConfiguration { return try! NIOSSLContext(configuration: tlsConfiguration) } return nil diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 43134d453..36efee949 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -972,9 +972,13 @@ class MockTaskQueuer: HTTPRequestScheduler { } extension RequestOptions { - static func forTests(idleReadTimeout: TimeAmount? = nil) -> Self { + static func forTests( + idleReadTimeout: TimeAmount? = nil, + dnsOverride: [String: String] = [:] + ) -> Self { RequestOptions( - idleReadTimeout: idleReadTimeout + idleReadTimeout: idleReadTimeout, + dnsOverride: dnsOverride ) } } diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem new file mode 100644 index 000000000..69af76e77 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBwzCCAUmgAwIBAgIVAIFK2HEjRjd9rH6Szp3jT52U4wYjMAoGCCqGSM49BAMD +MCoxFDASBgNVBAoMC1NlbGYgU2lnbmVkMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcN +MjMwMzI5MTE1ODQwWhcNMjQwMzI4MTE1ODQwWjAqMRQwEgYDVQQKDAtTZWxmIFNp +Z25lZDESMBAGA1UEAwwJbG9jYWxob3N0MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE +SiOrOD8CbOyvj0yg+ArayukRCjw9AAaW3lsrsiRsSaqRxDcZ7+uR5nt2FUXc25mD +Ap+adz4g5gigpIUaVQc69AgMavYFCHF3Tb0TF1D4yAFLk8GFuWqxHDuqCQaGoyS5 +oy8wLTAWBgNVHREEDzANggtleGFtcGxlLmNvbTATBgNVHSUEDDAKBggrBgEFBQcD +ATAKBggqhkjOPQQDAwNoADBlAjALdKj7fq0Hvv69KUdMGvpHBaqRq+4+X4T1gAm/ +Z09XPB3BAd9z3Ov7fMnc65iKRwICMQCxxu0rBJUmR9v1BINxA4S1EPH0S/U5ysTp +Wu1n1LZ3C5ooxMiO50cPuWupaB2LElY= +-----END CERTIFICATE----- \ No newline at end of file diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem new file mode 100644 index 000000000..775a5ea56 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem @@ -0,0 +1,6 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAbqzPBHiy/SoUXTlYl +F0q3AK+N5wvpb93vS8jdRYAY2BIKIQOurw4WLp0qVxKgYGqhZANiAARKI6s4PwJs +7K+PTKD4CtrK6REKPD0ABpbeWyuyJGxJqpHENxnv65Hme3YVRdzbmYMCn5p3PiDm +CKCkhRpVBzr0CAxq9gUIcXdNvRMXUPjIAUuTwYW5arEcO6oJBoajJLk= +-----END PRIVATE KEY----- \ No newline at end of file From 9cdc4290b81d0fb33178cdd014728dcf6330a2dd Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 3 Apr 2023 17:07:02 +0100 Subject: [PATCH 071/146] Accept bare 2023 in license header (#676) --- scripts/soundness.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/soundness.sh b/scripts/soundness.sh index 5170ec4f1..6d37546e8 100755 --- a/scripts/soundness.sh +++ b/scripts/soundness.sh @@ -18,7 +18,7 @@ here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" function replace_acceptable_years() { # this needs to replace all acceptable forms with 'YEARS' - sed -e 's/20[12][0-9]-20[12][0-9]/YEARS/' -e 's/2019/YEARS/' -e 's/2020/YEARS/' -e 's/2021/YEARS/' -e 's/2022/YEARS/' + sed -e 's/20[12][0-9]-20[12][0-9]/YEARS/' -e 's/20[12][0-9]/YEARS/' } printf "=> Checking linux tests... " From 91b26407ad04822f6d2dd7c61a989cb75642c1bd Mon Sep 17 00:00:00 2001 From: carolinacass <67160898+carolinacass@users.noreply.github.com> Date: Tue, 4 Apr 2023 16:56:58 +0100 Subject: [PATCH 072/146] Update collect to use content-length to make early checks Motivation: not accumulate too many bytes Modifications: Implementing collect function to use NIOCore version to prevent overflowing Co-authored-by: Cory Benfield --- .../AsyncAwait/HTTPClientResponse.swift | 71 +++++++++++++++---- .../AsyncAwait/Transaction.swift | 3 +- .../AsyncAwait/TransactionBody.swift | 4 +- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 41 +++++++---- .../HTTPClientResponseTests+XCTest.swift | 35 +++++++++ .../HTTPClientResponseTests.swift | 45 ++++++++++++ .../HTTPClientTestUtils.swift | 5 ++ Tests/LinuxMain.swift | 1 + 10 files changed, 175 insertions(+), 33 deletions(-) create mode 100644 Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift create mode 100644 Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index b68e8db8b..07904b681 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -32,18 +32,6 @@ public struct HTTPClientResponse: Sendable { /// The body of this HTTP response. public var body: Body - init( - bag: Transaction, - version: HTTPVersion, - status: HTTPResponseStatus, - headers: HTTPHeaders - ) { - self.version = version - self.status = status - self.headers = headers - self.body = Body(TransactionBody(bag)) - } - @inlinable public init( version: HTTPVersion = .http1_1, status: HTTPResponseStatus = .ok, @@ -55,6 +43,17 @@ public struct HTTPClientResponse: Sendable { self.headers = headers self.body = body } + + init( + bag: Transaction, + version: HTTPVersion, + status: HTTPResponseStatus, + headers: HTTPHeaders, + requestMethod: HTTPMethod + ) { + let contentLength = HTTPClientResponse.expectedContentLength(requestMethod: requestMethod, headers: headers, status: status) + self.init(version: version, status: status, headers: headers, body: .init(TransactionBody(bag, expectedContentLength: contentLength))) + } } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -83,6 +82,48 @@ extension HTTPClientResponse { @inlinable public func makeAsyncIterator() -> AsyncIterator { .init(storage: self.storage.makeAsyncIterator()) } + + @inlinable init(storage: Storage) { + self.storage = storage + } + + /// Accumulates `Body` of ``ByteBuffer``s into a single ``ByteBuffer``. + /// - Parameters: + /// - maxBytes: The maximum number of bytes this method is allowed to accumulate + /// - Throws: `NIOTooManyBytesError` if the the sequence contains more than `maxBytes`. + /// - Returns: the number of bytes collected over time + @inlinable public func collect(upTo maxBytes: Int) async throws -> ByteBuffer { + switch self.storage { + case .transaction(let transactionBody): + if let contentLength = transactionBody.expectedContentLength { + if contentLength > maxBytes { + throw NIOTooManyBytesError() + } + } + case .anyAsyncSequence: + break + } + + /// calling collect function within here in order to ensure the correct nested type + func collect(_ body: Body, maxBytes: Int) async throws -> ByteBuffer where Body.Element == ByteBuffer { + try await body.collect(upTo: maxBytes) + } + return try await collect(self, maxBytes: maxBytes) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientResponse { + static func expectedContentLength(requestMethod: HTTPMethod, headers: HTTPHeaders, status: HTTPResponseStatus) -> Int? { + if status == .notModified { + return 0 + } else if requestMethod == .HEAD { + return 0 + } else { + let contentLength = headers["content-length"].first.flatMap { Int($0, radix: 10) } + return contentLength + } } } @@ -132,10 +173,10 @@ extension HTTPClientResponse.Body.Storage.AsyncIterator: AsyncIteratorProtocol { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientResponse.Body { init(_ body: TransactionBody) { - self.init(.transaction(body)) + self.init(storage: .transaction(body)) } - @usableFromInline init(_ storage: Storage) { + @inlinable init(_ storage: Storage) { self.storage = storage } @@ -146,7 +187,7 @@ extension HTTPClientResponse.Body { @inlinable public static func stream( _ sequenceOfBytes: SequenceOfBytes ) -> Self where SequenceOfBytes: AsyncSequence & Sendable, SequenceOfBytes.Element == ByteBuffer { - self.init(.anyAsyncSequence(AnyAsyncSequence(sequenceOfBytes.singleIteratorPrecondition))) + Self(storage: .anyAsyncSequence(AnyAsyncSequence(sequenceOfBytes.singleIteratorPrecondition))) } public static func bytes(_ byteBuffer: ByteBuffer) -> Self { diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index d81fbfd28..8846f36a5 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -236,7 +236,8 @@ extension Transaction: HTTPExecutableRequest { bag: self, version: head.version, status: head.status, - headers: head.headers + headers: head.headers, + requestMethod: self.requestHead.method ) continuation.resume(returning: asyncResponse) } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift b/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift index 497a3cc72..23a8e505e 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift @@ -20,9 +20,11 @@ import NIOCore @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @usableFromInline final class TransactionBody: Sendable { @usableFromInline let transaction: Transaction + @usableFromInline let expectedContentLength: Int? - init(_ transaction: Transaction) { + init(_ transaction: Transaction, expectedContentLength: Int?) { self.transaction = transaction + self.expectedContentLength = expectedContentLength } deinit { diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index beb2ea458..9f25cc15c 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -645,7 +645,7 @@ public class HTTPClient { "ahc-el-preference": "\(eventLoopPreference)"]) let failedTask: Task? = self.stateLock.withLock { - switch state { + switch self.state { case .upAndRunning: return nil case .shuttingDown, .shutDown: diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index ce0e2846d..85ac04f5c 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -51,6 +51,7 @@ extension AsyncAwaitEndToEndTests { ("testRejectsInvalidCharactersInHeaderFieldValues_http1", testRejectsInvalidCharactersInHeaderFieldValues_http1), ("testRejectsInvalidCharactersInHeaderFieldValues_http2", testRejectsInvalidCharactersInHeaderFieldValues_http2), ("testUsingGetMethodInsteadOfWait", testUsingGetMethodInsteadOfWait), + ("testSimpleContentLengthErrorNoBody", testSimpleContentLengthErrorNoBody), ] } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index e80957079..af99fb0a4 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -114,7 +114,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], ["4"]) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -137,7 +137,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -160,7 +160,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -183,7 +183,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], ["4"]) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -210,7 +210,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -233,7 +233,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect() + try await response.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -580,7 +580,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } - guard let body = await XCTAssertNoThrowWithResult(try await response.body.collect()) else { return } + guard let body = await XCTAssertNoThrowWithResult(try await response.body.collect(upTo: 1024)) else { return } var maybeRequestInfo: RequestInfo? XCTAssertNoThrow(maybeRequestInfo = try JSONDecoder().decode(RequestInfo.self, from: body)) guard let requestInfo = maybeRequestInfo else { return } @@ -641,7 +641,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response1.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response1.body.collect() + try await response1.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) @@ -650,7 +650,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { ) else { return } XCTAssertEqual(response2.headers["content-length"], []) guard let body = await XCTAssertNoThrowWithResult( - try await response2.body.collect() + try await response2.body.collect(upTo: 1024) ) else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } @@ -803,13 +803,24 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(response.version, .http2) } } -} -extension AsyncSequence where Element == ByteBuffer { - func collect() async rethrows -> ByteBuffer { - try await self.reduce(into: ByteBuffer()) { accumulatingBuffer, nextBuffer in - var nextBuffer = nextBuffer - accumulatingBuffer.writeBuffer(&nextBuffer) + func testSimpleContentLengthErrorNoBody() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest { + let bin = HTTPBin(.http2(compress: false)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + let request = HTTPClientRequest(url: "https://localhost:\(bin.port)/content-length-without-body") + guard let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) else { return } + await XCTAssertThrowsError( + try await response.body.collect(upTo: 3) + ) { + XCTAssertEqualTypeAndValue($0, NIOTooManyBytesError()) + } } } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift new file mode 100644 index 000000000..0a1a7cab6 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// +// +// HTTPClientResponseTests+XCTest.swift +// +import XCTest + +/// +/// NOTE: This file was generated by generate_linux_tests.rb +/// +/// Do NOT edit this file directly as it will be regenerated automatically when needed. +/// + +extension HTTPClientResponseTests { + static var allTests: [(String, (HTTPClientResponseTests) -> () throws -> Void)] { + return [ + ("testSimpleResponse", testSimpleResponse), + ("testSimpleResponseNotModified", testSimpleResponseNotModified), + ("testSimpleResponseHeadRequestMethod", testSimpleResponseHeadRequestMethod), + ("testResponseNoContentLengthHeader", testResponseNoContentLengthHeader), + ("testResponseInvalidInteger", testResponseInvalidInteger), + ] + } +} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift new file mode 100644 index 000000000..bf0ecfeb9 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift @@ -0,0 +1,45 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +@testable import AsyncHTTPClient +import Logging +import NIOCore +import XCTest + +final class HTTPClientResponseTests: XCTestCase { + func testSimpleResponse() { + let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "1025"], status: .ok) + XCTAssertEqual(response, 1025) + } + + func testSimpleResponseNotModified() { + let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "1025"], status: .notModified) + XCTAssertEqual(response, 0) + } + + func testSimpleResponseHeadRequestMethod() { + let response = HTTPClientResponse.expectedContentLength(requestMethod: .HEAD, headers: ["content-length": "1025"], status: .ok) + XCTAssertEqual(response, 0) + } + + func testResponseNoContentLengthHeader() { + let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: [:], status: .ok) + XCTAssertEqual(response, nil) + } + + func testResponseInvalidInteger() { + let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "none"], status: .ok) + XCTAssertEqual(response, nil) + } +} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index c617555c6..1a3cbd968 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -945,6 +945,11 @@ internal final class HTTPBinHandler: ChannelInboundHandler { // We're forcing this closed now. self.shouldClose = true self.resps.append(builder) + case "/content-length-without-body": + var headers = self.responseHeaders + headers.replaceOrAdd(name: "content-length", value: "1234") + context.writeAndFlush(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + return default: context.write(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .notFound))), promise: nil) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift index ca8478326..886bf2b95 100644 --- a/Tests/LinuxMain.swift +++ b/Tests/LinuxMain.swift @@ -44,6 +44,7 @@ struct LinuxMain { testCase(HTTPClientNIOTSTests.allTests), testCase(HTTPClientReproTests.allTests), testCase(HTTPClientRequestTests.allTests), + testCase(HTTPClientResponseTests.allTests), testCase(HTTPClientSOCKSTests.allTests), testCase(HTTPClientTests.allTests), testCase(HTTPClientUncleanSSLConnectionShutdownTests.allTests), From 343cdf4639924302cbb6883ce898111b7111cb54 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 11 Apr 2023 13:35:46 +0100 Subject: [PATCH 073/146] Fix documentation and add support for CI-ing it (#679) * Fix documentation and add support for CI-ing it * Fixup license header --- .../AsyncAwait/HTTPClientResponse.swift | 2 +- Sources/AsyncHTTPClient/Docs.docc/index.md | 28 ++++++++++--------- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- Sources/AsyncHTTPClient/HTTPHandler.swift | 4 +-- docker/docker-compose.2004.55.yaml | 3 ++ docker/docker-compose.2004.56.yaml | 3 ++ docker/docker-compose.2204.57.yaml | 3 ++ docker/docker-compose.2204.58.yaml | 3 ++ docker/docker-compose.2204.main.yaml | 3 ++ docker/docker-compose.yaml | 4 +++ scripts/check-docs.sh | 23 +++++++++++++++ 11 files changed, 61 insertions(+), 17 deletions(-) create mode 100755 scripts/check-docs.sh diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index 07904b681..f37489a89 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -87,7 +87,7 @@ extension HTTPClientResponse { self.storage = storage } - /// Accumulates `Body` of ``ByteBuffer``s into a single ``ByteBuffer``. + /// Accumulates `Body` of `ByteBuffer`s into a single `ByteBuffer`. /// - Parameters: /// - maxBytes: The maximum number of bytes this method is allowed to accumulate /// - Throws: `NIOTooManyBytesError` if the the sequence contains more than `maxBytes`. diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md index 60e928e7d..3a05778ce 100644 --- a/Sources/AsyncHTTPClient/Docs.docc/index.md +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -2,6 +2,8 @@ This package provides simple HTTP Client library built on top of SwiftNIO. +## Overview + This library provides the following: - First class support for Swift Concurrency (since version 1.9.0) - Asynchronous and non-blocking request methods @@ -17,7 +19,7 @@ This library provides the following: --- -## Getting Started +### Getting Started #### Adding the dependency @@ -71,13 +73,13 @@ httpClient.get(url: "https://apple.com/").whenComplete { result in } ``` -You should always shut down ``HTTPClient`` instances you created using ``HTTPClient/shutdown()-96ayw()``. Please note that you must not call ``HTTPClient/shutdown()-96ayw()`` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. +You should always shut down ``HTTPClient`` instances you created using ``HTTPClient/shutdown()-9gcpw``. Please note that you must not call ``HTTPClient/shutdown()-9gcpw`` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. -### async/await examples +#### async/await examples Examples for the async/await API can be found in the [`Examples` folder](https://github.com/swift-server/async-http-client/tree/main/Examples) in the repository. -## Usage guide +### Usage guide The default HTTP Method is `GET`. In case you need to have more control over the method, or you want to add headers or body, use the ``HTTPClientRequest`` struct: @@ -134,14 +136,14 @@ httpClient.execute(request: request).whenComplete { result in } ``` -### Redirects following +#### Redirects following Enable follow-redirects behavior using the client configuration: ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: HTTPClient.Configuration(followRedirects: true)) ``` -### Timeouts +#### Timeouts Timeouts (connect and read) can also be set using the client configuration: ```swift let timeout = HTTPClient.Configuration.Timeout(connect: .seconds(1), read: .seconds(1)) @@ -153,11 +155,11 @@ or on a per-request basis: httpClient.execute(request: request, deadline: .now() + .milliseconds(1)) ``` -### Streaming +#### Streaming When dealing with larger amount of data, it's critical to stream the response body instead of aggregating in-memory. The following example demonstrates how to count the number of bytes in a streaming response body: -#### Using Swift Concurrency +##### Using Swift Concurrency ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) do { @@ -189,7 +191,7 @@ do { try await httpClient.shutdown() ``` -#### Using HTTPClientResponseDelegate and SwiftNIO EventLoopFuture +##### Using HTTPClientResponseDelegate and SwiftNIO EventLoopFuture ```swift import NIOCore @@ -251,7 +253,7 @@ httpClient.execute(request: request, delegate: delegate).futureResult.whenSucces } ``` -### File downloads +#### File downloads Based on the `HTTPClientResponseDelegate` example above you can build more complex delegates, the built-in `FileDownloadDelegate` is one of them. It allows streaming the downloaded data @@ -280,7 +282,7 @@ client.execute(request: request, delegate: delegate).futureResult } ``` -### Unix Domain Socket Paths +#### Unix Domain Socket Paths Connecting to servers bound to socket paths is easy: ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) @@ -314,7 +316,7 @@ let secureSocketPathBasedURL = URL( ) ``` -### Disabling HTTP/2 +#### Disabling HTTP/2 The exclusive use of HTTP/1 is possible by setting ``HTTPClient/Configuration/httpVersion-swift.property`` to ``HTTPClient/Configuration/HTTPVersion-swift.struct/http1Only`` on the ``HTTPClient/Configuration``: ```swift var configuration = HTTPClient.Configuration() @@ -325,7 +327,7 @@ let client = HTTPClient( ) ``` -## Security +### Security AsyncHTTPClient's security process is documented on [GitHub](https://github.com/swift-server/async-http-client/blob/main/SECURITY.md). diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 9f25cc15c..a116ea495 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -228,7 +228,7 @@ public class HTTPClient { /// Shuts down the ``HTTPClient`` and releases its resources. /// /// - note: You cannot use this method if you sharted the ``HTTPClient`` with - /// ``init(eventLoopGroupProvider: .createNew)`` because that will shut down the ``EventLoopGroup`` the + /// `init(eventLoopGroupProvider: .createNew)` because that will shut down the `EventLoopGroup` the /// returned future would run in. public func shutdown() -> EventLoopFuture { switch self.eventLoopGroupProvider { diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 278bf18a8..614c55f3a 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -388,7 +388,7 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { /// until it will abort the request and throw an ``ResponseTooBigError``. /// /// Default is 2^32. - /// - precondition: not allowed to exceed 2^32 because ``ByteBuffer`` can not store more bytes + /// - precondition: not allowed to exceed 2^32 because `ByteBuffer` can not store more bytes public let maxBodySize: Int public convenience init(request: HTTPClient.Request) { @@ -400,7 +400,7 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { /// - maxBodySize: Maximum size in bytes of the HTTP response body that ``ResponseAccumulator`` will accept /// until it will abort the request and throw an ``ResponseTooBigError``. /// Default is 2^32. - /// - precondition: maxBodySize is not allowed to exceed 2^32 because ``ByteBuffer`` can not store more bytes + /// - precondition: maxBodySize is not allowed to exceed 2^32 because `ByteBuffer` can not store more bytes /// - warning: You can use ``ResponseAccumulator`` for just one request. /// If you start another request, you need to initiate another ``ResponseAccumulator``. public init(request: HTTPClient.Request, maxBodySize: Int) { diff --git a/docker/docker-compose.2004.55.yaml b/docker/docker-compose.2004.55.yaml index 71b75c0fe..d181ef142 100644 --- a/docker/docker-compose.2004.55.yaml +++ b/docker/docker-compose.2004.55.yaml @@ -9,6 +9,9 @@ services: ubuntu_version: "focal" swift_version: "5.5" + documentation-check: + image: async-http-client:20.04-5.5 + test: image: async-http-client:20.04-5.5 command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" diff --git a/docker/docker-compose.2004.56.yaml b/docker/docker-compose.2004.56.yaml index ed61267a9..b496e8484 100644 --- a/docker/docker-compose.2004.56.yaml +++ b/docker/docker-compose.2004.56.yaml @@ -9,6 +9,9 @@ services: ubuntu_version: "focal" swift_version: "5.6" + documentation-check: + image: async-http-client:20.04-5.6 + test: image: async-http-client:20.04-5.6 environment: [] diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml index bea713bad..78eb83e1c 100644 --- a/docker/docker-compose.2204.57.yaml +++ b/docker/docker-compose.2204.57.yaml @@ -9,6 +9,9 @@ services: ubuntu_version: "jammy" swift_version: "5.7" + documentation-check: + image: async-http-client:22.04-5.7 + test: image: async-http-client:22.04-5.7 environment: [] diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index d5dc8432e..2d3a300d8 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -8,6 +8,9 @@ services: args: base_image: "swiftlang/swift:nightly-5.8-jammy" + documentation-check: + image: async-http-client:22.04-5.8 + test: image: async-http-client:22.04-5.8 environment: diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 9132ed322..8dfa4c921 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -8,6 +8,9 @@ services: args: base_image: "swiftlang/swift:nightly-main-jammy" + documentation-check: + image: async-http-client:22.04-main + test: image: async-http-client:22.04-main environment: diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 803fad9b1..9ac4a6eea 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -26,6 +26,10 @@ services: <<: *common command: /bin/bash -xcl "./scripts/soundness.sh" + documentation-check: + <<: *common + command: /bin/bash -xcl "./scripts/check-docs.sh" + test: <<: *common command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh new file mode 100755 index 000000000..61a13a56f --- /dev/null +++ b/scripts/check-docs.sh @@ -0,0 +1,23 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the AsyncHTTPClient open source project +## +## Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu + +raw_targets=$(sed -E -n -e 's/^.* - documentation_targets: \[(.*)\].*$/\1/p' .spi.yml) +targets=(${raw_targets//,/ }) + +for target in "${targets[@]}"; do + swift package plugin generate-documentation --target "$target" --warnings-as-errors --analyze --level detailed +done From 6c5058ee2cfc3327a0daad8696d8269d7b0c7530 Mon Sep 17 00:00:00 2001 From: George Barnett Date: Tue, 11 Apr 2023 14:49:44 +0100 Subject: [PATCH 074/146] Add a control to limit connection reuses (#678) Motivation: Sometimes it can be helpful to limit the number of times a connection can be used before discarding it. AHC has no such support for this at the moment. Modifications: - Add a `maximumUsesPerConnection` configuration option which defaults to `nil` (i.e. no limit). - For HTTP1 we count down uses in the state machine and close the connection if it hits zero. - For HTTP2, each use maps to a stream so we count down remaining uses in the state machine which we combine with max concurrent streams to limit how many streams are available per connection. We also count remaining uses in the HTTP2 idle handler: we treat no remaining uses as receiving a GOAWAY frame and notify the pool which then drains the streams and replaces the connection. Result: Users can control how many times each connection can be used. --- .../HTTP2/HTTP2Connection.swift | 7 +- .../HTTP2/HTTP2IdleHandler.swift | 52 +++++++---- .../HTTPConnectionPool+Factory.swift | 1 + .../ConnectionPool/HTTPConnectionPool.swift | 3 +- .../HTTPConnectionPool+HTTP1Connections.swift | 82 +++++++++++++----- ...HTTPConnectionPool+HTTP1StateMachine.swift | 23 +++-- .../HTTPConnectionPool+HTTP2Connections.swift | 86 +++++++++++++------ ...HTTPConnectionPool+HTTP2StateMachine.swift | 6 +- .../HTTPConnectionPool+StateMachine.swift | 10 ++- Sources/AsyncHTTPClient/HTTPClient.swift | 12 +++ .../HTTP2ConnectionTests.swift | 2 + .../HTTP2IdleHandlerTests+XCTest.swift | 1 + .../HTTP2IdleHandlerTests.swift | 34 ++++++++ .../HTTPClientTests+XCTest.swift | 3 + .../HTTPClientTests.swift | 51 +++++++++++ ...PConnectionPool+HTTP1ConnectionsTest.swift | 34 ++++---- .../HTTPConnectionPool+HTTP1StateTests.swift | 27 ++++-- ...PConnectionPool+HTTP2ConnectionsTest.swift | 38 ++++---- ...onnectionPool+HTTP2StateMachineTests.swift | 70 +++++++++------ .../Mocks/MockConnectionPool.swift | 6 +- 20 files changed, 394 insertions(+), 154 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index 0cad92cfe..2c3c3cc0a 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -81,6 +81,7 @@ final class HTTP2Connection { private var openStreams = Set() let id: HTTPConnectionPool.Connection.ID let decompression: HTTPClient.Decompression + let maximumConnectionUses: Int? var closeFuture: EventLoopFuture { self.channel.closeFuture @@ -89,11 +90,13 @@ final class HTTP2Connection { init(channel: Channel, connectionID: HTTPConnectionPool.Connection.ID, decompression: HTTPClient.Decompression, + maximumConnectionUses: Int?, delegate: HTTP2ConnectionDelegate, logger: Logger) { self.channel = channel self.id = connectionID self.decompression = decompression + self.maximumConnectionUses = maximumConnectionUses self.logger = logger self.multiplexer = HTTP2StreamMultiplexer( mode: .client, @@ -120,12 +123,14 @@ final class HTTP2Connection { connectionID: HTTPConnectionPool.Connection.ID, delegate: HTTP2ConnectionDelegate, decompression: HTTPClient.Decompression, + maximumConnectionUses: Int?, logger: Logger ) -> EventLoopFuture<(HTTP2Connection, Int)> { let connection = HTTP2Connection( channel: channel, connectionID: connectionID, decompression: decompression, + maximumConnectionUses: maximumConnectionUses, delegate: delegate, logger: logger ) @@ -192,7 +197,7 @@ final class HTTP2Connection { let sync = self.channel.pipeline.syncOperations let http2Handler = NIOHTTP2Handler(mode: .client, initialSettings: nioDefaultSettings) - let idleHandler = HTTP2IdleHandler(delegate: self, logger: self.logger) + let idleHandler = HTTP2IdleHandler(delegate: self, logger: self.logger, maximumConnectionUses: self.maximumConnectionUses) try sync.addHandler(http2Handler, position: .last) try sync.addHandler(idleHandler, position: .last) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift index c522b2425..06458cb7e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift @@ -35,9 +35,10 @@ final class HTTP2IdleHandler: ChannelDuplexH let logger: Logger let delegate: Delegate - private var state: StateMachine = .init() + private var state: StateMachine - init(delegate: Delegate, logger: Logger) { + init(delegate: Delegate, logger: Logger, maximumConnectionUses: Int? = nil) { + self.state = StateMachine(maximumUses: maximumConnectionUses) self.delegate = delegate self.logger = logger } @@ -140,19 +141,23 @@ extension HTTP2IdleHandler { } enum State { - case initialized - case connected - case active(openStreams: Int, maxStreams: Int) + case initialized(maximumUses: Int?) + case connected(remainingUses: Int?) + case active(openStreams: Int, maxStreams: Int, remainingUses: Int?) case closing(openStreams: Int, maxStreams: Int) case closed } - var state: State = .initialized + var state: State + + init(maximumUses: Int?) { + self.state = .initialized(maximumUses: maximumUses) + } mutating func channelActive() { switch self.state { - case .initialized: - self.state = .connected + case .initialized(let maximumUses): + self.state = .connected(remainingUses: maximumUses) case .connected, .active, .closing, .closed: break @@ -171,17 +176,17 @@ extension HTTP2IdleHandler { case .initialized: preconditionFailure("Invalid state: \(self.state)") - case .connected: + case .connected(let remainingUses): // a settings frame might have multiple entries for `maxConcurrentStreams`. We are // only interested in the last value! If no `maxConcurrentStreams` is set, we assume // the http/2 default of 100. let maxStreams = settings.last(where: { $0.parameter == .maxConcurrentStreams })?.value ?? 100 - self.state = .active(openStreams: 0, maxStreams: maxStreams) + self.state = .active(openStreams: 0, maxStreams: maxStreams, remainingUses: remainingUses) return .notifyConnectionNewMaxStreamsSettings(maxStreams) - case .active(openStreams: let openStreams, maxStreams: let maxStreams): + case .active(openStreams: let openStreams, maxStreams: let maxStreams, remainingUses: let remainingUses): if let newMaxStreams = settings.last(where: { $0.parameter == .maxConcurrentStreams })?.value, newMaxStreams != maxStreams { - self.state = .active(openStreams: openStreams, maxStreams: newMaxStreams) + self.state = .active(openStreams: openStreams, maxStreams: newMaxStreams, remainingUses: remainingUses) return .notifyConnectionNewMaxStreamsSettings(newMaxStreams) } return .nothing @@ -205,7 +210,7 @@ extension HTTP2IdleHandler { self.state = .closing(openStreams: 0, maxStreams: 0) return .notifyConnectionGoAwayReceived(close: true) - case .active(let openStreams, let maxStreams): + case .active(let openStreams, let maxStreams, _): self.state = .closing(openStreams: openStreams, maxStreams: maxStreams) return .notifyConnectionGoAwayReceived(close: openStreams == 0) @@ -228,7 +233,7 @@ extension HTTP2IdleHandler { self.state = .closing(openStreams: 0, maxStreams: 0) return .close - case .active(let openStreams, let maxStreams): + case .active(let openStreams, let maxStreams, _): if openStreams == 0 { self.state = .closed return .close @@ -247,10 +252,19 @@ extension HTTP2IdleHandler { case .initialized, .connected: preconditionFailure("Invalid state: \(self.state)") - case .active(var openStreams, let maxStreams): + case .active(var openStreams, let maxStreams, let remainingUses): openStreams += 1 - self.state = .active(openStreams: openStreams, maxStreams: maxStreams) - return .nothing + let remainingUses = remainingUses.map { $0 - 1 } + self.state = .active(openStreams: openStreams, maxStreams: maxStreams, remainingUses: remainingUses) + + if remainingUses == 0 { + // Treat running out of connection uses as if we received a GOAWAY frame. This + // will notify the delegate (i.e. connection pool) that the connection can no + // longer be used. + return self.goAwayReceived() + } else { + return .nothing + } case .closing(var openStreams, let maxStreams): // A stream might be opened, while we are closing because of race conditions. For @@ -271,10 +285,10 @@ extension HTTP2IdleHandler { case .initialized, .connected: preconditionFailure("Invalid state: \(self.state)") - case .active(var openStreams, let maxStreams): + case .active(var openStreams, let maxStreams, let remainingUses): openStreams -= 1 assert(openStreams >= 0) - self.state = .active(openStreams: openStreams, maxStreams: maxStreams) + self.state = .active(openStreams: openStreams, maxStreams: maxStreams, remainingUses: remainingUses) return .notifyConnectionStreamClosed(currentlyAvailable: maxStreams - openStreams) case .closing(var openStreams, let maxStreams): diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 48aedfd8e..1461a6620 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -84,6 +84,7 @@ extension HTTPConnectionPool.ConnectionFactory { connectionID: connectionID, delegate: http2ConnectionDelegate, decompression: self.clientConfiguration.decompression, + maximumConnectionUses: self.clientConfiguration.maximumUsesPerConnection, logger: logger ).whenComplete { result in switch result { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 593802a58..eac4cc21f 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -71,7 +71,8 @@ final class HTTPConnectionPool { self._state = StateMachine( idGenerator: idGenerator, maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool.concurrentHTTP1ConnectionsPerHostSoftLimit, - retryConnectionEstablishment: clientConfiguration.connectionPool.retryConnectionEstablishment + retryConnectionEstablishment: clientConfiguration.connectionPool.retryConnectionEstablishment, + maximumConnectionUses: clientConfiguration.maximumUsesPerConnection ) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift index cdbf02394..935cdb2f6 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift @@ -19,15 +19,15 @@ extension HTTPConnectionPool { private struct HTTP1ConnectionState { enum State { /// the connection is creating a connection. Valid transitions are to: .backingOff, .idle, and .closed - case starting + case starting(maximumUses: Int?) /// the connection is waiting to retry the establishing a connection. Valid transitions are to: .closed. /// This means, the connection can be removed from the connections without cancelling external /// state. The connection state can then be replaced by a new one. case backingOff /// the connection is idle for a new request. Valid transitions to: .leased and .closed - case idle(Connection, since: NIODeadline) + case idle(Connection, since: NIODeadline, remainingUses: Int?) /// the connection is leased and running for a request. Valid transitions to: .idle and .closed - case leased(Connection) + case leased(Connection, remainingUses: Int?) /// the connection is closed. final state. case closed } @@ -36,10 +36,10 @@ extension HTTPConnectionPool { let connectionID: Connection.ID let eventLoop: EventLoop - init(connectionID: Connection.ID, eventLoop: EventLoop) { + init(connectionID: Connection.ID, eventLoop: EventLoop, maximumUses: Int?) { self.connectionID = connectionID self.eventLoop = eventLoop - self.state = .starting + self.state = .starting(maximumUses: maximumUses) } var isConnecting: Bool { @@ -69,6 +69,19 @@ extension HTTPConnectionPool { } } + var idleAndNoRemainingUses: Bool { + switch self.state { + case .idle(_, since: _, remainingUses: let remainingUses): + if let remainingUses = remainingUses { + return remainingUses <= 0 + } else { + return false + } + case .backingOff, .starting, .leased, .closed: + return false + } + } + var canOrWillBeAbleToExecuteRequests: Bool { switch self.state { case .leased, .backingOff, .idle, .starting: @@ -89,7 +102,7 @@ extension HTTPConnectionPool { var idleSince: NIODeadline? { switch self.state { - case .idle(_, since: let idleSince): + case .idle(_, since: let idleSince, _): return idleSince case .backingOff, .starting, .leased, .closed: return nil @@ -107,8 +120,8 @@ extension HTTPConnectionPool { mutating func connected(_ connection: Connection) { switch self.state { - case .starting: - self.state = .idle(connection, since: .now()) + case .starting(maximumUses: let maxUses): + self.state = .idle(connection, since: .now(), remainingUses: maxUses) case .backingOff, .idle, .leased, .closed: preconditionFailure("Invalid state: \(self.state)") } @@ -126,8 +139,8 @@ extension HTTPConnectionPool { mutating func lease() -> Connection { switch self.state { - case .idle(let connection, since: _): - self.state = .leased(connection) + case .idle(let connection, since: _, remainingUses: let remainingUses): + self.state = .leased(connection, remainingUses: remainingUses.map { $0 - 1 }) return connection case .backingOff, .starting, .leased, .closed: preconditionFailure("Invalid state: \(self.state)") @@ -136,8 +149,8 @@ extension HTTPConnectionPool { mutating func release() { switch self.state { - case .leased(let connection): - self.state = .idle(connection, since: .now()) + case .leased(let connection, let remainingUses): + self.state = .idle(connection, since: .now(), remainingUses: remainingUses) case .backingOff, .starting, .idle, .closed: preconditionFailure("Invalid state: \(self.state)") } @@ -145,7 +158,7 @@ extension HTTPConnectionPool { mutating func close() -> Connection { switch self.state { - case .idle(let connection, since: _): + case .idle(let connection, since: _, remainingUses: _): self.state = .closed return connection case .backingOff, .starting, .leased, .closed: @@ -188,10 +201,10 @@ extension HTTPConnectionPool { return .removeConnection case .starting: return .keepConnection - case .idle(let connection, since: _): + case .idle(let connection, since: _, remainingUses: _): context.close.append(connection) return .removeConnection - case .leased(let connection): + case .leased(let connection, remainingUses: _): context.cancel.append(connection) return .keepConnection case .closed: @@ -212,7 +225,7 @@ extension HTTPConnectionPool { case .backingOff: context.backingOff.append((self.connectionID, self.eventLoop)) return .removeConnection - case .idle(let connection, since: _): + case .idle(let connection, since: _, remainingUses: _): // Idle connections can be removed right away context.close.append(connection) return .removeConnection @@ -243,13 +256,16 @@ extension HTTPConnectionPool { /// The index after which you will find the connections for requests with `EventLoop` /// requirements in `connections`. private var overflowIndex: Array.Index + /// The number of times each connection can be used before it is closed and replaced. + private let maximumConnectionUses: Int? - init(maximumConcurrentConnections: Int, generator: Connection.ID.Generator) { + init(maximumConcurrentConnections: Int, generator: Connection.ID.Generator, maximumConnectionUses: Int?) { self.connections = [] self.connections.reserveCapacity(maximumConcurrentConnections) self.overflowIndex = self.connections.endIndex self.maximumConcurrentConnections = maximumConcurrentConnections self.generator = generator + self.maximumConnectionUses = maximumConnectionUses } var stats: Stats { @@ -323,6 +339,8 @@ extension HTTPConnectionPool { /// The connection's use. Either general purpose or for requests with `EventLoop` /// requirements. var use: ConnectionUse + /// Whether the connection should be closed. + var shouldBeClosed: Bool } /// Information around the failed/closed connection. @@ -345,14 +363,22 @@ extension HTTPConnectionPool { mutating func createNewConnection(on eventLoop: EventLoop) -> Connection.ID { precondition(self.canGrow) - let connection = HTTP1ConnectionState(connectionID: self.generator.next(), eventLoop: eventLoop) + let connection = HTTP1ConnectionState( + connectionID: self.generator.next(), + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) self.connections.insert(connection, at: self.overflowIndex) self.overflowIndex = self.connections.index(after: self.overflowIndex) return connection.connectionID } mutating func createNewOverflowConnection(on eventLoop: EventLoop) -> Connection.ID { - let connection = HTTP1ConnectionState(connectionID: self.generator.next(), eventLoop: eventLoop) + let connection = HTTP1ConnectionState( + connectionID: self.generator.next(), + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) self.connections.append(connection) return connection.connectionID } @@ -484,7 +510,8 @@ extension HTTPConnectionPool { precondition(self.connections[index].isClosed) let newConnection = HTTP1ConnectionState( connectionID: self.generator.next(), - eventLoop: self.connections[index].eventLoop + eventLoop: self.connections[index].eventLoop, + maximumUses: self.maximumConnectionUses ) self.connections[index] = newConnection @@ -562,7 +589,11 @@ extension HTTPConnectionPool { backingOff: [(Connection.ID, EventLoop)] ) { for (connectionID, eventLoop) in starting { - let newConnection = HTTP1ConnectionState(connectionID: connectionID, eventLoop: eventLoop) + let newConnection = HTTP1ConnectionState( + connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) self.connections.insert(newConnection, at: self.overflowIndex) /// If we can grow, we mark the connection as a general purpose connection. /// Otherwise, it will be an overflow connection which is only used once for requests with a required event loop @@ -572,7 +603,11 @@ extension HTTPConnectionPool { } for (connectionID, eventLoop) in backingOff { - var backingOffConnection = HTTP1ConnectionState(connectionID: connectionID, eventLoop: eventLoop) + var backingOffConnection = HTTP1ConnectionState( + connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) // TODO: Maybe we want to add a static init for backing off connections to HTTP1ConnectionState backingOffConnection.failedToConnect() self.connections.insert(backingOffConnection, at: self.overflowIndex) @@ -690,7 +725,8 @@ extension HTTPConnectionPool { } else { use = .eventLoop(eventLoop) } - return IdleConnectionContext(eventLoop: eventLoop, use: use) + let hasNoRemainingUses = self.connections[index].idleAndNoRemainingUses + return IdleConnectionContext(eventLoop: eventLoop, use: use, shouldBeClosed: hasNoRemainingUses) } private func findIdleConnection(onPreferred preferredEL: EventLoop) -> Int? { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift index 669e43f13..2629b0ea2 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift @@ -38,11 +38,13 @@ extension HTTPConnectionPool { idGenerator: Connection.ID.Generator, maximumConcurrentConnections: Int, retryConnectionEstablishment: Bool, + maximumConnectionUses: Int?, lifecycleState: StateMachine.LifecycleState ) { self.connections = HTTP1Connections( maximumConcurrentConnections: maximumConcurrentConnections, - generator: idGenerator + generator: idGenerator, + maximumConnectionUses: maximumConnectionUses ) self.retryConnectionEstablishment = retryConnectionEstablishment @@ -397,11 +399,20 @@ extension HTTPConnectionPool { ) -> EstablishedAction { switch self.lifecycleState { case .running: - switch context.use { - case .generalPurpose: - return self.nextActionForIdleGeneralPurposeConnection(at: index, context: context) - case .eventLoop: - return self.nextActionForIdleEventLoopConnection(at: index, context: context) + // Close the connection if it's expired. + if context.shouldBeClosed { + let connection = self.connections.closeConnection(at: index) + return .init( + request: .none, + connection: .closeConnection(connection, isShutdown: .no) + ) + } else { + switch context.use { + case .generalPurpose: + return self.nextActionForIdleGeneralPurposeConnection(at: index, context: context) + case .eventLoop: + return self.nextActionForIdleEventLoopConnection(at: index, context: context) + } } case .shuttingDown(let unclean): assert(self.requests.isEmpty) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift index 7aa504d03..01d68b8e4 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift @@ -18,12 +18,12 @@ extension HTTPConnectionPool { private struct HTTP2ConnectionState { private enum State { /// the pool is establishing a connection. Valid transitions are to: .backingOff, .active and .closed - case starting + case starting(maximumUses: Int?) /// the connection is waiting to retry to establish a connection. Valid transitions are to .closed. /// From .closed a new connection state must be created for a retry. case backingOff /// the connection is active and is able to run requests. Valid transitions are to: .draining and .closed - case active(Connection, maxStreams: Int, usedStreams: Int, lastIdle: NIODeadline) + case active(Connection, maxStreams: Int, usedStreams: Int, lastIdle: NIODeadline, remainingUses: Int?) /// the connection is active and is running requests. No new requests must be scheduled. /// Valid transitions to: .draining and .closed case draining(Connection, maxStreams: Int, usedStreams: Int) @@ -71,8 +71,12 @@ extension HTTPConnectionPool { /// A request can be scheduled on the connection var isAvailable: Bool { switch self.state { - case .active(_, let maxStreams, let usedStreams, _): - return usedStreams < maxStreams + case .active(_, let maxStreams, let usedStreams, _, let remainingUses): + if let remainingUses = remainingUses { + return usedStreams < maxStreams && remainingUses > 0 + } else { + return usedStreams < maxStreams + } case .starting, .backingOff, .draining, .closed: return false } @@ -82,7 +86,7 @@ extension HTTPConnectionPool { /// Every idle connection is available, but not every available connection is idle. var isIdle: Bool { switch self.state { - case .active(_, _, let usedStreams, _): + case .active(_, _, let usedStreams, _, _): return usedStreams == 0 case .starting, .backingOff, .draining, .closed: return false @@ -112,9 +116,13 @@ extension HTTPConnectionPool { case .active, .draining, .backingOff, .closed: preconditionFailure("Invalid state: \(self.state)") - case .starting: - self.state = .active(conn, maxStreams: maxStreams, usedStreams: 0, lastIdle: .now()) - return maxStreams + case .starting(let maxUses): + self.state = .active(conn, maxStreams: maxStreams, usedStreams: 0, lastIdle: .now(), remainingUses: maxUses) + if let maxUses = maxUses { + return min(maxStreams, maxUses) + } else { + return maxStreams + } } } @@ -127,9 +135,14 @@ extension HTTPConnectionPool { case .starting, .backingOff, .closed: preconditionFailure("Invalid state for updating max concurrent streams: \(self.state)") - case .active(let conn, _, let usedStreams, let lastIdle): - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle) - return max(maxStreams - usedStreams, 0) + case .active(let conn, _, let usedStreams, let lastIdle, let remainingUses): + self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses) + let availableStreams = max(maxStreams - usedStreams, 0) + if let remainingUses = remainingUses { + return min(remainingUses, availableStreams) + } else { + return availableStreams + } case .draining(let conn, _, let usedStreams): self.state = .draining(conn, maxStreams: maxStreams, usedStreams: usedStreams) @@ -142,7 +155,7 @@ extension HTTPConnectionPool { case .starting, .backingOff, .closed: preconditionFailure("Invalid state for draining a connection: \(self.state)") - case .active(let conn, let maxStreams, let usedStreams, _): + case .active(let conn, let maxStreams, let usedStreams, _, _): self.state = .draining(conn, maxStreams: maxStreams, usedStreams: usedStreams) return conn.eventLoop @@ -176,10 +189,11 @@ extension HTTPConnectionPool { case .starting, .backingOff, .draining, .closed: preconditionFailure("Invalid state for leasing a stream: \(self.state)") - case .active(let conn, let maxStreams, var usedStreams, let lastIdle): + case .active(let conn, let maxStreams, var usedStreams, let lastIdle, let remainingUses): usedStreams += count precondition(usedStreams <= maxStreams, "tried to lease a connection which is not available") - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle) + precondition(remainingUses.map { $0 >= count } ?? true, "tried to lease streams from a connection which does not have enough remaining streams") + self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses.map { $0 - count }) return conn } } @@ -191,14 +205,20 @@ extension HTTPConnectionPool { case .starting, .backingOff, .closed: preconditionFailure("Invalid state: \(self.state)") - case .active(let conn, let maxStreams, var usedStreams, var lastIdle): + case .active(let conn, let maxStreams, var usedStreams, var lastIdle, let remainingUses): precondition(usedStreams > 0, "we cannot release more streams than we have leased") usedStreams &-= 1 if usedStreams == 0 { lastIdle = .now() } - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle) - return max(maxStreams &- usedStreams, 0) + + self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses) + let availableStreams = max(maxStreams &- usedStreams, 0) + if let remainingUses = remainingUses { + return min(availableStreams, remainingUses) + } else { + return availableStreams + } case .draining(let conn, let maxStreams, var usedStreams): precondition(usedStreams > 0, "we cannot release more streams than we have leased") @@ -210,7 +230,7 @@ extension HTTPConnectionPool { mutating func close() -> Connection { switch self.state { - case .active(let conn, _, 0, _): + case .active(let conn, _, 0, _, _): self.state = .closed return conn @@ -247,7 +267,7 @@ extension HTTPConnectionPool { context.connectBackoff.append(self.connectionID) return .removeConnection - case .active(let connection, _, let usedStreams, _): + case .active(let connection, _, let usedStreams, _, _): precondition(usedStreams >= 0) if usedStreams == 0 { context.close.append(connection) @@ -274,7 +294,7 @@ extension HTTPConnectionPool { case .backingOff: stats.backingOffConnections &+= 1 - case .active(_, let maxStreams, let usedStreams, _): + case .active(_, let maxStreams, let usedStreams, _, _): stats.availableStreams += max(maxStreams - usedStreams, 0) stats.leasedStreams += usedStreams stats.availableConnections &+= 1 @@ -304,7 +324,7 @@ extension HTTPConnectionPool { context.starting.append((self.connectionID, self.eventLoop)) return .removeConnection - case .active(let connection, _, let usedStreams, _): + case .active(let connection, _, let usedStreams, _, _): precondition(usedStreams >= 0) if usedStreams == 0 { context.close.append(connection) @@ -325,10 +345,10 @@ extension HTTPConnectionPool { } } - init(connectionID: Connection.ID, eventLoop: EventLoop) { + init(connectionID: Connection.ID, eventLoop: EventLoop, maximumUses: Int?) { self.connectionID = connectionID self.eventLoop = eventLoop - self.state = .starting + self.state = .starting(maximumUses: maximumUses) } } @@ -337,6 +357,8 @@ extension HTTPConnectionPool { private let generator: Connection.ID.Generator /// The connections states private var connections: [HTTP2ConnectionState] + /// The number of times each connection can be used before it is closed and replaced. + private let maximumConnectionUses: Int? var isEmpty: Bool { self.connections.isEmpty @@ -348,9 +370,10 @@ extension HTTPConnectionPool { } } - init(generator: Connection.ID.Generator) { + init(generator: Connection.ID.Generator, maximumConnectionUses: Int?) { self.generator = generator self.connections = [] + self.maximumConnectionUses = maximumConnectionUses } // MARK: Migration @@ -365,12 +388,16 @@ extension HTTPConnectionPool { backingOff: [(Connection.ID, EventLoop)] ) { for (connectionID, eventLoop) in starting { - let newConnection = HTTP2ConnectionState(connectionID: connectionID, eventLoop: eventLoop) + let newConnection = HTTP2ConnectionState(connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses) self.connections.append(newConnection) } for (connectionID, eventLoop) in backingOff { - var backingOffConnection = HTTP2ConnectionState(connectionID: connectionID, eventLoop: eventLoop) + var backingOffConnection = HTTP2ConnectionState(connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses) // TODO: Maybe we want to add a static init for backing off connections to HTTP2ConnectionState backingOffConnection.failedToConnect() self.connections.append(backingOffConnection) @@ -476,7 +503,9 @@ extension HTTPConnectionPool { "we should not create more than one connection per event loop" ) - let connection = HTTP2ConnectionState(connectionID: self.generator.next(), eventLoop: eventLoop) + let connection = HTTP2ConnectionState(connectionID: self.generator.next(), + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses) self.connections.append(connection) return connection.connectionID } @@ -661,7 +690,8 @@ extension HTTPConnectionPool { precondition(self.connections[index].isClosed) let newConnection = HTTP2ConnectionState( connectionID: self.generator.next(), - eventLoop: self.connections[index].eventLoop + eventLoop: self.connections[index].eventLoop, + maximumUses: self.maximumConnectionUses ) self.connections[index] = newConnection diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index 9964ccd05..83a7647f4 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -41,12 +41,14 @@ extension HTTPConnectionPool { init( idGenerator: Connection.ID.Generator, retryConnectionEstablishment: Bool, - lifecycleState: StateMachine.LifecycleState + lifecycleState: StateMachine.LifecycleState, + maximumConnectionUses: Int? ) { self.idGenerator = idGenerator self.requests = RequestQueue() - self.connections = HTTP2Connections(generator: idGenerator) + self.connections = HTTP2Connections(generator: idGenerator, + maximumConnectionUses: maximumConnectionUses) self.lifecycleState = lifecycleState self.retryConnectionEstablishment = retryConnectionEstablishment } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift index 0460849cc..a61471a69 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift @@ -99,19 +99,23 @@ extension HTTPConnectionPool { /// The property was introduced to fail fast during testing. /// Otherwise this should always be true and not turned off. private let retryConnectionEstablishment: Bool + let maximumConnectionUses: Int? init( idGenerator: Connection.ID.Generator, maximumConcurrentHTTP1Connections: Int, - retryConnectionEstablishment: Bool + retryConnectionEstablishment: Bool, + maximumConnectionUses: Int? ) { self.maximumConcurrentHTTP1Connections = maximumConcurrentHTTP1Connections self.retryConnectionEstablishment = retryConnectionEstablishment self.idGenerator = idGenerator + self.maximumConnectionUses = maximumConnectionUses let http1State = HTTP1StateMachine( idGenerator: idGenerator, maximumConcurrentConnections: maximumConcurrentHTTP1Connections, retryConnectionEstablishment: retryConnectionEstablishment, + maximumConnectionUses: maximumConnectionUses, lifecycleState: .running ) self.state = .http1(http1State) @@ -137,6 +141,7 @@ extension HTTPConnectionPool { idGenerator: self.idGenerator, maximumConcurrentConnections: self.maximumConcurrentHTTP1Connections, retryConnectionEstablishment: self.retryConnectionEstablishment, + maximumConnectionUses: self.maximumConnectionUses, lifecycleState: http2StateMachine.lifecycleState ) @@ -158,7 +163,8 @@ extension HTTPConnectionPool { var http2StateMachine = HTTP2StateMachine( idGenerator: self.idGenerator, retryConnectionEstablishment: self.retryConnectionEstablishment, - lifecycleState: http1StateMachine.lifecycleState + lifecycleState: http1StateMachine.lifecycleState, + maximumConnectionUses: self.maximumConnectionUses ) let migrationAction = http2StateMachine.migrateFromHTTP1( http1Connections: http1StateMachine.connections, diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index a116ea495..a916d3ade 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -758,6 +758,18 @@ public class HTTPClient { /// which is the recommended setting. Only set this to `false` when attempting to trigger a particular error path. public var networkFrameworkWaitForConnectivity: Bool + /// The maximum number of times each connection can be used before it is replaced with a new one. Use `nil` (the default) + /// if no limit should be applied to each connection. + /// + /// - Precondition: The value must be greater than zero. + public var maximumUsesPerConnection: Int? { + willSet { + if let newValue = newValue, newValue <= 0 { + fatalError("maximumUsesPerConnection must be greater than zero or nil") + } + } + } + public init( tlsConfiguration: TLSConfiguration? = nil, redirectConfiguration: RedirectConfiguration? = nil, diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index 951a64494..15e5cdff2 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -38,6 +38,7 @@ class HTTP2ConnectionTests: XCTestCase { connectionID: 0, delegate: TestHTTP2ConnectionDelegate(), decompression: .disabled, + maximumConnectionUses: nil, logger: logger ).wait()) } @@ -51,6 +52,7 @@ class HTTP2ConnectionTests: XCTestCase { channel: embedded, connectionID: 0, decompression: .disabled, + maximumConnectionUses: nil, delegate: TestHTTP2ConnectionDelegate(), logger: logger ) diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift index 1b9558105..a69530597 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift @@ -36,6 +36,7 @@ extension HTTP2IdleHandlerTests { ("testCloseEventWhileThereAreOpenStreams", testCloseEventWhileThereAreOpenStreams), ("testGoAwayWhileThereAreOpenStreams", testGoAwayWhileThereAreOpenStreams), ("testReceiveSettingsAndGoAwayAfterClientSideClose", testReceiveSettingsAndGoAwayAfterClientSideClose), + ("testConnectionUseLimitTriggersGoAway", testConnectionUseLimitTriggersGoAway), ] } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift index 355969c6a..611e31457 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift @@ -251,6 +251,40 @@ class HTTP2IdleHandlerTests: XCTestCase { XCTAssertNoThrow(try embedded.writeInbound(goAwayFrame)) XCTAssertEqual(delegate.goAwayReceived, false, "Expected go away to not be forwarded.") } + + func testConnectionUseLimitTriggersGoAway() { + let delegate = MockHTTP2IdleHandlerDelegate() + let idleHandler = HTTP2IdleHandler(delegate: delegate, logger: Logger(label: "test"), maximumConnectionUses: 5) + let embedded = EmbeddedChannel(handlers: [idleHandler]) + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + + let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 100)]))) + XCTAssertEqual(delegate.maxStreams, nil) + XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) + XCTAssertEqual(delegate.maxStreams, 100) + + for streamID in HTTP2StreamID(1)...Mode, maximumUses: Int, requests: Int) throws { + let bin = HTTPBin(mode) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + var configuration = HTTPClient.Configuration(certificateVerification: .none) + // Limit each connection to two uses before discarding them. The test will verify that the + // connection number indicated by the server increments every two requests. + configuration.maximumUsesPerConnection = maximumUses + + let client = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: configuration) + defer { XCTAssertNoThrow(try client.syncShutdown()) } + + let request = try HTTPClient.Request(url: bin.baseURL + "stats") + let decoder = JSONDecoder() + + // Do two requests per batch. Both should report the same connection number. + for requestNumber in stride(from: 0, to: requests, by: maximumUses) { + var responses = [RequestInfo]() + + for _ in 0.. Date: Wed, 12 Apr 2023 16:11:56 +0200 Subject: [PATCH 075/146] Replace `TransactionBody` with `NIOAsyncSequenceProducer` (#677) * save progress * Replace `TransactionBody` with `NIOAsyncSequenceProducer` * test * revert unnesscary changes * Add end-to-end test which currently fails because of https://github.com/apple/swift-nio/issues/2398 * soundness * Use latest swift-nio release * throw CancellationError on task cancelation * Fix Swift 5.5 & 5.6 --- Package.swift | 2 +- Package@swift-5.5.swift | 2 +- .../AnyAsyncSequenceProucerDelete.swift | 37 ++ .../AsyncAwait/HTTPClientResponse.swift | 40 +- .../AsyncAwait/Transaction+StateMachine.swift | 416 ++++-------------- .../AsyncAwait/Transaction.swift | 89 ++-- .../AsyncAwait/TransactionBody.swift | 68 --- .../AsyncAwaitEndToEndTests+XCTest.swift | 1 + .../AsyncAwaitEndToEndTests.swift | 31 +- .../Transaction+StateMachineTests.swift | 9 +- .../TransactionTests.swift | 9 +- 11 files changed, 221 insertions(+), 483 deletions(-) create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequenceProucerDelete.swift delete mode 100644 Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift diff --git a/Package.swift b/Package.swift index 83d5c65bc..1f2f0046f 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.50.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift index 02c91c7ef..0ad7d1f3b 100644 --- a/Package@swift-5.5.swift +++ b/Package@swift-5.5.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.42.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.50.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequenceProucerDelete.swift b/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequenceProucerDelete.swift new file mode 100644 index 000000000..1e35df7f2 --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/AnyAsyncSequenceProucerDelete.swift @@ -0,0 +1,37 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import NIOCore + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +@usableFromInline +struct AnyAsyncSequenceProducerDelegate: NIOAsyncSequenceProducerDelegate { + @usableFromInline + var delegate: NIOAsyncSequenceProducerDelegate + + @inlinable + init(_ delegate: Delegate) { + self.delegate = delegate + } + + @inlinable + func produceMore() { + self.delegate.produceMore() + } + + @inlinable + func didTerminate() { + self.delegate.didTerminate() + } +} diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index f37489a89..ee7f11592 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -45,14 +45,25 @@ public struct HTTPClientResponse: Sendable { } init( - bag: Transaction, + requestMethod: HTTPMethod, version: HTTPVersion, status: HTTPResponseStatus, headers: HTTPHeaders, - requestMethod: HTTPMethod + body: TransactionBody ) { - let contentLength = HTTPClientResponse.expectedContentLength(requestMethod: requestMethod, headers: headers, status: status) - self.init(version: version, status: status, headers: headers, body: .init(TransactionBody(bag, expectedContentLength: contentLength))) + self.init( + version: version, + status: status, + headers: headers, + body: .init(.transaction( + body, + expectedContentLength: HTTPClientResponse.expectedContentLength( + requestMethod: requestMethod, + headers: headers, + status: status + ) + )) + ) } } @@ -94,8 +105,8 @@ extension HTTPClientResponse { /// - Returns: the number of bytes collected over time @inlinable public func collect(upTo maxBytes: Int) async throws -> ByteBuffer { switch self.storage { - case .transaction(let transactionBody): - if let contentLength = transactionBody.expectedContentLength { + case .transaction(_, let expectedContentLength): + if let contentLength = expectedContentLength { if contentLength > maxBytes { throw NIOTooManyBytesError() } @@ -127,10 +138,19 @@ extension HTTPClientResponse { } } +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +@usableFromInline +typealias TransactionBody = NIOThrowingAsyncSequenceProducer< + ByteBuffer, + Error, + NIOAsyncSequenceProducerBackPressureStrategies.HighLowWatermark, + AnyAsyncSequenceProducerDelegate +> + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientResponse.Body { @usableFromInline enum Storage: Sendable { - case transaction(TransactionBody) + case transaction(TransactionBody, expectedContentLength: Int?) case anyAsyncSequence(AnyAsyncSequence) } } @@ -141,7 +161,7 @@ extension HTTPClientResponse.Body.Storage: AsyncSequence { @inlinable func makeAsyncIterator() -> AsyncIterator { switch self { - case .transaction(let transaction): + case .transaction(let transaction, _): return .transaction(transaction.makeAsyncIterator()) case .anyAsyncSequence(let anyAsyncSequence): return .anyAsyncSequence(anyAsyncSequence.makeAsyncIterator()) @@ -172,10 +192,6 @@ extension HTTPClientResponse.Body.Storage.AsyncIterator: AsyncIteratorProtocol { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientResponse.Body { - init(_ body: TransactionBody) { - self.init(storage: .transaction(body)) - } - @inlinable init(_ storage: Storage) { self.storage = storage } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 008f1f823..538424538 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -18,6 +18,7 @@ import NIOHTTP1 @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction { + @usableFromInline struct StateMachine { struct ExecutionContext { let executor: HTTPRequestExecutor @@ -30,7 +31,7 @@ extension Transaction { case queued(CheckedContinuation, HTTPRequestScheduler) case deadlineExceededWhileQueued(CheckedContinuation) case executing(ExecutionContext, RequestStreamState, ResponseStreamState) - case finished(error: Error?, TransactionBody.AsyncIterator.ID?) + case finished(error: Error?) } fileprivate enum RequestStreamState { @@ -41,20 +42,11 @@ extension Transaction { } fileprivate enum ResponseStreamState { - enum Next { - case askExecutorForMore - case error(Error) - case endOfFile - } - - // Waiting for response head. Valid transitions to: waitingForStream. + // Waiting for response head. Valid transitions to: streamingBody. case waitingForResponseHead - // We are waiting for the user to create a response body iterator and to call next on - // it for the first time. - case waitingForResponseIterator(CircularBuffer, next: Next) - case buffering(TransactionBody.AsyncIterator.ID, CircularBuffer, next: Next) - case waitingForRemote(TransactionBody.AsyncIterator.ID, CheckedContinuation) - case finished(TransactionBody.AsyncIterator.ID, CheckedContinuation) + // streaming response body. Valid transitions to: finished. + case streamingBody(TransactionBody.Source) + case finished } private var state: State @@ -92,7 +84,7 @@ extension Transaction { /// fail response before head received. scheduler and executor are exclusive here. case failResponseHead(CheckedContinuation, Error, HTTPRequestScheduler?, HTTPRequestExecutor?, bodyStreamContinuation: CheckedContinuation?) /// fail response after response head received. fail the response stream (aka call to `next()`) - case failResponseStream(CheckedContinuation, Error, HTTPRequestExecutor, bodyStreamContinuation: CheckedContinuation?) + case failResponseStream(TransactionBody.Source, Error, HTTPRequestExecutor, bodyStreamContinuation: CheckedContinuation?) case failRequestStreamContinuation(CheckedContinuation, Error) } @@ -100,11 +92,11 @@ extension Transaction { mutating func fail(_ error: Error) -> FailAction { switch self.state { case .initialized(let continuation): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .failResponseHead(continuation, error, nil, nil, bodyStreamContinuation: nil) case .queued(let continuation, let scheduler): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .failResponseHead(continuation, error, scheduler, nil, bodyStreamContinuation: nil) case .deadlineExceededWhileQueued(let continuation): let realError: Error = { @@ -118,73 +110,31 @@ extension Transaction { } }() - self.state = .finished(error: realError, nil) + self.state = .finished(error: realError) return .failResponseHead(continuation, realError, nil, nil, bodyStreamContinuation: nil) case .executing(let context, let requestStreamState, .waitingForResponseHead): switch requestStreamState { case .paused(continuation: .some(let continuation)): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .failResponseHead(context.continuation, error, nil, context.executor, bodyStreamContinuation: continuation) case .requestHeadSent, .finished, .producing, .paused(continuation: .none): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .failResponseHead(context.continuation, error, nil, context.executor, bodyStreamContinuation: nil) } - case .executing(let context, let requestStreamState, .waitingForResponseIterator(let buffer, next: .askExecutorForMore)), - .executing(let context, let requestStreamState, .waitingForResponseIterator(let buffer, next: .endOfFile)): + case .executing(let context, let requestStreamState, .streamingBody(let source)): + self.state = .finished(error: error) switch requestStreamState { - case .paused(.some(let continuation)): - self.state = .executing(context, .finished, .waitingForResponseIterator(buffer, next: .error(error))) - return .failRequestStreamContinuation(continuation, error) - - case .requestHeadSent, .producing, .paused(continuation: .none), .finished: - self.state = .executing(context, .finished, .waitingForResponseIterator(buffer, next: .error(error))) - return .none + case .paused(let bodyStreamContinuation): + return .failResponseStream(source, error, context.executor, bodyStreamContinuation: bodyStreamContinuation) + case .finished, .producing, .requestHeadSent: + return .failResponseStream(source, error, context.executor, bodyStreamContinuation: nil) } - case .executing(let context, let requestStreamState, .buffering(let streamID, let buffer, next: .askExecutorForMore)), - .executing(let context, let requestStreamState, .buffering(let streamID, let buffer, next: .endOfFile)): - switch requestStreamState { - case .paused(continuation: .some(let continuation)): - self.state = .executing(context, .finished, .buffering(streamID, buffer, next: .error(error))) - return .failRequestStreamContinuation(continuation, error) - - case .requestHeadSent, .paused(continuation: .none), .producing, .finished: - self.state = .executing(context, .finished, .buffering(streamID, buffer, next: .error(error))) - return .none - } - - case .executing(let context, let requestStreamState, .waitingForRemote(let streamID, let continuation)): - // We are in response streaming. The response stream is waiting for the next bytes - // from the server. We can fail the call to `next` immediately. - switch requestStreamState { - case .paused(continuation: .some(let bodyStreamContinuation)): - self.state = .finished(error: error, streamID) - return .failResponseStream(continuation, error, context.executor, bodyStreamContinuation: bodyStreamContinuation) - - case .requestHeadSent, .paused(continuation: .none), .producing, .finished: - self.state = .finished(error: error, streamID) - return .failResponseStream(continuation, error, context.executor, bodyStreamContinuation: nil) - } - - case .finished(error: _, _), - .executing(_, _, .waitingForResponseIterator(_, next: .error)), - .executing(_, _, .buffering(_, _, next: .error)): - // The request has already failed, succeeded, or the users is not interested in the - // response. There is no more way to reach the user code. Just drop the error. + case .finished(error: _), + .executing(_, _, .finished): return .none - - case .executing(let context, let requestStreamState, .finished(let streamID, let continuation)): - switch requestStreamState { - case .paused(continuation: .some(let bodyStreamContinuation)): - self.state = .finished(error: error, streamID) - return .failResponseStream(continuation, error, context.executor, bodyStreamContinuation: bodyStreamContinuation) - - case .requestHeadSent, .paused(continuation: .none), .producing, .finished: - self.state = .finished(error: error, streamID) - return .failResponseStream(continuation, error, context.executor, bodyStreamContinuation: nil) - } } } @@ -208,15 +158,14 @@ extension Transaction { return .none case .deadlineExceededWhileQueued(let continuation): let error = HTTPClientError.deadlineExceeded - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .cancelAndFail(executor, continuation, with: error) - case .finished(error: .some, .none): + case .finished(error: .some): return .cancel(executor) case .executing, - .finished(error: .none, _), - .finished(error: .some, .some): + .finished(error: .none): preconditionFailure("Invalid state: \(self.state)") } } @@ -346,9 +295,8 @@ extension Transaction { } enum FinishAction { - // forward the notice that the request stream has finished. If finalContinuation is not - // nil, succeed the continuation with nil to signal the requests end. - case forwardStreamFinished(HTTPRequestExecutor, finalContinuation: CheckedContinuation?) + // forward the notice that the request stream has finished. + case forwardStreamFinished(HTTPRequestExecutor) case none } @@ -368,15 +316,15 @@ extension Transaction { .executing(let context, .requestHeadSent, let responseState): switch responseState { - case .finished(let registeredStreamID, let continuation): + case .finished: // if the response stream has already finished before the request, we must succeed // the final continuation. - self.state = .finished(error: nil, registeredStreamID) - return .forwardStreamFinished(context.executor, finalContinuation: continuation) + self.state = .finished(error: nil) + return .forwardStreamFinished(context.executor) - case .waitingForResponseHead, .waitingForResponseIterator, .waitingForRemote, .buffering: + case .waitingForResponseHead, .streamingBody: self.state = .executing(context, .finished, responseState) - return .forwardStreamFinished(context.executor, finalContinuation: nil) + return .forwardStreamFinished(context.executor) } case .finished: @@ -387,276 +335,91 @@ extension Transaction { // MARK: - Response - enum ReceiveResponseHeadAction { - case succeedResponseHead(HTTPResponseHead, CheckedContinuation) + case succeedResponseHead(TransactionBody, CheckedContinuation) case none } - mutating func receiveResponseHead(_ head: HTTPResponseHead) -> ReceiveResponseHeadAction { + mutating func receiveResponseHead( + _ head: HTTPResponseHead, + delegate: Delegate + ) -> ReceiveResponseHeadAction { switch self.state { case .initialized, .queued, .deadlineExceededWhileQueued, - .executing(_, _, .waitingForResponseIterator), - .executing(_, _, .buffering), - .executing(_, _, .waitingForRemote): - preconditionFailure("How can we receive a response, if the request hasn't started yet.") + .executing(_, _, .streamingBody), + .executing(_, _, .finished): + preconditionFailure("invalid state \(self.state)") case .executing(let context, let requestState, .waitingForResponseHead): // The response head was received. Next we will wait for the consumer to create a // response body stream. - self.state = .executing(context, requestState, .waitingForResponseIterator(.init(), next: .askExecutorForMore)) - return .succeedResponseHead(head, context.continuation) + let body = TransactionBody.makeSequence( + backPressureStrategy: .init(lowWatermark: 1, highWatermark: 1), + delegate: AnyAsyncSequenceProducerDelegate(delegate) + ) + + self.state = .executing(context, requestState, .streamingBody(body.source)) + return .succeedResponseHead(body.sequence, context.continuation) - case .finished(error: .some, _): + case .finished(error: .some): // If the request failed before, we don't need to do anything in response to // receiving the response head. return .none - case .executing(_, _, .finished), - .finished(error: .none, _): + case .finished(error: .none): preconditionFailure("How can the request be finished without error, before receiving response head?") } } - enum ReceiveResponsePartAction { - case none - case succeedContinuation(CheckedContinuation, ByteBuffer) - } - - mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponsePartAction { - switch self.state { - case .initialized, .queued, .deadlineExceededWhileQueued: - preconditionFailure("Received a response body part, but request hasn't started yet. Invalid state: \(self.state)") - - case .executing(_, _, .waitingForResponseHead): - preconditionFailure("If we receive a response body, we must have received a head before") - - case .executing(_, _, .buffering(_, _, next: .endOfFile)): - preconditionFailure("If we have received an eof before, why did we get another body part?") - - case .executing(_, _, .buffering(_, _, next: .error)): - // we might still get pending buffers if the user has canceled the request - return .none - - case .executing(let context, let requestState, .buffering(let streamID, var currentBuffer, next: .askExecutorForMore)): - if currentBuffer.isEmpty { - currentBuffer = buffer - } else { - currentBuffer.append(contentsOf: buffer) - } - self.state = .executing(context, requestState, .buffering(streamID, currentBuffer, next: .askExecutorForMore)) - return .none - - case .executing(let executor, let requestState, .waitingForResponseIterator(var currentBuffer, next: let next)): - guard case .askExecutorForMore = next else { - preconditionFailure("If we have received an error or eof before, why did we get another body part? Next: \(next)") - } - - if currentBuffer.isEmpty { - currentBuffer = buffer - } else { - currentBuffer.append(contentsOf: buffer) - } - self.state = .executing(executor, requestState, .waitingForResponseIterator(currentBuffer, next: next)) - return .none - - case .executing(let executor, let requestState, .waitingForRemote(let streamID, let continuation)): - var buffer = buffer - let first = buffer.removeFirst() - self.state = .executing(executor, requestState, .buffering(streamID, buffer, next: .askExecutorForMore)) - return .succeedContinuation(continuation, first) - - case .finished: - // the request failed or was cancelled before, we can ignore further data - return .none - - case .executing(_, _, .finished): - preconditionFailure("Received response end. Must not receive further body parts after that. Invalid state: \(self.state)") - } - } - - enum ResponseBodyDeinitedAction { - case cancel(HTTPRequestExecutor) + enum ProduceMoreAction { case none + case requestMoreResponseBodyParts(HTTPRequestExecutor) } - mutating func responseBodyDeinited() -> ResponseBodyDeinitedAction { + mutating func produceMore() -> ProduceMoreAction { switch self.state { case .initialized, .queued, .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): - preconditionFailure("Got notice about a deinited response, before we even received a response. Invalid state: \(self.state)") - - case .executing(_, _, .waitingForResponseIterator(_, next: .endOfFile)): - self.state = .finished(error: nil, nil) - return .none + preconditionFailure("invalid state \(self.state)") - case .executing(let context, _, .waitingForResponseIterator(_, next: .askExecutorForMore)): - self.state = .finished(error: nil, nil) - return .cancel(context.executor) - - case .executing(_, _, .waitingForResponseIterator(_, next: .error(let error))): - self.state = .finished(error: error, nil) - return .none - - case .finished: - // body was released after the response was consumed - return .none - - case .executing(_, _, .buffering), - .executing(_, _, .waitingForRemote), + case .executing(let context, _, .streamingBody): + return .requestMoreResponseBodyParts(context.executor) + case .finished, .executing(_, _, .finished): - // user is consuming the stream with an iterator return .none } } - mutating func responseBodyIteratorDeinited(streamID: TransactionBody.AsyncIterator.ID) -> FailAction { - switch self.state { - case .initialized, .queued, .deadlineExceededWhileQueued, .executing(_, _, .waitingForResponseHead): - preconditionFailure("Got notice about a deinited response body iterator, before we even received a response. Invalid state: \(self.state)") - - case .executing(_, _, .buffering(let registeredStreamID, _, next: _)), - .executing(_, _, .waitingForRemote(let registeredStreamID, _)): - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - return self.fail(HTTPClientError.cancelled) - - case .executing(_, _, .waitingForResponseIterator), - .executing(_, _, .finished), - .finished: - // the iterator went out of memory after the request was done. nothing to do. - return .none - } - } - - enum ConsumeAction { - case succeedContinuation(CheckedContinuation, ByteBuffer?) - case failContinuation(CheckedContinuation, Error) - case askExecutorForMore(HTTPRequestExecutor) + enum ReceiveResponsePartAction { case none + case yieldResponseBodyParts(TransactionBody.Source, CircularBuffer, HTTPRequestExecutor) } - mutating func consumeNextResponsePart( - streamID: TransactionBody.AsyncIterator.ID, - continuation: CheckedContinuation - ) -> ConsumeAction { + mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponsePartAction { switch self.state { - case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, _, .waitingForResponseHead): - preconditionFailure("If we receive a response body, we must have received a head before") - - case .executing(_, _, .finished): - preconditionFailure("This is an invalid state at this point. We are waiting for the request stream to finish to succeed the response stream. By sending a fi") - - case .executing(let context, let requestState, .waitingForResponseIterator(var buffer, next: .askExecutorForMore)): - if buffer.isEmpty { - self.state = .executing(context, requestState, .waitingForRemote(streamID, continuation)) - return .askExecutorForMore(context.executor) - } else { - let toReturn = buffer.removeFirst() - self.state = .executing(context, requestState, .buffering(streamID, buffer, next: .askExecutorForMore)) - return .succeedContinuation(continuation, toReturn) - } - - case .executing(_, _, .waitingForResponseIterator(_, next: .error(let error))): - self.state = .finished(error: error, streamID) - return .failContinuation(continuation, error) - - case .executing(_, _, .waitingForResponseIterator(let buffer, next: .endOfFile)) where buffer.isEmpty: - self.state = .finished(error: nil, streamID) - return .succeedContinuation(continuation, nil) - - case .executing(let context, let requestState, .waitingForResponseIterator(var buffer, next: .endOfFile)): - assert(!buffer.isEmpty) - let toReturn = buffer.removeFirst() - self.state = .executing(context, requestState, .buffering(streamID, buffer, next: .endOfFile)) - return .succeedContinuation(continuation, toReturn) - - case .executing(let context, let requestState, .buffering(let registeredStreamID, var buffer, next: .askExecutorForMore)): - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - - if buffer.isEmpty { - self.state = .executing(context, requestState, .waitingForRemote(streamID, continuation)) - return .askExecutorForMore(context.executor) - } else { - let toReturn = buffer.removeFirst() - self.state = .executing(context, requestState, .buffering(streamID, buffer, next: .askExecutorForMore)) - return .succeedContinuation(continuation, toReturn) - } - - case .executing(_, _, .buffering(let registeredStreamID, _, next: .error(let error))): - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - self.state = .finished(error: error, registeredStreamID) - return .failContinuation(continuation, error) - - case .executing(_, _, .buffering(let registeredStreamID, let buffer, next: .endOfFile)) where buffer.isEmpty: - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - self.state = .finished(error: nil, registeredStreamID) - return .succeedContinuation(continuation, nil) - - case .executing(let context, let requestState, .buffering(let registeredStreamID, var buffer, next: .endOfFile)): - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - if let toReturn = buffer.popFirst() { - // As long as we have bytes in the local store, we can hand them to the user. - self.state = .executing(context, requestState, .buffering(streamID, buffer, next: .endOfFile)) - return .succeedContinuation(continuation, toReturn) - } - - switch requestState { - case .requestHeadSent, .paused, .producing: - // if the request isn't finished yet, we don't succeed the final response stream - // continuation. We will succeed it once the request has been fully send. - self.state = .executing(context, requestState, .finished(streamID, continuation)) - return .none - case .finished: - // if the request is finished, we can succeed the final continuation. - self.state = .finished(error: nil, streamID) - return .succeedContinuation(continuation, nil) - } - - case .executing(_, _, .waitingForRemote(let registeredStreamID, _)): - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - preconditionFailure("A body response continuation from this iterator already exists! Queuing calls to `next()` is not supported.") + case .initialized, .queued, .deadlineExceededWhileQueued: + preconditionFailure("Received a response body part, but request hasn't started yet. Invalid state: \(self.state)") - case .finished(error: .some(let error), let registeredStreamID): - if let registeredStreamID = registeredStreamID { - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - } else { - self.state = .finished(error: error, streamID) - } - return .failContinuation(continuation, error) + case .executing(_, _, .waitingForResponseHead): + preconditionFailure("If we receive a response body, we must have received a head before") - case .finished(error: .none, let registeredStreamID): - if let registeredStreamID = registeredStreamID { - self.verifyStreamIDIsEqual(registered: registeredStreamID, this: streamID) - } else { - self.state = .finished(error: .none, streamID) - } + case .executing(let context, _, .streamingBody(let source)): + return .yieldResponseBodyParts(source, buffer, context.executor) - return .succeedContinuation(continuation, nil) - } - } + case .finished: + // the request failed or was cancelled before, we can ignore further data + return .none - private func verifyStreamIDIsEqual( - registered: TransactionBody.AsyncIterator.ID, - this: TransactionBody.AsyncIterator.ID, - file: StaticString = #fileID, - line: UInt = #line - ) { - if registered != this { - preconditionFailure( - "Tried to use a second iterator on response body stream. Multiple iterators are not supported.", - file: file, line: line - ) + case .executing(_, _, .finished): + preconditionFailure("Received response end. Must not receive further body parts after that. Invalid state: \(self.state)") } } enum ReceiveResponseEndAction { - case succeedContinuation(CheckedContinuation, ByteBuffer) - case finishResponseStream(CheckedContinuation) + case finishResponseStream(TransactionBody.Source, finalBody: CircularBuffer?) case none } @@ -668,40 +431,13 @@ extension Transaction { .executing(_, _, .waitingForResponseHead): preconditionFailure("Received no response head, but received a response end. Invalid state: \(self.state)") - case .executing(let context, let requestState, .waitingForResponseIterator(var buffer, next: .askExecutorForMore)): - if let newChunks = newChunks, !newChunks.isEmpty { - buffer.append(contentsOf: newChunks) - } - self.state = .executing(context, requestState, .waitingForResponseIterator(buffer, next: .endOfFile)) - return .none - - case .executing(let context, let requestState, .waitingForRemote(let streamID, let continuation)): - if var newChunks = newChunks, !newChunks.isEmpty { - let first = newChunks.removeFirst() - self.state = .executing(context, requestState, .buffering(streamID, newChunks, next: .endOfFile)) - return .succeedContinuation(continuation, first) - } - - self.state = .finished(error: nil, streamID) - return .finishResponseStream(continuation) - - case .executing(let context, let requestState, .buffering(let streamID, var buffer, next: .askExecutorForMore)): - if let newChunks = newChunks, !newChunks.isEmpty { - buffer.append(contentsOf: newChunks) - } - self.state = .executing(context, requestState, .buffering(streamID, buffer, next: .endOfFile)) - return .none - + case .executing(let context, let requestState, .streamingBody(let source)): + self.state = .executing(context, requestState, .finished) + return .finishResponseStream(source, finalBody: newChunks) case .finished: // the request failed or was cancelled before, we can ignore all events return .none - case .executing(_, _, .buffering(_, _, next: .error)): - // we might still get pending buffers if the user has canceled the request - return .none - case .executing(_, _, .waitingForResponseIterator(_, next: .error)), - .executing(_, _, .waitingForResponseIterator(_, next: .endOfFile)), - .executing(_, _, .buffering(_, _, next: .endOfFile)), - .executing(_, _, .finished(_, _)): + case .executing(_, _, .finished): preconditionFailure("Already received an eof or error before. Must not receive further events. Invalid state: \(self.state)") } } @@ -722,7 +458,7 @@ extension Transaction { let error = HTTPClientError.deadlineExceeded switch self.state { case .initialized(let continuation): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .cancel( requestContinuation: continuation, scheduler: nil, @@ -740,7 +476,7 @@ extension Transaction { case .executing(let context, let requestStreamState, .waitingForResponseHead): switch requestStreamState { case .paused(continuation: .some(let continuation)): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .cancel( requestContinuation: context.continuation, scheduler: nil, @@ -748,7 +484,7 @@ extension Transaction { bodyStreamContinuation: continuation ) case .requestHeadSent, .finished, .producing, .paused(continuation: .none): - self.state = .finished(error: error, nil) + self.state = .finished(error: error) return .cancel( requestContinuation: context.continuation, scheduler: nil, diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 8846f36a5..a69285e85 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -48,7 +48,7 @@ import NIOSSL } func cancel() { - self.fail(HTTPClientError.cancelled) + self.fail(CancellationError()) } // MARK: Request body helpers @@ -130,9 +130,8 @@ import NIOSSL // an error/cancellation has happened. nothing to do. break - case .forwardStreamFinished(let executor, let succeedContinuation): + case .forwardStreamFinished(let executor): executor.finishRequestBodyStream(self, promise: nil) - succeedContinuation?.resume(returning: nil) } return } @@ -224,22 +223,22 @@ extension Transaction: HTTPExecutableRequest { func receiveResponseHead(_ head: HTTPResponseHead) { let action = self.stateLock.withLock { - self.state.receiveResponseHead(head) + self.state.receiveResponseHead(head, delegate: self) } switch action { case .none: break - case .succeedResponseHead(let head, let continuation): - let asyncResponse = HTTPClientResponse( - bag: self, + case .succeedResponseHead(let body, let continuation): + let response = HTTPClientResponse( + requestMethod: self.requestHead.method, version: head.version, status: head.status, headers: head.headers, - requestMethod: self.requestHead.method + body: body ) - continuation.resume(returning: asyncResponse) + continuation.resume(returning: response) } } @@ -250,8 +249,13 @@ extension Transaction: HTTPExecutableRequest { switch action { case .none: break - case .succeedContinuation(let continuation, let bytes): - continuation.resume(returning: bytes) + case .yieldResponseBodyParts(let source, let responseBodyParts, let executer): + switch source.yield(contentsOf: responseBodyParts) { + case .dropped, .stopProducing: + break + case .produceMore: + executer.demandResponseBodyStream(self) + } } } @@ -260,10 +264,12 @@ extension Transaction: HTTPExecutableRequest { self.state.succeedRequest(buffer) } switch succeedAction { - case .finishResponseStream(let continuation): - continuation.resume(returning: nil) - case .succeedContinuation(let continuation, let byteBuffer): - continuation.resume(returning: byteBuffer) + case .finishResponseStream(let source, let finalResponse): + if let finalResponse = finalResponse { + _ = source.yield(contentsOf: finalResponse) + } + source.finish() + case .none: break } @@ -287,9 +293,9 @@ extension Transaction: HTTPExecutableRequest { scheduler?.cancelRequest(self) // NOTE: scheduler and executor are exclusive here executor?.cancelRequest(self) - case .failResponseStream(let continuation, let error, let executor, let bodyStreamContinuation): - continuation.resume(throwing: error) - bodyStreamContinuation?.resume(throwing: error) + case .failResponseStream(let source, let error, let executor, let requestBodyStreamContinuation): + source.finish(error) + requestBodyStreamContinuation?.resume(throwing: error) executor.cancelRequest(self) case .failRequestStreamContinuation(let bodyStreamContinuation, let error): @@ -319,46 +325,23 @@ extension Transaction: HTTPExecutableRequest { } } -@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension Transaction { - func responseBodyDeinited() { - let deinitedAction = self.stateLock.withLock { - self.state.responseBodyDeinited() +@available(macOS 10.15, *) +extension Transaction: NIOAsyncSequenceProducerDelegate { + @usableFromInline + func produceMore() { + let action = self.stateLock.withLock { + self.state.produceMore() } - - switch deinitedAction { - case .cancel(let executor): - executor.cancelRequest(self) + switch action { case .none: break + case .requestMoreResponseBodyParts(let executer): + executer.demandResponseBodyStream(self) } } - func nextResponsePart(streamID: TransactionBody.AsyncIterator.ID) async throws -> ByteBuffer? { - try await withCheckedThrowingContinuation { continuation in - let action = self.stateLock.withLock { - self.state.consumeNextResponsePart(streamID: streamID, continuation: continuation) - } - switch action { - case .succeedContinuation(let continuation, let result): - continuation.resume(returning: result) - - case .failContinuation(let continuation, let error): - continuation.resume(throwing: error) - - case .askExecutorForMore(let executor): - executor.demandResponseBodyStream(self) - - case .none: - return - } - } - } - - func responseBodyIteratorDeinited(streamID: TransactionBody.AsyncIterator.ID) { - let action = self.stateLock.withLock { - self.state.responseBodyIteratorDeinited(streamID: streamID) - } - self.performFailAction(action) + @usableFromInline + func didTerminate() { + self.fail(HTTPClientError.cancelled) } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift b/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift deleted file mode 100644 index 23a8e505e..000000000 --- a/Sources/AsyncHTTPClient/AsyncAwait/TransactionBody.swift +++ /dev/null @@ -1,68 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2021-2022 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import NIOCore - -/// This is a class because we need to inform the transaction about the response body being deinitialized. -/// If the users has not called `makeAsyncIterator` on the body, before it is deinited, the http -/// request needs to be cancelled. -@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -@usableFromInline final class TransactionBody: Sendable { - @usableFromInline let transaction: Transaction - @usableFromInline let expectedContentLength: Int? - - init(_ transaction: Transaction, expectedContentLength: Int?) { - self.transaction = transaction - self.expectedContentLength = expectedContentLength - } - - deinit { - self.transaction.responseBodyDeinited() - } -} - -@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -extension TransactionBody: AsyncSequence { - @usableFromInline typealias Element = AsyncIterator.Element - - @usableFromInline final class AsyncIterator: AsyncIteratorProtocol { - @usableFromInline struct ID: Hashable { - private let objectID: ObjectIdentifier - - init(_ object: AsyncIterator) { - self.objectID = ObjectIdentifier(object) - } - } - - @usableFromInline var id: ID { ID(self) } - @usableFromInline let transaction: Transaction - - @inlinable init(transaction: Transaction) { - self.transaction = transaction - } - - deinit { - self.transaction.responseBodyIteratorDeinited(streamID: self.id) - } - - // TODO: this should be @inlinable - @usableFromInline func next() async throws -> ByteBuffer? { - try await self.transaction.nextResponsePart(streamID: self.id) - } - } - - @inlinable func makeAsyncIterator() -> AsyncIterator { - AsyncIterator(transaction: self.transaction) - } -} diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift index 85ac04f5c..3156a2a0d 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift @@ -36,6 +36,7 @@ extension AsyncAwaitEndToEndTests { ("testPostWithFragmentedAsyncSequenceOfByteBuffers", testPostWithFragmentedAsyncSequenceOfByteBuffers), ("testPostWithFragmentedAsyncSequenceOfLargeByteBuffers", testPostWithFragmentedAsyncSequenceOfLargeByteBuffers), ("testCanceling", testCanceling), + ("testCancelingResponseBody", testCancelingResponseBody), ("testDeadline", testDeadline), ("testImmediateDeadline", testImmediateDeadline), ("testConnectTimeout", testConnectTimeout), diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index af99fb0a4..3259fec9a 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -338,11 +338,40 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } task.cancel() await XCTAssertThrowsError(try await task.value) { error in - XCTAssertEqual(error as? HTTPClientError, .cancelled) + XCTAssertTrue(error is CancellationError, "unexpected error \(error)") } } } + func testCancelingResponseBody() { + guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + XCTAsyncTest(timeout: 5) { + let bin = HTTPBin(.http2(compress: false)) { _ in + HTTPEchoHandler() + } + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/handler") + request.method = .POST + let streamWriter = AsyncSequenceWriter() + request.body = .stream(streamWriter, length: .unknown) + let response = try await client.execute(request, deadline: .now() + .seconds(2), logger: logger) + streamWriter.write(.init(bytes: [1])) + let task = Task { + try await response.body.collect(upTo: 1024 * 1024) + } + task.cancel() + + await XCTAssertThrowsError(try await task.value) { error in + XCTAssertTrue(error is CancellationError, "unexpected error \(error)") + } + + streamWriter.end() + } + } + func testDeadline() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift index bb3a2f03e..3420f2ebd 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift @@ -18,6 +18,11 @@ import NIOEmbedded import NIOHTTP1 import XCTest +struct NoOpAsyncSequenceProducerDelegate: NIOAsyncSequenceProducerDelegate { + func produceMore() {} + func didTerminate() {} +} + final class Transaction_StateMachineTests: XCTestCase { func testRequestWasQueuedAfterWillExecuteRequestWasCalled() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } @@ -204,8 +209,8 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertEqual(state.willExecuteRequest(executor), .none) state.requestWasQueued(queuer) let head = HTTPResponseHead(version: .http1_1, status: .ok) - let receiveResponseHeadAction = state.receiveResponseHead(head) - guard case .succeedResponseHead(head, let continuation) = receiveResponseHeadAction else { + let receiveResponseHeadAction = state.receiveResponseHead(head, delegate: NoOpAsyncSequenceProducerDelegate()) + guard case .succeedResponseHead(_, let continuation) = receiveResponseHeadAction else { return XCTFail("Unexpected action: \(receiveResponseHeadAction)") } diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 1ef0bfed1..79a0b83af 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -52,8 +52,8 @@ final class TransactionTests: XCTestCase { } XCTAssertEqual(queuer.hitCancelCount, 0) - await XCTAssertThrowsError(try await responseTask.value) { - XCTAssertEqual($0 as? HTTPClientError, .cancelled) + await XCTAssertThrowsError(try await responseTask.value) { error in + XCTAssertTrue(error is CancellationError, "unexpected error \(error)") } XCTAssertEqual(queuer.hitCancelCount, 1) } @@ -146,9 +146,9 @@ final class TransactionTests: XCTestCase { let iterator = SharedIterator(response.body.filter { $0.readableBytes > 0 }) - for i in 0..<100 { - XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") + XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") + for i in 0..<100 { async let part = iterator.next() XCTAssertNoThrow(try executor.receiveResponseDemand()) @@ -159,7 +159,6 @@ final class TransactionTests: XCTestCase { XCTAssertEqual(result, ByteBuffer(integer: i)) } - XCTAssertFalse(executor.signalledDemandForResponseBody, "Demand was not signalled yet.") async let part = iterator.next() XCTAssertNoThrow(try executor.receiveResponseDemand()) executor.resetResponseStreamDemandSignal() From 5b4f03de0600da906f9e46e9f636ec26218da080 Mon Sep 17 00:00:00 2001 From: Yim Lee Date: Wed, 12 Apr 2023 17:52:06 -0700 Subject: [PATCH 076/146] Add docker-compose file for Swift 5.9 (#682) --- docker/docker-compose.2204.58.yaml | 3 ++- docker/docker-compose.2204.59.yaml | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 docker/docker-compose.2204.59.yaml diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index 2d3a300d8..89b410ae2 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -6,7 +6,8 @@ services: image: async-http-client:22.04-5.8 build: args: - base_image: "swiftlang/swift:nightly-5.8-jammy" + ubuntu_version: "jammy" + swift_version: "5.8" documentation-check: image: async-http-client:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml new file mode 100644 index 000000000..2c5a9e297 --- /dev/null +++ b/docker/docker-compose.2204.59.yaml @@ -0,0 +1,21 @@ +version: "3" + +services: + + runtime-setup: + image: async-http-client:22.04-5.9 + build: + args: + base_image: "swiftlang/swift:nightly-5.9-jammy" + + documentation-check: + image: async-http-client:22.04-5.9 + + test: + image: async-http-client:22.04-5.9 + environment: + - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error + #- SANITIZER_ARG=--sanitize=thread + + shell: + image: async-http-client:22.04-5.9 From 45626d33c75ba879ef0b44912175ab9dd38168f6 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 13 Apr 2023 11:21:13 +0200 Subject: [PATCH 077/146] Pass request `Task` to `FileDownloadDelegate` `reportHead` and `reportProgress` closures (#681) ### Motivation If the HTTP response status is not 2xx or the file being downloaded becomes too big, it is desirable to cancel the file download eagerly. This is currently quite hard because the `reportHead` and `reportProgress` closures do not have direct access to the `HTTPClient.Task`. ### Modifications - Pass `HTTPClient.Task` additionally to `reportHead` and `reportProgress` closures ### Result A file download can now easily be cancelled at any time. --- .../FileDownloadDelegate.swift | 88 +++++++++++++------ 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index c328c7211..9a351f3c1 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -31,8 +31,8 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { private let filePath: String private(set) var fileIOThreadPool: NIOThreadPool? - private let reportHead: ((HTTPResponseHead) -> Void)? - private let reportProgress: ((Progress) -> Void)? + private let reportHead: ((HTTPClient.Task, HTTPResponseHead) -> Void)? + private let reportProgress: ((HTTPClient.Task, Progress) -> Void)? private var fileHandleFuture: EventLoopFuture? private var writeFuture: EventLoopFuture? @@ -41,25 +41,37 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { /// /// - parameters: /// - path: Path to a file you'd like to write the download to. - /// - pool: A thread pool to use for asynchronous file I/O. + /// - pool: A thread pool to use for asynchronous file I/O. If nil, a shared thread pool will be used. Defaults to nil. /// - reportHead: A closure called when the response head is available. /// - reportProgress: A closure called when a body chunk has been downloaded, with /// the total byte count and download byte count passed to it as arguments. The callbacks /// will be invoked in the same threading context that the delegate itself is invoked, /// as controlled by `EventLoopPreference`. - public convenience init( + public init( path: String, - pool: NIOThreadPool, - reportHead: ((HTTPResponseHead) -> Void)? = nil, - reportProgress: ((Progress) -> Void)? = nil + pool: NIOThreadPool? = nil, + reportHead: ((HTTPClient.Task, HTTPResponseHead) -> Void)? = nil, + reportProgress: ((HTTPClient.Task, Progress) -> Void)? = nil ) throws { - try self.init(path: path, pool: .some(pool), reportHead: reportHead, reportProgress: reportProgress) + if let pool = pool { + self.fileIOThreadPool = pool + } else { + // we should use the shared thread pool from the HTTPClient which + // we will get from the `HTTPClient.Task` + self.fileIOThreadPool = nil + } + + self.filePath = path + + self.reportHead = reportHead + self.reportProgress = reportProgress } - /// Initializes a new file download delegate and uses the shared thread pool of the ``HTTPClient`` for file I/O. + /// Initializes a new file download delegate. /// /// - parameters: /// - path: Path to a file you'd like to write the download to. + /// - pool: A thread pool to use for asynchronous file I/O. /// - reportHead: A closure called when the response head is available. /// - reportProgress: A closure called when a body chunk has been downloaded, with /// the total byte count and download byte count passed to it as arguments. The callbacks @@ -67,37 +79,61 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { /// as controlled by `EventLoopPreference`. public convenience init( path: String, + pool: NIOThreadPool, reportHead: ((HTTPResponseHead) -> Void)? = nil, reportProgress: ((Progress) -> Void)? = nil ) throws { - try self.init(path: path, pool: nil, reportHead: reportHead, reportProgress: reportProgress) + try self.init( + path: path, + pool: .some(pool), + reportHead: reportHead.map { reportHead in + return { _, head in + reportHead(head) + } + }, + reportProgress: reportProgress.map { reportProgress in + return { _, head in + reportProgress(head) + } + } + ) } - private init( + /// Initializes a new file download delegate and uses the shared thread pool of the ``HTTPClient`` for file I/O. + /// + /// - parameters: + /// - path: Path to a file you'd like to write the download to. + /// - reportHead: A closure called when the response head is available. + /// - reportProgress: A closure called when a body chunk has been downloaded, with + /// the total byte count and download byte count passed to it as arguments. The callbacks + /// will be invoked in the same threading context that the delegate itself is invoked, + /// as controlled by `EventLoopPreference`. + public convenience init( path: String, - pool: NIOThreadPool?, reportHead: ((HTTPResponseHead) -> Void)? = nil, reportProgress: ((Progress) -> Void)? = nil ) throws { - if let pool = pool { - self.fileIOThreadPool = pool - } else { - // we should use the shared thread pool from the HTTPClient which - // we will get from the `HTTPClient.Task` - self.fileIOThreadPool = nil - } - - self.filePath = path - - self.reportHead = reportHead - self.reportProgress = reportProgress + try self.init( + path: path, + pool: nil, + reportHead: reportHead.map { reportHead in + return { _, head in + reportHead(head) + } + }, + reportProgress: reportProgress.map { reportProgress in + return { _, head in + reportProgress(head) + } + } + ) } public func didReceiveHead( task: HTTPClient.Task, _ head: HTTPResponseHead ) -> EventLoopFuture { - self.reportHead?(head) + self.reportHead?(task, head) if let totalBytesString = head.headers.first(name: "Content-Length"), let totalBytes = Int(totalBytesString) { @@ -121,7 +157,7 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { }() let io = NonBlockingFileIO(threadPool: threadPool) self.progress.receivedBytes += buffer.readableBytes - self.reportProgress?(self.progress) + self.reportProgress?(task, self.progress) let writeFuture: EventLoopFuture if let fileHandleFuture = self.fileHandleFuture { From b9029ef67cfa2e816a22cd3a2b210be2148340e4 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Thu, 13 Apr 2023 18:00:03 +0200 Subject: [PATCH 078/146] Add support for custom cancellation error (#683) --- Sources/AsyncHTTPClient/HTTPHandler.swift | 10 ++++++-- Sources/AsyncHTTPClient/RequestBag.swift | 8 +----- .../HTTP1ClientChannelHandlerTests.swift | 2 +- .../HTTP2ClientRequestHandlerTests.swift | 2 +- .../HTTPClientTests+XCTest.swift | 1 + .../HTTPClientTests.swift | 25 +++++++++++++++++++ .../HTTPConnectionPoolTests.swift | 2 +- .../RequestBagTests.swift | 12 ++++----- 8 files changed, 44 insertions(+), 18 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 614c55f3a..0c84ef6bc 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -688,7 +688,7 @@ extension URL { } protocol HTTPClientTaskDelegate { - func cancel() + func fail(_ error: Error) } extension HTTPClient { @@ -780,12 +780,18 @@ extension HTTPClient { /// Cancels the request execution. public func cancel() { + self.fail(reason: HTTPClientError.cancelled) + } + + /// Cancels the request execution with a custom `Error`. + /// - Parameter reason: the error that is used to fail the promise + public func fail(reason error: Error) { let taskDelegate = self.lock.withLock { () -> HTTPClientTaskDelegate? in self._isCancelled = true return self._taskDelegate } - taskDelegate?.cancel() + taskDelegate?.fail(error) } func succeed(promise: EventLoopPromise?, diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index 2b20193b4..c5472fc6f 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -394,7 +394,7 @@ final class RequestBag { } } -extension RequestBag: HTTPSchedulableRequest { +extension RequestBag: HTTPSchedulableRequest, HTTPClientTaskDelegate { var tlsConfiguration: TLSConfiguration? { self.request.tlsConfiguration } @@ -511,9 +511,3 @@ extension RequestBag: HTTPExecutableRequest { } } } - -extension RequestBag: HTTPClientTaskDelegate { - func cancel() { - self.fail(HTTPClientError.cancelled) - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index bdf897b3d..2aa010491 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -327,7 +327,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { XCTAssertNoThrow(try embedded.writeInbound(HTTPClientResponsePart.head(responseHead))) // canceling the request - requestBag.cancel() + requestBag.fail(HTTPClientError.cancelled) XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { XCTAssertEqual($0 as? HTTPClientError, .cancelled) } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index c0e5b6054..2b68fceb3 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -276,7 +276,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { XCTAssertNoThrow(try embedded.writeInbound(HTTPClientResponsePart.head(responseHead))) // canceling the request - requestBag.cancel() + requestBag.fail(HTTPClientError.cancelled) XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { XCTAssertEqual($0 as? HTTPClientError, .cancelled) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift index 06324d3fc..8f292239e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift @@ -52,6 +52,7 @@ extension HTTPClientTests { ("testStreaming", testStreaming), ("testFileDownload", testFileDownload), ("testFileDownloadError", testFileDownloadError), + ("testFileDownloadCustomError", testFileDownloadCustomError), ("testRemoteClose", testRemoteClose), ("testReadTimeout", testReadTimeout), ("testConnectTimeout", testConnectTimeout), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 70a44ed07..8cd0b3bba 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -569,6 +569,31 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(0, progress.receivedBytes) } + func testFileDownloadCustomError() throws { + let request = try Request(url: self.defaultHTTPBinURLPrefix + "get") + struct CustomError: Equatable, Error {} + + try TemporaryFileHelpers.withTemporaryFilePath { path in + let delegate = try FileDownloadDelegate(path: path, reportHead: { task, head in + XCTAssertEqual(head.status, .ok) + task.fail(reason: CustomError()) + }, reportProgress: { _, _ in + XCTFail("should never be called") + }) + XCTAssertThrowsError( + try self.defaultClient.execute( + request: request, + delegate: delegate + ) + .wait() + ) { error in + XCTAssertEqualTypeAndValue(error, CustomError()) + } + + XCTAssertFalse(TemporaryFileHelpers.fileExists(path: path)) + } + } + func testRemoteClose() { XCTAssertThrowsError(try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "close").wait()) { XCTAssertEqual($0 as? HTTPClientError, .remoteConnectionClosed) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift index e85571ad1..2cf222afe 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift @@ -334,7 +334,7 @@ class HTTPConnectionPoolTests: XCTestCase { pool.executeRequest(requestBag) XCTAssertNoThrow(try eventLoop.scheduleTask(in: .seconds(1)) {}.futureResult.wait()) - requestBag.cancel() + requestBag.fail(HTTPClientError.cancelled) XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { XCTAssertEqual($0 as? HTTPClientError, .cancelled) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 36efee949..54de39e12 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -220,7 +220,7 @@ final class RequestBagTests: XCTestCase { XCTAssert(bag.eventLoop === embeddedEventLoop) let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) - bag.cancel() + bag.fail(HTTPClientError.cancelled) bag.willExecuteRequest(executor) XCTAssertTrue(executor.isCancelled, "The request bag, should call cancel immediately on the executor") @@ -301,7 +301,7 @@ final class RequestBagTests: XCTestCase { bag.fail(MyError()) XCTAssertEqual(delegate.hitDidReceiveError, 1) - bag.cancel() + bag.fail(HTTPClientError.cancelled) XCTAssertEqual(delegate.hitDidReceiveError, 1) XCTAssertThrowsError(try bag.task.futureResult.wait()) { @@ -342,7 +342,7 @@ final class RequestBagTests: XCTestCase { XCTAssertEqual(delegate.hitDidSendRequestHead, 1) XCTAssertEqual(delegate.hitDidSendRequest, 1) - bag.cancel() + bag.fail(HTTPClientError.cancelled) XCTAssertTrue(executor.isCancelled, "The request bag, should call cancel immediately on the executor") XCTAssertThrowsError(try bag.task.futureResult.wait()) { @@ -376,7 +376,7 @@ final class RequestBagTests: XCTestCase { bag.requestWasQueued(queuer) XCTAssertEqual(queuer.hitCancelCount, 0) - bag.cancel() + bag.fail(HTTPClientError.cancelled) XCTAssertEqual(queuer.hitCancelCount, 1) XCTAssertThrowsError(try bag.task.futureResult.wait()) { @@ -445,9 +445,9 @@ final class RequestBagTests: XCTestCase { let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) executor.runRequest(bag) - // This simulates a race between the user cancelling the task (which invokes `RequestBag.cancel`) and the + // This simulates a race between the user cancelling the task (which invokes `RequestBag.fail(_:)`) and the // call to `resumeRequestBodyStream` (which comes from the `Channel` event loop and so may have to hop. - bag.cancel() + bag.fail(HTTPClientError.cancelled) bag.resumeRequestBodyStream() XCTAssertEqual(executor.isCancelled, true) From 333e60cc90f52973f7ee29cd8e3a7f6adfe79f4e Mon Sep 17 00:00:00 2001 From: Corey Date: Fri, 14 Apr 2023 09:03:14 -0400 Subject: [PATCH 079/146] Fix building transaction extension for Apple Platforms (#685) --- Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index a69285e85..d3793c990 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -325,7 +325,7 @@ extension Transaction: HTTPExecutableRequest { } } -@available(macOS 10.15, *) +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction: NIOAsyncSequenceProducerDelegate { @usableFromInline func produceMore() { From d62c475401df66ad65fc652c625fa8c43f505338 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 14 Apr 2023 18:05:09 +0200 Subject: [PATCH 080/146] Drop Swift 5.5 (#686) --- Package@swift-5.5.swift | 75 ------ README.md | 5 +- .../AsyncAwait/HTTPClientRequest.swift | 110 -------- .../HTTP1/HTTP1ClientChannelHandler.swift | 2 - Sources/AsyncHTTPClient/Docs.docc/index.md | 6 - Sources/AsyncHTTPClient/HTTPClient.swift | 18 -- Sources/AsyncHTTPClient/HTTPHandler.swift | 32 +-- Sources/AsyncHTTPClient/Utils.swift | 6 - .../AsyncAwaitEndToEndTests+XCTest.swift | 58 ----- ...zeConfigValueIsRespectedTests+XCTest.swift | 31 --- ...TTP1ClientChannelHandlerTests+XCTest.swift | 39 --- ...P1ConnectionStateMachineTests+XCTest.swift | 48 ---- .../HTTP1ConnectionTests+XCTest.swift | 42 ---- ...HTTP1ProxyConnectHandlerTests+XCTest.swift | 35 --- ...TTP2ClientRequestHandlerTests+XCTest.swift | 36 --- .../HTTP2ClientTests+XCTest.swift | 43 ---- .../HTTP2ConnectionTests+XCTest.swift | 36 --- .../HTTP2IdleHandlerTests+XCTest.swift | 42 ---- .../HTTPClient+SOCKSTests+XCTest.swift | 35 --- .../HTTPClientCookieTests+XCTest.swift | 44 ---- ...ntInformationalResponsesTests+XCTest.swift | 32 --- .../HTTPClientInternalTests+XCTest.swift | 42 ---- .../HTTPClientNIOTSTests+XCTest.swift | 36 --- .../HTTPClientRequestTests+XCTest.swift | 43 ---- .../HTTPClientResponseTests+XCTest.swift | 35 --- .../HTTPClientTests+XCTest.swift | 156 ------------ ...eanSSLConnectionShutdownTests+XCTest.swift | 36 --- ...TPConnectionPool+FactoryTests+XCTest.swift | 34 --- ...tionPool+HTTP1ConnectionsTest+XCTest.swift | 47 ---- ...onnectionPool+HTTP1StateTests+XCTest.swift | 47 ---- ...tionPool+HTTP2ConnectionsTest+XCTest.swift | 49 ---- ...onPool+HTTP2StateMachineTests+XCTest.swift | 51 ---- ...TPConnectionPool+ManagerTests+XCTest.swift | 33 --- ...nectionPool+RequestQueueTests+XCTest.swift | 31 --- .../HTTPConnectionPoolTests+XCTest.swift | 39 --- .../HTTPRequestStateMachineTests+XCTest.swift | 67 ----- .../IdleTimeoutNoReuseTests+XCTest.swift | 31 --- .../LRUCacheTests+XCTest.swift | 33 --- ...NoBytesSentOverBodyLimitTests+XCTest.swift | 31 --- ...oolIdleConnectionsAndGetTests+XCTest.swift | 31 --- .../RequestBagTests+XCTest.swift | 46 ---- .../RequestValidationTests+XCTest.swift | 52 ---- .../ResponseDelayGetTests+XCTest.swift | 31 --- .../SOCKSEventsHandlerTests+XCTest.swift | 35 --- .../SSLContextCacheTests+XCTest.swift | 33 --- .../StressGetHttpsTests+XCTest.swift | 31 --- .../TLSEventsHandlerTests+XCTest.swift | 34 --- ...Transaction+StateMachineTests+XCTest.swift | 37 --- .../TransactionTests+XCTest.swift | 40 --- .../TransactionTests.swift | 5 +- Tests/LinuxMain.swift | 76 ------ docker/docker-compose.2004.55.yaml | 22 -- scripts/generate_linux_tests.rb | 236 ------------------ scripts/soundness.sh | 12 - 54 files changed, 5 insertions(+), 2332 deletions(-) delete mode 100644 Package@swift-5.5.swift delete mode 100644 Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP1ConnectionTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientCookieTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientRequestTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/LRUCacheTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/RequestValidationTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/SSLContextCacheTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/TLSEventsHandlerTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift delete mode 100644 Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift delete mode 100644 Tests/LinuxMain.swift delete mode 100644 docker/docker-compose.2004.55.yaml delete mode 100755 scripts/generate_linux_tests.rb diff --git a/Package@swift-5.5.swift b/Package@swift-5.5.swift deleted file mode 100644 index 0ad7d1f3b..000000000 --- a/Package@swift-5.5.swift +++ /dev/null @@ -1,75 +0,0 @@ -// swift-tools-version:5.5 -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -import PackageDescription - -let package = Package( - name: "async-http-client", - products: [ - .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), - ], - dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.50.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), - .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), - .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), - .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), - .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), - .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), - ], - targets: [ - .target(name: "CAsyncHTTPClient"), - .target( - name: "AsyncHTTPClient", - dependencies: [ - .target(name: "CAsyncHTTPClient"), - .product(name: "NIO", package: "swift-nio"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOPosix", package: "swift-nio"), - .product(name: "NIOHTTP1", package: "swift-nio"), - .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOHTTP2", package: "swift-nio-http2"), - .product(name: "NIOSSL", package: "swift-nio-ssl"), - .product(name: "NIOHTTPCompression", package: "swift-nio-extras"), - .product(name: "NIOSOCKS", package: "swift-nio-extras"), - .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), - .product(name: "Logging", package: "swift-log"), - .product(name: "Atomics", package: "swift-atomics"), - ] - ), - .testTarget( - name: "AsyncHTTPClientTests", - dependencies: [ - .target(name: "AsyncHTTPClient"), - .product(name: "NIOCore", package: "swift-nio"), - .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), - .product(name: "NIOEmbedded", package: "swift-nio"), - .product(name: "NIOFoundationCompat", package: "swift-nio"), - .product(name: "NIOTestUtils", package: "swift-nio"), - .product(name: "NIOSSL", package: "swift-nio-ssl"), - .product(name: "NIOHTTP2", package: "swift-nio-http2"), - .product(name: "NIOSOCKS", package: "swift-nio-extras"), - .product(name: "Logging", package: "swift-log"), - .product(name: "Atomics", package: "swift-atomics"), - ], - resources: [ - .copy("Resources/self_signed_cert.pem"), - .copy("Resources/self_signed_key.pem"), - .copy("Resources/example.com.cert.pem"), - .copy("Resources/example.com.private-key.pem"), - ] - ), - ] -) diff --git a/README.md b/README.md index 27354d8da..e969c54ff 100644 --- a/README.md +++ b/README.md @@ -323,11 +323,12 @@ Please have a look at [SECURITY.md](SECURITY.md) for AsyncHTTPClient's security ## Supported Versions -The most recent versions of AsyncHTTPClient support Swift 5.5.2 and newer. The minimum Swift version supported by AsyncHTTPClient releases are detailed below: +The most recent versions of AsyncHTTPClient support Swift 5.6 and newer. The minimum Swift version supported by AsyncHTTPClient releases are detailed below: AsyncHTTPClient | Minimum Swift Version --------------------|---------------------- `1.0.0 ..< 1.5.0` | 5.0 `1.5.0 ..< 1.10.0` | 5.2 `1.10.0 ..< 1.13.0` | 5.4 -`1.13.0 ...` | 5.5.2 +`1.13.0 ..< 1.18.0` | 5.5.2 +`1.18.0 ...` | 5.6 diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index 6f3637f77..278be7f84 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -91,7 +91,6 @@ extension HTTPClientRequest.Body { self.init(.byteBuffer(byteBuffer)) } - #if swift(>=5.6) /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -106,21 +105,6 @@ extension HTTPClientRequest.Body { ) -> Self where Bytes.Element == UInt8 { Self._bytes(bytes) } - #else - /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. - /// - /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory - /// usage of this construction will be double the size of the original collection. The construction - /// of the `ByteBuffer` will be delayed until it's needed. - /// - /// - parameter bytes: The bytes of the request body. - @inlinable - public static func bytes( - _ bytes: Bytes - ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes) - } - #endif @inlinable static func _bytes( @@ -139,7 +123,6 @@ extension HTTPClientRequest.Body { }) } - #if swift(>=5.6) /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -165,32 +148,6 @@ extension HTTPClientRequest.Body { ) -> Self where Bytes.Element == UInt8 { Self._bytes(bytes, length: length) } - #else - /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. - /// - /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory - /// usage of this construction will be double the size of the original collection. The construction - /// of the `ByteBuffer` will be delayed until it's needed. - /// - /// Unlike ``bytes(_:)-1uns7``, this construction does not assume that the body can be replayed. As a result, - /// if a redirect is encountered that would need us to replay the request body, the redirect will instead - /// not be followed. Prefer ``bytes(_:)-1uns7`` wherever possible. - /// - /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths - /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. - /// - /// - parameters: - /// - bytes: The bytes of the request body. - /// - length: The length of the request body. - @inlinable - public static func bytes( - _ bytes: Bytes, - length: Length - ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes, length: length) - } - #endif @inlinable static func _bytes( @@ -210,7 +167,6 @@ extension HTTPClientRequest.Body { }) } - #if swift(>=5.6) /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. /// /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory @@ -232,28 +188,6 @@ extension HTTPClientRequest.Body { ) -> Self where Bytes.Element == UInt8 { Self._bytes(bytes, length: length) } - #else - /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. - /// - /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory - /// usage of this construction will be double the size of the original collection. The construction - /// of the `ByteBuffer` will be delayed until it's needed. - /// - /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths - /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. - /// - /// - parameters: - /// - bytes: The bytes of the request body. - /// - length: The length of the request body. - @inlinable - public static func bytes( - _ bytes: Bytes, - length: Length - ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes, length: length) - } - #endif @inlinable static func _bytes( @@ -273,7 +207,6 @@ extension HTTPClientRequest.Body { }) } - #if swift(>=5.6) /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. /// /// This construction will stream the upload one `ByteBuffer` at a time. @@ -293,26 +226,6 @@ extension HTTPClientRequest.Body { ) -> Self where SequenceOfBytes.Element == ByteBuffer { Self._stream(sequenceOfBytes, length: length) } - #else - /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. - /// - /// This construction will stream the upload one `ByteBuffer` at a time. - /// - /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths - /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. - /// - /// - parameters: - /// - sequenceOfBytes: The bytes of the request body. - /// - length: The length of the request body. - @inlinable - public static func stream( - _ sequenceOfBytes: SequenceOfBytes, - length: Length - ) -> Self where SequenceOfBytes.Element == ByteBuffer { - Self._stream(sequenceOfBytes, length: length) - } - #endif @inlinable static func _stream( @@ -328,7 +241,6 @@ extension HTTPClientRequest.Body { return body } - #if swift(>=5.6) /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. /// /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for @@ -350,28 +262,6 @@ extension HTTPClientRequest.Body { ) -> Self where Bytes.Element == UInt8 { Self._stream(bytes, length: length) } - #else - /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. - /// - /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for - /// `AsyncSequence`s where larger chunks are buffered up and available without actually suspending, such - /// as those provided by `FileHandle`. - /// - /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths - /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. - /// - /// - parameters: - /// - bytes: The bytes of the request body. - /// - length: The length of the request body. - @inlinable - public static func stream( - _ bytes: Bytes, - length: Length - ) -> Self where Bytes.Element == UInt8 { - Self._stream(bytes, length: length) - } - #endif @inlinable static func _stream( diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 8af70ac23..63cb70b99 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -433,10 +433,8 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } } -#if swift(>=5.6) @available(*, unavailable) extension HTTP1ClientChannelHandler: Sendable {} -#endif extension HTTP1ClientChannelHandler: HTTPRequestExecutor { func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md index 3a05778ce..66a9d1135 100644 --- a/Sources/AsyncHTTPClient/Docs.docc/index.md +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -13,12 +13,6 @@ This library provides the following: - Automatic HTTP/2 over HTTPS (since version 1.7.0) - Cookie parsing (but not storage) ---- - -**NOTE**: You will need [Xcode 13.2](https://apps.apple.com/gb/app/xcode/id497799835?mt=12) or [Swift 5.5.2](https://swift.org/download/#swift-552) to try out `AsyncHTTPClient`s new async/await APIs. - ---- - ### Getting Started #### Adding the dependency diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index a916d3ade..de6b57087 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -200,7 +200,6 @@ public class HTTPClient { } } - #if swift(>=5.6) /// Shuts down the client and event loop gracefully. /// /// This function is clearly an outlier in that it uses a completion @@ -213,17 +212,6 @@ public class HTTPClient { ) { self.shutdown(requiresCleanClose: false, queue: queue, callback) } - #else - /// Shuts down the client and event loop gracefully. - /// - /// This function is clearly an outlier in that it uses a completion - /// callback instead of an EventLoopFuture. The reason for that is that NIO's EventLoopFutures will call back on an event loop. - /// The virtue of this function is to shut the event loop down. To work around that we call back on a DispatchQueue - /// instead. - public func shutdown(queue: DispatchQueue = .global(), _ callback: @escaping (Error?) -> Void) { - self.shutdown(requiresCleanClose: false, queue: queue, callback) - } - #endif /// Shuts down the ``HTTPClient`` and releases its resources. /// @@ -917,11 +905,7 @@ public class HTTPClient { case enabled(limit: NIOHTTPDecompression.DecompressionLimit) } - #if swift(>=5.6) typealias ShutdownCallback = @Sendable (Error?) -> Void - #else - typealias ShutdownCallback = (Error?) -> Void - #endif enum State { case upAndRunning @@ -934,10 +918,8 @@ public class HTTPClient { extension HTTPClient.Configuration: Sendable {} #endif -#if swift(>=5.6) extension HTTPClient.EventLoopGroupProvider: Sendable {} extension HTTPClient.EventLoopPreference: Sendable {} -#endif // HTTPClient is thread-safe because its shared mutable state is protected through a lock extension HTTPClient: @unchecked Sendable {} diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 0c84ef6bc..c5a3b3a4a 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -49,16 +49,11 @@ extension HTTPClient { /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `length`. public var length: Int? - #if swift(>=5.6) + /// Body chunk provider. public var stream: @Sendable (StreamWriter) -> EventLoopFuture @usableFromInline typealias StreamCallback = @Sendable (StreamWriter) -> EventLoopFuture - #else - public var stream: (StreamWriter) -> EventLoopFuture - - @usableFromInline typealias StreamCallback = (StreamWriter) -> EventLoopFuture - #endif @inlinable init(length: Int?, stream: @escaping StreamCallback) { @@ -76,7 +71,6 @@ extension HTTPClient { } } - #if swift(>=5.6) /// Create and stream body using ``StreamWriter``. /// /// - parameters: @@ -87,19 +81,7 @@ extension HTTPClient { public static func stream(length: Int? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { return Body(length: length, stream: stream) } - #else - /// Create and stream body using ``StreamWriter``. - /// - /// - parameters: - /// - length: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` - /// header is set with the given `length`. - /// - stream: Body chunk provider. - public static func stream(length: Int? = nil, _ stream: @escaping (StreamWriter) -> EventLoopFuture) -> Body { - return Body(length: length, stream: stream) - } - #endif - #if swift(>=5.6) /// Create and stream body using a collection of bytes. /// /// - parameters: @@ -111,18 +93,6 @@ extension HTTPClient { writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) } } - #else - /// Create and stream body using a collection of bytes. - /// - /// - parameters: - /// - data: Body binary representation. - @inlinable - public static func bytes(_ bytes: Bytes) -> Body where Bytes: RandomAccessCollection, Bytes.Element == UInt8 { - return Body(length: bytes.count) { writer in - writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) - } - } - #endif /// Create and stream body using `String`. /// diff --git a/Sources/AsyncHTTPClient/Utils.swift b/Sources/AsyncHTTPClient/Utils.swift index 3bbb97904..f8618ea17 100644 --- a/Sources/AsyncHTTPClient/Utils.swift +++ b/Sources/AsyncHTTPClient/Utils.swift @@ -23,16 +23,10 @@ public final class HTTPClientCopyingDelegate: HTTPClientResponseDelegate { let chunkHandler: (ByteBuffer) -> EventLoopFuture - #if swift(>=5.6) @preconcurrency public init(chunkHandler: @Sendable @escaping (ByteBuffer) -> EventLoopFuture) { self.chunkHandler = chunkHandler } - #else - public init(chunkHandler: @escaping (ByteBuffer) -> EventLoopFuture) { - self.chunkHandler = chunkHandler - } - #endif public func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { return self.chunkHandler(buffer) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift deleted file mode 100644 index 3156a2a0d..000000000 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests+XCTest.swift +++ /dev/null @@ -1,58 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// AsyncAwaitEndToEndTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension AsyncAwaitEndToEndTests { - static var allTests: [(String, (AsyncAwaitEndToEndTests) -> () throws -> Void)] { - return [ - ("testSimpleGet", testSimpleGet), - ("testSimplePost", testSimplePost), - ("testPostWithByteBuffer", testPostWithByteBuffer), - ("testPostWithSequenceOfUInt8", testPostWithSequenceOfUInt8), - ("testPostWithCollectionOfUInt8", testPostWithCollectionOfUInt8), - ("testPostWithRandomAccessCollectionOfUInt8", testPostWithRandomAccessCollectionOfUInt8), - ("testPostWithAsyncSequenceOfByteBuffers", testPostWithAsyncSequenceOfByteBuffers), - ("testPostWithAsyncSequenceOfUInt8", testPostWithAsyncSequenceOfUInt8), - ("testPostWithFragmentedAsyncSequenceOfByteBuffers", testPostWithFragmentedAsyncSequenceOfByteBuffers), - ("testPostWithFragmentedAsyncSequenceOfLargeByteBuffers", testPostWithFragmentedAsyncSequenceOfLargeByteBuffers), - ("testCanceling", testCanceling), - ("testCancelingResponseBody", testCancelingResponseBody), - ("testDeadline", testDeadline), - ("testImmediateDeadline", testImmediateDeadline), - ("testConnectTimeout", testConnectTimeout), - ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), - ("testDnsOverride", testDnsOverride), - ("testInvalidURL", testInvalidURL), - ("testRedirectChangesHostHeader", testRedirectChangesHostHeader), - ("testShutdown", testShutdown), - ("testCancelingBodyDoesNotCrash", testCancelingBodyDoesNotCrash), - ("testAsyncSequenceReuse", testAsyncSequenceReuse), - ("testRejectsInvalidCharactersInHeaderFieldNames_http1", testRejectsInvalidCharactersInHeaderFieldNames_http1), - ("testRejectsInvalidCharactersInHeaderFieldNames_http2", testRejectsInvalidCharactersInHeaderFieldNames_http2), - ("testRejectsInvalidCharactersInHeaderFieldValues_http1", testRejectsInvalidCharactersInHeaderFieldValues_http1), - ("testRejectsInvalidCharactersInHeaderFieldValues_http2", testRejectsInvalidCharactersInHeaderFieldValues_http2), - ("testUsingGetMethodInsteadOfWait", testUsingGetMethodInsteadOfWait), - ("testSimpleContentLengthErrorNoBody", testSimpleContentLengthErrorNoBody), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift deleted file mode 100644 index f76fea3c4..000000000 --- a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// ConnectionPoolSizeConfigValueIsRespectedTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension ConnectionPoolSizeConfigValueIsRespectedTests { - static var allTests: [(String, (ConnectionPoolSizeConfigValueIsRespectedTests) -> () throws -> Void)] { - return [ - ("testConnectionPoolSizeConfigValueIsRespected", testConnectionPoolSizeConfigValueIsRespected), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift deleted file mode 100644 index 2502e6fb7..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests+XCTest.swift +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP1ClientChannelHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP1ClientChannelHandlerTests { - static var allTests: [(String, (HTTP1ClientChannelHandlerTests) -> () throws -> Void)] { - return [ - ("testResponseBackpressure", testResponseBackpressure), - ("testWriteBackpressure", testWriteBackpressure), - ("testClientHandlerCancelsRequestIfWeWantToShutdown", testClientHandlerCancelsRequestIfWeWantToShutdown), - ("testIdleReadTimeout", testIdleReadTimeout), - ("testIdleReadTimeoutIsCanceledIfRequestIsCanceled", testIdleReadTimeoutIsCanceledIfRequestIsCanceled), - ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand), - ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), - ("testHandlerClosesChannelIfLastActionIsSendEndAndItFails", testHandlerClosesChannelIfLastActionIsSendEndAndItFails), - ("testChannelBecomesNonWritableDuringHeaderWrite", testChannelBecomesNonWritableDuringHeaderWrite), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests+XCTest.swift deleted file mode 100644 index 76a37936b..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests+XCTest.swift +++ /dev/null @@ -1,48 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP1ConnectionStateMachineTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP1ConnectionStateMachineTests { - static var allTests: [(String, (HTTP1ConnectionStateMachineTests) -> () throws -> Void)] { - return [ - ("testPOSTRequestWithWriteAndReadBackpressure", testPOSTRequestWithWriteAndReadBackpressure), - ("testResponseReadingWithBackpressure", testResponseReadingWithBackpressure), - ("testAConnectionCloseHeaderInTheRequestLeadsToConnectionCloseAfterRequest", testAConnectionCloseHeaderInTheRequestLeadsToConnectionCloseAfterRequest), - ("testAHTTP1_0ResponseWithoutKeepAliveHeaderLeadsToConnectionCloseAfterRequest", testAHTTP1_0ResponseWithoutKeepAliveHeaderLeadsToConnectionCloseAfterRequest), - ("testAHTTP1_0ResponseWithKeepAliveHeaderLeadsToConnectionBeingKeptAlive", testAHTTP1_0ResponseWithKeepAliveHeaderLeadsToConnectionBeingKeptAlive), - ("testAConnectionCloseHeaderInTheResponseLeadsToConnectionCloseAfterRequest", testAConnectionCloseHeaderInTheResponseLeadsToConnectionCloseAfterRequest), - ("testNIOTriggersChannelActiveTwice", testNIOTriggersChannelActiveTwice), - ("testIdleConnectionBecomesInactive", testIdleConnectionBecomesInactive), - ("testConnectionGoesAwayWhileInRequest", testConnectionGoesAwayWhileInRequest), - ("testRequestWasCancelledWhileUploadingData", testRequestWasCancelledWhileUploadingData), - ("testCancelRequestIsIgnoredWhenConnectionIsIdle", testCancelRequestIsIgnoredWhenConnectionIsIdle), - ("testReadsAreForwardedIfConnectionIsClosing", testReadsAreForwardedIfConnectionIsClosing), - ("testChannelReadsAreIgnoredIfConnectionIsClosing", testChannelReadsAreIgnoredIfConnectionIsClosing), - ("testRequestIsCancelledWhileWaitingForWritable", testRequestIsCancelledWhileWaitingForWritable), - ("testConnectionIsClosedIfErrorHappensWhileInRequest", testConnectionIsClosedIfErrorHappensWhileInRequest), - ("testConnectionIsClosedAfterSwitchingProtocols", testConnectionIsClosedAfterSwitchingProtocols), - ("testWeDontCrashAfterEarlyHintsAndConnectionClose", testWeDontCrashAfterEarlyHintsAndConnectionClose), - ("testWeDontCrashInRaceBetweenSchedulingNewRequestAndConnectionClose", testWeDontCrashInRaceBetweenSchedulingNewRequestAndConnectionClose), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests+XCTest.swift deleted file mode 100644 index 95b3e5dac..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests+XCTest.swift +++ /dev/null @@ -1,42 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP1ConnectionTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP1ConnectionTests { - static var allTests: [(String, (HTTP1ConnectionTests) -> () throws -> Void)] { - return [ - ("testCreateNewConnectionWithDecompression", testCreateNewConnectionWithDecompression), - ("testCreateNewConnectionWithoutDecompression", testCreateNewConnectionWithoutDecompression), - ("testCreateNewConnectionFailureClosedIO", testCreateNewConnectionFailureClosedIO), - ("testGETRequest", testGETRequest), - ("testConnectionClosesOnCloseHeader", testConnectionClosesOnCloseHeader), - ("testConnectionClosesOnRandomlyAppearingCloseHeader", testConnectionClosesOnRandomlyAppearingCloseHeader), - ("testConnectionClosesAfterTheRequestWithoutHavingSentAnCloseHeader", testConnectionClosesAfterTheRequestWithoutHavingSentAnCloseHeader), - ("testConnectionIsClosedAfterSwitchingProtocols", testConnectionIsClosedAfterSwitchingProtocols), - ("testConnectionDropAfterEarlyHints", testConnectionDropAfterEarlyHints), - ("testConnectionIsClosedIfResponseIsReceivedBeforeRequest", testConnectionIsClosedIfResponseIsReceivedBeforeRequest), - ("testDoubleHTTPResponseLine", testDoubleHTTPResponseLine), - ("testDownloadStreamingBackpressure", testDownloadStreamingBackpressure), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests+XCTest.swift deleted file mode 100644 index 15c432037..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests+XCTest.swift +++ /dev/null @@ -1,35 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP1ProxyConnectHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP1ProxyConnectHandlerTests { - static var allTests: [(String, (HTTP1ProxyConnectHandlerTests) -> () throws -> Void)] { - return [ - ("testProxyConnectWithoutAuthorizationSuccess", testProxyConnectWithoutAuthorizationSuccess), - ("testProxyConnectWithAuthorization", testProxyConnectWithAuthorization), - ("testProxyConnectWithoutAuthorizationFailure500", testProxyConnectWithoutAuthorizationFailure500), - ("testProxyConnectWithoutAuthorizationButAuthorizationNeeded", testProxyConnectWithoutAuthorizationButAuthorizationNeeded), - ("testProxyConnectReceivesBody", testProxyConnectReceivesBody), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift deleted file mode 100644 index 221a63211..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests+XCTest.swift +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP2ClientRequestHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP2ClientRequestHandlerTests { - static var allTests: [(String, (HTTP2ClientRequestHandlerTests) -> () throws -> Void)] { - return [ - ("testResponseBackpressure", testResponseBackpressure), - ("testWriteBackpressure", testWriteBackpressure), - ("testIdleReadTimeout", testIdleReadTimeout), - ("testIdleReadTimeoutIsCanceledIfRequestIsCanceled", testIdleReadTimeoutIsCanceledIfRequestIsCanceled), - ("testWriteHTTPHeadFails", testWriteHTTPHeadFails), - ("testChannelBecomesNonWritableDuringHeaderWrite", testChannelBecomesNonWritableDuringHeaderWrite), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift deleted file mode 100644 index 915791cdf..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests+XCTest.swift +++ /dev/null @@ -1,43 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP2ClientTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP2ClientTests { - static var allTests: [(String, (HTTP2ClientTests) -> () throws -> Void)] { - return [ - ("testSimpleGet", testSimpleGet), - ("testStreamRequestBodyWithoutKnowledgeAboutLength", testStreamRequestBodyWithoutKnowledgeAboutLength), - ("testStreamRequestBodyWithFalseKnowledgeAboutLength", testStreamRequestBodyWithFalseKnowledgeAboutLength), - ("testConcurrentRequests", testConcurrentRequests), - ("testConcurrentRequestsFromDifferentThreads", testConcurrentRequestsFromDifferentThreads), - ("testConcurrentRequestsWorkWithRequiredEventLoop", testConcurrentRequestsWorkWithRequiredEventLoop), - ("testUncleanShutdownCancelsExecutingAndQueuedTasks", testUncleanShutdownCancelsExecutingAndQueuedTasks), - ("testCancelingRunningRequest", testCancelingRunningRequest), - ("testReadTimeout", testReadTimeout), - ("testH2CanHandleRequestsThatHaveAlreadyHitTheDeadline", testH2CanHandleRequestsThatHaveAlreadyHitTheDeadline), - ("testStressCancelingRunningRequestFromDifferentThreads", testStressCancelingRunningRequestFromDifferentThreads), - ("testPlatformConnectErrorIsForwardedOnTimeout", testPlatformConnectErrorIsForwardedOnTimeout), - ("testMassiveDownload", testMassiveDownload), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift deleted file mode 100644 index f26ca7d38..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests+XCTest.swift +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP2ConnectionTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP2ConnectionTests { - static var allTests: [(String, (HTTP2ConnectionTests) -> () throws -> Void)] { - return [ - ("testCreateNewConnectionFailureClosedIO", testCreateNewConnectionFailureClosedIO), - ("testConnectionToleratesShutdownEventsAfterAlreadyClosed", testConnectionToleratesShutdownEventsAfterAlreadyClosed), - ("testSimpleGetRequest", testSimpleGetRequest), - ("testEveryDoneRequestLeadsToAStreamAvailableCall", testEveryDoneRequestLeadsToAStreamAvailableCall), - ("testCancelAllRunningRequests", testCancelAllRunningRequests), - ("testChildStreamsAreRemovedFromTheOpenChannelListOnceTheRequestIsDone", testChildStreamsAreRemovedFromTheOpenChannelListOnceTheRequestIsDone), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift deleted file mode 100644 index a69530597..000000000 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests+XCTest.swift +++ /dev/null @@ -1,42 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTP2IdleHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTP2IdleHandlerTests { - static var allTests: [(String, (HTTP2IdleHandlerTests) -> () throws -> Void)] { - return [ - ("testReceiveSettingsWithMaxConcurrentStreamSetting", testReceiveSettingsWithMaxConcurrentStreamSetting), - ("testReceiveSettingsWithoutMaxConcurrentStreamSetting", testReceiveSettingsWithoutMaxConcurrentStreamSetting), - ("testEmptySettingsDontOverwriteMaxConcurrentStreamSetting", testEmptySettingsDontOverwriteMaxConcurrentStreamSetting), - ("testOverwriteMaxConcurrentStreamSetting", testOverwriteMaxConcurrentStreamSetting), - ("testGoAwayReceivedBeforeSettings", testGoAwayReceivedBeforeSettings), - ("testGoAwayReceivedAfterSettings", testGoAwayReceivedAfterSettings), - ("testCloseEventBeforeFirstSettings", testCloseEventBeforeFirstSettings), - ("testCloseEventWhileNoOpenStreams", testCloseEventWhileNoOpenStreams), - ("testCloseEventWhileThereAreOpenStreams", testCloseEventWhileThereAreOpenStreams), - ("testGoAwayWhileThereAreOpenStreams", testGoAwayWhileThereAreOpenStreams), - ("testReceiveSettingsAndGoAwayAfterClientSideClose", testReceiveSettingsAndGoAwayAfterClientSideClose), - ("testConnectionUseLimitTriggersGoAway", testConnectionUseLimitTriggersGoAway), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests+XCTest.swift deleted file mode 100644 index 40ef6e0ff..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests+XCTest.swift +++ /dev/null @@ -1,35 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClient+SOCKSTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientSOCKSTests { - static var allTests: [(String, (HTTPClientSOCKSTests) -> () throws -> Void)] { - return [ - ("testProxySOCKS", testProxySOCKS), - ("testProxySOCKSBogusAddress", testProxySOCKSBogusAddress), - ("testProxySOCKSFailureNoServer", testProxySOCKSFailureNoServer), - ("testProxySOCKSFailureInvalidServer", testProxySOCKSFailureInvalidServer), - ("testProxySOCKSMisbehavingServer", testProxySOCKSMisbehavingServer), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientCookieTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientCookieTests+XCTest.swift deleted file mode 100644 index 7ecf54d4d..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientCookieTests+XCTest.swift +++ /dev/null @@ -1,44 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientCookieTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientCookieTests { - static var allTests: [(String, (HTTPClientCookieTests) -> () throws -> Void)] { - return [ - ("testCookie", testCookie), - ("testEmptyValueCookie", testEmptyValueCookie), - ("testCookieDefaults", testCookieDefaults), - ("testCookieInit", testCookieInit), - ("testMalformedCookies", testMalformedCookies), - ("testExpires", testExpires), - ("testMaxAge", testMaxAge), - ("testDomain", testDomain), - ("testPath", testPath), - ("testSecure", testSecure), - ("testHttpOnly", testHttpOnly), - ("testCookieExpiresDateParsing", testCookieExpiresDateParsing), - ("testQuotedCookies", testQuotedCookies), - ("testCookieExpiresDateParsingWithNonEnglishLocale", testCookieExpiresDateParsingWithNonEnglishLocale), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests+XCTest.swift deleted file mode 100644 index 63d7f85e2..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests+XCTest.swift +++ /dev/null @@ -1,32 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientInformationalResponsesTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientReproTests { - static var allTests: [(String, (HTTPClientReproTests) -> () throws -> Void)] { - return [ - ("testServerSends100ContinueFirst", testServerSends100ContinueFirst), - ("testServerSendsSwitchingProtocols", testServerSendsSwitchingProtocols), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift deleted file mode 100644 index 9114df259..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests+XCTest.swift +++ /dev/null @@ -1,42 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientInternalTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientInternalTests { - static var allTests: [(String, (HTTPClientInternalTests) -> () throws -> Void)] { - return [ - ("testProxyStreaming", testProxyStreaming), - ("testProxyStreamingFailure", testProxyStreamingFailure), - ("testRequestURITrailingSlash", testRequestURITrailingSlash), - ("testChannelAndDelegateOnDifferentEventLoops", testChannelAndDelegateOnDifferentEventLoops), - ("testResponseFutureIsOnCorrectEL", testResponseFutureIsOnCorrectEL), - ("testUncleanCloseThrows", testUncleanCloseThrows), - ("testUploadStreamingIsCalledOnTaskEL", testUploadStreamingIsCalledOnTaskEL), - ("testTaskPromiseBoundToEL", testTaskPromiseBoundToEL), - ("testConnectErrorCalloutOnCorrectEL", testConnectErrorCalloutOnCorrectEL), - ("testInternalRequestURI", testInternalRequestURI), - ("testHasSuffix", testHasSuffix), - ("testSharedThreadPoolIsIdenticalForAllDelegates", testSharedThreadPoolIsIdenticalForAllDelegates), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift deleted file mode 100644 index 77f4298ba..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests+XCTest.swift +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientNIOTSTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientNIOTSTests { - static var allTests: [(String, (HTTPClientNIOTSTests) -> () throws -> Void)] { - return [ - ("testCorrectEventLoopGroup", testCorrectEventLoopGroup), - ("testTLSFailError", testTLSFailError), - ("testConnectionFailsFastError", testConnectionFailsFastError), - ("testConnectionFailError", testConnectionFailError), - ("testTLSVersionError", testTLSVersionError), - ("testTrustRootCertificateLoadFail", testTrustRootCertificateLoadFail), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests+XCTest.swift deleted file mode 100644 index 30d93f7de..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests+XCTest.swift +++ /dev/null @@ -1,43 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientRequestTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientRequestTests { - static var allTests: [(String, (HTTPClientRequestTests) -> () throws -> Void)] { - return [ - ("testCustomHeadersAreRespected", testCustomHeadersAreRespected), - ("testUnixScheme", testUnixScheme), - ("testHTTPUnixScheme", testHTTPUnixScheme), - ("testHTTPSUnixScheme", testHTTPSUnixScheme), - ("testGetWithoutBody", testGetWithoutBody), - ("testPostWithoutBody", testPostWithoutBody), - ("testPostWithEmptyByteBuffer", testPostWithEmptyByteBuffer), - ("testPostWithByteBuffer", testPostWithByteBuffer), - ("testPostWithSequenceOfUnknownLength", testPostWithSequenceOfUnknownLength), - ("testPostWithSequenceWithFixedLength", testPostWithSequenceWithFixedLength), - ("testPostWithRandomAccessCollection", testPostWithRandomAccessCollection), - ("testPostWithAsyncSequenceOfUnknownLength", testPostWithAsyncSequenceOfUnknownLength), - ("testPostWithAsyncSequenceWithKnownLength", testPostWithAsyncSequenceWithKnownLength), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift deleted file mode 100644 index 0a1a7cab6..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests+XCTest.swift +++ /dev/null @@ -1,35 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientResponseTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientResponseTests { - static var allTests: [(String, (HTTPClientResponseTests) -> () throws -> Void)] { - return [ - ("testSimpleResponse", testSimpleResponse), - ("testSimpleResponseNotModified", testSimpleResponseNotModified), - ("testSimpleResponseHeadRequestMethod", testSimpleResponseHeadRequestMethod), - ("testResponseNoContentLengthHeader", testResponseNoContentLengthHeader), - ("testResponseInvalidInteger", testResponseInvalidInteger), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift deleted file mode 100644 index 8f292239e..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests+XCTest.swift +++ /dev/null @@ -1,156 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientTests { - static var allTests: [(String, (HTTPClientTests) -> () throws -> Void)] { - return [ - ("testRequestURI", testRequestURI), - ("testBadRequestURI", testBadRequestURI), - ("testSchemaCasing", testSchemaCasing), - ("testURLSocketPathInitializers", testURLSocketPathInitializers), - ("testBadUnixWithBaseURL", testBadUnixWithBaseURL), - ("testConvenienceExecuteMethods", testConvenienceExecuteMethods), - ("testConvenienceExecuteMethodsOverSocket", testConvenienceExecuteMethodsOverSocket), - ("testConvenienceExecuteMethodsOverSecureSocket", testConvenienceExecuteMethodsOverSecureSocket), - ("testGet", testGet), - ("testGetWithDifferentEventLoopBackpressure", testGetWithDifferentEventLoopBackpressure), - ("testPost", testPost), - ("testPostWithGenericBody", testPostWithGenericBody), - ("testPostWithFoundationDataBody", testPostWithFoundationDataBody), - ("testGetHttps", testGetHttps), - ("testGetHttpsWithIP", testGetHttpsWithIP), - ("testGetHTTPSWorksOnMTELGWithIP", testGetHTTPSWorksOnMTELGWithIP), - ("testGetHttpsWithIPv6", testGetHttpsWithIPv6), - ("testGetHTTPSWorksOnMTELGWithIPv6", testGetHTTPSWorksOnMTELGWithIPv6), - ("testPostHttps", testPostHttps), - ("testHttpRedirect", testHttpRedirect), - ("testHttpHostRedirect", testHttpHostRedirect), - ("testPercentEncoded", testPercentEncoded), - ("testPercentEncodedBackslash", testPercentEncodedBackslash), - ("testMultipleContentLengthHeaders", testMultipleContentLengthHeaders), - ("testStreaming", testStreaming), - ("testFileDownload", testFileDownload), - ("testFileDownloadError", testFileDownloadError), - ("testFileDownloadCustomError", testFileDownloadCustomError), - ("testRemoteClose", testRemoteClose), - ("testReadTimeout", testReadTimeout), - ("testConnectTimeout", testConnectTimeout), - ("testDeadline", testDeadline), - ("testCancel", testCancel), - ("testStressCancel", testStressCancel), - ("testHTTPClientAuthorization", testHTTPClientAuthorization), - ("testProxyPlaintext", testProxyPlaintext), - ("testProxyTLS", testProxyTLS), - ("testProxyPlaintextWithCorrectlyAuthorization", testProxyPlaintextWithCorrectlyAuthorization), - ("testProxyPlaintextWithIncorrectlyAuthorization", testProxyPlaintextWithIncorrectlyAuthorization), - ("testUploadStreaming", testUploadStreaming), - ("testEventLoopArgument", testEventLoopArgument), - ("testDecompression", testDecompression), - ("testDecompressionHTTP2", testDecompressionHTTP2), - ("testDecompressionLimit", testDecompressionLimit), - ("testLoopDetectionRedirectLimit", testLoopDetectionRedirectLimit), - ("testCountRedirectLimit", testCountRedirectLimit), - ("testRedirectToTheInitialURLDoesThrowOnFirstRedirect", testRedirectToTheInitialURLDoesThrowOnFirstRedirect), - ("testMultipleConcurrentRequests", testMultipleConcurrentRequests), - ("testWorksWith500Error", testWorksWith500Error), - ("testWorksWithHTTP10Response", testWorksWithHTTP10Response), - ("testWorksWhenServerClosesConnectionAfterReceivingRequest", testWorksWhenServerClosesConnectionAfterReceivingRequest), - ("testSubsequentRequestsWorkWithServerSendingConnectionClose", testSubsequentRequestsWorkWithServerSendingConnectionClose), - ("testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose", testSubsequentRequestsWorkWithServerAlternatingBetweenKeepAliveAndClose), - ("testStressGetHttpsSSLError", testStressGetHttpsSSLError), - ("testSelfSignedCertificateIsRejectedWithCorrectError", testSelfSignedCertificateIsRejectedWithCorrectError), - ("testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded", testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded), - ("testFailingConnectionIsReleased", testFailingConnectionIsReleased), - ("testStressGetClose", testStressGetClose), - ("testManyConcurrentRequestsWork", testManyConcurrentRequestsWork), - ("testRepeatedRequestsWorkWhenServerAlwaysCloses", testRepeatedRequestsWorkWhenServerAlwaysCloses), - ("testShutdownBeforeTasksCompletion", testShutdownBeforeTasksCompletion), - ("testUncleanShutdownActuallyShutsDown", testUncleanShutdownActuallyShutsDown), - ("testUncleanShutdownCancelsTasks", testUncleanShutdownCancelsTasks), - ("testDoubleShutdown", testDoubleShutdown), - ("testTaskFailsWhenClientIsShutdown", testTaskFailsWhenClientIsShutdown), - ("testRaceNewRequestsVsShutdown", testRaceNewRequestsVsShutdown), - ("testVaryingLoopPreference", testVaryingLoopPreference), - ("testMakeSecondRequestDuringCancelledCallout", testMakeSecondRequestDuringCancelledCallout), - ("testMakeSecondRequestDuringSuccessCallout", testMakeSecondRequestDuringSuccessCallout), - ("testMakeSecondRequestWhilstFirstIsOngoing", testMakeSecondRequestWhilstFirstIsOngoing), - ("testUDSBasic", testUDSBasic), - ("testUDSSocketAndPath", testUDSSocketAndPath), - ("testHTTPPlusUNIX", testHTTPPlusUNIX), - ("testHTTPSPlusUNIX", testHTTPSPlusUNIX), - ("testUseExistingConnectionOnDifferentEL", testUseExistingConnectionOnDifferentEL), - ("testWeRecoverFromServerThatClosesTheConnectionOnUs", testWeRecoverFromServerThatClosesTheConnectionOnUs), - ("testPoolClosesIdleConnections", testPoolClosesIdleConnections), - ("testAvoidLeakingTLSHandshakeCompletionPromise", testAvoidLeakingTLSHandshakeCompletionPromise), - ("testAsyncShutdown", testAsyncShutdown), - ("testAsyncShutdownDefaultQueue", testAsyncShutdownDefaultQueue), - ("testValidationErrorsAreSurfaced", testValidationErrorsAreSurfaced), - ("testUploadsReallyStream", testUploadsReallyStream), - ("testUploadStreamingCallinToleratedFromOtsideEL", testUploadStreamingCallinToleratedFromOtsideEL), - ("testWeHandleUsSendingACloseHeaderCorrectly", testWeHandleUsSendingACloseHeaderCorrectly), - ("testWeHandleUsReceivingACloseHeaderCorrectly", testWeHandleUsReceivingACloseHeaderCorrectly), - ("testWeHandleUsSendingACloseHeaderAmongstOtherConnectionHeadersCorrectly", testWeHandleUsSendingACloseHeaderAmongstOtherConnectionHeadersCorrectly), - ("testWeHandleUsReceivingACloseHeaderAmongstOtherConnectionHeadersCorrectly", testWeHandleUsReceivingACloseHeaderAmongstOtherConnectionHeadersCorrectly), - ("testLoggingCorrectlyAttachesRequestInformationEvenAfterDuringRedirect", testLoggingCorrectlyAttachesRequestInformationEvenAfterDuringRedirect), - ("testLoggingCorrectlyAttachesRequestInformation", testLoggingCorrectlyAttachesRequestInformation), - ("testNothingIsLoggedAtInfoOrHigher", testNothingIsLoggedAtInfoOrHigher), - ("testAllMethodsLog", testAllMethodsLog), - ("testClosingIdleConnectionsInPoolLogsInTheBackground", testClosingIdleConnectionsInPoolLogsInTheBackground), - ("testUploadStreamingNoLength", testUploadStreamingNoLength), - ("testConnectErrorPropagatedToDelegate", testConnectErrorPropagatedToDelegate), - ("testDelegateCallinsTolerateRandomEL", testDelegateCallinsTolerateRandomEL), - ("testContentLengthTooLongFails", testContentLengthTooLongFails), - ("testContentLengthTooShortFails", testContentLengthTooShortFails), - ("testBodyUploadAfterEndFails", testBodyUploadAfterEndFails), - ("testDoubleError", testDoubleError), - ("testSSLHandshakeErrorPropagation", testSSLHandshakeErrorPropagation), - ("testSSLHandshakeErrorPropagationDelayedClose", testSSLHandshakeErrorPropagationDelayedClose), - ("testWeCloseConnectionsWhenConnectionCloseSetByServer", testWeCloseConnectionsWhenConnectionCloseSetByServer), - ("testBiDirectionalStreaming", testBiDirectionalStreaming), - ("testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLength", testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLength), - ("testResponseAccumulatorMaxBodySizeLimitNotExceedingWithContentLength", testResponseAccumulatorMaxBodySizeLimitNotExceedingWithContentLength), - ("testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLengthButMethodIsHead", testResponseAccumulatorMaxBodySizeLimitExceedingWithContentLengthButMethodIsHead), - ("testResponseAccumulatorMaxBodySizeLimitExceedingWithTransferEncodingChuncked", testResponseAccumulatorMaxBodySizeLimitExceedingWithTransferEncodingChuncked), - ("testResponseAccumulatorMaxBodySizeLimitNotExceedingWithTransferEncodingChuncked", testResponseAccumulatorMaxBodySizeLimitNotExceedingWithTransferEncodingChuncked), - ("testBiDirectionalStreamingEarly200", testBiDirectionalStreamingEarly200), - ("testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests", testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests), - ("testCloseConnectionAfterEarly2XXWhenStreaming", testCloseConnectionAfterEarly2XXWhenStreaming), - ("testSynchronousHandshakeErrorReporting", testSynchronousHandshakeErrorReporting), - ("testFileDownloadChunked", testFileDownloadChunked), - ("testCloseWhileBackpressureIsExertedIsFine", testCloseWhileBackpressureIsExertedIsFine), - ("testErrorAfterCloseWhileBackpressureExerted", testErrorAfterCloseWhileBackpressureExerted), - ("testRequestSpecificTLS", testRequestSpecificTLS), - ("testRequestWithHeaderTransferEncodingIdentityDoesNotFail", testRequestWithHeaderTransferEncodingIdentityDoesNotFail), - ("testMassiveDownload", testMassiveDownload), - ("testShutdownWithFutures", testShutdownWithFutures), - ("testMassiveHeaderHTTP1", testMassiveHeaderHTTP1), - ("testMassiveHeaderHTTP2", testMassiveHeaderHTTP2), - ("testCancelingHTTP1RequestAfterHeaderSend", testCancelingHTTP1RequestAfterHeaderSend), - ("testCancelingHTTP2RequestAfterHeaderSend", testCancelingHTTP2RequestAfterHeaderSend), - ("testMaxConnectionReusesHTTP1", testMaxConnectionReusesHTTP1), - ("testMaxConnectionReusesHTTP2", testMaxConnectionReusesHTTP2), - ("testMaxConnectionReusesExceedsMaxConcurrentStreamsForHTTP2", testMaxConnectionReusesExceedsMaxConcurrentStreamsForHTTP2), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests+XCTest.swift deleted file mode 100644 index d95346673..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests+XCTest.swift +++ /dev/null @@ -1,36 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPClientUncleanSSLConnectionShutdownTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPClientUncleanSSLConnectionShutdownTests { - static var allTests: [(String, (HTTPClientUncleanSSLConnectionShutdownTests) -> () throws -> Void)] { - return [ - ("testEOFFramedSuccess", testEOFFramedSuccess), - ("testContentLength", testContentLength), - ("testContentLengthButTruncated", testContentLengthButTruncated), - ("testTransferEncoding", testTransferEncoding), - ("testTransferEncodingButTruncated", testTransferEncodingButTruncated), - ("testConnectionDrop", testConnectionDrop), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests+XCTest.swift deleted file mode 100644 index 898b2b867..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests+XCTest.swift +++ /dev/null @@ -1,34 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+FactoryTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_FactoryTests { - static var allTests: [(String, (HTTPConnectionPool_FactoryTests) -> () throws -> Void)] { - return [ - ("testConnectionCreationTimesoutIfDeadlineIsInThePast", testConnectionCreationTimesoutIfDeadlineIsInThePast), - ("testSOCKSConnectionCreationTimesoutIfRemoteIsUnresponsive", testSOCKSConnectionCreationTimesoutIfRemoteIsUnresponsive), - ("testHTTPProxyConnectionCreationTimesoutIfRemoteIsUnresponsive", testHTTPProxyConnectionCreationTimesoutIfRemoteIsUnresponsive), - ("testTLSConnectionCreationTimesoutIfRemoteIsUnresponsive", testTLSConnectionCreationTimesoutIfRemoteIsUnresponsive), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest+XCTest.swift deleted file mode 100644 index 21eb3029e..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest+XCTest.swift +++ /dev/null @@ -1,47 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+HTTP1ConnectionsTest+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_HTTP1ConnectionsTests { - static var allTests: [(String, (HTTPConnectionPool_HTTP1ConnectionsTests) -> () throws -> Void)] { - return [ - ("testCreatingConnections", testCreatingConnections), - ("testCreatingConnectionAndFailing", testCreatingConnectionAndFailing), - ("testLeaseConnectionOnPreferredAndAvailableEL", testLeaseConnectionOnPreferredAndAvailableEL), - ("testLeaseConnectionOnPreferredButUnavailableEL", testLeaseConnectionOnPreferredButUnavailableEL), - ("testLeaseConnectionOnRequiredButUnavailableEL", testLeaseConnectionOnRequiredButUnavailableEL), - ("testLeaseConnectionOnRequiredAndAvailableEL", testLeaseConnectionOnRequiredAndAvailableEL), - ("testCloseConnectionIfIdle", testCloseConnectionIfIdle), - ("testCloseConnectionIfIdleButLeasedRaceCondition", testCloseConnectionIfIdleButLeasedRaceCondition), - ("testCloseConnectionIfIdleButClosedRaceCondition", testCloseConnectionIfIdleButClosedRaceCondition), - ("testShutdown", testShutdown), - ("testMigrationFromHTTP2", testMigrationFromHTTP2), - ("testMigrationFromHTTP2WithPendingRequestsWithRequiredEventLoop", testMigrationFromHTTP2WithPendingRequestsWithRequiredEventLoop), - ("testMigrationFromHTTP2WithPendingRequestsWithPreferredEventLoop", testMigrationFromHTTP2WithPendingRequestsWithPreferredEventLoop), - ("testMigrationFromHTTP2WithAlreadyLeasedHTTP1Connection", testMigrationFromHTTP2WithAlreadyLeasedHTTP1Connection), - ("testMigrationFromHTTP2WithMoreStartingConnectionsThanMaximumAllowedConccurentConnections", testMigrationFromHTTP2WithMoreStartingConnectionsThanMaximumAllowedConccurentConnections), - ("testMigrationFromHTTP2StartsEnoghOverflowConnectionsForRequiredEventLoopRequests", testMigrationFromHTTP2StartsEnoghOverflowConnectionsForRequiredEventLoopRequests), - ("testMigrationFromHTTP1ToHTTP2AndBackToHTTP1", testMigrationFromHTTP1ToHTTP2AndBackToHTTP1), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift deleted file mode 100644 index d50ab9893..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests+XCTest.swift +++ /dev/null @@ -1,47 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+HTTP1StateTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_HTTP1StateMachineTests { - static var allTests: [(String, (HTTPConnectionPool_HTTP1StateMachineTests) -> () throws -> Void)] { - return [ - ("testCreatingAndFailingConnections", testCreatingAndFailingConnections), - ("testCreatingAndFailingConnectionsWithoutRetry", testCreatingAndFailingConnectionsWithoutRetry), - ("testConnectionFailureBackoff", testConnectionFailureBackoff), - ("testCancelRequestWorks", testCancelRequestWorks), - ("testExecuteOnShuttingDownPool", testExecuteOnShuttingDownPool), - ("testRequestsAreQueuedIfAllConnectionsAreInUseAndRequestsAreDequeuedInOrder", testRequestsAreQueuedIfAllConnectionsAreInUseAndRequestsAreDequeuedInOrder), - ("testBestConnectionIsPicked", testBestConnectionIsPicked), - ("testConnectionAbortIsIgnoredIfThereAreNoQueuedRequests", testConnectionAbortIsIgnoredIfThereAreNoQueuedRequests), - ("testConnectionCloseLeadsToTumbleWeedIfThereNoQueuedRequests", testConnectionCloseLeadsToTumbleWeedIfThereNoQueuedRequests), - ("testConnectionAbortLeadsToNewConnectionsIfThereAreQueuedRequests", testConnectionAbortLeadsToNewConnectionsIfThereAreQueuedRequests), - ("testParkedConnectionTimesOut", testParkedConnectionTimesOut), - ("testConnectionPoolFullOfParkedConnectionsIsShutdownImmediately", testConnectionPoolFullOfParkedConnectionsIsShutdownImmediately), - ("testParkedConnectionTimesOutButIsAlsoClosedByRemote", testParkedConnectionTimesOutButIsAlsoClosedByRemote), - ("testConnectionBackoffVsShutdownRace", testConnectionBackoffVsShutdownRace), - ("testRequestThatTimesOutIsFailedWithLastConnectionCreationError", testRequestThatTimesOutIsFailedWithLastConnectionCreationError), - ("testRequestThatTimesOutBeforeAConnectionIsEstablishedIsFailedWithConnectTimeoutError", testRequestThatTimesOutBeforeAConnectionIsEstablishedIsFailedWithConnectTimeoutError), - ("testRequestThatTimesOutAfterAConnectionWasEstablishedSuccessfullyTimesOutWithGenericError", testRequestThatTimesOutAfterAConnectionWasEstablishedSuccessfullyTimesOutWithGenericError), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest+XCTest.swift deleted file mode 100644 index 95cade669..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest+XCTest.swift +++ /dev/null @@ -1,49 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+HTTP2ConnectionsTest+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_HTTP2ConnectionsTests { - static var allTests: [(String, (HTTPConnectionPool_HTTP2ConnectionsTests) -> () throws -> Void)] { - return [ - ("testCreatingConnections", testCreatingConnections), - ("testCreatingConnectionAndFailing", testCreatingConnectionAndFailing), - ("testFailConnectionRace", testFailConnectionRace), - ("testLeaseConnectionOfPreferredButUnavailableEL", testLeaseConnectionOfPreferredButUnavailableEL), - ("testLeaseConnectionOnRequiredButUnavailableEL", testLeaseConnectionOnRequiredButUnavailableEL), - ("testCloseConnectionIfIdle", testCloseConnectionIfIdle), - ("testCloseConnectionIfIdleButLeasedRaceCondition", testCloseConnectionIfIdleButLeasedRaceCondition), - ("testCloseConnectionIfIdleButClosedRaceCondition", testCloseConnectionIfIdleButClosedRaceCondition), - ("testCloseConnectionIfIdleRace", testCloseConnectionIfIdleRace), - ("testShutdown", testShutdown), - ("testLeasingAllConnections", testLeasingAllConnections), - ("testGoAway", testGoAway), - ("testNewMaxConcurrentStreamsSetting", testNewMaxConcurrentStreamsSetting), - ("testEventsAfterConnectionIsClosed", testEventsAfterConnectionIsClosed), - ("testLeaseOnPreferredEventLoopWithoutAnyAvailable", testLeaseOnPreferredEventLoopWithoutAnyAvailable), - ("testMigrationFromHTTP1", testMigrationFromHTTP1), - ("testMigrationToHTTP1", testMigrationToHTTP1), - ("testMigrationFromHTTP1WithPendingRequestsWithRequiredEventLoop", testMigrationFromHTTP1WithPendingRequestsWithRequiredEventLoop), - ("testMigrationFromHTTP1WithAlreadyEstablishedHTTP2Connection", testMigrationFromHTTP1WithAlreadyEstablishedHTTP2Connection), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift deleted file mode 100644 index 12b031cc0..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift +++ /dev/null @@ -1,51 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+HTTP2StateMachineTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_HTTP2StateMachineTests { - static var allTests: [(String, (HTTPConnectionPool_HTTP2StateMachineTests) -> () throws -> Void)] { - return [ - ("testCreatingOfConnection", testCreatingOfConnection), - ("testConnectionFailureBackoff", testConnectionFailureBackoff), - ("testConnectionFailureWhileShuttingDown", testConnectionFailureWhileShuttingDown), - ("testConnectionFailureWithoutRetry", testConnectionFailureWithoutRetry), - ("testCancelRequestWorks", testCancelRequestWorks), - ("testExecuteOnShuttingDownPool", testExecuteOnShuttingDownPool), - ("testHTTP1ToHTTP2MigrationAndShutdownIfFirstConnectionIsHTTP1", testHTTP1ToHTTP2MigrationAndShutdownIfFirstConnectionIsHTTP1), - ("testSchedulingAndCancelingOfIdleTimeout", testSchedulingAndCancelingOfIdleTimeout), - ("testConnectionTimeout", testConnectionTimeout), - ("testConnectionEstablishmentFailure", testConnectionEstablishmentFailure), - ("testGoAwayOnIdleConnection", testGoAwayOnIdleConnection), - ("testGoAwayWithLeasedStream", testGoAwayWithLeasedStream), - ("testGoAwayWithPendingRequestsStartsNewConnection", testGoAwayWithPendingRequestsStartsNewConnection), - ("testMigrationFromHTTP1ToHTTP2", testMigrationFromHTTP1ToHTTP2), - ("testMigrationFromHTTP1ToHTTP2WhileShuttingDown", testMigrationFromHTTP1ToHTTP2WhileShuttingDown), - ("testMigrationFromHTTP1ToHTTP2WithAlreadyStartedHTTP1Connections", testMigrationFromHTTP1ToHTTP2WithAlreadyStartedHTTP1Connections), - ("testHTTP2toHTTP1Migration", testHTTP2toHTTP1Migration), - ("testHTTP2toHTTP1MigrationDuringShutdown", testHTTP2toHTTP1MigrationDuringShutdown), - ("testConnectionIsImmediatelyCreatedAfterBackoffTimerFires", testConnectionIsImmediatelyCreatedAfterBackoffTimerFires), - ("testMaxConcurrentStreamsIsRespected", testMaxConcurrentStreamsIsRespected), - ("testEventsAfterConnectionIsClosed", testEventsAfterConnectionIsClosed), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests+XCTest.swift deleted file mode 100644 index 93945f63c..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests+XCTest.swift +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+ManagerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_ManagerTests { - static var allTests: [(String, (HTTPConnectionPool_ManagerTests) -> () throws -> Void)] { - return [ - ("testManagerHappyPath", testManagerHappyPath), - ("testShutdownManagerThatHasSeenNoConnections", testShutdownManagerThatHasSeenNoConnections), - ("testExecutingARequestOnAShutdownPoolManager", testExecutingARequestOnAShutdownPoolManager), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests+XCTest.swift deleted file mode 100644 index 2511ba267..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPool+RequestQueueTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPool_RequestQueueTests { - static var allTests: [(String, (HTTPConnectionPool_RequestQueueTests) -> () throws -> Void)] { - return [ - ("testCountAndIsEmptyWorks", testCountAndIsEmptyWorks), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests+XCTest.swift deleted file mode 100644 index acdc0ab26..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests+XCTest.swift +++ /dev/null @@ -1,39 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPConnectionPoolTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPConnectionPoolTests { - static var allTests: [(String, (HTTPConnectionPoolTests) -> () throws -> Void)] { - return [ - ("testOnlyOneConnectionIsUsedForSubSequentRequests", testOnlyOneConnectionIsUsedForSubSequentRequests), - ("testConnectionsForEventLoopRequirementsAreClosed", testConnectionsForEventLoopRequirementsAreClosed), - ("testConnectionPoolGrowsToMaxConcurrentConnections", testConnectionPoolGrowsToMaxConcurrentConnections), - ("testConnectionCreationIsRetriedUntilRequestIsFailed", testConnectionCreationIsRetriedUntilRequestIsFailed), - ("testConnectionCreationIsRetriedUntilPoolIsShutdown", testConnectionCreationIsRetriedUntilPoolIsShutdown), - ("testConnectionCreationIsRetriedUntilRequestIsCancelled", testConnectionCreationIsRetriedUntilRequestIsCancelled), - ("testConnectionShutdownIsCalledOnActiveConnections", testConnectionShutdownIsCalledOnActiveConnections), - ("testConnectionPoolStressResistanceHTTP1", testConnectionPoolStressResistanceHTTP1), - ("testBackoffBehavesSensibly", testBackoffBehavesSensibly), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift deleted file mode 100644 index ad85bd71e..000000000 --- a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests+XCTest.swift +++ /dev/null @@ -1,67 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// HTTPRequestStateMachineTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension HTTPRequestStateMachineTests { - static var allTests: [(String, (HTTPRequestStateMachineTests) -> () throws -> Void)] { - return [ - ("testSimpleGETRequest", testSimpleGETRequest), - ("testPOSTRequestWithWriterBackpressure", testPOSTRequestWithWriterBackpressure), - ("testPOSTContentLengthIsTooLong", testPOSTContentLengthIsTooLong), - ("testPOSTContentLengthIsTooShort", testPOSTContentLengthIsTooShort), - ("testRequestBodyStreamIsCancelledIfServerRespondsWith301", testRequestBodyStreamIsCancelledIfServerRespondsWith301), - ("testStreamPartReceived_whenCancelled", testStreamPartReceived_whenCancelled), - ("testRequestBodyStreamIsCancelledIfServerRespondsWith301WhileWriteBackpressure", testRequestBodyStreamIsCancelledIfServerRespondsWith301WhileWriteBackpressure), - ("testRequestBodyStreamIsContinuedIfServerRespondsWith200", testRequestBodyStreamIsContinuedIfServerRespondsWith200), - ("testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200", testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200), - ("testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerRespondedWith200", testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerRespondedWith200), - ("testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerSendHeadWithStatus200", testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerSendHeadWithStatus200), - ("testRequestIsNotSendUntilChannelIsWritable", testRequestIsNotSendUntilChannelIsWritable), - ("testConnectionBecomesInactiveWhileWaitingForWritable", testConnectionBecomesInactiveWhileWaitingForWritable), - ("testResponseReadingWithBackpressure", testResponseReadingWithBackpressure), - ("testChannelReadCompleteTriggersButNoBodyDataWasReceivedSoFar", testChannelReadCompleteTriggersButNoBodyDataWasReceivedSoFar), - ("testResponseReadingWithBackpressureEndOfResponseAllowsReadEventsToTriggerDirectly", testResponseReadingWithBackpressureEndOfResponseAllowsReadEventsToTriggerDirectly), - ("testCancellingARequestInStateInitializedKeepsTheConnectionAlive", testCancellingARequestInStateInitializedKeepsTheConnectionAlive), - ("testCancellingARequestBeforeBeingSendKeepsTheConnectionAlive", testCancellingARequestBeforeBeingSendKeepsTheConnectionAlive), - ("testConnectionBecomesWritableBeforeFirstRequest", testConnectionBecomesWritableBeforeFirstRequest), - ("testCancellingARequestThatIsSent", testCancellingARequestThatIsSent), - ("testRemoteSuddenlyClosesTheConnection", testRemoteSuddenlyClosesTheConnection), - ("testReadTimeoutLeadsToFailureWithEverythingAfterBeingIgnored", testReadTimeoutLeadsToFailureWithEverythingAfterBeingIgnored), - ("testResponseWithStatus1XXAreIgnored", testResponseWithStatus1XXAreIgnored), - ("testReadTimeoutThatFiresToLateIsIgnored", testReadTimeoutThatFiresToLateIsIgnored), - ("testCancellationThatIsInvokedToLateIsIgnored", testCancellationThatIsInvokedToLateIsIgnored), - ("testErrorWhileRunningARequestClosesTheStream", testErrorWhileRunningARequestClosesTheStream), - ("testCanReadHTTP1_0ResponseWithoutBody", testCanReadHTTP1_0ResponseWithoutBody), - ("testCanReadHTTP1_0ResponseWithBody", testCanReadHTTP1_0ResponseWithBody), - ("testFailHTTP1_0RequestThatIsStillUploading", testFailHTTP1_0RequestThatIsStillUploading), - ("testFailHTTP1RequestWithoutContentLengthWithNIOSSLErrorUncleanShutdown", testFailHTTP1RequestWithoutContentLengthWithNIOSSLErrorUncleanShutdown), - ("testNIOSSLErrorUncleanShutdownShouldBeTreatedAsRemoteConnectionCloseWhileInWaitingForHeadState", testNIOSSLErrorUncleanShutdownShouldBeTreatedAsRemoteConnectionCloseWhileInWaitingForHeadState), - ("testArbitraryErrorShouldBeTreatedAsARequestFailureWhileInWaitingForHeadState", testArbitraryErrorShouldBeTreatedAsARequestFailureWhileInWaitingForHeadState), - ("testFailHTTP1RequestWithContentLengthWithNIOSSLErrorUncleanShutdownButIgnoreIt", testFailHTTP1RequestWithContentLengthWithNIOSSLErrorUncleanShutdownButIgnoreIt), - ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand), - ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForRead", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForRead), - ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForReadAndDemand", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForReadAndDemand), - ("testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForReadAndDemandMultipleTimes", testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForReadAndDemandMultipleTimes), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests+XCTest.swift b/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests+XCTest.swift deleted file mode 100644 index b496d527c..000000000 --- a/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// IdleTimeoutNoReuseTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension TestIdleTimeoutNoReuse { - static var allTests: [(String, (TestIdleTimeoutNoReuse) -> () throws -> Void)] { - return [ - ("testIdleTimeoutNoReuse", testIdleTimeoutNoReuse), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/LRUCacheTests+XCTest.swift b/Tests/AsyncHTTPClientTests/LRUCacheTests+XCTest.swift deleted file mode 100644 index a0231bf0d..000000000 --- a/Tests/AsyncHTTPClientTests/LRUCacheTests+XCTest.swift +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// LRUCacheTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension LRUCacheTests { - static var allTests: [(String, (LRUCacheTests) -> () throws -> Void)] { - return [ - ("testBasicsWork", testBasicsWork), - ("testCachesTheRightThings", testCachesTheRightThings), - ("testAppendingTheSameDoesNotEvictButUpdates", testAppendingTheSameDoesNotEvictButUpdates), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift deleted file mode 100644 index 9ed1ca2a8..000000000 --- a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// NoBytesSentOverBodyLimitTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension NoBytesSentOverBodyLimitTests { - static var allTests: [(String, (NoBytesSentOverBodyLimitTests) -> () throws -> Void)] { - return [ - ("testNoBytesSentOverBodyLimit", testNoBytesSentOverBodyLimit), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift deleted file mode 100644 index b4aa20dad..000000000 --- a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// RacePoolIdleConnectionsAndGetTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension RacePoolIdleConnectionsAndGetTests { - static var allTests: [(String, (RacePoolIdleConnectionsAndGetTests) -> () throws -> Void)] { - return [ - ("testRacePoolIdleConnectionsAndGet", testRacePoolIdleConnectionsAndGet), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift deleted file mode 100644 index 53c152c06..000000000 --- a/Tests/AsyncHTTPClientTests/RequestBagTests+XCTest.swift +++ /dev/null @@ -1,46 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// RequestBagTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension RequestBagTests { - static var allTests: [(String, (RequestBagTests) -> () throws -> Void)] { - return [ - ("testWriteBackpressureWorks", testWriteBackpressureWorks), - ("testTaskIsFailedIfWritingFails", testTaskIsFailedIfWritingFails), - ("testCancelFailsTaskBeforeRequestIsSent", testCancelFailsTaskBeforeRequestIsSent), - ("testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart", testDeadlineExceededFailsTaskEvenIfRaceBetweenCancelingSchedulerAndRequestStart), - ("testCancelHasNoEffectAfterDeadlineExceededFailsTask", testCancelHasNoEffectAfterDeadlineExceededFailsTask), - ("testCancelFailsTaskAfterRequestIsSent", testCancelFailsTaskAfterRequestIsSent), - ("testCancelFailsTaskWhenTaskIsQueued", testCancelFailsTaskWhenTaskIsQueued), - ("testFailsTaskWhenTaskIsWaitingForMoreFromServer", testFailsTaskWhenTaskIsWaitingForMoreFromServer), - ("testChannelBecomingWritableDoesntCrashCancelledTask", testChannelBecomingWritableDoesntCrashCancelledTask), - ("testDidReceiveBodyPartFailedPromise", testDidReceiveBodyPartFailedPromise), - ("testHTTPUploadIsCancelledEvenThoughRequestSucceeds", testHTTPUploadIsCancelledEvenThoughRequestSucceeds), - ("testRaceBetweenConnectionCloseAndDemandMoreData", testRaceBetweenConnectionCloseAndDemandMoreData), - ("testRedirectWith3KBBody", testRedirectWith3KBBody), - ("testRedirectWith4KBBodyAnnouncedInResponseHead", testRedirectWith4KBBodyAnnouncedInResponseHead), - ("testRedirectWith4KBBodyNotAnnouncedInResponseHead", testRedirectWith4KBBodyNotAnnouncedInResponseHead), - ("testWeDontLeakTheRequestIfTheRequestWriterWasCapturedByAPromise", testWeDontLeakTheRequestIfTheRequestWriterWasCapturedByAPromise), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/RequestValidationTests+XCTest.swift b/Tests/AsyncHTTPClientTests/RequestValidationTests+XCTest.swift deleted file mode 100644 index 3a93d70ec..000000000 --- a/Tests/AsyncHTTPClientTests/RequestValidationTests+XCTest.swift +++ /dev/null @@ -1,52 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// RequestValidationTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension RequestValidationTests { - static var allTests: [(String, (RequestValidationTests) -> () throws -> Void)] { - return [ - ("testContentLengthHeaderIsRemovedFromGETIfNoBody", testContentLengthHeaderIsRemovedFromGETIfNoBody), - ("testContentLengthHeaderIsAddedToPOSTAndPUTWithNoBody", testContentLengthHeaderIsAddedToPOSTAndPUTWithNoBody), - ("testContentLengthHeaderIsChangedIfBodyHasDifferentLength", testContentLengthHeaderIsChangedIfBodyHasDifferentLength), - ("testTRACERequestMustNotHaveBody", testTRACERequestMustNotHaveBody), - ("testGET_HEAD_DELETE_CONNECTRequestCanHaveBody", testGET_HEAD_DELETE_CONNECTRequestCanHaveBody), - ("testInvalidHeaderFieldNames", testInvalidHeaderFieldNames), - ("testValidHeaderFieldNames", testValidHeaderFieldNames), - ("testMetadataDetectConnectionClose", testMetadataDetectConnectionClose), - ("testMetadataDefaultIsConnectionCloseIsFalse", testMetadataDefaultIsConnectionCloseIsFalse), - ("testNoHeadersNoBody", testNoHeadersNoBody), - ("testNoHeadersHasBody", testNoHeadersHasBody), - ("testContentLengthHeaderNoBody", testContentLengthHeaderNoBody), - ("testContentLengthHeaderHasBody", testContentLengthHeaderHasBody), - ("testTransferEncodingHeaderNoBody", testTransferEncodingHeaderNoBody), - ("testTransferEncodingHeaderHasBody", testTransferEncodingHeaderHasBody), - ("testBothHeadersNoBody", testBothHeadersNoBody), - ("testBothHeadersHasBody", testBothHeadersHasBody), - ("testHostHeaderIsSetCorrectlyInCreateRequestHead", testHostHeaderIsSetCorrectlyInCreateRequestHead), - ("testTraceMethodIsNotAllowedToHaveAFixedLengthBody", testTraceMethodIsNotAllowedToHaveAFixedLengthBody), - ("testTraceMethodIsNotAllowedToHaveADynamicLengthBody", testTraceMethodIsNotAllowedToHaveADynamicLengthBody), - ("testTransferEncodingsAreOverwrittenIfBodyLengthIsFixed", testTransferEncodingsAreOverwrittenIfBodyLengthIsFixed), - ("testTransferEncodingsAreOverwrittenIfBodyLengthIsDynamic", testTransferEncodingsAreOverwrittenIfBodyLengthIsDynamic), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift deleted file mode 100644 index 5d3c8bfb1..000000000 --- a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// ResponseDelayGetTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension ResponseDelayGetTests { - static var allTests: [(String, (ResponseDelayGetTests) -> () throws -> Void)] { - return [ - ("testResponseDelayGet", testResponseDelayGet), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests+XCTest.swift deleted file mode 100644 index 0338adf3c..000000000 --- a/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests+XCTest.swift +++ /dev/null @@ -1,35 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// SOCKSEventsHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension SOCKSEventsHandlerTests { - static var allTests: [(String, (SOCKSEventsHandlerTests) -> () throws -> Void)] { - return [ - ("testHandlerHappyPath", testHandlerHappyPath), - ("testHandlerFailsFutureWhenRemovedWithoutEvent", testHandlerFailsFutureWhenRemovedWithoutEvent), - ("testHandlerFailsFutureWhenHandshakeFails", testHandlerFailsFutureWhenHandshakeFails), - ("testHandlerClosesConnectionIfHandshakeTimesout", testHandlerClosesConnectionIfHandshakeTimesout), - ("testHandlerWorksIfDeadlineIsInPast", testHandlerWorksIfDeadlineIsInPast), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/SSLContextCacheTests+XCTest.swift b/Tests/AsyncHTTPClientTests/SSLContextCacheTests+XCTest.swift deleted file mode 100644 index d98f5a853..000000000 --- a/Tests/AsyncHTTPClientTests/SSLContextCacheTests+XCTest.swift +++ /dev/null @@ -1,33 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// SSLContextCacheTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension SSLContextCacheTests { - static var allTests: [(String, (SSLContextCacheTests) -> () throws -> Void)] { - return [ - ("testRequestingSSLContextWorks", testRequestingSSLContextWorks), - ("testCacheWorks", testCacheWorks), - ("testCacheDoesNotReturnWrongEntry", testCacheDoesNotReturnWrongEntry), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift b/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift deleted file mode 100644 index faf64cb19..000000000 --- a/Tests/AsyncHTTPClientTests/StressGetHttpsTests+XCTest.swift +++ /dev/null @@ -1,31 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// StressGetHttpsTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension StressGetHttpsTests { - static var allTests: [(String, (StressGetHttpsTests) -> () throws -> Void)] { - return [ - ("testStressGetHttps", testStressGetHttps), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests+XCTest.swift b/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests+XCTest.swift deleted file mode 100644 index 062132f4e..000000000 --- a/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests+XCTest.swift +++ /dev/null @@ -1,34 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// TLSEventsHandlerTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension TLSEventsHandlerTests { - static var allTests: [(String, (TLSEventsHandlerTests) -> () throws -> Void)] { - return [ - ("testHandlerHappyPath", testHandlerHappyPath), - ("testHandlerFailsFutureWhenRemovedWithoutEvent", testHandlerFailsFutureWhenRemovedWithoutEvent), - ("testHandlerFailsFutureWhenHandshakeFails", testHandlerFailsFutureWhenHandshakeFails), - ("testHandlerIgnoresShutdownCompletedEvent", testHandlerIgnoresShutdownCompletedEvent), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift deleted file mode 100644 index f86344137..000000000 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests+XCTest.swift +++ /dev/null @@ -1,37 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// Transaction+StateMachineTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension Transaction_StateMachineTests { - static var allTests: [(String, (Transaction_StateMachineTests) -> () throws -> Void)] { - return [ - ("testRequestWasQueuedAfterWillExecuteRequestWasCalled", testRequestWasQueuedAfterWillExecuteRequestWasCalled), - ("testRequestBodyStreamWasPaused", testRequestBodyStreamWasPaused), - ("testQueuedRequestGetsRemovedWhenDeadlineExceeded", testQueuedRequestGetsRemovedWhenDeadlineExceeded), - ("testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect", testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect), - ("testScheduledRequestGetsRemovedWhenDeadlineExceeded", testScheduledRequestGetsRemovedWhenDeadlineExceeded), - ("testDeadlineExceededRaceWithRequestWillExecute", testDeadlineExceededRaceWithRequestWillExecute), - ("testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded", testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift b/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift deleted file mode 100644 index de63914a9..000000000 --- a/Tests/AsyncHTTPClientTests/TransactionTests+XCTest.swift +++ /dev/null @@ -1,40 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// TransactionTests+XCTest.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -extension TransactionTests { - static var allTests: [(String, (TransactionTests) -> () throws -> Void)] { - return [ - ("testCancelAsyncRequest", testCancelAsyncRequest), - ("testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask", testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask), - ("testResponseStreamingWorks", testResponseStreamingWorks), - ("testIgnoringResponseBodyWorks", testIgnoringResponseBodyWorks), - ("testWriteBackpressureWorks", testWriteBackpressureWorks), - ("testSimpleGetRequest", testSimpleGetRequest), - ("testSimplePostRequest", testSimplePostRequest), - ("testPostStreamFails", testPostStreamFails), - ("testResponseStreamFails", testResponseStreamFails), - ("testBiDirectionalStreamingHTTP2", testBiDirectionalStreamingHTTP2), - ] - } -} diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 79a0b83af..19454681c 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -580,10 +580,7 @@ actor SharedIterator where Wrapped.Element: Sendable { self.nextCallInProgress = true var iter = self.wrappedIterator defer { - // auto-closure of `precondition(_:)` messes with actor isolation analyses in Swift 5.5 - // we therefore need to move the access to `self.nextCallInProgress` out of the auto-closure - let nextCallInProgress = nextCallInProgress - precondition(nextCallInProgress == true) + precondition(self.nextCallInProgress == true) self.nextCallInProgress = false self.wrappedIterator = iter } diff --git a/Tests/LinuxMain.swift b/Tests/LinuxMain.swift deleted file mode 100644 index 886bf2b95..000000000 --- a/Tests/LinuxMain.swift +++ /dev/null @@ -1,76 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// LinuxMain.swift -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - -#if os(Linux) || os(FreeBSD) -@testable import AsyncHTTPClientTests - -@main -struct LinuxMain { - static func main() { - XCTMain([ - testCase(AsyncAwaitEndToEndTests.allTests), - testCase(ConnectionPoolSizeConfigValueIsRespectedTests.allTests), - testCase(HTTP1ClientChannelHandlerTests.allTests), - testCase(HTTP1ConnectionStateMachineTests.allTests), - testCase(HTTP1ConnectionTests.allTests), - testCase(HTTP1ProxyConnectHandlerTests.allTests), - testCase(HTTP2ClientRequestHandlerTests.allTests), - testCase(HTTP2ClientTests.allTests), - testCase(HTTP2ConnectionTests.allTests), - testCase(HTTP2IdleHandlerTests.allTests), - testCase(HTTPClientCookieTests.allTests), - testCase(HTTPClientInternalTests.allTests), - testCase(HTTPClientNIOTSTests.allTests), - testCase(HTTPClientReproTests.allTests), - testCase(HTTPClientRequestTests.allTests), - testCase(HTTPClientResponseTests.allTests), - testCase(HTTPClientSOCKSTests.allTests), - testCase(HTTPClientTests.allTests), - testCase(HTTPClientUncleanSSLConnectionShutdownTests.allTests), - testCase(HTTPConnectionPoolTests.allTests), - testCase(HTTPConnectionPool_FactoryTests.allTests), - testCase(HTTPConnectionPool_HTTP1ConnectionsTests.allTests), - testCase(HTTPConnectionPool_HTTP1StateMachineTests.allTests), - testCase(HTTPConnectionPool_HTTP2ConnectionsTests.allTests), - testCase(HTTPConnectionPool_HTTP2StateMachineTests.allTests), - testCase(HTTPConnectionPool_ManagerTests.allTests), - testCase(HTTPConnectionPool_RequestQueueTests.allTests), - testCase(HTTPRequestStateMachineTests.allTests), - testCase(LRUCacheTests.allTests), - testCase(NoBytesSentOverBodyLimitTests.allTests), - testCase(RacePoolIdleConnectionsAndGetTests.allTests), - testCase(RequestBagTests.allTests), - testCase(RequestValidationTests.allTests), - testCase(ResponseDelayGetTests.allTests), - testCase(SOCKSEventsHandlerTests.allTests), - testCase(SSLContextCacheTests.allTests), - testCase(StressGetHttpsTests.allTests), - testCase(TLSEventsHandlerTests.allTests), - testCase(TestIdleTimeoutNoReuse.allTests), - testCase(TransactionTests.allTests), - testCase(Transaction_StateMachineTests.allTests), - ]) - } -} -#endif diff --git a/docker/docker-compose.2004.55.yaml b/docker/docker-compose.2004.55.yaml deleted file mode 100644 index d181ef142..000000000 --- a/docker/docker-compose.2004.55.yaml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:20.04-5.5 - build: - args: - ubuntu_version: "focal" - swift_version: "5.5" - - documentation-check: - image: async-http-client:20.04-5.5 - - test: - image: async-http-client:20.04-5.5 - command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors $${SANITIZER_ARG-}" - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:20.04-5.5 diff --git a/scripts/generate_linux_tests.rb b/scripts/generate_linux_tests.rb deleted file mode 100755 index fe5726f6c..000000000 --- a/scripts/generate_linux_tests.rb +++ /dev/null @@ -1,236 +0,0 @@ -#!/usr/bin/env ruby - -# -# process_test_files.rb -# -# Copyright 2016 Tony Stone -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Created by Tony Stone on 5/4/16. -# -require 'getoptlong' -require 'fileutils' -require 'pathname' - -include FileUtils - -# -# This ruby script will auto generate LinuxMain.swift and the +XCTest.swift extension files for Swift Package Manager on Linux platforms. -# -# See https://github.com/apple/swift-corelibs-xctest/blob/master/Documentation/Linux.md -# -def header(fileName) - string = <<-eos -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -// -// -// -import XCTest - -/// -/// NOTE: This file was generated by generate_linux_tests.rb -/// -/// Do NOT edit this file directly as it will be regenerated automatically when needed. -/// - eos - - string - .sub('', File.basename(fileName)) - .sub('', Time.now.to_s) -end - -def createExtensionFile(fileName, classes) - extensionFile = fileName.sub! '.swift', '+XCTest.swift' - print 'Creating file: ' + extensionFile + "\n" - - File.open(extensionFile, 'w') do |file| - file.write header(extensionFile) - file.write "\n" - - for classArray in classes - file.write 'extension ' + classArray[0] + " {\n" - file.write ' static var allTests: [(String, (' + classArray[0] + ") -> () throws -> Void)] {\n" - file.write " return [\n" - - for funcName in classArray[1] - file.write ' ("' + funcName + '", ' + funcName + "),\n" - end - - file.write " ]\n" - file.write " }\n" - file.write "}\n" - end - end -end - -def createLinuxMain(testsDirectory, allTestSubDirectories, files) - fileName = testsDirectory + '/LinuxMain.swift' - print 'Creating file: ' + fileName + "\n" - - File.open(fileName, 'w') do |file| - file.write header(fileName) - file.write "\n" - - file.write "#if os(Linux) || os(FreeBSD)\n" - for testSubDirectory in allTestSubDirectories.sort { |x, y| x <=> y } - file.write '@testable import ' + testSubDirectory + "\n" - end - file.write "\n" - file.write "@main\n" - file.write "struct LinuxMain {\n" - file.write " static func main() {\n" - file.write " XCTMain([\n" - - testCases = [] - for classes in files - for classArray in classes - testCases << classArray[0] - end - end - - for testCase in testCases.sort { |x, y| x <=> y } - file.write ' testCase(' + testCase + ".allTests),\n" - end - file.write " ])\n" - file.write " }\n" - file.write "}\n" - file.write "#endif\n" - end -end - -def parseSourceFile(fileName) - puts 'Parsing file: ' + fileName + "\n" - - classes = [] - currentClass = nil - inIfLinux = false - inElse = false - ignore = false - - # - # Read the file line by line - # and parse to find the class - # names and func names - # - File.readlines(fileName).each do |line| - if inIfLinux - if /\#else/.match(line) - inElse = true - ignore = true - else - if /\#end/.match(line) - inElse = false - inIfLinux = false - ignore = false - end - end - else - if /\#if[ \t]+os\(Linux\)/.match(line) - inIfLinux = true - ignore = false - end - end - - next if ignore - # Match class or func - match = line[/class[ \t]+[a-zA-Z0-9_]*(?=[ \t]*:[ \t]*XCTestCase)|func[ \t]+test[a-zA-Z0-9_]*(?=[ \t]*\(\))/, 0] - if match - - if match[/class/, 0] == 'class' - className = match.sub(/^class[ \t]+/, '') - # - # Create a new class / func structure - # and add it to the classes array. - # - currentClass = [className, []] - classes << currentClass - else # Must be a func - funcName = match.sub(/^func[ \t]+/, '') - # - # Add each func name the the class / func - # structure created above. - # - currentClass[1] << funcName - end - end - end - classes -end - -# -# Main routine -# -# - -testsDirectory = 'Tests' - -options = GetoptLong.new(['--tests-dir', GetoptLong::OPTIONAL_ARGUMENT]) -options.quiet = true - -begin - options.each do |option, value| - case option - when '--tests-dir' - testsDirectory = value - end - end -rescue GetoptLong::InvalidOption -end - -allTestSubDirectories = [] -allFiles = [] - -Dir[testsDirectory + '/*'].each do |subDirectory| - next unless File.directory?(subDirectory) - directoryHasClasses = false - Dir[subDirectory + '/*Test{s,}.swift'].each do |fileName| - next unless File.file? fileName - fileClasses = parseSourceFile(fileName) - - # - # If there are classes in the - # test source file, create an extension - # file for it. - # - next unless fileClasses.count > 0 - createExtensionFile(fileName, fileClasses) - directoryHasClasses = true - allFiles << fileClasses - end - - if directoryHasClasses - allTestSubDirectories << Pathname.new(subDirectory).split.last.to_s - end -end - -# -# Last step is the create a LinuxMain.swift file that -# references all the classes and funcs in the source files. -# -if allFiles.count > 0 - createLinuxMain(testsDirectory, allTestSubDirectories, allFiles) -end -# eof diff --git a/scripts/soundness.sh b/scripts/soundness.sh index 6d37546e8..216eab206 100755 --- a/scripts/soundness.sh +++ b/scripts/soundness.sh @@ -21,18 +21,6 @@ function replace_acceptable_years() { sed -e 's/20[12][0-9]-20[12][0-9]/YEARS/' -e 's/20[12][0-9]/YEARS/' } -printf "=> Checking linux tests... " -FIRST_OUT="$(git status --porcelain)" -ruby "$here/../scripts/generate_linux_tests.rb" > /dev/null -SECOND_OUT="$(git status --porcelain)" -if [[ "$FIRST_OUT" != "$SECOND_OUT" ]]; then - printf "\033[0;31mmissing changes!\033[0m\n" - git --no-pager diff - exit 1 -else - printf "\033[0;32mokay.\033[0m\n" -fi - printf "=> Checking for unacceptable language... " # This greps for unacceptable terminology. The square bracket[s] are so that # "git grep" doesn't find the lines that greps :). From 78db67e5bf4a8543075787f228e8920097319281 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 17 May 2023 10:37:35 +0200 Subject: [PATCH 081/146] Tolerate new request after connection error happened (#688) * Tolerate new request after connection error happened * fix tests --- .../HTTP1/HTTP1ConnectionStateMachine.swift | 50 +++++++++---------- .../HTTP1ConnectionStateMachineTests.swift | 16 ++++++ 2 files changed, 41 insertions(+), 25 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index a908ded9a..eb4182593 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -106,14 +106,14 @@ struct HTTP1ConnectionStateMachine { return .wait case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } mutating func channelInactive() -> Action { switch self.state { case .initialized: - preconditionFailure("A channel that isn't active, must not become inactive") + fatalError("A channel that isn't active, must not become inactive") case .inRequest(var requestStateMachine, close: _): return self.avoidingStateMachineCoW { state -> Action in @@ -130,7 +130,7 @@ struct HTTP1ConnectionStateMachine { return .wait case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } @@ -155,7 +155,7 @@ struct HTTP1ConnectionStateMachine { return .fireChannelError(error, closeConnection: false) case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } @@ -173,7 +173,7 @@ struct HTTP1ConnectionStateMachine { } case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } @@ -182,15 +182,15 @@ struct HTTP1ConnectionStateMachine { metadata: RequestFramingMetadata ) -> Action { switch self.state { - case .initialized, .closing, .inRequest: + case .initialized, .inRequest: // These states are unreachable as the connection pool state machine has put the // connection into these states. In other words the connection pool state machine must // be aware about these states before the connection itself. For this reason the // connection pool state machine must not send a new request to the connection, if the // connection is `.initialized`, `.closing` or `.inRequest` - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") - case .closed: + case .closing, .closed: // The remote may have closed the connection and the connection pool state machine // was not updated yet because of a race condition. New request vs. marking connection // as closed. @@ -208,13 +208,13 @@ struct HTTP1ConnectionStateMachine { return self.state.modify(with: action) case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } mutating func requestStreamPartReceived(_ part: IOData, promise: EventLoopPromise?) -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in @@ -226,7 +226,7 @@ struct HTTP1ConnectionStateMachine { mutating func requestStreamFinished(promise: EventLoopPromise?) -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in @@ -239,7 +239,7 @@ struct HTTP1ConnectionStateMachine { mutating func requestCancelled(closeConnection: Bool) -> Action { switch self.state { case .initialized: - preconditionFailure("This event must only happen, if the connection is leased. During startup this is impossible. Invalid state: \(self.state)") + fatalError("This event must only happen, if the connection is leased. During startup this is impossible. Invalid state: \(self.state)") case .idle: if closeConnection { @@ -260,7 +260,7 @@ struct HTTP1ConnectionStateMachine { return .wait case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } @@ -269,7 +269,7 @@ struct HTTP1ConnectionStateMachine { mutating func read() -> Action { switch self.state { case .initialized: - preconditionFailure("Why should we read something, if we are not connected yet") + fatalError("Why should we read something, if we are not connected yet") case .idle: return .read case .inRequest(var requestStateMachine, let close): @@ -284,14 +284,14 @@ struct HTTP1ConnectionStateMachine { return .read case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } mutating func channelRead(_ part: HTTPClientResponsePart) -> Action { switch self.state { case .initialized, .idle: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") case .inRequest(var requestStateMachine, var close): return self.avoidingStateMachineCoW { state -> Action in @@ -310,7 +310,7 @@ struct HTTP1ConnectionStateMachine { return .wait case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } @@ -327,13 +327,13 @@ struct HTTP1ConnectionStateMachine { } case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } mutating func demandMoreResponseBodyParts() -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in @@ -345,7 +345,7 @@ struct HTTP1ConnectionStateMachine { mutating func idleReadTimeoutTriggered() -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } return self.avoidingStateMachineCoW { state -> Action in @@ -423,7 +423,7 @@ extension HTTP1ConnectionStateMachine.State { return .forwardResponseBodyParts(parts) case .succeedRequest(let finalAction, let finalParts): guard case .inRequest(_, close: let close) = self else { - preconditionFailure("Invalid state: \(self)") + fatalError("Invalid state: \(self)") } let newFinalAction: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction @@ -443,9 +443,9 @@ extension HTTP1ConnectionStateMachine.State { case .failRequest(let error, let finalAction): switch self { case .initialized: - preconditionFailure("Invalid state: \(self)") + fatalError("Invalid state: \(self)") case .idle: - preconditionFailure("How can we fail a task, if we are idle") + fatalError("How can we fail a task, if we are idle") case .inRequest(_, close: let close): if case .close(let promise) = finalAction { self = .closing @@ -465,7 +465,7 @@ extension HTTP1ConnectionStateMachine.State { return .failRequest(error, .none) case .modifying: - preconditionFailure("Invalid state: \(self)") + fatalError("Invalid state: \(self)") } case .read: @@ -497,7 +497,7 @@ extension HTTP1ConnectionStateMachine: CustomStringConvertible { case .closed: return ".closed" case .modifying: - preconditionFailure("Invalid state: \(self.state)") + fatalError("Invalid state: \(self.state)") } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index ce8e6ed17..e256aa49e 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -200,6 +200,19 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { XCTAssertEqual(state.requestCancelled(closeConnection: false), .failRequest(HTTPClientError.cancelled, .close(nil))) } + func testNewRequestAfterErrorHappened() { + var state = HTTP1ConnectionStateMachine() + XCTAssertEqual(state.channelActive(isWritable: false), .fireChannelActive) + struct MyError: Error, Equatable {} + XCTAssertEqual(state.errorHappened(MyError()), .fireChannelError(MyError(), closeConnection: true)) + let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: ["content-length": "4"]) + let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) + let action = state.runNewRequest(head: requestHead, metadata: metadata) + guard case .failRequest = action else { + return XCTFail("unexpected action \(action)") + } + } + func testCancelRequestIsIgnoredWhenConnectionIsIdle() { var state = HTTP1ConnectionStateMachine() XCTAssertEqual(state.channelActive(isWritable: true), .fireChannelActive) @@ -303,6 +316,8 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { case (.fireChannelInactive, .fireChannelInactive): return true + case (.fireChannelError(_, let lhsCloseConnection), .fireChannelError(_, let rhsCloseConnection)): + return lhsCloseConnection == rhsCloseConnection case (.sendRequestHead(let lhsHead, let lhsStartBody), .sendRequestHead(let rhsHead, let rhsStartBody)): return lhsHead == rhsHead && lhsStartBody == rhsStartBody @@ -377,6 +392,7 @@ extension HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction: Equatable return lhsPromise?.futureResult == rhsPromise?.futureResult case (.none, .none): return true + default: return false } From 960af0dcf4aaf143c9f2383ebb014d655e6920a3 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 6 Jun 2023 17:22:31 +0100 Subject: [PATCH 082/146] Adopt the Swift CoC (#691) Motivation: We're centralizing on the Swift code of conduct, so we'll x-reference that instead of holding our own. Modifications: Hyperlink out to Swift. Result: Shared CoC across the projects. --- CODE_OF_CONDUCT.md | 54 ++-------------------------------------------- 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c7a248828..76501d7d6 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -1,55 +1,5 @@ # Code of Conduct -To be a truly great community, AsyncHTTPClient needs to welcome developers from all walks of life, -with different backgrounds, and with a wide range of experience. A diverse and friendly -community will have more great ideas, more unique perspectives, and produce more great -code. We will work diligently to make the AsyncHTTPClient community welcoming to everyone. -To give clarity of what is expected of our members, AsyncHTTPClient has adopted the code of conduct -defined by [contributor-covenant.org](https://www.contributor-covenant.org). This document is used across many open source -communities, and we think it articulates our values well. The full text is copied below: +The code of conduct for this project can be found at https://swift.org/code-of-conduct. -### Contributor Code of Conduct v1.3 -As contributors and maintainers of this project, and in the interest of fostering an open and -welcoming community, we pledge to respect all people who contribute through reporting -issues, posting feature requests, updating documentation, submitting pull requests or patches, -and other activities. - -We are committed to making participation in this project a harassment-free experience for -everyone, regardless of level of experience, gender, gender identity and expression, sexual -orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or -nationality. - -Examples of unacceptable behavior by participants include: -- The use of sexualized language or imagery -- Personal attacks -- Trolling or insulting/derogatory comments -- Public or private harassment -- Publishing otherโ€™s private information, such as physical or electronic addresses, without explicit permission -- Other unethical or unprofessional conduct - -Project maintainers have the right and responsibility to remove, edit, or reject comments, -commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of -Conduct, or to ban temporarily or permanently any contributor for other behaviors that they -deem inappropriate, threatening, offensive, or harmful. - -By adopting this Code of Conduct, project maintainers commit themselves to fairly and -consistently applying these principles to every aspect of managing this project. Project -maintainers who do not follow or enforce the Code of Conduct may be permanently removed -from the project team. - -This code of conduct applies both within project spaces and in public spaces when an -individual is representing the project or its community. - -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by -contacting a project maintainer at [conduct@swiftserver.group](mailto:conduct@swiftserver.group). All complaints will be reviewed and -investigated and will result in a response that is deemed necessary and appropriate to the -circumstances. Maintainers are obligated to maintain confidentiality with regard to the reporter -of an incident. - -*This policy is adapted from the Contributor Code of Conduct [version 1.3.0](https://contributor-covenant.org/version/1/3/0/).* - -### Reporting -A working group of community members is committed to promptly addressing any [reported issues](mailto:conduct@swiftserver.group). -Working group members are volunteers appointed by the project lead, with a -preference for individuals with varied backgrounds and perspectives. Membership is expected -to change regularly, and may grow or shrink. + From c7fb77557c10de6d2a7ef0bb5389b59252d396ec Mon Sep 17 00:00:00 2001 From: brenno <37243584+brennobemoura@users.noreply.github.com> Date: Mon, 26 Jun 2023 04:22:30 -0300 Subject: [PATCH 083/146] Replace os() with canImport(Darwin) (#693) * Replace os() with canImport(Darwin) * Remove compilation conditional since it'll always be true --- Sources/AsyncHTTPClient/Base64.swift | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/Sources/AsyncHTTPClient/Base64.swift b/Sources/AsyncHTTPClient/Base64.swift index dbbf742ab..eed511a8c 100644 --- a/Sources/AsyncHTTPClient/Base64.swift +++ b/Sources/AsyncHTTPClient/Base64.swift @@ -146,11 +146,6 @@ extension String { } } -// Frustratingly, Swift 5.3 shipped before the macOS 11 SDK did, so we cannot gate the availability of -// this declaration on having the 5.3 compiler. This has caused a number of build issues. While updating -// to newer Xcodes does work, we can save ourselves some hassle and just wait until 5.4 to get this -// enhancement on Apple platforms. -#if (compiler(>=5.3) && !(os(macOS) || os(iOS) || os(tvOS) || os(watchOS))) || compiler(>=5.4) extension String { @inlinable @@ -163,12 +158,3 @@ extension String { } } } -#else -extension String { - @inlinable - init(customUnsafeUninitializedCapacity capacity: Int, - initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int) rethrows { - try self.init(backportUnsafeUninitializedCapacity: capacity, initializingUTF8With: initializer) - } -} -#endif From 668c193451944ba8b917852a90772563227354bc Mon Sep 17 00:00:00 2001 From: brenno <37243584+brennobemoura@users.noreply.github.com> Date: Mon, 26 Jun 2023 07:13:50 -0300 Subject: [PATCH 084/146] Removed duplicated code (#694) Co-authored-by: Franz Busch --- .../NIOTransportServices/TLSConfiguration.swift | 5 ----- 1 file changed, 5 deletions(-) diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index f79954da7..06ae5e146 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -121,11 +121,6 @@ extension TLSConfiguration { } } - // the certificate chain - if self.certificateChain.count > 0 { - preconditionFailure("TLSConfiguration.certificateChain is not supported. \(useMTELGExplainer)") - } - // cipher suites if self.cipherSuites.count > 0 { // TODO: Requires NIOSSL to provide list of cipher values before we can continue From df66c67cada9550cc688dfb60348b7a0875bc69d Mon Sep 17 00:00:00 2001 From: brenno <37243584+brennobemoura@users.noreply.github.com> Date: Fri, 30 Jun 2023 07:18:05 -0300 Subject: [PATCH 085/146] Fixed typo (#695) --- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 1a3cbd968..fc0879de3 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -510,7 +510,7 @@ internal final class HTTPBin where let responseEncoder = HTTPResponseEncoder() let requestDecoder = ByteToMessageHandler(HTTPRequestDecoder(leftOverBytesStrategy: .forwardBytes)) - let proxySimulator = HTTPProxySimulator(promise: promise, expectedAuhorization: expectedAuthorization) + let proxySimulator = HTTPProxySimulator(promise: promise, expectedAuthorization: expectedAuthorization) try sync.addHandler(responseEncoder) try sync.addHandler(requestDecoder) @@ -660,13 +660,13 @@ final class HTTPProxySimulator: ChannelInboundHandler, RemovableChannelHandler { // the promise to succeed, once the proxy connection is setup let promise: EventLoopPromise - let expectedAuhorization: String? + let expectedAuthorization: String? var head: HTTPResponseHead - init(promise: EventLoopPromise, expectedAuhorization: String?) { + init(promise: EventLoopPromise, expectedAuthorization: String?) { self.promise = promise - self.expectedAuhorization = expectedAuhorization + self.expectedAuthorization = expectedAuthorization self.head = HTTPResponseHead(version: .init(major: 1, minor: 1), status: .ok, headers: .init([("Content-Length", "0")])) } @@ -679,9 +679,9 @@ final class HTTPProxySimulator: ChannelInboundHandler, RemovableChannelHandler { return } - if let expectedAuhorization = self.expectedAuhorization { + if let expectedAuthorization = self.expectedAuthorization { guard let authorization = head.headers["proxy-authorization"].first, - expectedAuhorization == authorization else { + expectedAuthorization == authorization else { self.head.status = .proxyAuthenticationRequired return } From 7935de10c0ac2612f0fe60d5f6419f21f7483d90 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 14 Jul 2023 10:01:30 +0100 Subject: [PATCH 086/146] Fix flaky `AsyncAwaitEndToEndTests.testImmediateDeadline` test (#698) --- .../AsyncAwaitEndToEndTests.swift | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 3259fec9a..994e331fb 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -389,8 +389,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { guard let error = error as? HTTPClientError else { return XCTFail("unexpected error \(error)") } - // a race between deadline and connect timer can result in either error - XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error)) + // a race between deadline and connect timer can result in either error. + // If closing happens really fast we might shutdown the pipeline before we fail the request. + // If the pipeline is closed we may receive a `.remoteConnectionClosed`. + XCTAssertTrue([.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), "unexpected error \(error)") } } } @@ -412,8 +414,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { guard let error = error as? HTTPClientError else { return XCTFail("unexpected error \(error)") } - // a race between deadline and connect timer can result in either error - XCTAssertTrue([.deadlineExceeded, .connectTimeout].contains(error), "unexpected error \(error)") + // a race between deadline and connect timer can result in either error. + // If closing happens really fast we might shutdown the pipeline before we fail the request. + // If the pipeline is closed we may receive a `.remoteConnectionClosed`. + XCTAssertTrue([.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), "unexpected error \(error)") } } } From e1c85a65d82a5fcd49e74e6afb80362d058ca066 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Fri, 14 Jul 2023 11:34:27 +0100 Subject: [PATCH 087/146] Add timeout to `RequestBagTests.testCancelFailsTaskAfterRequestIsSent` test (#699) We have seen that the `RequestBagTests.testCancelFailsTaskAfterRequestIsSent` can timeout the CI. To better understand where it fails we can add a timeout that will throw with a timeout error and fail the error check below it. This will make will allow us in the future to better analyse the failure and will no longer timeout the CI. --- Tests/AsyncHTTPClientTests/RequestBagTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 54de39e12..75d57ba26 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -345,7 +345,7 @@ final class RequestBagTests: XCTestCase { bag.fail(HTTPClientError.cancelled) XCTAssertTrue(executor.isCancelled, "The request bag, should call cancel immediately on the executor") - XCTAssertThrowsError(try bag.task.futureResult.wait()) { + XCTAssertThrowsError(try bag.task.futureResult.timeout(after: .seconds(10)).wait()) { XCTAssertEqual($0 as? HTTPClientError, .cancelled) } } From 62c06d47c8d21c91335e9f8998589e4ce31411e6 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Thu, 10 Aug 2023 11:33:18 +0100 Subject: [PATCH 088/146] Remove tests relying on OS-dependent behaviour (#703) Motivation URL parsing has changed in macOS Sonoma and associated releases. Our tests were reliant on the old behaviour. The behaviour is controlled by the OS on which the program was linked, which makes it impossible for us to programmatically work out which result we should see. The affected tests are not actually useful, we don't really care how the URLs parse, so we can safely just remove them. Modifications Remove the affected tests. Result Tests pass! --- Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 6 ------ 1 file changed, 6 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8cd0b3bba..f04f84d0a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -155,12 +155,6 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(url.path, "/file/path") XCTAssertEqual(url.absoluteString, "https+unix://%2Ftmp%2Ffile%20with%20spaces%E3%81%A8%E6%BC%A2%E5%AD%97/file/path") } - - let url9 = URL(httpURLWithSocketPath: "/tmp/file", uri: " ") - XCTAssertNil(url9) - - let url10 = URL(httpsURLWithSocketPath: "/tmp/file", uri: " ") - XCTAssertNil(url10) } func testBadUnixWithBaseURL() { From 8c90405f0cc9160373920a6d70ceffbddbe2d5a6 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Mon, 14 Aug 2023 14:59:33 +0100 Subject: [PATCH 089/146] use NIOSingletons EventLoops/NIOThreadPool instead of spawning new (#697) Co-authored-by: Johannes Weiss --- Package.swift | 4 +- README.md | 27 ++- Sources/AsyncHTTPClient/Docs.docc/index.md | 56 +++++-- Sources/AsyncHTTPClient/HTTPClient.swift | 157 +++++++++--------- .../AsyncAwaitEndToEndTests.swift | 6 +- .../HTTP2ClientTests.swift | 9 +- ...TTPClientInformationalResponsesTests.swift | 4 +- .../HTTPClientNIOTSTests.swift | 2 +- .../HTTPClientTests.swift | 18 ++ 9 files changed, 166 insertions(+), 117 deletions(-) diff --git a/Package.swift b/Package.swift index 1f2f0046f..7e80f853e 100644 --- a/Package.swift +++ b/Package.swift @@ -21,11 +21,11 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.50.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), - .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.11.4"), + .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), diff --git a/README.md b/README.md index e969c54ff..806cc2960 100644 --- a/README.md +++ b/README.md @@ -27,14 +27,10 @@ and `AsyncHTTPClient` dependency to your target: The code snippet below illustrates how to make a simple GET request to a remote server. -Please note that the example will spawn a new `EventLoopGroup` which will _create fresh threads_ which is a very costly operation. In a real-world application that uses [SwiftNIO](https://github.com/apple/swift-nio) for other parts of your application (for example a web server), please prefer `eventLoopGroupProvider: .shared(myExistingEventLoopGroup)` to share the `EventLoopGroup` used by AsyncHTTPClient with other parts of your application. - -If your application does not use SwiftNIO yet, it is acceptable to use `eventLoopGroupProvider: .createNew` but please make sure to share the returned `HTTPClient` instance throughout your whole application. Do not create a large number of `HTTPClient` instances with `eventLoopGroupProvider: .createNew`, this is very wasteful and might exhaust the resources of your program. - ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) /// MARK: - Using Swift Concurrency let request = HTTPClientRequest(url: "https://apple.com/") @@ -78,7 +74,7 @@ The default HTTP Method is `GET`. In case you need to have more control over the ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { var request = HTTPClientRequest(url: "https://apple.com/") request.method = .POST @@ -103,9 +99,10 @@ try await httpClient.shutdown() ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) defer { - try? httpClient.syncShutdown() + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() } var request = try HTTPClient.Request(url: "https://apple.com/", method: .POST) @@ -129,7 +126,7 @@ httpClient.execute(request: request).whenComplete { result in ### Redirects following Enable follow-redirects behavior using the client configuration: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: HTTPClient.Configuration(followRedirects: true)) ``` @@ -137,7 +134,7 @@ let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, Timeouts (connect and read) can also be set using the client configuration: ```swift let timeout = HTTPClient.Configuration.Timeout(connect: .seconds(1), read: .seconds(1)) -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: HTTPClient.Configuration(timeout: timeout)) ``` or on a per-request basis: @@ -151,7 +148,7 @@ The following example demonstrates how to count the number of bytes in a streami #### Using Swift Concurrency ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { let request = HTTPClientRequest(url: "https://apple.com/") let response = try await httpClient.execute(request, timeout: .seconds(30)) @@ -251,7 +248,7 @@ asynchronously, while reporting the download progress at the same time, like in example: ```swift -let client = HTTPClient(eventLoopGroupProvider: .createNew) +let client = HTTPClient(eventLoopGroupProvider: .singleton) let request = try HTTPClient.Request( url: "https://swift.org/builds/development/ubuntu1804/latest-build.yml" ) @@ -275,7 +272,7 @@ client.execute(request: request, delegate: delegate).futureResult ### Unix Domain Socket Paths Connecting to servers bound to socket paths is easy: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) httpClient.execute( .GET, socketPath: "/tmp/myServer.socket", @@ -285,7 +282,7 @@ httpClient.execute( Connecting over TLS to a unix domain socket path is possible as well: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) httpClient.execute( .POST, secureSocketPath: "/tmp/myServer.socket", @@ -312,7 +309,7 @@ The exclusive use of HTTP/1 is possible by setting `httpVersion` to `.http1Only` var configuration = HTTPClient.Configuration() configuration.httpVersion = .http1Only let client = HTTPClient( - eventLoopGroupProvider: .createNew, + eventLoopGroupProvider: .singleton, configuration: configuration ) ``` diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md index 66a9d1135..82e859b03 100644 --- a/Sources/AsyncHTTPClient/Docs.docc/index.md +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -31,14 +31,14 @@ and `AsyncHTTPClient` dependency to your target: The code snippet below illustrates how to make a simple GET request to a remote server. -Please note that the example will spawn a new `EventLoopGroup` which will _create fresh threads_ which is a very costly operation. In a real-world application that uses [SwiftNIO](https://github.com/apple/swift-nio) for other parts of your application (for example a web server), please prefer `eventLoopGroupProvider: .shared(myExistingEventLoopGroup)` to share the `EventLoopGroup` used by AsyncHTTPClient with other parts of your application. - -If your application does not use SwiftNIO yet, it is acceptable to use `eventLoopGroupProvider: .createNew` but please make sure to share the returned `HTTPClient` instance throughout your whole application. Do not create a large number of `HTTPClient` instances with `eventLoopGroupProvider: .createNew`, this is very wasteful and might exhaust the resources of your program. - ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} /// MARK: - Using Swift Concurrency let request = HTTPClientRequest(url: "https://apple.com/") @@ -82,7 +82,12 @@ The default HTTP Method is `GET`. In case you need to have more control over the ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} + do { var request = HTTPClientRequest(url: "https://apple.com/") request.method = .POST @@ -107,9 +112,10 @@ try await httpClient.shutdown() ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) defer { - try? httpClient.syncShutdown() + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() } var request = try HTTPClient.Request(url: "https://apple.com/", method: .POST) @@ -133,7 +139,7 @@ httpClient.execute(request: request).whenComplete { result in #### Redirects following Enable follow-redirects behavior using the client configuration: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: HTTPClient.Configuration(followRedirects: true)) ``` @@ -141,7 +147,7 @@ let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, Timeouts (connect and read) can also be set using the client configuration: ```swift let timeout = HTTPClient.Configuration.Timeout(connect: .seconds(1), read: .seconds(1)) -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew, +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: HTTPClient.Configuration(timeout: timeout)) ``` or on a per-request basis: @@ -155,7 +161,12 @@ The following example demonstrates how to count the number of bytes in a streami ##### Using Swift Concurrency ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} + do { let request = HTTPClientRequest(url: "https://apple.com/") let response = try await httpClient.execute(request, timeout: .seconds(30)) @@ -255,7 +266,12 @@ asynchronously, while reporting the download progress at the same time, like in example: ```swift -let client = HTTPClient(eventLoopGroupProvider: .createNew) +let client = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} + let request = try HTTPClient.Request( url: "https://swift.org/builds/development/ubuntu1804/latest-build.yml" ) @@ -279,7 +295,12 @@ client.execute(request: request, delegate: delegate).futureResult #### Unix Domain Socket Paths Connecting to servers bound to socket paths is easy: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} + httpClient.execute( .GET, socketPath: "/tmp/myServer.socket", @@ -289,7 +310,12 @@ httpClient.execute( Connecting over TLS to a unix domain socket path is possible as well: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) +let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) +defer { + // Shutdown is guaranteed to work if it's done precisely once (which is the case here). + try! httpClient.syncShutdown() +} + httpClient.execute( .POST, secureSocketPath: "/tmp/myServer.socket", @@ -316,7 +342,7 @@ The exclusive use of HTTP/1 is possible by setting ``HTTPClient/Configuration/ht var configuration = HTTPClient.Configuration() configuration.httpVersion = .http1Only let client = HTTPClient( - eventLoopGroupProvider: .createNew, + eventLoopGroupProvider: .singleton, configuration: configuration ) ``` diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index de6b57087..53da87e75 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -44,7 +44,7 @@ let globalRequestID = ManagedAtomic(0) /// Example: /// /// ```swift -/// let client = HTTPClient(eventLoopGroupProvider: .createNew) +/// let client = HTTPClient(eventLoopGroupProvider: .singleton) /// client.get(url: "https://swift.org", deadline: .now() + .seconds(1)).whenComplete { result in /// switch result { /// case .failure(let error): @@ -69,7 +69,6 @@ public class HTTPClient { /// /// All HTTP transactions will occur on loops owned by this group. public let eventLoopGroup: EventLoopGroup - let eventLoopGroupProvider: EventLoopGroupProvider let configuration: Configuration let poolManager: HTTPConnectionPool.Manager @@ -94,29 +93,50 @@ public class HTTPClient { backgroundActivityLogger: HTTPClient.loggingDisabled) } + /// Create an ``HTTPClient`` with specified `EventLoopGroup` and configuration. + /// + /// - parameters: + /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. + /// - configuration: Client configuration. + public convenience init(eventLoopGroup: EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration()) { + self.init(eventLoopGroupProvider: .shared(eventLoopGroup), + configuration: configuration, + backgroundActivityLogger: HTTPClient.loggingDisabled) + } + /// Create an ``HTTPClient`` with specified `EventLoopGroup` provider and configuration. /// /// - parameters: /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. /// - configuration: Client configuration. - public required init(eventLoopGroupProvider: EventLoopGroupProvider, - configuration: Configuration = Configuration(), - backgroundActivityLogger: Logger) { - self.eventLoopGroupProvider = eventLoopGroupProvider - switch self.eventLoopGroupProvider { + public convenience init(eventLoopGroupProvider: EventLoopGroupProvider, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger) { + let eventLoopGroup: any EventLoopGroup + + switch eventLoopGroupProvider { case .shared(let group): - self.eventLoopGroup = group - case .createNew: - #if canImport(Network) - if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) { - self.eventLoopGroup = NIOTSEventLoopGroup() - } else { - self.eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) - } - #else - self.eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) - #endif + eventLoopGroup = group + default: // handle `.createNew` without a deprecation warning + eventLoopGroup = HTTPClient.defaultEventLoopGroup } + + self.init(eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: backgroundActivityLogger) + } + + /// Create an ``HTTPClient`` with specified `EventLoopGroup` and configuration. + /// + /// - parameters: + /// - eventLoopGroup: The `EventLoopGroup` that the ``HTTPClient`` will use. + /// - configuration: Client configuration. + /// - backgroundActivityLogger: The `Logger` that will be used to log background any activity that's not associated with a request. + public required init(eventLoopGroup: any EventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger) { + self.eventLoopGroup = eventLoopGroup self.configuration = configuration self.poolManager = HTTPConnectionPool.Manager( eventLoopGroup: self.eventLoopGroup, @@ -214,55 +234,16 @@ public class HTTPClient { } /// Shuts down the ``HTTPClient`` and releases its resources. - /// - /// - note: You cannot use this method if you sharted the ``HTTPClient`` with - /// `init(eventLoopGroupProvider: .createNew)` because that will shut down the `EventLoopGroup` the - /// returned future would run in. public func shutdown() -> EventLoopFuture { - switch self.eventLoopGroupProvider { - case .shared(let group): - let promise = group.any().makePromise(of: Void.self) - self.shutdown(queue: .global()) { error in - if let error = error { - promise.fail(error) - } else { - promise.succeed(()) - } - } - return promise.futureResult - case .createNew: - preconditionFailure("Cannot use the shutdown() method which returns a future when owning the EventLoopGroup. Please use the one of the other shutdown methods.") - } - } - - private func shutdownEventLoop(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { - self.stateLock.withLock { - switch self.eventLoopGroupProvider { - case .shared: - self.state = .shutDown - queue.async { - callback(nil) - } - case .createNew: - switch self.state { - case .shuttingDown: - self.state = .shutDown - self.eventLoopGroup.shutdownGracefully(queue: queue, callback) - case .shutDown, .upAndRunning: - assertionFailure("The only valid state at this point is \(String(describing: State.shuttingDown))") - } - } - } - } - - private func shutdownFileIOThreadPool(queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { - self.fileIOThreadPoolLock.withLock { - guard let fileIOThreadPool = fileIOThreadPool else { - callback(nil) - return + let promise = self.eventLoopGroup.any().makePromise(of: Void.self) + self.shutdown(queue: .global()) { error in + if let error = error { + promise.fail(error) + } else { + promise.succeed(()) } - fileIOThreadPool.shutdownGracefully(queue: queue, callback) } + return promise.futureResult } private func shutdown(requiresCleanClose: Bool, queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { @@ -293,11 +274,11 @@ public class HTTPClient { let error: Error? = (requiresClean && unclean) ? HTTPClientError.uncleanShutdown : nil return (callback, error) } - self.shutdownFileIOThreadPool(queue: queue) { ioThreadPoolError in - self.shutdownEventLoop(queue: queue) { error in - let reportedError = error ?? ioThreadPoolError ?? uncleanError - callback(reportedError) - } + self.stateLock.withLock { + self.state = .shutDown + } + queue.async { + callback(uncleanError) } } } @@ -305,11 +286,8 @@ public class HTTPClient { private func makeOrGetFileIOThreadPool() -> NIOThreadPool { self.fileIOThreadPoolLock.withLock { - guard let fileIOThreadPool = fileIOThreadPool else { - let fileIOThreadPool = NIOThreadPool(numberOfThreads: System.coreCount) - fileIOThreadPool.start() - self.fileIOThreadPool = fileIOThreadPool - return fileIOThreadPool + guard let fileIOThreadPool = self.fileIOThreadPool else { + return NIOThreadPool.singleton } return fileIOThreadPool } @@ -853,7 +831,12 @@ public class HTTPClient { public enum EventLoopGroupProvider { /// `EventLoopGroup` will be provided by the user. Owner of this group is responsible for its lifecycle. case shared(EventLoopGroup) - /// `EventLoopGroup` will be created by the client. When ``HTTPClient/syncShutdown()`` is called, the created `EventLoopGroup` will be shut down as well. + /// The original intention of this was that ``HTTPClient`` would create and own its own `EventLoopGroup` to + /// facilitate use in programs that are not already using SwiftNIO. + /// Since https://github.com/apple/swift-nio/pull/2471 however, SwiftNIO does provide a global, shared singleton + /// `EventLoopGroup`s that we can use. ``HTTPClient`` is no longer able to create & own its own + /// `EventLoopGroup` which solves a whole host of issues around shutdown. + @available(*, deprecated, renamed: "singleton", message: "Please use the singleton EventLoopGroup explicitly") case createNew } @@ -914,6 +897,30 @@ public class HTTPClient { } } +extension HTTPClient.EventLoopGroupProvider { + /// Shares ``HTTPClient/defaultEventLoopGroup`` which is a singleton `EventLoopGroup` suitable for the platform. + public static var singleton: Self { + return .shared(HTTPClient.defaultEventLoopGroup) + } +} + +extension HTTPClient { + /// Returns the default `EventLoopGroup` singleton, automatically selecting the best for the platform. + /// + /// This will select the concrete `EventLoopGroup` depending which platform this is running on. + public static var defaultEventLoopGroup: EventLoopGroup { + #if canImport(Network) + if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) { + return NIOTSEventLoopGroup.singleton + } else { + return MultiThreadedEventLoopGroup.singleton + } + #else + return MultiThreadedEventLoopGroup.singleton + #endif + } +} + #if swift(>=5.7) extension HTTPClient.Configuration: Sendable {} #endif diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 994e331fb..d26f5ae24 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -20,7 +20,7 @@ import NIOSSL import XCTest private func makeDefaultHTTPClient( - eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .createNew + eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .singleton ) -> HTTPClient { var config = HTTPClient.Configuration() config.tlsConfiguration = .clientDefault @@ -504,7 +504,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let config = HTTPClient.Configuration() .enableFastFailureModeForTesting() - let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) + let localClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://localhost:\(port)") await XCTAssertThrowsError(try await localClient.execute(request, deadline: .now() + .seconds(2))) { error in @@ -570,7 +570,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { // this is the actual configuration under test config.dnsOverride = ["example.com": "localhost"] - let localClient = HTTPClient(eventLoopGroupProvider: .createNew, configuration: config) + let localClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echohostheader") let response = await XCTAssertNoThrowWithResult(try await localClient.execute(request, deadline: .now() + .seconds(2))) diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 7d0aaad0c..71eb0c44b 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -25,7 +25,7 @@ import XCTest class HTTP2ClientTests: XCTestCase { func makeDefaultHTTPClient( - eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .createNew + eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .singleton ) -> HTTPClient { var config = HTTPClient.Configuration() config.tlsConfiguration = .clientDefault @@ -40,7 +40,7 @@ class HTTP2ClientTests: XCTestCase { func makeClientWithActiveHTTP2Connection( to bin: HTTPBin, - eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .createNew + eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .singleton ) -> HTTPClient { let client = self.makeDefaultHTTPClient(eventLoopGroupProvider: eventLoopGroupProvider) var response: HTTPClient.Response? @@ -301,7 +301,7 @@ class HTTP2ClientTests: XCTestCase { config.httpVersion = .automatic config.timeout.read = .milliseconds(100) let client = HTTPClient( - eventLoopGroupProvider: .createNew, + eventLoopGroupProvider: .singleton, configuration: config, backgroundActivityLogger: Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) ) @@ -322,7 +322,8 @@ class HTTP2ClientTests: XCTestCase { config.tlsConfiguration = tlsConfig config.httpVersion = .automatic let client = HTTPClient( - eventLoopGroupProvider: .createNew, + // TODO: Test fails if the provided ELG is a multi-threaded NIOTSEventLoopGroup (probably racy) + eventLoopGroupProvider: .shared(bin.group), configuration: config, backgroundActivityLogger: Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) ) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests.swift index f57d5fd10..3ca6e1054 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInformationalResponsesTests.swift @@ -37,7 +37,7 @@ final class HTTPClientReproTests: XCTestCase { } } - let client = HTTPClient(eventLoopGroupProvider: .createNew) + let client = HTTPClient(eventLoopGroupProvider: .singleton) defer { XCTAssertNoThrow(try client.syncShutdown()) } let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in @@ -91,7 +91,7 @@ final class HTTPClientReproTests: XCTestCase { } } - let client = HTTPClient(eventLoopGroupProvider: .createNew) + let client = HTTPClient(eventLoopGroupProvider: .singleton) defer { XCTAssertNoThrow(try client.syncShutdown()) } let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 3db4385cd..be03f6a6a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -38,7 +38,7 @@ class HTTPClientNIOTSTests: XCTestCase { } func testCorrectEventLoopGroup() { - let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) + let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index f04f84d0a..218abaf82 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3507,4 +3507,22 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { requests: 300 ) } + + func testClientWithDefaultSingletonELG() throws { + let client = HTTPClient() + defer { + XCTAssertNoThrow(try client.shutdown().wait()) + } + let response = try client.get(url: self.defaultHTTPBinURLPrefix + "get").wait() + XCTAssertEqual(.ok, response.status) + } + + func testClientWithELGInit() throws { + let client = HTTPClient(eventLoopGroup: MultiThreadedEventLoopGroup.singleton) + defer { + XCTAssertNoThrow(try client.shutdown().wait()) + } + let response = try client.get(url: self.defaultHTTPBinURLPrefix + "get").wait() + XCTAssertEqual(.ok, response.status) + } } From 16f7e62c08c6969899ce6cc277041e868364e5cf Mon Sep 17 00:00:00 2001 From: Natik Gadzhi Date: Mon, 14 Aug 2023 12:28:11 -0700 Subject: [PATCH 090/146] Add unit tests for NWWaitingHandler, closes #589 (#702) * Add unit tests for NWWaitingHandler Motivation: Closes #589. Since we already have a public initializer for NIOTSNetworkEvents.WaitingForConnectivity, we should add unit tests for the handler now that it's straightforward. Modifications: The tests are in their own file `Tests/NWWaitingHandlerTests.swift`. * Bump swift-nio-transport-services to 1.13.0 * SwiftFormat@0.48.8 * Apply suggestions from code review Co-authored-by: David Nadoba --------- Co-authored-by: David Nadoba --- .../NWWaitingHandlerTests.swift | 79 +++++++++++++++++++ 1 file changed, 79 insertions(+) create mode 100644 Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift diff --git a/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift new file mode 100644 index 000000000..967a7da40 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift @@ -0,0 +1,79 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if canImport(Network) +@testable import AsyncHTTPClient +import Network +import NIOCore +import NIOEmbedded +import NIOSSL +import NIOTransportServices +import XCTest + +@available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) +class NWWaitingHandlerTests: XCTestCase { + class MockRequester: HTTPConnectionRequester { + var waitingForConnectivityCalled = false + var connectionID: AsyncHTTPClient.HTTPConnectionPool.Connection.ID? + var transientError: NWError? + + func http1ConnectionCreated(_: AsyncHTTPClient.HTTP1Connection) {} + + func http2ConnectionCreated(_: AsyncHTTPClient.HTTP2Connection, maximumStreams: Int) {} + + func failedToCreateHTTPConnection(_: AsyncHTTPClient.HTTPConnectionPool.Connection.ID, error: Error) {} + + func waitingForConnectivity(_ connectionID: AsyncHTTPClient.HTTPConnectionPool.Connection.ID, error: Error) { + self.waitingForConnectivityCalled = true + self.connectionID = connectionID + self.transientError = error as? NWError + } + } + + func testWaitingHandlerInvokesWaitingForConnectivity() { + let requester = MockRequester() + let connectionID: AsyncHTTPClient.HTTPConnectionPool.Connection.ID = 1 + let waitingEventHandler = NWWaitingHandler(requester: requester, connectionID: connectionID) + let embedded = EmbeddedChannel(handlers: [waitingEventHandler]) + + embedded.pipeline.fireUserInboundEventTriggered(NIOTSNetworkEvents.WaitingForConnectivity(transientError: .dns(1))) + + XCTAssertTrue(requester.waitingForConnectivityCalled, "Expected the handler to invoke .waitingForConnectivity on the requester") + XCTAssertEqual(requester.connectionID, connectionID, "Expected the handler to pass connectionID to requester") + XCTAssertEqual(requester.transientError, NWError.dns(1)) + } + + func testWaitingHandlerDoesNotInvokeWaitingForConnectionOnUnrelatedErrors() { + let requester = MockRequester() + let waitingEventHandler = NWWaitingHandler(requester: requester, connectionID: 1) + let embedded = EmbeddedChannel(handlers: [waitingEventHandler]) + embedded.pipeline.fireUserInboundEventTriggered(NIOTSNetworkEvents.BetterPathAvailable()) + + XCTAssertFalse(requester.waitingForConnectivityCalled, "Should not call .waitingForConnectivity on unrelated events") + } + + func testWaitingHandlerPassesTheEventDownTheContext() { + let requester = MockRequester() + let waitingEventHandler = NWWaitingHandler(requester: requester, connectionID: 1) + let tlsEventsHandler = TLSEventsHandler(deadline: nil) + let embedded = EmbeddedChannel(handlers: [waitingEventHandler, tlsEventsHandler]) + + embedded.pipeline.fireErrorCaught(NIOSSLError.handshakeFailed(BoringSSLError.wantConnect)) + XCTAssertThrowsError(try XCTUnwrap(tlsEventsHandler.tlsEstablishedFuture).wait()) { + XCTAssertEqualTypeAndValue($0, NIOSSLError.handshakeFailed(BoringSSLError.wantConnect)) + } + } +} + +#endif From 75d7f63abfad0378877cd80039b68fc18cca80e6 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 6 Sep 2023 11:08:09 +0100 Subject: [PATCH 091/146] Automatically chunk large request bodies (#710) * ChunkedCollection * Use swift-algorithms * SwiftFormat * test chunking * add documentation * SwiftFormat * fix old swift versions * fix older swift versions second attempt * fix old swift versions third attempt * fix old swift versions fourth attempt * update documentation --- Package.swift | 3 + .../AsyncAwait/HTTPClientRequest.swift | 224 ++++++++++++------ Sources/AsyncHTTPClient/HTTPHandler.swift | 34 ++- .../HTTPClientRequestTests.swift | 197 ++++++++++++--- 4 files changed, 349 insertions(+), 109 deletions(-) diff --git a/Package.swift b/Package.swift index 7e80f853e..db458583c 100644 --- a/Package.swift +++ b/Package.swift @@ -29,6 +29,7 @@ let package = Package( .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), + .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"), ], targets: [ .target(name: "CAsyncHTTPClient"), @@ -48,6 +49,7 @@ let package = Package( .product(name: "NIOTransportServices", package: "swift-nio-transport-services"), .product(name: "Logging", package: "swift-log"), .product(name: "Atomics", package: "swift-atomics"), + .product(name: "Algorithms", package: "swift-algorithms"), ] ), .testTarget( @@ -64,6 +66,7 @@ let package = Package( .product(name: "NIOSOCKS", package: "swift-nio-extras"), .product(name: "Logging", package: "swift-log"), .product(name: "Atomics", package: "swift-atomics"), + .product(name: "Algorithms", package: "swift-algorithms"), ], resources: [ .copy("Resources/self_signed_cert.pem"), diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index 278be7f84..a5b0a0061 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -12,9 +12,23 @@ // //===----------------------------------------------------------------------===// +import Algorithms import NIOCore import NIOHTTP1 +@usableFromInline +let bagOfBytesToByteBufferConversionChunkSize = 1024 * 1024 * 4 + +#if arch(arm) || arch(i386) +// on 32-bit platforms we can't make use of a whole UInt32.max (as it doesn't fit in an Int) +@usableFromInline +let byteBufferMaxSize = Int.max +#else +// on 64-bit platforms we're good +@usableFromInline +let byteBufferMaxSize = Int(UInt32.max) +#endif + /// A representation of an HTTP request for the Swift Concurrency HTTPClient API. /// /// This object is similar to ``HTTPClient/Request``, but used for the Swift Concurrency API. @@ -93,9 +107,9 @@ extension HTTPClientRequest.Body { /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `RandomAccessCollection` of bytes. /// - /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory - /// usage of this construction will be double the size of the original collection. The construction - /// of the `ByteBuffer` will be delayed until it's needed. + /// This construction will flatten the `bytes` into a `ByteBuffer` in chunks of ~4MB. + /// As a result, the peak memory usage of this construction will be a small multiple of ~4MB. + /// The construction of the `ByteBuffer` will be delayed until it's needed. /// /// - parameter bytes: The bytes of the request body. @inlinable @@ -103,24 +117,7 @@ extension HTTPClientRequest.Body { public static func bytes( _ bytes: Bytes ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes) - } - - @inlinable - static func _bytes( - _ bytes: Bytes - ) -> Self where Bytes.Element == UInt8 { - self.init(.sequence( - length: .known(bytes.count), - canBeConsumedMultipleTimes: true - ) { allocator in - if let buffer = bytes.withContiguousStorageIfAvailable({ allocator.buffer(bytes: $0) }) { - // fastpath - return buffer - } - // potentially really slow path - return allocator.buffer(bytes: bytes) - }) + self.bytes(bytes, length: .known(bytes.count)) } /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. @@ -146,32 +143,77 @@ extension HTTPClientRequest.Body { _ bytes: Bytes, length: Length ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes, length: length) + Self._bytes( + bytes, + length: length, + bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, + byteBufferMaxSize: byteBufferMaxSize + ) } + /// internal method to test chunking @inlinable - static func _bytes( + @preconcurrency + static func _bytes( _ bytes: Bytes, - length: Length + length: Length, + bagOfBytesToByteBufferConversionChunkSize: Int, + byteBufferMaxSize: Int ) -> Self where Bytes.Element == UInt8 { - self.init(.sequence( - length: length.storage, - canBeConsumedMultipleTimes: false - ) { allocator in - if let buffer = bytes.withContiguousStorageIfAvailable({ allocator.buffer(bytes: $0) }) { - // fastpath - return buffer + // fast path + let body: Self? = bytes.withContiguousStorageIfAvailable { bufferPointer -> Self in + // `some Sequence` is special as it can't be efficiently chunked lazily. + // Therefore we need to do the chunking eagerly if it implements the fast path withContiguousStorageIfAvailable + // If we do it eagerly, it doesn't make sense to do a bunch of small chunks, so we only chunk if it exceeds + // the maximum size of a ByteBuffer. + if bufferPointer.count <= byteBufferMaxSize { + let buffer = ByteBuffer(bytes: bufferPointer) + return Self(.sequence( + length: length.storage, + canBeConsumedMultipleTimes: true, + makeCompleteBody: { _ in buffer } + )) + } else { + // we need to copy `bufferPointer` eagerly as the pointer is only valid during the call to `withContiguousStorageIfAvailable` + let buffers: Array = bufferPointer.chunks(ofCount: byteBufferMaxSize).map { ByteBuffer(bytes: $0) } + return Self(.asyncSequence( + length: length.storage, + makeAsyncIterator: { + var iterator = buffers.makeIterator() + return { _ in + iterator.next() + } + } + )) + } + } + if let body = body { + return body + } + + // slow path + return Self(.asyncSequence( + length: length.storage + ) { + var iterator = bytes.makeIterator() + return { allocator in + var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) + while buffer.writableBytes > 0, let byte = iterator.next() { + buffer.writeInteger(byte) + } + if buffer.readableBytes > 0 { + return buffer + } + return nil } - // potentially really slow path - return allocator.buffer(bytes: bytes) }) } /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. /// - /// This construction will flatten the bytes into a `ByteBuffer`. As a result, the peak memory - /// usage of this construction will be double the size of the original collection. The construction - /// of the `ByteBuffer` will be delayed until it's needed. + /// This construction will flatten the `bytes` into a `ByteBuffer` in chunks of ~4MB. + /// As a result, the peak memory usage of this construction will be a small multiple of ~4MB. + /// The construction of the `ByteBuffer` will be delayed until it's needed. /// /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload @@ -186,25 +228,27 @@ extension HTTPClientRequest.Body { _ bytes: Bytes, length: Length ) -> Self where Bytes.Element == UInt8 { - Self._bytes(bytes, length: length) - } - - @inlinable - static func _bytes( - _ bytes: Bytes, - length: Length - ) -> Self where Bytes.Element == UInt8 { - self.init(.sequence( - length: length.storage, - canBeConsumedMultipleTimes: true - ) { allocator in - if let buffer = bytes.withContiguousStorageIfAvailable({ allocator.buffer(bytes: $0) }) { - // fastpath - return buffer - } - // potentially really slow path - return allocator.buffer(bytes: bytes) - }) + if bytes.count <= bagOfBytesToByteBufferConversionChunkSize { + return self.init(.sequence( + length: length.storage, + canBeConsumedMultipleTimes: true + ) { allocator in + allocator.buffer(bytes: bytes) + }) + } else { + return self.init(.asyncSequence( + length: length.storage, + makeAsyncIterator: { + var iterator = bytes.chunks(ofCount: bagOfBytesToByteBufferConversionChunkSize).makeIterator() + return { allocator in + guard let chunk = iterator.next() else { + return nil + } + return allocator.buffer(bytes: chunk) + } + } + )) + } } /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of `ByteBuffer`s. @@ -223,14 +267,6 @@ extension HTTPClientRequest.Body { public static func stream( _ sequenceOfBytes: SequenceOfBytes, length: Length - ) -> Self where SequenceOfBytes.Element == ByteBuffer { - Self._stream(sequenceOfBytes, length: length) - } - - @inlinable - static func _stream( - _ sequenceOfBytes: SequenceOfBytes, - length: Length ) -> Self where SequenceOfBytes.Element == ByteBuffer { let body = self.init(.asyncSequence(length: length.storage) { var iterator = sequenceOfBytes.makeAsyncIterator() @@ -243,7 +279,7 @@ extension HTTPClientRequest.Body { /// Create an ``HTTPClientRequest/Body-swift.struct`` from an `AsyncSequence` of bytes. /// - /// This construction will consume 1kB chunks from the `Bytes` and send them at once. This optimizes for + /// This construction will consume 4MB chunks from the `Bytes` and send them at once. This optimizes for /// `AsyncSequence`s where larger chunks are buffered up and available without actually suspending, such /// as those provided by `FileHandle`. /// @@ -259,19 +295,11 @@ extension HTTPClientRequest.Body { public static func stream( _ bytes: Bytes, length: Length - ) -> Self where Bytes.Element == UInt8 { - Self._stream(bytes, length: length) - } - - @inlinable - static func _stream( - _ bytes: Bytes, - length: Length ) -> Self where Bytes.Element == UInt8 { let body = self.init(.asyncSequence(length: length.storage) { var iterator = bytes.makeAsyncIterator() return { allocator -> ByteBuffer? in - var buffer = allocator.buffer(capacity: 1024) // TODO: Magic number + var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) while buffer.writableBytes > 0, let byte = try await iterator.next() { buffer.writeInteger(byte) } @@ -313,3 +341,53 @@ extension HTTPClientRequest.Body { internal var storage: RequestBodyLength } } + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientRequest.Body: AsyncSequence { + public typealias Element = ByteBuffer + + @inlinable + public func makeAsyncIterator() -> AsyncIterator { + switch self.mode { + case .asyncSequence(_, let makeAsyncIterator): + return .init(storage: .makeNext(makeAsyncIterator())) + case .sequence(_, _, let makeCompleteBody): + return .init(storage: .byteBuffer(makeCompleteBody(AsyncIterator.allocator))) + case .byteBuffer(let byteBuffer): + return .init(storage: .byteBuffer(byteBuffer)) + } + } +} + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientRequest.Body { + public struct AsyncIterator: AsyncIteratorProtocol { + @usableFromInline + static let allocator = ByteBufferAllocator() + + @usableFromInline + enum Storage { + case byteBuffer(ByteBuffer?) + case makeNext((ByteBufferAllocator) async throws -> ByteBuffer?) + } + + @usableFromInline + var storage: Storage + + @inlinable + init(storage: Storage) { + self.storage = storage + } + + @inlinable + public mutating func next() async throws -> ByteBuffer? { + switch self.storage { + case .byteBuffer(let buffer): + self.storage = .byteBuffer(nil) + return buffer + case .makeNext(let makeNext): + return try await makeNext(Self.allocator) + } + } + } +} diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index c5a3b3a4a..33e68995e 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Algorithms import Foundation import Logging import NIOConcurrencyHelpers @@ -44,6 +45,27 @@ extension HTTPClient { public func write(_ data: IOData) -> EventLoopFuture { return self.closure(data) } + + @inlinable + func writeChunks(of bytes: Bytes, maxChunkSize: Int) -> EventLoopFuture where Bytes.Element == UInt8 { + let iterator = UnsafeMutableTransferBox(bytes.chunks(ofCount: maxChunkSize).makeIterator()) + guard let chunk = iterator.wrappedValue.next() else { + return self.write(IOData.byteBuffer(.init())) + } + + @Sendable // can't use closure here as we recursively call ourselves which closures can't do + func writeNextChunk(_ chunk: Bytes.SubSequence) -> EventLoopFuture { + if let nextChunk = iterator.wrappedValue.next() { + return self.write(.byteBuffer(ByteBuffer(bytes: chunk))).flatMap { + writeNextChunk(nextChunk) + } + } else { + return self.write(.byteBuffer(ByteBuffer(bytes: chunk))) + } + } + + return writeNextChunk(chunk) + } } /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` @@ -90,7 +112,11 @@ extension HTTPClient { @inlinable public static func bytes(_ bytes: Bytes) -> Body where Bytes: RandomAccessCollection, Bytes: Sendable, Bytes.Element == UInt8 { return Body(length: bytes.count) { writer in - writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) + if bytes.count <= bagOfBytesToByteBufferConversionChunkSize { + return writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) + } else { + return writer.writeChunks(of: bytes, maxChunkSize: bagOfBytesToByteBufferConversionChunkSize) + } } } @@ -100,7 +126,11 @@ extension HTTPClient { /// - string: Body `String` representation. public static func string(_ string: String) -> Body { return Body(length: string.utf8.count) { writer in - writer.write(.byteBuffer(ByteBuffer(string: string))) + if string.utf8.count <= bagOfBytesToByteBufferConversionChunkSize { + return writer.write(.byteBuffer(ByteBuffer(string: string))) + } else { + return writer.writeChunks(of: string.utf8, maxChunkSize: bagOfBytesToByteBufferConversionChunkSize) + } } } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index aa1071de6..3536160fd 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Algorithms @testable import AsyncHTTPClient import NIOCore import XCTest @@ -393,7 +394,7 @@ class HTTPClientRequestTests: XCTestCase { request.method = .POST let asyncSequence = ByteBuffer(string: "post body") .readableBytesView - .chunked(maxChunkSize: 2) + .chunks(ofCount: 2) .async .map { ByteBuffer($0) } @@ -433,7 +434,7 @@ class HTTPClientRequestTests: XCTestCase { request.method = .POST let asyncSequence = ByteBuffer(string: "post body") .readableBytesView - .chunked(maxChunkSize: 2) + .chunks(ofCount: 2) .async .map { ByteBuffer($0) } @@ -465,6 +466,166 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(buffer, .init(string: "post body")) } } + + func testChunkingRandomAccessCollection() async throws { + let body = try await HTTPClientRequest.Body.bytes( + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + ).collect() + + let expectedChunks = [ + ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize), + ] + + XCTAssertEqual(body, expectedChunks) + } + + func testChunkingCollection() async throws { + let body = try await HTTPClientRequest.Body.bytes( + ( + String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) + ).utf8, + length: .known(bagOfBytesToByteBufferConversionChunkSize * 3) + ).collect() + + let expectedChunks = [ + ByteBuffer(repeating: UInt8(ascii: "0"), count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: UInt8(ascii: "1"), count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: UInt8(ascii: "2"), count: bagOfBytesToByteBufferConversionChunkSize), + ] + + XCTAssertEqual(body, expectedChunks) + } + + func testChunkingSequenceThatDoesNotImplementWithContiguousStorageIfAvailable() async throws { + let bagOfBytesToByteBufferConversionChunkSize = 8 + let body = try await HTTPClientRequest.Body._bytes( + AnySequence( + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + ), + length: .known(bagOfBytesToByteBufferConversionChunkSize * 3), + bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, + byteBufferMaxSize: byteBufferMaxSize + ).collect() + + let expectedChunks = [ + ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize), + ] + + XCTAssertEqual(body, expectedChunks) + } + + #if swift(>=5.7) + func testChunkingSequenceFastPath() async throws { + func makeBytes() -> some Sequence & Sendable { + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + } + let body = try await HTTPClientRequest.Body.bytes( + makeBytes(), + length: .known(bagOfBytesToByteBufferConversionChunkSize * 3) + ).collect() + + var firstChunk = ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + firstChunk.writeImmutableBuffer(ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize)) + firstChunk.writeImmutableBuffer(ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize)) + let expectedChunks = [ + firstChunk, + ] + + XCTAssertEqual(body, expectedChunks) + } + + func testChunkingSequenceFastPathExceedingByteBufferMaxSize() async throws { + let bagOfBytesToByteBufferConversionChunkSize = 8 + let byteBufferMaxSize = 16 + func makeBytes() -> some Sequence & Sendable { + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + } + let body = try await HTTPClientRequest.Body._bytes( + makeBytes(), + length: .known(bagOfBytesToByteBufferConversionChunkSize * 3), + bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, + byteBufferMaxSize: byteBufferMaxSize + ).collect() + + var firstChunk = ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + firstChunk.writeImmutableBuffer(ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize)) + let secondChunk = ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + let expectedChunks = [ + firstChunk, + secondChunk, + ] + + XCTAssertEqual(body, expectedChunks) + } + #endif + + func testBodyStringChunking() throws { + let body = try HTTPClient.Body.string( + String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) + ).collect().wait() + + let expectedChunks = [ + ByteBuffer(repeating: UInt8(ascii: "0"), count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: UInt8(ascii: "1"), count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: UInt8(ascii: "2"), count: bagOfBytesToByteBufferConversionChunkSize), + ] + + XCTAssertEqual(body, expectedChunks) + } + + func testBodyChunkingRandomAccessCollection() throws { + let body = try HTTPClient.Body.bytes( + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + ).collect().wait() + + let expectedChunks = [ + ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize), + ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize), + ] + + XCTAssertEqual(body, expectedChunks) + } +} + +extension AsyncSequence { + func collect() async throws -> [Element] { + try await self.reduce(into: []) { $0 += CollectionOfOne($1) } + } +} + +extension HTTPClient.Body { + func collect() -> EventLoopFuture<[ByteBuffer]> { + let eelg = EmbeddedEventLoopGroup(loops: 1) + let el = eelg.next() + var body = [ByteBuffer]() + let writer = StreamWriter { + switch $0 { + case .byteBuffer(let byteBuffer): + body.append(byteBuffer) + case .fileRegion: + fatalError("file region not supported") + } + return el.makeSucceededVoidFuture() + } + return self.stream(writer).map { _ in body } + } } private struct LengthMismatch: Error { @@ -502,35 +663,3 @@ extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { } } } - -struct ChunkedSequence: Sequence { - struct Iterator: IteratorProtocol { - fileprivate var remainingElements: Wrapped.SubSequence - fileprivate let maxChunkSize: Int - mutating func next() -> Wrapped.SubSequence? { - guard !self.remainingElements.isEmpty else { - return nil - } - let chunk = self.remainingElements.prefix(self.maxChunkSize) - self.remainingElements = self.remainingElements.dropFirst(self.maxChunkSize) - return chunk - } - } - - fileprivate let wrapped: Wrapped - fileprivate let maxChunkSize: Int - - func makeIterator() -> Iterator { - .init(remainingElements: self.wrapped[...], maxChunkSize: self.maxChunkSize) - } -} - -extension ChunkedSequence: Sendable where Wrapped: Sendable {} - -extension Collection { - /// Lazily splits `self` into `SubSequence`s with `maxChunkSize` elements. - /// - Parameter maxChunkSize: size of each chunk except the last one which can be smaller if not enough elements are remaining. - func chunked(maxChunkSize: Int) -> ChunkedSequence { - .init(wrapped: self, maxChunkSize: maxChunkSize) - } -} From 4e74fefd1cc4e747e7989cbe6f125553d3f526e0 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 6 Sep 2023 13:49:25 +0100 Subject: [PATCH 092/146] Support custom `backgroundActivityLogger` with using the default ELG. (#711) --- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 53da87e75..f39a86c95 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -133,7 +133,7 @@ public class HTTPClient { /// - eventLoopGroup: The `EventLoopGroup` that the ``HTTPClient`` will use. /// - configuration: Client configuration. /// - backgroundActivityLogger: The `Logger` that will be used to log background any activity that's not associated with a request. - public required init(eventLoopGroup: any EventLoopGroup, + public required init(eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, configuration: Configuration = Configuration(), backgroundActivityLogger: Logger) { self.eventLoopGroup = eventLoopGroup From de7c84a6071de3114e5302c6585828f33c411a4f Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Wed, 4 Oct 2023 17:43:21 +0100 Subject: [PATCH 093/146] Bump minimum Swift version to 5.7 (#712) Motivation: Now that Swift 5.9 is GM we should update the supported versions and remove 5.6 Modifications: * Update `Package.swift` * Remove `#if swift(>=5.7)` guards * Delete the 5.6 docker compose file and make a 5.10 one * Update docs Result: Remove support for Swift 5.6, add 5.10 --- Package.swift | 2 +- README.md | 3 ++- Sources/AsyncHTTPClient/HTTPClient.swift | 11 ---------- Sources/AsyncHTTPClient/HTTPHandler.swift | 10 --------- .../HTTPClientRequestTests.swift | 2 -- docker/docker-compose.2004.56.yaml | 21 ------------------- docker/docker-compose.2204.510.yaml | 21 +++++++++++++++++++ docker/docker-compose.2204.59.yaml | 3 ++- 8 files changed, 26 insertions(+), 47 deletions(-) delete mode 100644 docker/docker-compose.2004.56.yaml create mode 100644 docker/docker-compose.2204.510.yaml diff --git a/Package.swift b/Package.swift index db458583c..23a733d75 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.6 +// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the AsyncHTTPClient open source project diff --git a/README.md b/README.md index 806cc2960..45a847fea 100644 --- a/README.md +++ b/README.md @@ -328,4 +328,5 @@ AsyncHTTPClient | Minimum Swift Version `1.5.0 ..< 1.10.0` | 5.2 `1.10.0 ..< 1.13.0` | 5.4 `1.13.0 ..< 1.18.0` | 5.5.2 -`1.18.0 ...` | 5.6 +`1.18.0 ..< 1.20.0` | 5.6 +`1.20.0 ...` | 5.7 diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index f39a86c95..46d238b23 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -166,7 +166,6 @@ public class HTTPClient { } } - #if swift(>=5.7) /// Shuts down the client and `EventLoopGroup` if it was created by the client. /// /// This method blocks the thread indefinitely, prefer using ``shutdown()-96ayw``. @@ -174,14 +173,6 @@ public class HTTPClient { public func syncShutdown() throws { try self.syncShutdown(requiresCleanClose: false) } - #else - /// Shuts down the client and `EventLoopGroup` if it was created by the client. - /// - /// This method blocks the thread indefinitely, prefer using ``shutdown()-96ayw``. - public func syncShutdown() throws { - try self.syncShutdown(requiresCleanClose: false) - } - #endif /// Shuts down the client and `EventLoopGroup` if it was created by the client. /// @@ -921,9 +912,7 @@ extension HTTPClient { } } -#if swift(>=5.7) extension HTTPClient.Configuration: Sendable {} -#endif extension HTTPClient.EventLoopGroupProvider: Sendable {} extension HTTPClient.EventLoopPreference: Sendable {} diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 33e68995e..98415a124 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -750,7 +750,6 @@ extension HTTPClient { return self.promise.futureResult } - #if swift(>=5.7) /// Waits for execution of this request to complete. /// /// - returns: The value of ``futureResult`` when it completes. @@ -759,15 +758,6 @@ extension HTTPClient { public func wait() throws -> Response { return try self.promise.futureResult.wait() } - #else - /// Waits for execution of this request to complete. - /// - /// - returns: The value of ``futureResult`` when it completes. - /// - throws: The error value of ``futureResult`` if it errors. - public func wait() throws -> Response { - return try self.promise.futureResult.wait() - } - #endif /// Provides the result of this request. /// diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 3536160fd..f0a329f2a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -522,7 +522,6 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(body, expectedChunks) } - #if swift(>=5.7) func testChunkingSequenceFastPath() async throws { func makeBytes() -> some Sequence & Sendable { Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + @@ -569,7 +568,6 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertEqual(body, expectedChunks) } - #endif func testBodyStringChunking() throws { let body = try HTTPClient.Body.string( diff --git a/docker/docker-compose.2004.56.yaml b/docker/docker-compose.2004.56.yaml deleted file mode 100644 index b496e8484..000000000 --- a/docker/docker-compose.2004.56.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:20.04-5.6 - build: - args: - ubuntu_version: "focal" - swift_version: "5.6" - - documentation-check: - image: async-http-client:20.04-5.6 - - test: - image: async-http-client:20.04-5.6 - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:20.04-5.6 diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml new file mode 100644 index 000000000..fdc3d2bdd --- /dev/null +++ b/docker/docker-compose.2204.510.yaml @@ -0,0 +1,21 @@ +version: "3" + +services: + + runtime-setup: + image: async-http-client:22.04-5.10 + build: + args: + base_image: "swiftlang/swift:nightly-5.10-jammy" + + documentation-check: + image: async-http-client:22.04-5.10 + + test: + image: async-http-client:22.04-5.10 + environment: + - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error + #- SANITIZER_ARG=--sanitize=thread + + shell: + image: async-http-client:22.04-5.10 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index 2c5a9e297..b125fff39 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -6,7 +6,8 @@ services: image: async-http-client:22.04-5.9 build: args: - base_image: "swiftlang/swift:nightly-5.9-jammy" + ubuntu_version: "jammy" + swift_version: "5.9" documentation-check: image: async-http-client:22.04-5.9 From 4c07d3bbf63fcd8e62e3b7599c7eb743a15b10c4 Mon Sep 17 00:00:00 2001 From: David Nadoba Date: Wed, 4 Oct 2023 19:47:57 +0200 Subject: [PATCH 094/146] Fix flaky test `TransactionTests.testCancelAsyncRequest` (#707) * Fix flaky test `TransactionTests.testCancelAsyncRequest` Resolves #706. The continuation is resumed with the cancelation error before we cancel the scheduled request. https://github.com/swift-server/async-http-client/blob/62c06d47c8d21c91335e9f8998589e4ce31411e6/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift#L290C10-L294 Therefore this test is inherently flaky. The fix waits up to one second for the scheduled request to be canceled. * self.fulfillment(of:) is not available on Linux * Fix formatting * Fix 5.6 --- Tests/AsyncHTTPClientTests/RequestBagTests.swift | 11 ++++++++--- Tests/AsyncHTTPClientTests/TransactionTests.swift | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 75d57ba26..e2a959589 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -961,13 +961,18 @@ class UploadCountingDelegate: HTTPClientResponseDelegate { } } -class MockTaskQueuer: HTTPRequestScheduler { +final class MockTaskQueuer: HTTPRequestScheduler { private(set) var hitCancelCount = 0 - init() {} + let onCancelRequest: (@Sendable (HTTPSchedulableRequest) -> Void)? - func cancelRequest(_: HTTPSchedulableRequest) { + init(onCancelRequest: (@Sendable (HTTPSchedulableRequest) -> Void)? = nil) { + self.onCancelRequest = onCancelRequest + } + + func cancelRequest(_ request: HTTPSchedulableRequest) { self.hitCancelCount += 1 + self.onCancelRequest?(request) } } diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 19454681c..13cb87eb5 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -27,6 +27,9 @@ typealias PreparedRequest = HTTPClientRequest.Prepared final class TransactionTests: XCTestCase { func testCancelAsyncRequest() { guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } + // creating the `XCTestExpectation` off the main thread crashes on Linux with Swift 5.6 + // therefore we create it here as a workaround which works fine + let scheduledRequestCanceled = self.expectation(description: "scheduled request canceled") XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -43,19 +46,25 @@ final class TransactionTests: XCTestCase { preferredEventLoop: embeddedEventLoop ) - let queuer = MockTaskQueuer() + let queuer = MockTaskQueuer { _ in + scheduledRequestCanceled.fulfill() + } transaction.requestWasQueued(queuer) + XCTAssertEqual(queuer.hitCancelCount, 0) Task.detached { try await Task.sleep(nanoseconds: 5 * 1000 * 1000) transaction.cancel() } - XCTAssertEqual(queuer.hitCancelCount, 0) await XCTAssertThrowsError(try await responseTask.value) { error in XCTAssertTrue(error is CancellationError, "unexpected error \(error)") } - XCTAssertEqual(queuer.hitCancelCount, 1) + + // self.fulfillment(of:) is not available on Linux + _ = { + self.wait(for: [scheduledRequestCanceled], timeout: 1) + }() } } From d766674c7e9c12f3f9f75ddef31af32ff6606fd0 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Mon, 16 Oct 2023 15:00:58 +0100 Subject: [PATCH 095/146] HTTPClientRequest: allow custom TLS config (#709) --- .../AsyncAwait/HTTPClient+execute.swift | 2 +- .../HTTPClientRequest+Prepared.swift | 7 ++- .../AsyncAwait/HTTPClientRequest.swift | 8 ++++ .../AsyncAwait/Transaction.swift | 2 +- .../HTTPClientTests.swift | 45 +++++++++++++++++++ 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index 5f0a5f7c5..ef858443e 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -78,7 +78,7 @@ extension HTTPClient { // this loop is there to follow potential redirects while true { let preparedRequest = try HTTPClientRequest.Prepared(currentRequest, dnsOverride: configuration.dnsOverride) - let response = try await executeCancellable(preparedRequest, deadline: deadline, logger: logger) + let response = try await self.executeCancellable(preparedRequest, deadline: deadline, logger: logger) guard var redirectState = currentRedirectState else { // a `nil` redirectState means we should not follow redirects diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index 489ba5626..360e91b89 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -15,6 +15,7 @@ import struct Foundation.URL import NIOCore import NIOHTTP1 +import NIOSSL @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest { @@ -37,6 +38,7 @@ extension HTTPClientRequest { var requestFramingMetadata: RequestFramingMetadata var head: HTTPRequestHead var body: Body? + var tlsConfiguration: TLSConfiguration? } } @@ -58,7 +60,7 @@ extension HTTPClientRequest.Prepared { self.init( url: url, - poolKey: .init(url: deconstructedURL, tlsConfiguration: nil, dnsOverride: dnsOverride), + poolKey: .init(url: deconstructedURL, tlsConfiguration: request.tlsConfiguration, dnsOverride: dnsOverride), requestFramingMetadata: metadata, head: .init( version: .http1_1, @@ -66,7 +68,8 @@ extension HTTPClientRequest.Prepared { uri: deconstructedURL.uri, headers: headers ), - body: request.body.map { .init($0) } + body: request.body.map { .init($0) }, + tlsConfiguration: request.tlsConfiguration ) } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index a5b0a0061..4ed79e38c 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -15,6 +15,7 @@ import Algorithms import NIOCore import NIOHTTP1 +import NIOSSL @usableFromInline let bagOfBytesToByteBufferConversionChunkSize = 1024 * 1024 * 4 @@ -32,6 +33,9 @@ let byteBufferMaxSize = Int(UInt32.max) /// A representation of an HTTP request for the Swift Concurrency HTTPClient API. /// /// This object is similar to ``HTTPClient/Request``, but used for the Swift Concurrency API. +/// +/// - note: For many ``HTTPClientRequest/body-swift.property`` configurations, this type is _not_ a value type +/// (https://github.com/swift-server/async-http-client/issues/708). @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) public struct HTTPClientRequest: Sendable { /// The request URL, including scheme, hostname, and optionally port. @@ -46,11 +50,15 @@ public struct HTTPClientRequest: Sendable { /// The request body, if any. public var body: Body? + /// Request-specific TLS configuration, defaults to no request-specific TLS configuration. + public var tlsConfiguration: TLSConfiguration? + public init(url: String) { self.url = url self.method = .GET self.headers = .init() self.body = .none + self.tlsConfiguration = nil } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index d3793c990..6d9192642 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -146,7 +146,7 @@ import NIOSSL @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction: HTTPSchedulableRequest { var poolKey: ConnectionPool.Key { self.request.poolKey } - var tlsConfiguration: TLSConfiguration? { return nil } + var tlsConfiguration: TLSConfiguration? { return self.request.tlsConfiguration } var requiredEventLoop: EventLoop? { return nil } func requestWasQueued(_ scheduler: HTTPRequestScheduler) { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 218abaf82..c6a67c155 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3525,4 +3525,49 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let response = try client.get(url: self.defaultHTTPBinURLPrefix + "get").wait() XCTAssertEqual(.ok, response.status) } + + func testAsyncExecuteWithCustomTLS() async throws { + let httpsBin = HTTPBin(.http1_1(ssl: true)) + defer { + XCTAssertNoThrow(try httpsBin.shutdown()) + } + + // A client with default TLS settings, i.e. it won't accept `httpsBin`'s fake self-signed cert + let client = HTTPClient(eventLoopGroup: MultiThreadedEventLoopGroup.singleton) + defer { + XCTAssertNoThrow(try client.shutdown().wait()) + } + + var request = HTTPClientRequest(url: "https://localhost:\(httpsBin.port)/get") + + // For now, let's allow bad TLS certs + request.tlsConfiguration = TLSConfiguration.clientDefault + // ! is safe, assigned above + request.tlsConfiguration!.certificateVerification = .none + + let response1 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + XCTAssertEqual(.ok, response1.status) + + // For the second request, we reset the TLS config + request.tlsConfiguration = nil + do { + let response2 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + XCTFail("shouldn't succeed, self-signed cert: \(response2)") + } catch { + switch error as? NIOSSLError { + case .some(.handshakeFailed(_)): + () // ok + default: + XCTFail("unexpected error: \(error)") + } + } + + // And finally we allow it again. + request.tlsConfiguration = TLSConfiguration.clientDefault + // ! is safe, assigned above + request.tlsConfiguration!.certificateVerification = .none + + let response3 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + XCTAssertEqual(.ok, response3.status) + } } From 4824907382973f91b1cc681e54a15cf68f4bf4df Mon Sep 17 00:00:00 2001 From: Mahdi Bahrami Date: Wed, 25 Oct 2023 18:49:44 +0330 Subject: [PATCH 096/146] Fix wrong/outdated `connect` timeout documentation (#714) --- Sources/AsyncHTTPClient/HTTPClient.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 46d238b23..db8ed3d97 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -923,7 +923,7 @@ extension HTTPClient: @unchecked Sendable {} extension HTTPClient.Configuration { /// Timeout configuration. public struct Timeout: Sendable { - /// Specifies connect timeout. If no connect timeout is given, a default 30 seconds timeout will applied. + /// Specifies connect timeout. If no connect timeout is given, a default 10 seconds timeout will be applied. public var connect: TimeAmount? /// Specifies read timeout. public var read: TimeAmount? From c70e0856797ef826ff13627790241011f831975f Mon Sep 17 00:00:00 2001 From: Peter Adams <63288215+PeterAdams-A@users.noreply.github.com> Date: Fri, 3 Nov 2023 13:58:47 +0000 Subject: [PATCH 097/146] testPlatformConnectErrorIsForwardedOnTimeout port reuse (#716) Motivation: The above test has been seen to fail with port already in use. The test assumes that a port can be bound to twice in a row. It's possible that another process takes the port between the two binds - this can be stopped by binding a second time before stopping the first. Modifications: Allow port reuse and reorder the test to bind a second time before releasing the first. Result: Test should no longer be flaky. --- Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift | 10 ++++++---- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 7 +++++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 71eb0c44b..97f0385ea 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -376,7 +376,7 @@ class HTTP2ClientTests: XCTestCase { } func testPlatformConnectErrorIsForwardedOnTimeout() { - let bin = HTTPBin(.http2(compress: false)) + let bin = HTTPBin(.http2(compress: false), reusePort: true) let clientGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) let el1 = clientGroup.next() let el2 = clientGroup.next() @@ -404,20 +404,22 @@ class HTTP2ClientTests: XCTestCase { XCTAssertEqual(.ok, response1?.status) XCTAssertEqual(response1?.version, .http2) let serverPort = bin.port - XCTAssertNoThrow(try bin.shutdown()) - // client is now in HTTP/2 state and the HTTPBin is closed - // start a new server on the old port which closes all connections immediately + let serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) defer { XCTAssertNoThrow(try serverGroup.syncShutdownGracefully()) } var maybeServer: Channel? XCTAssertNoThrow(maybeServer = try ServerBootstrap(group: serverGroup) .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) + .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), value: 1) .childChannelInitializer { channel in channel.close() } .childChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) .bind(host: "127.0.0.1", port: serverPort) .wait()) + // shutting down the old server closes all connections immediately + XCTAssertNoThrow(try bin.shutdown()) + // client is now in HTTP/2 state and the HTTPBin is closed guard let server = maybeServer else { return } defer { XCTAssertNoThrow(try server.close().wait()) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index fc0879de3..2ebccdafe 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -438,6 +438,7 @@ internal final class HTTPBin where _ mode: Mode = .http1_1(ssl: false, compress: false), proxy: Proxy = .none, bindTarget: BindTarget = .localhostIPv4RandomPort, + reusePort: Bool = false, handlerFactory: @escaping (Int) -> (RequestHandler) ) { self.mode = mode @@ -460,6 +461,7 @@ internal final class HTTPBin where self.serverChannel = try! ServerBootstrap(group: self.group) .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEADDR), value: 1) + .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), value: reusePort ? 1 : 0) .serverChannelInitializer { channel in channel.pipeline.addHandler(self.activeConnCounterHandler) }.childChannelInitializer { channel in @@ -642,9 +644,10 @@ extension HTTPBin where RequestHandler == HTTPBinHandler { convenience init( _ mode: Mode = .http1_1(ssl: false, compress: false), proxy: Proxy = .none, - bindTarget: BindTarget = .localhostIPv4RandomPort + bindTarget: BindTarget = .localhostIPv4RandomPort, + reusePort: Bool = false ) { - self.init(mode, proxy: proxy, bindTarget: bindTarget) { HTTPBinHandler(connectionID: $0) } + self.init(mode, proxy: proxy, bindTarget: bindTarget, reusePort: reusePort) { HTTPBinHandler(connectionID: $0) } } } From d2d35663a2286ebc673fe9311b3c0358d5dc50d5 Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Mon, 18 Dec 2023 09:06:06 -0300 Subject: [PATCH 098/146] Add an idle write timeout (#718) --- .../AsyncAwait/Transaction+StateMachine.swift | 1 + .../HTTP1/HTTP1ClientChannelHandler.swift | 167 +++++++++++++++++- .../HTTP1/HTTP1ConnectionStateMachine.swift | 12 ++ .../HTTP2/HTTP2ClientRequestHandler.swift | 77 +++++++- .../HTTPRequestStateMachine.swift | 22 +++ .../ConnectionPool/RequestOptions.swift | 12 +- Sources/AsyncHTTPClient/HTTPClient.swift | 39 +++- .../HTTP1ClientChannelHandlerTests.swift | 141 ++++++++++++++- .../HTTP2ClientRequestHandlerTests.swift | 133 ++++++++++++++ .../RequestBagTests.swift | 2 + 10 files changed, 585 insertions(+), 21 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 538424538..ad49332c0 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -356,6 +356,7 @@ extension Transaction { // response body stream. let body = TransactionBody.makeSequence( backPressureStrategy: .init(lowWatermark: 1, highWatermark: 1), + finishOnDeinit: true, delegate: AnyAsyncSequenceProducerDelegate(delegate) ) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 63cb70b99..ba3a09cc9 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -42,9 +42,17 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { if let idleReadTimeout = newRequest.requestOptions.idleReadTimeout { self.idleReadTimeoutStateMachine = .init(timeAmount: idleReadTimeout) } + + if let idleWriteTimeout = newRequest.requestOptions.idleWriteTimeout { + self.idleWriteTimeoutStateMachine = .init( + timeAmount: idleWriteTimeout, + isWritabilityEnabled: self.channelContext?.channel.isWritable ?? false + ) + } } else { self.logger = self.backgroundLogger self.idleReadTimeoutStateMachine = nil + self.idleWriteTimeoutStateMachine = nil } } } @@ -57,6 +65,14 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { /// We check in the task if the timer ID has changed in the meantime and do not execute any action if has changed. private var currentIdleReadTimeoutTimerID: Int = 0 + private var idleWriteTimeoutStateMachine: IdleWriteStateMachine? + private var idleWriteTimeoutTimer: Scheduled? + + /// Cancelling a task in NIO does *not* guarantee that the task will not execute under certain race conditions. + /// We therefore give each timer an ID and increase the ID every time we reset or cancel it. + /// We check in the task if the timer ID has changed in the meantime and do not execute any action if has changed. + private var currentIdleWriteTimeoutTimerID: Int = 0 + private let backgroundLogger: Logger private var logger: Logger private let eventLoop: EventLoop @@ -106,6 +122,10 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { "ahc-channel-writable": "\(context.channel.isWritable)", ]) + if let timeoutAction = self.idleWriteTimeoutStateMachine?.channelWritabilityChanged(context: context) { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.writabilityChanged(writable: context.channel.isWritable) self.run(action, context: context) context.fireChannelWritabilityChanged() @@ -150,6 +170,11 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { self.request = req self.logger.debug("Request was scheduled on connection") + + if let timeoutAction = self.idleWriteTimeoutStateMachine?.write() { + self.runTimeoutAction(timeoutAction, context: context) + } + req.willExecuteRequest(self) let action = self.state.runNewRequest( @@ -196,8 +221,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { request.resumeRequestBodyStream() } if startIdleTimer { - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) + if let readTimeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(readTimeoutAction, context: context) + } + + if let writeTimeoutAction = self.idleWriteTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(writeTimeoutAction, context: context) } } case .sendBodyPart(let part, let writePromise): @@ -206,8 +235,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { case .sendRequestEnd(let writePromise): context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) + if let readTimeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(readTimeoutAction, context: context) + } + + if let writeTimeoutAction = self.idleWriteTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(writeTimeoutAction, context: context) } case .pauseRequestBodyStream: @@ -380,6 +413,40 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } } + private func runTimeoutAction(_ action: IdleWriteStateMachine.Action, context: ChannelHandlerContext) { + switch action { + case .startIdleWriteTimeoutTimer(let timeAmount): + assert(self.idleWriteTimeoutTimer == nil, "Expected there is no timeout timer so far.") + + let timerID = self.currentIdleWriteTimeoutTimerID + self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { + guard self.currentIdleWriteTimeoutTimerID == timerID else { return } + let action = self.state.idleWriteTimeoutTriggered() + self.run(action, context: context) + } + case .resetIdleWriteTimeoutTimer(let timeAmount): + if let oldTimer = self.idleWriteTimeoutTimer { + oldTimer.cancel() + } + + self.currentIdleWriteTimeoutTimerID &+= 1 + let timerID = self.currentIdleWriteTimeoutTimerID + self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { + guard self.currentIdleWriteTimeoutTimerID == timerID else { return } + let action = self.state.idleWriteTimeoutTriggered() + self.run(action, context: context) + } + case .clearIdleWriteTimeoutTimer: + if let oldTimer = self.idleWriteTimeoutTimer { + self.idleWriteTimeoutTimer = nil + self.currentIdleWriteTimeoutTimerID &+= 1 + oldTimer.cancel() + } + case .none: + break + } + } + // MARK: Private HTTPRequestExecutor private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { @@ -393,6 +460,10 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { return } + if let timeoutAction = self.idleWriteTimeoutStateMachine?.write() { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.requestStreamPartReceived(data, promise: promise) self.run(action, context: context) } @@ -428,6 +499,10 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { self.logger.trace("Request was cancelled") + if let timeoutAction = self.idleWriteTimeoutStateMachine?.cancelRequest() { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.requestCancelled(closeConnection: true) self.run(action, context: context) } @@ -540,3 +615,87 @@ struct IdleReadStateMachine { } } } + +struct IdleWriteStateMachine { + enum Action { + case startIdleWriteTimeoutTimer(TimeAmount) + case resetIdleWriteTimeoutTimer(TimeAmount) + case clearIdleWriteTimeoutTimer + case none + } + + enum State { + case waitingForRequestEnd + case waitingForWritabilityEnabled + case requestEndSent + } + + private var state: State + private let timeAmount: TimeAmount + + init(timeAmount: TimeAmount, isWritabilityEnabled: Bool) { + self.timeAmount = timeAmount + if isWritabilityEnabled { + self.state = .waitingForRequestEnd + } else { + self.state = .waitingForWritabilityEnabled + } + } + + mutating func cancelRequest() -> Action { + switch self.state { + case .waitingForRequestEnd, .waitingForWritabilityEnabled: + self.state = .requestEndSent + return .clearIdleWriteTimeoutTimer + case .requestEndSent: + return .none + } + } + + mutating func write() -> Action { + switch self.state { + case .waitingForRequestEnd: + return .resetIdleWriteTimeoutTimer(self.timeAmount) + case .waitingForWritabilityEnabled: + return .none + case .requestEndSent: + preconditionFailure("If the request end has been sent, we can't write more data.") + } + } + + mutating func requestEndSent() -> Action { + switch self.state { + case .waitingForRequestEnd: + self.state = .requestEndSent + return .clearIdleWriteTimeoutTimer + case .waitingForWritabilityEnabled: + preconditionFailure("If the channel is not writable, we can't have sent the request end.") + case .requestEndSent: + return .none + } + } + + mutating func channelWritabilityChanged(context: ChannelHandlerContext) -> Action { + if context.channel.isWritable { + switch self.state { + case .waitingForRequestEnd: + preconditionFailure("If waiting for more data, the channel was already writable.") + case .waitingForWritabilityEnabled: + self.state = .waitingForRequestEnd + return .startIdleWriteTimeoutTimer(self.timeAmount) + case .requestEndSent: + return .none + } + } else { + switch self.state { + case .waitingForRequestEnd: + self.state = .waitingForWritabilityEnabled + return .clearIdleWriteTimeoutTimer + case .waitingForWritabilityEnabled: + preconditionFailure("If the channel was writable before, then we should have been waiting for more data.") + case .requestEndSent: + return .none + } + } + } +} diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index eb4182593..ed4594183 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -355,6 +355,18 @@ struct HTTP1ConnectionStateMachine { } } + mutating func idleWriteTimeoutTriggered() -> Action { + guard case .inRequest(var requestStateMachine, let close) = self.state else { + preconditionFailure("Invalid state: \(self.state)") + } + + return self.avoidingStateMachineCoW { state -> Action in + let action = requestStateMachine.idleWriteTimeoutTriggered() + state = .inRequest(requestStateMachine, close: close) + return state.modify(with: action) + } + } + mutating func headSent() -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { return .wait diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 0e8e819e8..4c69bc5dd 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -35,8 +35,16 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { private var request: HTTPExecutableRequest? { didSet { - if let newRequest = self.request, let idleReadTimeout = newRequest.requestOptions.idleReadTimeout { - self.idleReadTimeoutStateMachine = .init(timeAmount: idleReadTimeout) + if let newRequest = self.request { + if let idleReadTimeout = newRequest.requestOptions.idleReadTimeout { + self.idleReadTimeoutStateMachine = .init(timeAmount: idleReadTimeout) + } + if let idleWriteTimeout = newRequest.requestOptions.idleWriteTimeout { + self.idleWriteTimeoutStateMachine = .init( + timeAmount: idleWriteTimeout, + isWritabilityEnabled: self.channelContext?.channel.isWritable ?? false + ) + } } else { self.idleReadTimeoutStateMachine = nil } @@ -46,6 +54,9 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { private var idleReadTimeoutStateMachine: IdleReadStateMachine? private var idleReadTimeoutTimer: Scheduled? + private var idleWriteTimeoutStateMachine: IdleWriteStateMachine? + private var idleWriteTimeoutTimer: Scheduled? + init(eventLoop: EventLoop) { self.eventLoop = eventLoop } @@ -77,6 +88,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } func channelWritabilityChanged(context: ChannelHandlerContext) { + if let timeoutAction = self.idleWriteTimeoutStateMachine?.channelWritabilityChanged(context: context) { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.writabilityChanged(writable: context.channel.isWritable) self.run(action, context: context) } @@ -110,6 +125,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // a single request. self.request = request + if let timeoutAction = self.idleWriteTimeoutStateMachine?.write() { + self.runTimeoutAction(timeoutAction, context: context) + } + request.willExecuteRequest(self) let action = self.state.startRequest( @@ -153,8 +172,12 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { request.resumeRequestBodyStream() } if startIdleTimer { - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) + if let readTimeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(readTimeoutAction, context: context) + } + + if let writeTimeoutAction = self.idleWriteTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(writeTimeoutAction, context: context) } } case .pauseRequestBodyStream: @@ -168,8 +191,12 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { case .sendRequestEnd(let writePromise): context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: writePromise) - if let timeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { - self.runTimeoutAction(timeoutAction, context: context) + if let readTimeoutAction = self.idleReadTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(readTimeoutAction, context: context) + } + + if let writeTimeoutAction = self.idleWriteTimeoutStateMachine?.requestEndSent() { + self.runTimeoutAction(writeTimeoutAction, context: context) } case .read: @@ -295,6 +322,36 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } } + private func runTimeoutAction(_ action: IdleWriteStateMachine.Action, context: ChannelHandlerContext) { + switch action { + case .startIdleWriteTimeoutTimer(let timeAmount): + assert(self.idleWriteTimeoutTimer == nil, "Expected there is no timeout timer so far.") + + self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { + guard self.idleWriteTimeoutTimer != nil else { return } + let action = self.state.idleWriteTimeoutTriggered() + self.run(action, context: context) + } + case .resetIdleWriteTimeoutTimer(let timeAmount): + if let oldTimer = self.idleWriteTimeoutTimer { + oldTimer.cancel() + } + + self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { + guard self.idleWriteTimeoutTimer != nil else { return } + let action = self.state.idleWriteTimeoutTriggered() + self.run(action, context: context) + } + case .clearIdleWriteTimeoutTimer: + if let oldTimer = self.idleWriteTimeoutTimer { + self.idleWriteTimeoutTimer = nil + oldTimer.cancel() + } + case .none: + break + } + } + // MARK: Private HTTPRequestExecutor private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { @@ -308,6 +365,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { return } + if let timeoutAction = self.idleWriteTimeoutStateMachine?.write() { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.requestStreamPartReceived(data, promise: promise) self.run(action, context: context) } @@ -338,6 +399,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { return } + if let timeoutAction = self.idleWriteTimeoutStateMachine?.cancelRequest() { + self.runTimeoutAction(timeoutAction, context: context) + } + let action = self.state.requestCancelled() self.run(action, context: context) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift index 4835feac3..b575ae094 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift @@ -704,6 +704,28 @@ struct HTTPRequestStateMachine { } } + mutating func idleWriteTimeoutTriggered() -> Action { + switch self.state { + case .initialized, + .waitForChannelToBecomeWritable: + preconditionFailure("We only schedule idle write timeouts while the request is being sent. Invalid state: \(self.state)") + + case .running(.streaming, _): + let error = HTTPClientError.writeTimeout + self.state = .failed(error) + return .failRequest(error, .close(nil)) + + case .running(.endSent, _): + preconditionFailure("Invalid state. This state should be: .finished") + + case .finished, .failed: + return .wait + + case .modifying: + preconditionFailure("Invalid state: \(self.state)") + } + } + private mutating func startSendingRequest(head: HTTPRequestHead, metadata: RequestFramingMetadata) -> Action { let length = metadata.body.expectedLength if length == 0 { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift index c46f1289c..903f962e5 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestOptions.swift @@ -17,11 +17,18 @@ import NIOCore struct RequestOptions { /// The maximal `TimeAmount` that is allowed to pass between `channelRead`s from the Channel. var idleReadTimeout: TimeAmount? - + /// The maximal `TimeAmount` that is allowed to pass between `write`s into the Channel. + var idleWriteTimeout: TimeAmount? + /// DNS overrides. var dnsOverride: [String: String] - init(idleReadTimeout: TimeAmount?, dnsOverride: [String: String]) { + init( + idleReadTimeout: TimeAmount?, + idleWriteTimeout: TimeAmount?, + dnsOverride: [String: String] + ) { self.idleReadTimeout = idleReadTimeout + self.idleWriteTimeout = idleWriteTimeout self.dnsOverride = dnsOverride } } @@ -30,6 +37,7 @@ extension RequestOptions { static func fromClientConfiguration(_ configuration: HTTPClient.Configuration) -> Self { RequestOptions( idleReadTimeout: configuration.timeout.read, + idleWriteTimeout: configuration.timeout.write, dnsOverride: configuration.dnsOverride ) } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index db8ed3d97..6fc94de5c 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -79,7 +79,7 @@ public class HTTPClient { private var state: State private let stateLock = NIOLock() - internal static let loggingDisabled = Logger(label: "AHC-do-not-log", factory: { _ in SwiftLogNoOpLogHandler() }) + static let loggingDisabled = Logger(label: "AHC-do-not-log", factory: { _ in SwiftLogNoOpLogHandler() }) /// Create an ``HTTPClient`` with specified `EventLoopGroup` provider and configuration. /// @@ -184,7 +184,7 @@ public class HTTPClient { /// throw the appropriate error if needed. For instance, if its internal connection pool has any non-released connections, /// this indicate shutdown was called too early before tasks were completed or explicitly canceled. /// In general, setting this parameter to `true` should make it easier and faster to catch related programming errors. - internal func syncShutdown(requiresCleanClose: Bool) throws { + func syncShutdown(requiresCleanClose: Bool) throws { if let eventLoop = MultiThreadedEventLoopGroup.currentEventLoop { preconditionFailure(""" BUG DETECTED: syncShutdown() must not be called when on an EventLoop. @@ -927,8 +927,10 @@ extension HTTPClient.Configuration { public var connect: TimeAmount? /// Specifies read timeout. public var read: TimeAmount? + /// Specifies the maximum amount of time without bytes being written by the client before closing the connection. + public var write: TimeAmount? - /// internal connection creation timeout. Defaults the connect timeout to always contain a value. + /// Internal connection creation timeout. Defaults the connect timeout to always contain a value. var connectionCreationTimeout: TimeAmount { self.connect ?? .seconds(10) } @@ -938,7 +940,25 @@ extension HTTPClient.Configuration { /// - parameters: /// - connect: `connect` timeout. Will default to 10 seconds, if no value is provided. /// - read: `read` timeout. - public init(connect: TimeAmount? = nil, read: TimeAmount? = nil) { + public init( + connect: TimeAmount? = nil, + read: TimeAmount? = nil + ) { + self.connect = connect + self.read = read + } + + /// Create timeout. + /// + /// - parameters: + /// - connect: `connect` timeout. Will default to 10 seconds, if no value is provided. + /// - read: `read` timeout. + /// - write: `write` timeout. + public init( + connect: TimeAmount? = nil, + read: TimeAmount? = nil, + write: TimeAmount + ) { self.connect = connect self.read = read } @@ -1007,7 +1027,7 @@ extension HTTPClient.Configuration { } public struct HTTPVersion: Sendable, Hashable { - internal enum Configuration { + enum Configuration { case http1Only case automatic } @@ -1018,7 +1038,7 @@ extension HTTPClient.Configuration { /// HTTP/2 is used if we connect to a server with HTTPS and the server supports HTTP/2, otherwise we use HTTP/1 public static let automatic: Self = .init(configuration: .automatic) - internal var configuration: Configuration + var configuration: Configuration } } @@ -1032,6 +1052,7 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { case emptyScheme case unsupportedScheme(String) case readTimeout + case writeTimeout case remoteConnectionClosed case cancelled case identityCodingIncorrectlyPresent @@ -1090,6 +1111,8 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { return "Unsupported scheme" case .readTimeout: return "Read timeout" + case .writeTimeout: + return "Write timeout" case .remoteConnectionClosed: return "Remote connection closed" case .cancelled: @@ -1155,8 +1178,10 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { public static let emptyScheme = HTTPClientError(code: .emptyScheme) /// Provided URL scheme is not supported, supported schemes are: `http` and `https` public static func unsupportedScheme(_ scheme: String) -> HTTPClientError { return HTTPClientError(code: .unsupportedScheme(scheme)) } - /// Request timed out. + /// Request timed out while waiting for response. public static let readTimeout = HTTPClientError(code: .readTimeout) + /// Request timed out. + public static let writeTimeout = HTTPClientError(code: .writeTimeout) /// Remote connection was closed unexpectedly. public static let remoteConnectionClosed = HTTPClientError(code: .remoteConnectionClosed) /// Request was cancelled. diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index 2aa010491..f6a2840d9 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -337,6 +337,135 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { embedded.embeddedEventLoop.advanceTime(by: .milliseconds(250)) } + func testIdleWriteTimeout() { + let embedded = EmbeddedChannel() + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + return testWriter.start(writer: writer) + })) + + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + testUtils.connection.executeRequest(requestBag) + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) + } + } + + func testIdleWriteTimeoutWritabilityChanged() { + let embedded = EmbeddedChannel() + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + embedded.isWritable = false + embedded.pipeline.fireChannelWritabilityChanged() + // This should not trigger any errors or timeouts, because the timer isn't running + // as the channel is not writable. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) + + // Now that the channel will become writable, this should trigger a timeout. + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + + return testWriter.start(writer: writer) + })) + + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + testUtils.connection.executeRequest(requestBag) + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) + } + } + + func testIdleWriteTimeoutIsCancelledIfRequestIsCancelled() { + let embedded = EmbeddedChannel() + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 1) + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 2) { writer in + return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) + })) + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + testUtils.connection.executeRequest(requestBag) + + // canceling the request + requestBag.fail(HTTPClientError.cancelled) + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .cancelled) + } + + // the idle write timeout should be cleared because we canceled the request + // therefore advancing the time should not trigger a crash + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(250)) + } + func testFailHTTPRequestWithContentLengthBecauseOfChannelInactiveWaitingForDemand() { let embedded = EmbeddedChannel() var maybeTestUtils: HTTP1TestTools? @@ -576,7 +705,7 @@ class TestBackpressureWriter { self.finishPromise = eventLoop.makePromise(of: Void.self) } - func start(writer: HTTPClient.Body.StreamWriter) -> EventLoopFuture { + func start(writer: HTTPClient.Body.StreamWriter, expectedErrors: [HTTPClientError] = []) -> EventLoopFuture { func recursive() { XCTAssert(self.eventLoop.inEventLoop) XCTAssert(self.channelIsWritable) @@ -591,7 +720,15 @@ class TestBackpressureWriter { case .success: recursive() case .failure(let error): - XCTFail("Unexpected error: \(error)") + let isExpectedError = expectedErrors.contains { httpError in + if let castError = error as? HTTPClientError { + return castError == httpError + } + return false + } + if !isExpectedError { + XCTFail("Unexpected error: \(error)") + } } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index 2b68fceb3..545ba1e3c 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -286,6 +286,139 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { embedded.embeddedEventLoop.advanceTime(by: .milliseconds(250)) } + func testIdleWriteTimeout() { + let embedded = EmbeddedChannel() + let requestHandler = HTTP2ClientRequestHandler(eventLoop: embedded.eventLoop) + XCTAssertNoThrow(try embedded.pipeline.syncOperations.addHandlers([requestHandler])) + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + let logger = Logger(label: "test") + + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + return testWriter.start(writer: writer) + })) + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + embedded.write(requestBag, promise: nil) + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) + } + } + + func testIdleWriteTimeoutWritabilityChanged() { + let embedded = EmbeddedChannel() + let readEventHandler = ReadEventHitHandler() + let requestHandler = HTTP2ClientRequestHandler(eventLoop: embedded.eventLoop) + XCTAssertNoThrow(try embedded.pipeline.syncOperations.addHandlers([readEventHandler, requestHandler])) + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + let logger = Logger(label: "test") + + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + embedded.isWritable = false + embedded.pipeline.fireChannelWritabilityChanged() + // This should not trigger any errors or timeouts, because the timer isn't running + // as the channel is not writable. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) + + // Now that the channel will become writable, this should trigger a timeout. + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + + return testWriter.start(writer: writer) + })) + + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + embedded.write(requestBag, promise: nil) + + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) + } + } + + func testIdleWriteTimeoutIsCanceledIfRequestIsCanceled() { + let embedded = EmbeddedChannel() + let readEventHandler = ReadEventHitHandler() + let requestHandler = HTTP2ClientRequestHandler(eventLoop: embedded.eventLoop) + XCTAssertNoThrow(try embedded.pipeline.syncOperations.addHandlers([readEventHandler, requestHandler])) + XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) + let logger = Logger(label: "test") + + let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 2) { writer in + return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) + })) + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + testWriter.writabilityChanged(true) + embedded.pipeline.fireChannelWritabilityChanged() + embedded.write(requestBag, promise: nil) + + // canceling the request + requestBag.fail(HTTPClientError.cancelled) + XCTAssertThrowsError(try requestBag.task.futureResult.wait()) { + XCTAssertEqual($0 as? HTTPClientError, .cancelled) + } + + // the idle read timeout should be cleared because we canceled the request + // therefore advancing the time should not trigger a crash + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(250)) + } + func testWriteHTTPHeadFails() { struct WriteError: Error, Equatable {} diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index e2a959589..610e429f5 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -979,10 +979,12 @@ final class MockTaskQueuer: HTTPRequestScheduler { extension RequestOptions { static func forTests( idleReadTimeout: TimeAmount? = nil, + idleWriteTimeout: TimeAmount? = nil, dnsOverride: [String: String] = [:] ) -> Self { RequestOptions( idleReadTimeout: idleReadTimeout, + idleWriteTimeout: idleWriteTimeout, dnsOverride: dnsOverride ) } From a4904fcc6b0a2a8c5666844839ad64b611f77cb2 Mon Sep 17 00:00:00 2001 From: Si Beaumont Date: Mon, 18 Dec 2023 12:52:50 +0000 Subject: [PATCH 099/146] Add missing availability guards in tests (#719) ## Motivation Some of the test code was missing availability guards for Apple platforms, resulting in build failures for these platforms, e.g. ``` error: 'AsyncSequence' is only available in iOS 13.0 or newer ``` ## Modifications Add missing availability guards. I've tried to keep them as scoped as possible. ## Result Tests can now build for run on iOS and other Apple platforms. --- .../TLSConfiguration.swift | 2 +- .../AsyncAwaitEndToEndTests.swift | 27 +------------------ .../HTTPClientRequestTests.swift | 17 ++---------- .../HTTPClientResponseTests.swift | 1 + .../HTTPClientTests.swift | 1 + .../NWWaitingHandlerTests.swift | 2 +- .../Transaction+StateMachineTests.swift | 8 +----- .../TransactionTests.swift | 11 +------- .../XCTest+AsyncAwait.swift | 2 +- 9 files changed, 10 insertions(+), 61 deletions(-) diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index 06ae5e146..cb6bd43bd 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -57,7 +57,7 @@ extension TLSVersion { } } -@available(macOS 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) +@available(macOS 10.14, iOS 12.0, tvOS 12.0, watchOS 5.0, *) extension TLSConfiguration { /// Dispatch queue used by Network framework TLS to control certificate verification static var tlsDispatchQueue = DispatchQueue(label: "TLSDispatch") diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index d26f5ae24..3faa082b1 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -33,6 +33,7 @@ private func makeDefaultHTTPClient( ) } +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class AsyncAwaitEndToEndTests: XCTestCase { var clientGroup: EventLoopGroup! var serverGroup: EventLoopGroup! @@ -56,7 +57,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testSimpleGet() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -77,7 +77,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testSimplePost() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -98,7 +97,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithByteBuffer() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -121,7 +119,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithSequenceOfUInt8() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -144,7 +141,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithCollectionOfUInt8() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -167,7 +163,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithRandomAccessCollectionOfUInt8() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -190,7 +185,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithAsyncSequenceOfByteBuffers() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -217,7 +211,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithAsyncSequenceOfUInt8() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -240,7 +233,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithFragmentedAsyncSequenceOfByteBuffers() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -280,7 +272,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testPostWithFragmentedAsyncSequenceOfLargeByteBuffers() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -321,7 +312,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testCanceling() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -344,7 +334,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testCancelingResponseBody() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() @@ -373,7 +362,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testDeadline() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -398,7 +386,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testImmediateDeadline() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -423,7 +410,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testConnectTimeout() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 60) { #if os(Linux) // 198.51.100.254 is reserved for documentation only and therefore should not accept any TCP connection @@ -480,7 +466,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testSelfSignedCertificateIsRejectedWithCorrectErrorIfRequestDeadlineIsExceeded() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { /// key + cert was created with the follwing command: /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' @@ -526,7 +511,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testDnsOverride() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { /// key + cert was created with the following code (depends on swift-certificates) /// ``` @@ -584,7 +568,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testInvalidURL() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 5) { let client = makeDefaultHTTPClient() defer { XCTAssertNoThrow(try client.syncShutdown()) } @@ -598,7 +581,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testRedirectChangesHostHeader() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -625,7 +607,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testShutdown() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let client = makeDefaultHTTPClient() try await client.shutdown() @@ -637,7 +618,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { /// Regression test for https://github.com/swift-server/async-http-client/issues/612 func testCancelingBodyDoesNotCrash() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let client = makeDefaultHTTPClient() defer { XCTAssertNoThrow(try client.syncShutdown()) } @@ -654,7 +634,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testAsyncSequenceReuse() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -698,7 +677,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } private func _rejectsInvalidCharactersInHeaderFieldNames(mode: HTTPBin.Mode) { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(mode) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -759,7 +737,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } private func _rejectsInvalidCharactersInHeaderFieldValues(mode: HTTPBin.Mode) { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(mode) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -818,7 +795,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testUsingGetMethodInsteadOfWait() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } @@ -838,7 +814,6 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func testSimpleContentLengthErrorNoBody() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) defer { XCTAssertNoThrow(try bin.shutdown()) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index f0a329f2a..b0b1be1d8 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -17,15 +17,13 @@ import Algorithms import NIOCore import XCTest +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) class HTTPClientRequestTests: XCTestCase { - @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) private typealias Request = HTTPClientRequest - @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) private typealias PreparedRequest = HTTPClientRequest.Prepared func testCustomHeadersAreRespected() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "https://example.com/get") request.headers = [ @@ -60,7 +58,6 @@ class HTTPClientRequestTests: XCTestCase { } func testUnixScheme() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "unix://%2Fexample%2Ffolder.sock/some_path") request.headers = ["custom-header": "custom-value"] @@ -90,7 +87,6 @@ class HTTPClientRequestTests: XCTestCase { } func testHTTPUnixScheme() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http+unix://%2Fexample%2Ffolder.sock/some_path") request.headers = ["custom-header": "custom-value"] @@ -120,7 +116,6 @@ class HTTPClientRequestTests: XCTestCase { } func testHTTPSUnixScheme() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "https+unix://%2Fexample%2Ffolder.sock/some_path") request.headers = ["custom-header": "custom-value"] @@ -150,7 +145,6 @@ class HTTPClientRequestTests: XCTestCase { } func testGetWithoutBody() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let request = Request(url: "https://example.com/get") var preparedRequest: PreparedRequest? @@ -179,7 +173,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithoutBody() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -213,7 +206,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithEmptyByteBuffer() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -248,7 +240,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithByteBuffer() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -282,7 +273,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithSequenceOfUnknownLength() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -317,7 +307,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithSequenceWithFixedLength() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -353,7 +342,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithRandomAccessCollection() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -388,7 +376,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithAsyncSequenceOfUnknownLength() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -428,7 +415,6 @@ class HTTPClientRequestTests: XCTestCase { } func testPostWithAsyncSequenceWithKnownLength() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { var request = Request(url: "http://example.com/post") request.method = .POST @@ -602,6 +588,7 @@ class HTTPClientRequestTests: XCTestCase { } } +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension AsyncSequence { func collect() async throws -> [Element] { try await self.reduce(into: []) { $0 += CollectionOfOne($1) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift index bf0ecfeb9..2c6c9afac 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift @@ -17,6 +17,7 @@ import Logging import NIOCore import XCTest +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class HTTPClientResponseTests: XCTestCase { func testSimpleResponse() { let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "1025"], status: .ok) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index c6a67c155..291d522fb 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3526,6 +3526,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(.ok, response.status) } + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func testAsyncExecuteWithCustomTLS() async throws { let httpsBin = HTTPBin(.http1_1(ssl: true)) defer { diff --git a/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift index 967a7da40..ff9e7f45d 100644 --- a/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift @@ -21,7 +21,7 @@ import NIOSSL import NIOTransportServices import XCTest -@available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *) +@available(macOS 10.14, iOS 12.0, tvOS 12.0, watchOS 5.0, *) class NWWaitingHandlerTests: XCTestCase { class MockRequester: HTTPConnectionRequester { var waitingForConnectivityCalled = false diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift index 3420f2ebd..a8d3d5a5e 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift @@ -23,9 +23,9 @@ struct NoOpAsyncSequenceProducerDelegate: NIOAsyncSequenceProducerDelegate { func didTerminate() {} } +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class Transaction_StateMachineTests: XCTestCase { func testRequestWasQueuedAfterWillExecuteRequestWasCalled() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -53,7 +53,6 @@ final class Transaction_StateMachineTests: XCTestCase { } func testRequestBodyStreamWasPaused() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -75,7 +74,6 @@ final class Transaction_StateMachineTests: XCTestCase { func testQueuedRequestGetsRemovedWhenDeadlineExceeded() { struct MyError: Error, Equatable {} - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { var state = Transaction.StateMachine(continuation) @@ -106,7 +104,6 @@ final class Transaction_StateMachineTests: XCTestCase { func testDeadlineExceededAndFullyFailedRequestCanBeCanceledWithNoEffect() { struct MyError: Error, Equatable {} - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { var state = Transaction.StateMachine(continuation) @@ -141,7 +138,6 @@ final class Transaction_StateMachineTests: XCTestCase { } func testScheduledRequestGetsRemovedWhenDeadlineExceeded() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -166,7 +162,6 @@ final class Transaction_StateMachineTests: XCTestCase { } func testDeadlineExceededRaceWithRequestWillExecute() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { @@ -198,7 +193,6 @@ final class Transaction_StateMachineTests: XCTestCase { } func testRequestWithHeadReceivedGetNotCancelledWhenDeadlineExceeded() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } let eventLoop = EmbeddedEventLoop() XCTAsyncTest { func workaround(_ continuation: CheckedContinuation) { diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 13cb87eb5..a8a2bb30e 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -24,9 +24,9 @@ import XCTest @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) typealias PreparedRequest = HTTPClientRequest.Prepared +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class TransactionTests: XCTestCase { func testCancelAsyncRequest() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } // creating the `XCTestExpectation` off the main thread crashes on Linux with Swift 5.6 // therefore we create it here as a workaround which works fine let scheduledRequestCanceled = self.expectation(description: "scheduled request canceled") @@ -69,7 +69,6 @@ final class TransactionTests: XCTestCase { } func testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -118,7 +117,6 @@ final class TransactionTests: XCTestCase { } func testResponseStreamingWorks() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -178,7 +176,6 @@ final class TransactionTests: XCTestCase { } func testIgnoringResponseBodyWorks() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -227,7 +224,6 @@ final class TransactionTests: XCTestCase { } func testWriteBackpressureWorks() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -299,7 +295,6 @@ final class TransactionTests: XCTestCase { } func testSimpleGetRequest() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) let eventLoop = eventLoopGroup.next() @@ -355,7 +350,6 @@ final class TransactionTests: XCTestCase { } func testSimplePostRequest() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -393,7 +387,6 @@ final class TransactionTests: XCTestCase { } func testPostStreamFails() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -436,7 +429,6 @@ final class TransactionTests: XCTestCase { } func testResponseStreamFails() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest(timeout: 30) { let embeddedEventLoop = EmbeddedEventLoop() defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } @@ -499,7 +491,6 @@ final class TransactionTests: XCTestCase { } func testBiDirectionalStreamingHTTP2() { - guard #available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) else { return } XCTAsyncTest { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) let eventLoop = eventLoopGroup.next() diff --git a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift index bf297413c..e1d2e4592 100644 --- a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift +++ b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift @@ -30,7 +30,6 @@ import XCTest extension XCTestCase { - @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) /// Cross-platform XCTest support for async-await tests. /// /// Currently the Linux implementation of XCTest doesn't have async-await support. @@ -39,6 +38,7 @@ extension XCTestCase { /// /// - NOTE: Support for Linux is tracked by https://bugs.swift.org/browse/SR-14403. /// - NOTE: Implementation currently in progress: https://github.com/apple/swift-corelibs-xctest/pull/326 + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func XCTAsyncTest( expectationDescription: String = "Async operation", timeout: TimeInterval = 30, From ffe36fcf54319770b4aa478f8134b4968b97b0e1 Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Mon, 18 Dec 2023 10:16:27 -0300 Subject: [PATCH 100/146] Fix potential race conditions when cancelling read/write idle timers (#720) --- .../HTTP1/HTTP1ClientChannelHandler.swift | 6 +---- .../HTTP2/HTTP2ClientRequestHandler.swift | 22 +++++++++++++++---- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index ba3a09cc9..04de8b352 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -60,17 +60,13 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { private var idleReadTimeoutStateMachine: IdleReadStateMachine? private var idleReadTimeoutTimer: Scheduled? - /// Cancelling a task in NIO does *not* guarantee that the task will not execute under certain race conditions. - /// We therefore give each timer an ID and increase the ID every time we reset or cancel it. - /// We check in the task if the timer ID has changed in the meantime and do not execute any action if has changed. - private var currentIdleReadTimeoutTimerID: Int = 0 - private var idleWriteTimeoutStateMachine: IdleWriteStateMachine? private var idleWriteTimeoutTimer: Scheduled? /// Cancelling a task in NIO does *not* guarantee that the task will not execute under certain race conditions. /// We therefore give each timer an ID and increase the ID every time we reset or cancel it. /// We check in the task if the timer ID has changed in the meantime and do not execute any action if has changed. + private var currentIdleReadTimeoutTimerID: Int = 0 private var currentIdleWriteTimeoutTimerID: Int = 0 private let backgroundLogger: Logger diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 4c69bc5dd..1520ff414 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -57,6 +57,12 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { private var idleWriteTimeoutStateMachine: IdleWriteStateMachine? private var idleWriteTimeoutTimer: Scheduled? + /// Cancelling a task in NIO does *not* guarantee that the task will not execute under certain race conditions. + /// We therefore give each timer an ID and increase the ID every time we reset or cancel it. + /// We check in the task if the timer ID has changed in the meantime and do not execute any action if has changed. + private var currentIdleReadTimeoutTimerID: Int = 0 + private var currentIdleWriteTimeoutTimerID: Int = 0 + init(eventLoop: EventLoop) { self.eventLoop = eventLoop } @@ -295,8 +301,9 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { case .startIdleReadTimeoutTimer(let timeAmount): assert(self.idleReadTimeoutTimer == nil, "Expected there is no timeout timer so far.") + let timerID = self.currentIdleReadTimeoutTimerID self.idleReadTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { - guard self.idleReadTimeoutTimer != nil else { return } + guard self.currentIdleReadTimeoutTimerID == timerID else { return } let action = self.state.idleReadTimeoutTriggered() self.run(action, context: context) } @@ -306,14 +313,17 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { oldTimer.cancel() } + self.currentIdleReadTimeoutTimerID &+= 1 + let timerID = self.currentIdleReadTimeoutTimerID self.idleReadTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { - guard self.idleReadTimeoutTimer != nil else { return } + guard self.currentIdleReadTimeoutTimerID == timerID else { return } let action = self.state.idleReadTimeoutTriggered() self.run(action, context: context) } case .clearIdleReadTimeoutTimer: if let oldTimer = self.idleReadTimeoutTimer { self.idleReadTimeoutTimer = nil + self.currentIdleReadTimeoutTimerID &+= 1 oldTimer.cancel() } @@ -327,8 +337,9 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { case .startIdleWriteTimeoutTimer(let timeAmount): assert(self.idleWriteTimeoutTimer == nil, "Expected there is no timeout timer so far.") + let timerID = self.currentIdleWriteTimeoutTimerID self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { - guard self.idleWriteTimeoutTimer != nil else { return } + guard self.currentIdleWriteTimeoutTimerID == timerID else { return } let action = self.state.idleWriteTimeoutTriggered() self.run(action, context: context) } @@ -337,14 +348,17 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { oldTimer.cancel() } + self.currentIdleWriteTimeoutTimerID &+= 1 + let timerID = self.currentIdleWriteTimeoutTimerID self.idleWriteTimeoutTimer = self.eventLoop.scheduleTask(in: timeAmount) { - guard self.idleWriteTimeoutTimer != nil else { return } + guard self.currentIdleWriteTimeoutTimerID == timerID else { return } let action = self.state.idleWriteTimeoutTriggered() self.run(action, context: context) } case .clearIdleWriteTimeoutTimer: if let oldTimer = self.idleWriteTimeoutTimer { self.idleWriteTimeoutTimer = nil + self.currentIdleWriteTimeoutTimerID &+= 1 oldTimer.cancel() } case .none: From 75fce63d9ba179dc06b057dadc36e5745703a71d Mon Sep 17 00:00:00 2001 From: Nishant Dani Date: Wed, 20 Dec 2023 15:15:30 -0800 Subject: [PATCH 101/146] Update Package.swift (#722) Fix to resolve linker error in xcode because of missing import. https://github.com/swift-server/async-http-client/issues/721 --- Package.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Package.swift b/Package.swift index 23a733d75..be52e1f13 100644 --- a/Package.swift +++ b/Package.swift @@ -38,6 +38,7 @@ let package = Package( dependencies: [ .target(name: "CAsyncHTTPClient"), .product(name: "NIO", package: "swift-nio"), + .product(name: "NIOTLS", package: "swift-nio"), .product(name: "NIOCore", package: "swift-nio"), .product(name: "NIOPosix", package: "swift-nio"), .product(name: "NIOHTTP1", package: "swift-nio"), @@ -56,6 +57,7 @@ let package = Package( name: "AsyncHTTPClientTests", dependencies: [ .target(name: "AsyncHTTPClient"), + .product(name: "NIOTLS", package: "swift-nio"), .product(name: "NIOCore", package: "swift-nio"), .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), .product(name: "NIOEmbedded", package: "swift-nio"), From 5ccda442f103792d67680aefc8d0a87392fbd66c Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Thu, 21 Dec 2023 09:51:54 -0300 Subject: [PATCH 102/146] Use the given connection pool idle timeout in the HTTPClient.Configuration inits (#723) --- Sources/AsyncHTTPClient/HTTPClient.swift | 4 ++-- .../HTTPClientTests.swift | 20 ++++++++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 6fc94de5c..d6d02c94f 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -775,7 +775,7 @@ public class HTTPClient { self.init(tlsConfiguration: tlsConfig, redirectConfiguration: redirectConfiguration, timeout: timeout, - connectionPool: ConnectionPool(), + connectionPool: ConnectionPool(idleTimeout: maximumAllowedIdleTimeInConnectionPool), proxy: proxy, ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, decompression: decompression) @@ -794,7 +794,7 @@ public class HTTPClient { self.init(tlsConfiguration: tlsConfig, redirectConfiguration: redirectConfiguration, timeout: timeout, - connectionPool: ConnectionPool(), + connectionPool: ConnectionPool(idleTimeout: connectionPool), proxy: proxy, ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, decompression: decompression) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 291d522fb..075530f4e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1847,11 +1847,29 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testPoolClosesIdleConnections() { + let configuration = HTTPClient.Configuration( + certificateVerification: .none, + maximumAllowedIdleTimeInConnectionPool: .milliseconds(100) + ) + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(connectionPool: .init(idleTimeout: .milliseconds(100)))) + configuration: configuration) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } + + // Make sure that the idle timeout of the connection pool is properly propagated + // to the connection pool itself, when using both inits. + XCTAssertEqual(configuration.connectionPool.idleTimeout, .milliseconds(100)) + XCTAssertEqual( + configuration.connectionPool.idleTimeout, + HTTPClient.Configuration( + certificateVerification: .none, + connectionPool: .milliseconds(100), + backgroundActivityLogger: nil + ).connectionPool.idleTimeout + ) + XCTAssertNoThrow(try localClient.get(url: self.defaultHTTPBinURLPrefix + "get").wait()) Thread.sleep(forTimeInterval: 0.2) XCTAssertEqual(self.defaultHTTPBin.activeConnections, 0) From 291438696abdd48d2a83b52465c176efbd94512b Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Fri, 12 Jan 2024 12:18:35 +0000 Subject: [PATCH 103/146] Update minimum swift-nio version (#725) --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index be52e1f13..13ddba19c 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.58.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.62.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), From 09b7eb751e297396f44b270b26754f9c85b105d1 Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Sat, 20 Jan 2024 00:09:35 +0000 Subject: [PATCH 104/146] Add support for Musl. (#726) Motivation: We would like to make this work for Musl so that we can build fully statically linked binaries that use AsyncHTTPClient. Modifications: Define `_GNU_SOURCE` as a compiler argument; doing it in a source file doesn't work properly with modular headers. Add imports of `Musl` in appropriate places. `Musl` doesn't have `strptime_l`, so avoid using that. Result: async-http-client will build for Musl. --- Package.swift | 7 ++++++- Sources/AsyncHTTPClient/ConnectionPool.swift | 2 ++ .../State Machine/HTTPConnectionPool+Backoff.swift | 2 ++ Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift | 2 ++ Sources/CAsyncHTTPClient/CAsyncHTTPClient.c | 5 ++++- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 2 ++ 6 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index 13ddba19c..dceb42501 100644 --- a/Package.swift +++ b/Package.swift @@ -32,7 +32,12 @@ let package = Package( .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"), ], targets: [ - .target(name: "CAsyncHTTPClient"), + .target( + name: "CAsyncHTTPClient", + cSettings: [ + .define("_GNU_SOURCE"), + ] + ), .target( name: "AsyncHTTPClient", dependencies: [ diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index b27e3fb97..8cca70750 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -16,6 +16,8 @@ import NIOSSL #if canImport(Darwin) import Darwin.C +#elseif canImport(Musl) +import Musl #elseif os(Linux) || os(FreeBSD) || os(Android) import Glibc #else diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift index 4aec9f6fe..cc7c7cfa1 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift @@ -15,6 +15,8 @@ import NIOCore #if canImport(Darwin) import func Darwin.pow +#elseif canImport(Musl) +import func Musl.pow #else import func Glibc.pow #endif diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index c35540114..b2e9d7b05 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -15,6 +15,8 @@ import NIOHTTP1 #if canImport(Darwin) import Darwin +#elseif canImport(Musl) +import Musl #elseif canImport(Glibc) import Glibc #endif diff --git a/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c b/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c index 2a09d04c9..5dfdc08a5 100644 --- a/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c +++ b/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c @@ -15,7 +15,6 @@ #if __APPLE__ #include #elif __linux__ - #define _GNU_SOURCE #include #endif @@ -32,7 +31,11 @@ bool swiftahc_cshims_strptime(const char * string, const char * format, struct t bool swiftahc_cshims_strptime_l(const char * string, const char * format, struct tm * result, void * locale) { // The pointer cast is fine as long we make sure it really points to a locale_t. +#ifdef __musl__ + const char * firstNonProcessed = strptime(string, format, result); +#else const char * firstNonProcessed = strptime_l(string, format, result, (locale_t)locale); +#endif if (firstNonProcessed) { return *firstNonProcessed == 0; } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 2ebccdafe..2d37b1387 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -30,6 +30,8 @@ import NIOTransportServices import XCTest #if canImport(Darwin) import Darwin +#elseif canImport(Musl) +import Musl #elseif canImport(Glibc) import Glibc #endif From e6a630de7523c60977c102d90577adfa0337359c Mon Sep 17 00:00:00 2001 From: George Barnett Date: Mon, 11 Mar 2024 08:41:35 +0000 Subject: [PATCH 105/146] Raise minimum Swift version to 5.8 (#729) --- Package.swift | 2 +- README.md | 3 ++- docker/docker-compose.2204.510.yaml | 3 ++- docker/docker-compose.2204.57.yaml | 21 --------------------- 4 files changed, 5 insertions(+), 24 deletions(-) delete mode 100644 docker/docker-compose.2204.57.yaml diff --git a/Package.swift b/Package.swift index dceb42501..dae0d91c7 100644 --- a/Package.swift +++ b/Package.swift @@ -1,4 +1,4 @@ -// swift-tools-version:5.7 +// swift-tools-version:5.8 //===----------------------------------------------------------------------===// // // This source file is part of the AsyncHTTPClient open source project diff --git a/README.md b/README.md index 45a847fea..26be89420 100644 --- a/README.md +++ b/README.md @@ -329,4 +329,5 @@ AsyncHTTPClient | Minimum Swift Version `1.10.0 ..< 1.13.0` | 5.4 `1.13.0 ..< 1.18.0` | 5.5.2 `1.18.0 ..< 1.20.0` | 5.6 -`1.20.0 ...` | 5.7 +`1.20.0 ..< 1.21.0` | 5.7 +`1.21.0 ...` | 5.8 diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml index fdc3d2bdd..8dbf21183 100644 --- a/docker/docker-compose.2204.510.yaml +++ b/docker/docker-compose.2204.510.yaml @@ -6,7 +6,8 @@ services: image: async-http-client:22.04-5.10 build: args: - base_image: "swiftlang/swift:nightly-5.10-jammy" + ubuntu_version: "jammy" + swift_version: "5.10" documentation-check: image: async-http-client:22.04-5.10 diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml deleted file mode 100644 index 78eb83e1c..000000000 --- a/docker/docker-compose.2204.57.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:22.04-5.7 - build: - args: - ubuntu_version: "jammy" - swift_version: "5.7" - - documentation-check: - image: async-http-client:22.04-5.7 - - test: - image: async-http-client:22.04-5.7 - environment: [] - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:22.04-5.7 From 83f015bf9432fcfab58991d40bd6cb6c478fde8d Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Thu, 21 Mar 2024 16:18:22 +0000 Subject: [PATCH 106/146] Fix write timeout not being initialised (#730) --- Sources/AsyncHTTPClient/HTTPClient.swift | 1 + .../HTTPClientTests.swift | 40 ++++++++++++++++--- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index d6d02c94f..683532933 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -961,6 +961,7 @@ extension HTTPClient.Configuration { ) { self.connect = connect self.read = read + self.write = write } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 075530f4e..cdf9aa219 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -20,6 +20,7 @@ import Network import Logging import NIOConcurrencyHelpers import NIOCore +import NIOEmbedded import NIOFoundationCompat import NIOHTTP1 import NIOHTTPCompression @@ -607,6 +608,35 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } } + func testWriteTimeout() throws { + let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(timeout: HTTPClient.Configuration.Timeout(write: .nanoseconds(10)))) + + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + } + + // Create a request that writes a chunk, then waits longer than the configured write timeout, + // and then writes again. This should trigger a write timeout error. + let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "post", + method: .POST, + headers: ["transfer-encoding": "chunked"], + body: .stream { streamWriter in + _ = streamWriter.write(.byteBuffer(.init())) + + let promise = self.clientGroup.next().makePromise(of: Void.self) + self.clientGroup.next().scheduleTask(in: .milliseconds(3)) { + streamWriter.write(.byteBuffer(.init())).cascade(to: promise) + } + + return promise.futureResult + }) + + XCTAssertThrowsError(try localClient.execute(request: request).wait()) { + XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) + } + } + func testConnectTimeout() throws { #if os(Linux) // 198.51.100.254 is reserved for documentation only and therefore should not accept any TCP connection @@ -1230,8 +1260,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! - let configuration = TLSConfiguration.makeServerConfiguration( - certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, + let configuration = try TLSConfiguration.makeServerConfiguration( + certificateChain: NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, privateKey: .file(keyPath) ) let sslContext = try NIOSSLContext(configuration: configuration) @@ -1270,8 +1300,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! - let configuration = TLSConfiguration.makeServerConfiguration( - certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, + let configuration = try TLSConfiguration.makeServerConfiguration( + certificateChain: NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, privateKey: .file(keyPath) ) let sslContext = try NIOSSLContext(configuration: configuration) @@ -2728,7 +2758,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func uploader(_ streamWriter: HTTPClient.Body.StreamWriter) -> EventLoopFuture { let done = streamWriter.write(.byteBuffer(ByteBuffer(string: "X"))) - done.recover { error -> Void in + done.recover { error in XCTFail("unexpected error \(error)") }.whenSuccess { // This is executed when we have already sent the end of the request. From 36292f9d57e4caf69a550639e249fceecfa32fb3 Mon Sep 17 00:00:00 2001 From: hamzahrmalik Date: Fri, 29 Mar 2024 08:12:13 +0000 Subject: [PATCH 107/146] Renew certificates in tests (#731) --- .../AsyncAwaitEndToEndTests.swift | 8 ++++++-- .../Resources/example.com.cert.pem | 14 +++++++------- .../Resources/example.com.private-key.pem | 8 ++++---- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 3faa082b1..a30a8cf91 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -514,6 +514,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAsyncTest(timeout: 5) { /// key + cert was created with the following code (depends on swift-certificates) /// ``` + /// import X509 + /// import CryptoKit + /// import Foundation + /// /// let privateKey = P384.Signing.PrivateKey() /// let name = try DistinguishedName { /// OrganizationName("Self Signed") @@ -524,13 +528,13 @@ final class AsyncAwaitEndToEndTests: XCTestCase { /// serialNumber: .init(), /// publicKey: .init(privateKey.publicKey), /// notValidBefore: Date(), - /// notValidAfter: Date() + .days(365), + /// notValidAfter: Date().advanced(by: 365 * 24 * 3600), /// issuer: name, /// subject: name, /// signatureAlgorithm: .ecdsaWithSHA384, /// extensions: try .init { /// SubjectAlternativeNames([.dnsName("example.com")]) - /// ExtendedKeyUsage([.serverAuth]) + /// try ExtendedKeyUsage([.serverAuth]) /// }, /// issuerPrivateKey: .init(privateKey) /// ) diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem index 69af76e77..f6314d47a 100644 --- a/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.cert.pem @@ -1,12 +1,12 @@ -----BEGIN CERTIFICATE----- -MIIBwzCCAUmgAwIBAgIVAIFK2HEjRjd9rH6Szp3jT52U4wYjMAoGCCqGSM49BAMD +MIIBxDCCAUmgAwIBAgIVAPY31L1kyEnjO1E4inpE7+SYRO9mMAoGCCqGSM49BAMD MCoxFDASBgNVBAoMC1NlbGYgU2lnbmVkMRIwEAYDVQQDDAlsb2NhbGhvc3QwHhcN -MjMwMzI5MTE1ODQwWhcNMjQwMzI4MTE1ODQwWjAqMRQwEgYDVQQKDAtTZWxmIFNp +MjQwMzI4MjI0MDUyWhcNMjUwMzI4MjI0MDUyWjAqMRQwEgYDVQQKDAtTZWxmIFNp Z25lZDESMBAGA1UEAwwJbG9jYWxob3N0MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAE -SiOrOD8CbOyvj0yg+ArayukRCjw9AAaW3lsrsiRsSaqRxDcZ7+uR5nt2FUXc25mD -Ap+adz4g5gigpIUaVQc69AgMavYFCHF3Tb0TF1D4yAFLk8GFuWqxHDuqCQaGoyS5 +o2i+uiLtMu0Jzsk3oEUnfoM9n44/aV9UeOXxyDs57i2E13HrJeWIXACetybkB+Q8 +Poab6ohbskTwrS7WN3tFgoGdRBCKQow/rTECdezR/fdz2cGADaBN+CNMuFSnFSr5 oy8wLTAWBgNVHREEDzANggtleGFtcGxlLmNvbTATBgNVHSUEDDAKBggrBgEFBQcD -ATAKBggqhkjOPQQDAwNoADBlAjALdKj7fq0Hvv69KUdMGvpHBaqRq+4+X4T1gAm/ -Z09XPB3BAd9z3Ov7fMnc65iKRwICMQCxxu0rBJUmR9v1BINxA4S1EPH0S/U5ysTp -Wu1n1LZ3C5ooxMiO50cPuWupaB2LElY= +ATAKBggqhkjOPQQDAwNpADBmAjEAwF5OlUBOloDTIAxgaSSvHBMSVOE1rY5hUlkT +kQ+dQFeUe3Fn+Er5ohvkt+qVOQ5yAjEAt9s5b/Iz+JmWxKKUyExHob6QHEuuHmJy +AKdrn20Ply60bb8qxGYHhwhoyV2MZYVV -----END CERTIFICATE----- \ No newline at end of file diff --git a/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem index 775a5ea56..7cf27cc35 100644 --- a/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem +++ b/Tests/AsyncHTTPClientTests/Resources/example.com.private-key.pem @@ -1,6 +1,6 @@ -----BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDAbqzPBHiy/SoUXTlYl -F0q3AK+N5wvpb93vS8jdRYAY2BIKIQOurw4WLp0qVxKgYGqhZANiAARKI6s4PwJs -7K+PTKD4CtrK6REKPD0ABpbeWyuyJGxJqpHENxnv65Hme3YVRdzbmYMCn5p3PiDm -CKCkhRpVBzr0CAxq9gUIcXdNvRMXUPjIAUuTwYW5arEcO6oJBoajJLk= +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDhC5OSjPQeYRm4irIH +z4EyM/NbJsX39SlI6J4/q0Syt0BwojgJKhCWfeveanbIjbWhZANiAASjaL66Iu0y +7QnOyTegRSd+gz2fjj9pX1R45fHIOznuLYTXcesl5YhcAJ63JuQH5Dw+hpvqiFuy +RPCtLtY3e0WCgZ1EEIpCjD+tMQJ17NH993PZwYANoE34I0y4VKcVKvk= -----END PRIVATE KEY----- \ No newline at end of file From e0977cf29052908856b0366ca8f8aeaa5bd5cac1 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Wed, 3 Apr 2024 13:54:00 +0100 Subject: [PATCH 108/146] HTTPClient.shared a globally shared singleton & .browserLike configuration (#705) Co-authored-by: Johannes Weiss --- README.md | 42 +++++-------- .../Configuration+BrowserLike.swift | 41 +++++++++++++ Sources/AsyncHTTPClient/Docs.docc/index.md | 60 +++---------------- Sources/AsyncHTTPClient/HTTPClient.swift | 41 +++++++++---- Sources/AsyncHTTPClient/Singleton.swift | 35 +++++++++++ .../HTTPClientTests.swift | 11 ++++ 6 files changed, 138 insertions(+), 92 deletions(-) create mode 100644 Sources/AsyncHTTPClient/Configuration+BrowserLike.swift create mode 100644 Sources/AsyncHTTPClient/Singleton.swift diff --git a/README.md b/README.md index 26be89420..871eb910b 100644 --- a/README.md +++ b/README.md @@ -30,11 +30,9 @@ The code snippet below illustrates how to make a simple GET request to a remote ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) - /// MARK: - Using Swift Concurrency let request = HTTPClientRequest(url: "https://apple.com/") -let response = try await httpClient.execute(request, timeout: .seconds(30)) +let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) print("HTTP head", response) if response.status == .ok { let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB @@ -45,7 +43,7 @@ if response.status == .ok { /// MARK: - Using SwiftNIO EventLoopFuture -httpClient.get(url: "https://apple.com/").whenComplete { result in +HTTPClient.shared.get(url: "https://apple.com/").whenComplete { result in switch result { case .failure(let error): // process error @@ -59,7 +57,8 @@ httpClient.get(url: "https://apple.com/").whenComplete { result in } ``` -You should always shut down `HTTPClient` instances you created using `try httpClient.shutdown()`. Please note that you must not call `httpClient.shutdown` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. +If you create your own `HTTPClient` instances, you should shut them down using `httpClient.shutdown()` when you're done using them. Failing to do so will leak resources. + Please note that you must not call `httpClient.shutdown` before all requests of the HTTP client have finished, or else the in-flight requests will likely fail because their network connections are interrupted. ### async/await examples @@ -74,14 +73,13 @@ The default HTTP Method is `GET`. In case you need to have more control over the ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { var request = HTTPClientRequest(url: "https://apple.com/") request.method = .POST request.headers.add(name: "User-Agent", value: "Swift HTTPClient") request.body = .bytes(ByteBuffer(string: "some data")) - let response = try await httpClient.execute(request, timeout: .seconds(30)) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) if response.status == .ok { // handle response } else { @@ -90,8 +88,6 @@ do { } catch { // handle error } -// it's important to shutdown the httpClient after all requests are done, even if one failed -try await httpClient.shutdown() ``` #### Using SwiftNIO EventLoopFuture @@ -99,17 +95,11 @@ try await httpClient.shutdown() ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - var request = try HTTPClient.Request(url: "https://apple.com/", method: .POST) request.headers.add(name: "User-Agent", value: "Swift HTTPClient") request.body = .string("some-body") -httpClient.execute(request: request).whenComplete { result in +HTTPClient.shared.execute(request: request).whenComplete { result in switch result { case .failure(let error): // process error @@ -124,7 +114,9 @@ httpClient.execute(request: request).whenComplete { result in ``` ### Redirects following -Enable follow-redirects behavior using the client configuration: + +The globally shared instance `HTTPClient.shared` follows redirects by default. If you create your own `HTTPClient`, you can enable the follow-redirects behavior using the client configuration: + ```swift let httpClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: HTTPClient.Configuration(followRedirects: true)) @@ -148,10 +140,9 @@ The following example demonstrates how to count the number of bytes in a streami #### Using Swift Concurrency ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { let request = HTTPClientRequest(url: "https://apple.com/") - let response = try await httpClient.execute(request, timeout: .seconds(30)) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) print("HTTP head", response) // if defined, the content-length headers announces the size of the body @@ -174,8 +165,6 @@ do { } catch { print("request failed:", error) } -// it is important to shutdown the httpClient after all requests are done, even if one failed -try await httpClient.shutdown() ``` #### Using HTTPClientResponseDelegate and SwiftNIO EventLoopFuture @@ -235,7 +224,7 @@ class CountingDelegate: HTTPClientResponseDelegate { let request = try HTTPClient.Request(url: "https://apple.com/") let delegate = CountingDelegate() -httpClient.execute(request: request, delegate: delegate).futureResult.whenSuccess { count in +HTTPClient.shared.execute(request: request, delegate: delegate).futureResult.whenSuccess { count in print(count) } ``` @@ -248,7 +237,6 @@ asynchronously, while reporting the download progress at the same time, like in example: ```swift -let client = HTTPClient(eventLoopGroupProvider: .singleton) let request = try HTTPClient.Request( url: "https://swift.org/builds/development/ubuntu1804/latest-build.yml" ) @@ -260,7 +248,7 @@ let delegate = try FileDownloadDelegate(path: "/tmp/latest-build.yml", reportPro print("Downloaded \($0.receivedBytes) bytes so far") }) -client.execute(request: request, delegate: delegate).futureResult +HTTPClient.shared.execute(request: request, delegate: delegate).futureResult .whenSuccess { progress in if let totalBytes = progress.totalBytes { print("Final total bytes count: \(totalBytes)") @@ -272,8 +260,7 @@ client.execute(request: request, delegate: delegate).futureResult ### Unix Domain Socket Paths Connecting to servers bound to socket paths is easy: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -httpClient.execute( +HTTPClient.shared.execute( .GET, socketPath: "/tmp/myServer.socket", urlPath: "/path/to/resource" @@ -282,8 +269,7 @@ httpClient.execute( Connecting over TLS to a unix domain socket path is possible as well: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -httpClient.execute( +HTTPClient.shared.execute( .POST, secureSocketPath: "/tmp/myServer.socket", urlPath: "/path/to/resource", diff --git a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift new file mode 100644 index 000000000..7af13514c --- /dev/null +++ b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift @@ -0,0 +1,41 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +extension HTTPClient.Configuration { + /// The ``HTTPClient/Configuration`` for ``HTTPClient/shared`` which tries to mimic the platform's default or prevalent browser as closely as possible. + /// + /// Don't rely on specific values of this configuration as they're subject to change. You can rely on them being somewhat sensible though. + /// + /// - note: At present, this configuration is nowhere close to a real browser configuration but in case of disagreements we will choose values that match + /// the default browser as closely as possible. + /// + /// Platform's default/prevalent browsers that we're trying to match (these might change over time): + /// - macOS: Safari + /// - iOS: Safari + /// - Android: Google Chrome + /// - Linux (non-Android): Google Chrome + public static var singletonConfiguration: HTTPClient.Configuration { + // To start with, let's go with these values. Obtained from Firefox's config. + return HTTPClient.Configuration( + certificateVerification: .fullVerification, + redirectConfiguration: .follow(max: 20, allowCycles: false), + timeout: Timeout(connect: .seconds(90), read: .seconds(90)), + connectionPool: .seconds(600), + proxy: nil, + ignoreUncleanSSLShutdown: false, + decompression: .enabled(limit: .ratio(10)), + backgroundActivityLogger: nil + ) + } +} diff --git a/Sources/AsyncHTTPClient/Docs.docc/index.md b/Sources/AsyncHTTPClient/Docs.docc/index.md index 82e859b03..37033e043 100644 --- a/Sources/AsyncHTTPClient/Docs.docc/index.md +++ b/Sources/AsyncHTTPClient/Docs.docc/index.md @@ -34,12 +34,6 @@ The code snippet below illustrates how to make a simple GET request to a remote ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - /// MARK: - Using Swift Concurrency let request = HTTPClientRequest(url: "https://apple.com/") let response = try await httpClient.execute(request, timeout: .seconds(30)) @@ -53,7 +47,7 @@ if response.status == .ok { /// MARK: - Using SwiftNIO EventLoopFuture -httpClient.get(url: "https://apple.com/").whenComplete { result in +HTTPClient.shared.get(url: "https://apple.com/").whenComplete { result in switch result { case .failure(let error): // process error @@ -82,19 +76,13 @@ The default HTTP Method is `GET`. In case you need to have more control over the ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - do { var request = HTTPClientRequest(url: "https://apple.com/") request.method = .POST request.headers.add(name: "User-Agent", value: "Swift HTTPClient") request.body = .bytes(ByteBuffer(string: "some data")) - let response = try await httpClient.execute(request, timeout: .seconds(30)) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) if response.status == .ok { // handle response } else { @@ -103,8 +91,6 @@ do { } catch { // handle error } -// it's important to shutdown the httpClient after all requests are done, even if one failed -try await httpClient.shutdown() ``` #### Using SwiftNIO EventLoopFuture @@ -112,17 +98,11 @@ try await httpClient.shutdown() ```swift import AsyncHTTPClient -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - var request = try HTTPClient.Request(url: "https://apple.com/", method: .POST) request.headers.add(name: "User-Agent", value: "Swift HTTPClient") request.body = .string("some-body") -httpClient.execute(request: request).whenComplete { result in +HTTPClient.shared.execute(request: request).whenComplete { result in switch result { case .failure(let error): // process error @@ -161,15 +141,9 @@ The following example demonstrates how to count the number of bytes in a streami ##### Using Swift Concurrency ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - do { let request = HTTPClientRequest(url: "https://apple.com/") - let response = try await httpClient.execute(request, timeout: .seconds(30)) + let response = try await HTTPClient.shared.execute(request, timeout: .seconds(30)) print("HTTP head", response) // if defined, the content-length headers announces the size of the body @@ -192,8 +166,6 @@ do { } catch { print("request failed:", error) } -// it is important to shutdown the httpClient after all requests are done, even if one failed -try await httpClient.shutdown() ``` ##### Using HTTPClientResponseDelegate and SwiftNIO EventLoopFuture @@ -266,12 +238,6 @@ asynchronously, while reporting the download progress at the same time, like in example: ```swift -let client = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - let request = try HTTPClient.Request( url: "https://swift.org/builds/development/ubuntu1804/latest-build.yml" ) @@ -283,7 +249,7 @@ let delegate = try FileDownloadDelegate(path: "/tmp/latest-build.yml", reportPro print("Downloaded \($0.receivedBytes) bytes so far") }) -client.execute(request: request, delegate: delegate).futureResult +HTTPClient.shared.execute(request: request, delegate: delegate).futureResult .whenSuccess { progress in if let totalBytes = progress.totalBytes { print("Final total bytes count: \(totalBytes)") @@ -295,13 +261,7 @@ client.execute(request: request, delegate: delegate).futureResult #### Unix Domain Socket Paths Connecting to servers bound to socket paths is easy: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - -httpClient.execute( +HTTPClient.shared.execute( .GET, socketPath: "/tmp/myServer.socket", urlPath: "/path/to/resource" @@ -310,13 +270,7 @@ httpClient.execute( Connecting over TLS to a unix domain socket path is possible as well: ```swift -let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) -defer { - // Shutdown is guaranteed to work if it's done precisely once (which is the case here). - try! httpClient.syncShutdown() -} - -httpClient.execute( +HTTPClient.shared.execute( .POST, secureSocketPath: "/tmp/myServer.socket", urlPath: "/path/to/resource", diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 683532933..6c5a9af20 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -44,8 +44,7 @@ let globalRequestID = ManagedAtomic(0) /// Example: /// /// ```swift -/// let client = HTTPClient(eventLoopGroupProvider: .singleton) -/// client.get(url: "https://swift.org", deadline: .now() + .seconds(1)).whenComplete { result in +/// HTTPClient.shared.get(url: "https://swift.org", deadline: .now() + .seconds(1)).whenComplete { result in /// switch result { /// case .failure(let error): /// // process error @@ -58,12 +57,6 @@ let globalRequestID = ManagedAtomic(0) /// } /// } /// ``` -/// -/// It is important to close the client instance, for example in a defer statement, after use to cleanly shutdown the underlying NIO `EventLoopGroup`: -/// -/// ```swift -/// try client.syncShutdown() -/// ``` public class HTTPClient { /// The `EventLoopGroup` in use by this ``HTTPClient``. /// @@ -78,6 +71,7 @@ public class HTTPClient { private var state: State private let stateLock = NIOLock() + private let canBeShutDown: Bool static let loggingDisabled = Logger(label: "AHC-do-not-log", factory: { _ in SwiftLogNoOpLogHandler() }) @@ -133,9 +127,20 @@ public class HTTPClient { /// - eventLoopGroup: The `EventLoopGroup` that the ``HTTPClient`` will use. /// - configuration: Client configuration. /// - backgroundActivityLogger: The `Logger` that will be used to log background any activity that's not associated with a request. - public required init(eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, - configuration: Configuration = Configuration(), - backgroundActivityLogger: Logger) { + public convenience init(eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger) { + self.init(eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: backgroundActivityLogger, + canBeShutDown: true) + } + + internal required init(eventLoopGroup: EventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger, + canBeShutDown: Bool) { + self.canBeShutDown = canBeShutDown self.eventLoopGroup = eventLoopGroup self.configuration = configuration self.poolManager = HTTPConnectionPool.Manager( @@ -238,6 +243,12 @@ public class HTTPClient { } private func shutdown(requiresCleanClose: Bool, queue: DispatchQueue, _ callback: @escaping ShutdownCallback) { + guard self.canBeShutDown else { + queue.async { + callback(HTTPClientError.shutdownUnsupported) + } + return + } do { try self.stateLock.withLock { guard case .upAndRunning = self.state else { @@ -1081,6 +1092,7 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { case getConnectionFromPoolTimeout case deadlineExceeded case httpEndReceivedAfterHeadWith1xx + case shutdownUnsupported } private var code: Code @@ -1164,6 +1176,8 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { return "Deadline exceeded" case .httpEndReceivedAfterHeadWith1xx: return "HTTP end received after head with 1xx" + case .shutdownUnsupported: + return "The global singleton HTTP client cannot be shut down" } } @@ -1230,6 +1244,11 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { return HTTPClientError(code: .serverOfferedUnsupportedApplicationProtocol(proto)) } + /// The globally shared singleton ``HTTPClient`` cannot be shut down. + public static var shutdownUnsupported: HTTPClientError { + return HTTPClientError(code: .shutdownUnsupported) + } + /// The request deadline was exceeded. The request was cancelled because of this. public static let deadlineExceeded = HTTPClientError(code: .deadlineExceeded) diff --git a/Sources/AsyncHTTPClient/Singleton.swift b/Sources/AsyncHTTPClient/Singleton.swift new file mode 100644 index 000000000..149f7586f --- /dev/null +++ b/Sources/AsyncHTTPClient/Singleton.swift @@ -0,0 +1,35 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +extension HTTPClient { + /// A globally shared, singleton ``HTTPClient``. + /// + /// The returned client uses the following settings: + /// - configuration is ``HTTPClient/Configuration/singletonConfiguration`` (matching the platform's default/prevalent browser as well as possible) + /// - `EventLoopGroup` is ``HTTPClient/defaultEventLoopGroup`` (matching the platform default) + /// - logging is disabled + public static var shared: HTTPClient { + return globallySharedHTTPClient + } +} + +private let globallySharedHTTPClient: HTTPClient = { + let httpClient = HTTPClient( + eventLoopGroup: HTTPClient.defaultEventLoopGroup, + configuration: .singletonConfiguration, + backgroundActivityLogger: HTTPClient.loggingDisabled, + canBeShutDown: false + ) + return httpClient +}() diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index cdf9aa219..2f1b7035a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3575,6 +3575,17 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + func testSingletonClientWorks() throws { + let response = try HTTPClient.shared.get(url: self.defaultHTTPBinURLPrefix + "get").wait() + XCTAssertEqual(.ok, response.status) + } + + func testSingletonClientCannotBeShutDown() { + XCTAssertThrowsError(try HTTPClient.shared.shutdown().wait()) { error in + XCTAssertEqual(.shutdownUnsupported, error as? HTTPClientError) + } + } + func testAsyncExecuteWithCustomTLS() async throws { let httpsBin = HTTPBin(.http1_1(ssl: true)) defer { From fb308ee72f3d4c082a507033f94afa7395963ef3 Mon Sep 17 00:00:00 2001 From: hamzahrmalik Date: Fri, 5 Apr 2024 11:16:39 +0100 Subject: [PATCH 109/146] Move availability guard to correct test (#734) It was previously accidentally moved to a different test, which does not need it Async tests need the guard for macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0 --- Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 2f1b7035a..1bfca1d30 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3574,7 +3574,6 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(.ok, response.status) } - @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func testSingletonClientWorks() throws { let response = try HTTPClient.shared.get(url: self.defaultHTTPBinURLPrefix + "get").wait() XCTAssertEqual(.ok, response.status) @@ -3586,6 +3585,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } } + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) func testAsyncExecuteWithCustomTLS() async throws { let httpsBin = HTTPBin(.http1_1(ssl: true)) defer { From a22083713ee90808d527d0baa290c2fb13ca3096 Mon Sep 17 00:00:00 2001 From: Gustavo Cairo Date: Wed, 1 May 2024 12:49:09 +0100 Subject: [PATCH 110/146] Disable SETTINGS_ENABLE_PUSH HTTP/2 setting (#741) --- .../HTTP2/HTTP2Connection.swift | 4 +++- .../HTTP2ConnectionTests.swift | 23 +++++++++++++++++++ .../HTTPClientTestUtils.swift | 7 ++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index 2c3c3cc0a..ab43558c0 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -29,6 +29,8 @@ struct HTTP2PushNotSupportedError: Error {} struct HTTP2ReceivedGoAwayBeforeSettingsError: Error {} final class HTTP2Connection { + internal static let defaultSettings = nioDefaultSettings + [HTTP2Setting(parameter: .enablePush, value: 0)] + let channel: Channel let multiplexer: HTTP2StreamMultiplexer let logger: Logger @@ -196,7 +198,7 @@ final class HTTP2Connection { // can be scheduled on this connection. let sync = self.channel.pipeline.syncOperations - let http2Handler = NIOHTTP2Handler(mode: .client, initialSettings: nioDefaultSettings) + let http2Handler = NIOHTTP2Handler(mode: .client, initialSettings: Self.defaultSettings) let idleHandler = HTTP2IdleHandler(delegate: self, logger: self.logger, maximumConnectionUses: self.maximumConnectionUses) try sync.addHandler(http2Handler, position: .last) diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index 15e5cdff2..2e82fafba 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -18,6 +18,7 @@ import NIOConcurrencyHelpers import NIOCore import NIOEmbedded import NIOHTTP1 +import NIOHTTP2 import NIOPosix import NIOSSL import NIOTestUtils @@ -338,6 +339,28 @@ class HTTP2ConnectionTests: XCTestCase { } XCTAssertLessThan(retryCount, maxRetries) } + + func testServerPushIsDisabled() { + let embedded = EmbeddedChannel() + let logger = Logger(label: "test.http2.connection") + let connection = HTTP2Connection( + channel: embedded, + connectionID: 0, + decompression: .disabled, + maximumConnectionUses: nil, + delegate: TestHTTP2ConnectionDelegate(), + logger: logger + ) + _ = connection._start0() + + let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([]))) + XCTAssertNoThrow(try connection.channel.writeAndFlush(settingsFrame).wait()) + + let pushPromiseFrame = HTTP2Frame(streamID: 0, payload: .pushPromise(.init(pushedStreamID: 1, headers: [:]))) + XCTAssertThrowsError(try connection.channel.writeAndFlush(pushPromiseFrame).wait()) { error in + XCTAssertNotNil(error as? NIOHTTP2Errors.PushInViolationOfSetting) + } + } } class TestConnectionCreator { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 2d37b1387..7f28040c2 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -12,7 +12,7 @@ // //===----------------------------------------------------------------------===// -import AsyncHTTPClient +@testable import AsyncHTTPClient import Atomics import Foundation import Logging @@ -361,10 +361,7 @@ internal final class HTTPBin where var httpSettings: HTTP2Settings { switch self { case .http1_1, .http2(_, _, nil), .refuse: - return [ - HTTP2Setting(parameter: .maxConcurrentStreams, value: 10), - HTTP2Setting(parameter: .maxHeaderListSize, value: HPACKDecoder.defaultMaxHeaderListSize), - ] + return HTTP2Connection.defaultSettings case .http2(_, _, .some(let customSettings)): return customSettings } From 2fa5b34b3959113f5b5a7c08bbb6eac882863c69 Mon Sep 17 00:00:00 2001 From: Wes Cruver Date: Thu, 9 May 2024 05:14:40 -0700 Subject: [PATCH 111/146] Update Examples to use `.singleton` (#742) --- Examples/GetHTML/GetHTML.swift | 2 +- Examples/GetJSON/GetJSON.swift | 2 +- Examples/StreamingByteCounter/StreamingByteCounter.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Examples/GetHTML/GetHTML.swift b/Examples/GetHTML/GetHTML.swift index dfefa922b..98d6eb3c6 100644 --- a/Examples/GetHTML/GetHTML.swift +++ b/Examples/GetHTML/GetHTML.swift @@ -18,7 +18,7 @@ import NIOCore @main struct GetHTML { static func main() async throws { - let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) + let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { let request = HTTPClientRequest(url: "https://apple.com") let response = try await httpClient.execute(request, timeout: .seconds(30)) diff --git a/Examples/GetJSON/GetJSON.swift b/Examples/GetJSON/GetJSON.swift index ae58ffeaa..8f77c4a89 100644 --- a/Examples/GetJSON/GetJSON.swift +++ b/Examples/GetJSON/GetJSON.swift @@ -33,7 +33,7 @@ struct Comic: Codable { @main struct GetJSON { static func main() async throws { - let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) + let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { let request = HTTPClientRequest(url: "https://xkcd.com/info.0.json") let response = try await httpClient.execute(request, timeout: .seconds(30)) diff --git a/Examples/StreamingByteCounter/StreamingByteCounter.swift b/Examples/StreamingByteCounter/StreamingByteCounter.swift index dc340d14b..ecfb48776 100644 --- a/Examples/StreamingByteCounter/StreamingByteCounter.swift +++ b/Examples/StreamingByteCounter/StreamingByteCounter.swift @@ -18,7 +18,7 @@ import NIOCore @main struct StreamingByteCounter { static func main() async throws { - let httpClient = HTTPClient(eventLoopGroupProvider: .createNew) + let httpClient = HTTPClient(eventLoopGroupProvider: .singleton) do { let request = HTTPClientRequest(url: "https://apple.com") let response = try await httpClient.execute(request, timeout: .seconds(30)) From 0ae99db85b2b9d1e79b362bd31fd1ffe492f7c47 Mon Sep 17 00:00:00 2001 From: Mahdi Bahrami Date: Thu, 9 May 2024 15:53:45 +0330 Subject: [PATCH 112/146] Increase decompression limit ratio 10 -> 25 (#740) Co-authored-by: Franz Busch Co-authored-by: Cory Benfield --- Sources/AsyncHTTPClient/Configuration+BrowserLike.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift index 7af13514c..b1ee8d5a9 100644 --- a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift +++ b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift @@ -34,7 +34,7 @@ extension HTTPClient.Configuration { connectionPool: .seconds(600), proxy: nil, ignoreUncleanSSLShutdown: false, - decompression: .enabled(limit: .ratio(10)), + decompression: .enabled(limit: .ratio(25)), backgroundActivityLogger: nil ) } From e27aef494daefb51359788699166369127b85161 Mon Sep 17 00:00:00 2001 From: Andreas Ley Date: Tue, 18 Jun 2024 15:04:29 +0200 Subject: [PATCH 113/146] Make ConnectionPool's `retryConnectionEstablishment` public (#744) * Make ConnectionPool's `retryConnectionEstablishment` public * Unified tests to consistently use `enableFastFailureModeForTesting()` * Add `retryConnectionEstablishment` as optional parameter to the initializer of `HTTPClient.Configuration.ConnectionPool` * Reverted change to initializer to prevent API stability breakage * Add parameterless initializer for `HTTPClient.Configuration.ConnectionPool` * Moved default values for `HTTPClient.Configuration.ConnectionPool` to the property declarations, so they only have to be specified at one point * Removed superfluous spaces Co-authored-by: Cory Benfield * Re-added missing line break --------- Co-authored-by: Cory Benfield --- Sources/AsyncHTTPClient/HTTPClient.swift | 13 +++++++------ .../HTTPClientNIOTSTests.swift | 15 ++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 6c5a9af20..c52263318 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -1012,11 +1012,11 @@ extension HTTPClient.Configuration { public struct ConnectionPool: Hashable, Sendable { /// Specifies amount of time connections are kept idle in the pool. After this time has passed without a new /// request the connections are closed. - public var idleTimeout: TimeAmount + public var idleTimeout: TimeAmount = .seconds(60) /// The maximum number of connections that are kept alive in the connection pool per host. If requests with /// an explicit eventLoopRequirement are sent, this number might be exceeded due to overflow connections. - public var concurrentHTTP1ConnectionsPerHostSoftLimit: Int + public var concurrentHTTP1ConnectionsPerHostSoftLimit: Int = 8 /// If true, ``HTTPClient`` will try to create new connections on connection failure with an exponential backoff. /// Requests will only fail after the ``HTTPClient/Configuration/Timeout-swift.struct/connect`` timeout exceeded. @@ -1025,16 +1025,17 @@ extension HTTPClient.Configuration { /// - warning: We highly recommend leaving this on. /// It is very common that connections establishment is flaky at scale. /// ``HTTPClient`` will automatically mitigate these kind of issues if this flag is turned on. - var retryConnectionEstablishment: Bool + public var retryConnectionEstablishment: Bool = true - public init(idleTimeout: TimeAmount = .seconds(60)) { - self.init(idleTimeout: idleTimeout, concurrentHTTP1ConnectionsPerHostSoftLimit: 8) + public init() {} + + public init(idleTimeout: TimeAmount) { + self.idleTimeout = idleTimeout } public init(idleTimeout: TimeAmount, concurrentHTTP1ConnectionsPerHostSoftLimit: Int) { self.idleTimeout = idleTimeout self.concurrentHTTP1ConnectionsPerHostSoftLimit = concurrentHTTP1ConnectionsPerHostSoftLimit - self.retryConnectionEstablishment = true } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index be03f6a6a..3bbac632b 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -55,9 +55,8 @@ class HTTPClientNIOTSTests: XCTestCase { guard isTestingNIOTS() else { return } let httpBin = HTTPBin(.http1_1(ssl: true)) - var config = HTTPClient.Configuration() - config.networkFrameworkWaitForConnectivity = false - config.connectionPool.retryConnectionEstablishment = false + let config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) defer { @@ -84,9 +83,8 @@ class HTTPClientNIOTSTests: XCTestCase { guard isTestingNIOTS() else { return } #if canImport(Network) let httpBin = HTTPBin(.http1_1(ssl: false)) - var config = HTTPClient.Configuration() - config.networkFrameworkWaitForConnectivity = false - config.connectionPool.retryConnectionEstablishment = false + let config = HTTPClient.Configuration() + .enableFastFailureModeForTesting() let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) @@ -140,9 +138,8 @@ class HTTPClientNIOTSTests: XCTestCase { tlsConfig.minimumTLSVersion = .tlsv11 tlsConfig.maximumTLSVersion = .tlsv1 - var clientConfig = HTTPClient.Configuration(tlsConfiguration: tlsConfig) - clientConfig.networkFrameworkWaitForConnectivity = false - clientConfig.connectionPool.retryConnectionEstablishment = false + let clientConfig = HTTPClient.Configuration(tlsConfiguration: tlsConfig) + .enableFastFailureModeForTesting() let httpClient = HTTPClient( eventLoopGroupProvider: .shared(self.clientGroup), configuration: clientConfig From 4316ecae091a20472c55974b55d74c50928d12af Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Fri, 28 Jun 2024 10:33:04 +0100 Subject: [PATCH 114/146] Add support for request body to be larger than 2GB on 32-bit devices (#746) ### Motivation: - The properties that store the request body length and the cumulative number of bytes sent as part of a request are of type `Int`. - On 32-bit devices, when sending requests larger than `Int32.max`, these properties overflow and cause a crash. - To solve this problem, the properties should use the explicit `Int64` type. ### Modifications: - Changed the type of the `known` field of the `RequestBodyLength` enum to `Int64`. - Changed the type of `expectedBodyLength` and `sentBodyBytes` in `HTTPRequestStateMachine` to `Int64?` and `Int64` respectively. - Deprecated the `public var length: Int?` property of `HTTPClient.Body` and backed it with a new property: `contentLength: Int64?` - Added a new initializer and "overloaded" the `stream` function in `HTTPClient.Body` to work with the new `contentLength` property. - **Note:** The newly added `stream` function has different parameter names (`length` -> `contentLength` and `stream` -> `bodyStream`) to avoid ambiguity problems. - Added a test case that streams a 3GB request -- verified this fails with the types of the properties set explicitly to `Int32`. ### Result: - 32-bit devices can send requests larger than 2GB without integer overflow issues. --- .../HTTPClientRequest+Prepared.swift | 2 +- .../AsyncAwait/HTTPClientRequest.swift | 16 +++-- .../HTTPRequestStateMachine.swift | 8 +-- .../ConnectionPool/RequestBodyLength.swift | 2 +- .../RequestFramingMetadata.swift | 2 +- Sources/AsyncHTTPClient/HTTPHandler.swift | 39 ++++++++--- .../AsyncAwaitEndToEndTests.swift | 56 ++++++++++++++++ .../HTTP1ClientChannelHandlerTests.swift | 10 +-- .../HTTP1ConnectionTests.swift | 4 +- .../HTTP2ClientRequestHandlerTests.swift | 8 +-- .../HTTP2ClientTests.swift | 4 +- .../HTTPClientInternalTests.swift | 10 +-- .../HTTPClientRequestTests.swift | 24 +++---- .../HTTPClientTests.swift | 64 +++++++++--------- .../NoBytesSentOverBodyLimitTests.swift | 2 +- .../RequestBagTests.swift | 65 +++++++++++-------- .../TransactionTests.swift | 2 +- 17 files changed, 208 insertions(+), 110 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index 360e91b89..2d5e3e2e0 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -95,7 +95,7 @@ extension RequestBodyLength { case .none: self = .known(0) case .byteBuffer(let buffer): - self = .known(buffer.readableBytes) + self = .known(Int64(buffer.readableBytes)) case .sequence(let length, _, _), .asyncSequence(let length, _): self = length } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index 4ed79e38c..ad81bfa32 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -125,7 +125,7 @@ extension HTTPClientRequest.Body { public static func bytes( _ bytes: Bytes ) -> Self where Bytes.Element == UInt8 { - self.bytes(bytes, length: .known(bytes.count)) + self.bytes(bytes, length: .known(Int64(bytes.count))) } /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Sequence` of bytes. @@ -140,7 +140,7 @@ extension HTTPClientRequest.Body { /// /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)-9q0ge`` will use `Content-Length`. /// /// - parameters: /// - bytes: The bytes of the request body. @@ -225,7 +225,7 @@ extension HTTPClientRequest.Body { /// /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)-9q0ge`` will use `Content-Length`. /// /// - parameters: /// - bytes: The bytes of the request body. @@ -265,7 +265,7 @@ extension HTTPClientRequest.Body { /// /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)-9q0ge`` will use `Content-Length`. /// /// - parameters: /// - sequenceOfBytes: The bytes of the request body. @@ -293,7 +293,7 @@ extension HTTPClientRequest.Body { /// /// Caution should be taken with this method to ensure that the `length` is correct. Incorrect lengths /// will cause unnecessary runtime failures. Setting `length` to ``Length/unknown`` will trigger the upload - /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)`` will use `Content-Length`. + /// to use `chunked` `Transfer-Encoding`, while using ``Length/known(_:)-9q0ge`` will use `Content-Length`. /// /// - parameters: /// - bytes: The bytes of the request body. @@ -341,7 +341,13 @@ extension HTTPClientRequest.Body { public static let unknown: Self = .init(storage: .unknown) /// The size of the request body is known and exactly `count` bytes + @available(*, deprecated, message: "Use `known(_ count: Int64)` with an explicit Int64 argument instead") public static func known(_ count: Int) -> Self { + .init(storage: .known(Int64(count))) + } + + /// The size of the request body is known and exactly `count` bytes + public static func known(_ count: Int64) -> Self { .init(storage: .known(count)) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift index b575ae094..533062036 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift @@ -58,7 +58,7 @@ struct HTTPRequestStateMachine { /// The request is streaming its request body. `expectedBodyLength` has a value, if the request header contained /// a `"content-length"` header field. If the request header contained a `"transfer-encoding" = "chunked"` /// header field, the `expectedBodyLength` is `nil`. - case streaming(expectedBodyLength: Int?, sentBodyBytes: Int, producer: ProducerControlState) + case streaming(expectedBodyLength: Int64?, sentBodyBytes: Int64, producer: ProducerControlState) /// The request has sent its request body and end. case endSent } @@ -308,13 +308,13 @@ struct HTTPRequestStateMachine { // pause. The reason for this is as follows: There might be thread synchronization // situations in which the producer might not have received the plea to pause yet. - if let expected = expectedBodyLength, sentBodyBytes + part.readableBytes > expected { + if let expected = expectedBodyLength, sentBodyBytes + Int64(part.readableBytes) > expected { let error = HTTPClientError.bodyLengthMismatch self.state = .failed(error) return .failRequest(error, .close(promise)) } - sentBodyBytes += part.readableBytes + sentBodyBytes += Int64(part.readableBytes) let requestState: RequestState = .streaming( expectedBodyLength: expectedBodyLength, @@ -768,7 +768,7 @@ struct HTTPRequestStateMachine { } extension RequestFramingMetadata.Body { - var expectedLength: Int? { + var expectedLength: Int64? { switch self { case .fixedSize(let length): return length case .stream: return nil diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift index 83f0e6edf..58ba694a7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestBodyLength.swift @@ -20,5 +20,5 @@ internal enum RequestBodyLength: Hashable, Sendable { /// size of the request body is not known before starting the request case unknown /// size of the request body is fixed and exactly `count` bytes - case known(_ count: Int) + case known(_ count: Int64) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/RequestFramingMetadata.swift b/Sources/AsyncHTTPClient/ConnectionPool/RequestFramingMetadata.swift index 98080e364..033060a99 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/RequestFramingMetadata.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/RequestFramingMetadata.swift @@ -15,7 +15,7 @@ struct RequestFramingMetadata: Hashable { enum Body: Hashable { case stream - case fixedSize(Int) + case fixedSize(Int64) } var connectionClose: Bool diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 98415a124..c8a485023 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -70,7 +70,19 @@ extension HTTPClient { /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `length`. - public var length: Int? + @available(*, deprecated, renamed: "contentLength") + public var length: Int? { + get { + self.contentLength.flatMap { Int($0) } + } + set { + self.contentLength = newValue.flatMap { Int64($0) } + } + } + + /// Body size. If nil,`Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` + /// header is set with the given `contentLength`. + public var contentLength: Int64? /// Body chunk provider. public var stream: @Sendable (StreamWriter) -> EventLoopFuture @@ -78,8 +90,8 @@ extension HTTPClient { @usableFromInline typealias StreamCallback = @Sendable (StreamWriter) -> EventLoopFuture @inlinable - init(length: Int?, stream: @escaping StreamCallback) { - self.length = length + init(contentLength: Int64?, stream: @escaping StreamCallback) { + self.contentLength = contentLength.flatMap { $0 } self.stream = stream } @@ -88,7 +100,7 @@ extension HTTPClient { /// - parameters: /// - buffer: Body `ByteBuffer` representation. public static func byteBuffer(_ buffer: ByteBuffer) -> Body { - return Body(length: buffer.readableBytes) { writer in + return Body(contentLength: Int64(buffer.readableBytes)) { writer in writer.write(.byteBuffer(buffer)) } } @@ -100,8 +112,19 @@ extension HTTPClient { /// header is set with the given `length`. /// - stream: Body chunk provider. @preconcurrency + @available(*, deprecated, renamed: "stream(contentLength:bodyStream:)") public static func stream(length: Int? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { - return Body(length: length, stream: stream) + return Body(contentLength: length.flatMap { Int64($0) }, stream: stream) + } + + /// Create and stream body using ``StreamWriter``. + /// + /// - parameters: + /// - contentLength: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` + /// header is set with the given `contentLength`. + /// - bodyStream: Body chunk provider. + public static func stream(contentLength: Int64? = nil, bodyStream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { + return Body(contentLength: contentLength, stream: bodyStream) } /// Create and stream body using a collection of bytes. @@ -111,7 +134,7 @@ extension HTTPClient { @preconcurrency @inlinable public static func bytes(_ bytes: Bytes) -> Body where Bytes: RandomAccessCollection, Bytes: Sendable, Bytes.Element == UInt8 { - return Body(length: bytes.count) { writer in + return Body(contentLength: Int64(bytes.count)) { writer in if bytes.count <= bagOfBytesToByteBufferConversionChunkSize { return writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) } else { @@ -125,7 +148,7 @@ extension HTTPClient { /// - parameters: /// - string: Body `String` representation. public static func string(_ string: String) -> Body { - return Body(length: string.utf8.count) { writer in + return Body(contentLength: Int64(string.utf8.count)) { writer in if string.utf8.count <= bagOfBytesToByteBufferConversionChunkSize { return writer.write(.byteBuffer(ByteBuffer(string: string))) } else { @@ -858,7 +881,7 @@ extension RequestBodyLength { self = .known(0) return } - guard let length = body.length else { + guard let length = body.contentLength else { self = .unknown return } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index a30a8cf91..626f5b4ae 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -184,6 +184,62 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } + struct AsyncSequenceByteBufferGenerator: AsyncSequence, Sendable, AsyncIteratorProtocol { + typealias Element = ByteBuffer + + let chunkSize: Int + let totalChunks: Int + let buffer: ByteBuffer + var chunksGenerated: Int = 0 + + init(chunkSize: Int, totalChunks: Int) { + self.chunkSize = chunkSize + self.totalChunks = totalChunks + self.buffer = ByteBuffer(repeating: 1, count: self.chunkSize) + } + + mutating func next() async throws -> ByteBuffer? { + guard self.chunksGenerated < self.totalChunks else { return nil } + + self.chunksGenerated += 1 + return self.buffer + } + + func makeAsyncIterator() -> AsyncSequenceByteBufferGenerator { + return self + } + } + + func testEchoStreamThatHas3GBInTotal() async throws { + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + let bin = HTTPBin(.http1_1()) { _ in HTTPEchoHandler() } + defer { XCTAssertNoThrow(try bin.shutdown()) } + + let client: HTTPClient = makeDefaultHTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) + defer { XCTAssertNoThrow(try client.syncShutdown()) } + + let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + + var request = HTTPClientRequest(url: "http://localhost:\(bin.port)/") + request.method = .POST + + let sequence = AsyncSequenceByteBufferGenerator( + chunkSize: 4_194_304, // 4MB chunk + totalChunks: 768 // Total = 3GB + ) + request.body = .stream(sequence, length: .unknown) + + let response: HTTPClientResponse = try await client.execute(request, deadline: .now() + .seconds(30), logger: logger) + XCTAssertEqual(response.headers["content-length"], []) + + var receivedBytes: Int64 = 0 + for try await part in response.body { + receivedBytes += Int64(part.readableBytes) + } + XCTAssertEqual(receivedBytes, 3_221_225_472) // 3GB + } + func testPostWithAsyncSequenceOfByteBuffers() { XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) { _ in HTTPEchoHandler() } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index f6a2840d9..f4f2d67f8 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -113,7 +113,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 100) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 100) { writer in testWriter.start(writer: writer) })) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } @@ -345,7 +345,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) return testWriter.start(writer: writer) @@ -384,7 +384,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in embedded.isWritable = false embedded.pipeline.fireChannelWritabilityChanged() // This should not trigger any errors or timeouts, because the timer isn't running @@ -432,7 +432,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 2) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 2) { writer in return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) })) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } @@ -595,7 +595,7 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in testWriter.start(writer: writer) })) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift index 3ff73de06..5ea8bb77c 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift @@ -116,8 +116,8 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( url: "http://localhost/hello/swift", method: .POST, - body: .stream(length: 4) { writer -> EventLoopFuture in - func recursive(count: UInt8, promise: EventLoopPromise) { + body: .stream(contentLength: 4) { writer -> EventLoopFuture in + @Sendable func recursive(count: UInt8, promise: EventLoopPromise) { guard count < 4 else { return promise.succeed(()) } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index 545ba1e3c..2428199a4 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -115,7 +115,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 50) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 100) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 100) { writer in testWriter.start(writer: writer) })) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } @@ -295,7 +295,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) return testWriter.start(writer: writer) @@ -335,7 +335,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 10) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in embedded.isWritable = false embedded.pipeline.fireChannelWritabilityChanged() // This should not trigger any errors or timeouts, because the timer isn't running @@ -385,7 +385,7 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(length: 2) { writer in + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 2) { writer in return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) })) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 97f0385ea..889cd38b9 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -68,7 +68,7 @@ class HTTP2ClientTests: XCTestCase { let client = self.makeDefaultHTTPClient() defer { XCTAssertNoThrow(try client.syncShutdown()) } var response: HTTPClient.Response? - let body = HTTPClient.Body.stream(length: nil) { writer in + let body = HTTPClient.Body.stream(contentLength: nil) { writer in writer.write(.byteBuffer(ByteBuffer(integer: UInt64(0)))).flatMap { writer.write(.byteBuffer(ByteBuffer(integer: UInt64(0)))) } @@ -84,7 +84,7 @@ class HTTP2ClientTests: XCTestCase { defer { XCTAssertNoThrow(try bin.shutdown()) } let client = self.makeDefaultHTTPClient() defer { XCTAssertNoThrow(try client.syncShutdown()) } - let body = HTTPClient.Body.stream(length: 12) { writer in + let body = HTTPClient.Body.stream(contentLength: 12) { writer in writer.write(.byteBuffer(ByteBuffer(integer: UInt64(0)))).flatMap { writer.write(.byteBuffer(ByteBuffer(integer: UInt64(0)))) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index 6f412a30d..80446251c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -52,7 +52,7 @@ class HTTPClientInternalTests: XCTestCase { XCTAssertNoThrow(try httpBin.shutdown()) } - let body: HTTPClient.Body = .stream(length: 50) { writer in + let body: HTTPClient.Body = .stream(contentLength: 50) { writer in do { var request = try Request(url: "http://localhost:\(httpBin.port)/events/10/1") request.headers.add(name: "Accept", value: "text/event-stream") @@ -81,13 +81,13 @@ class HTTPClientInternalTests: XCTestCase { XCTAssertNoThrow(try httpBin.shutdown()) } - var body: HTTPClient.Body = .stream(length: 50) { _ in + var body: HTTPClient.Body = .stream(contentLength: 50) { _ in httpClient.eventLoopGroup.next().makeFailedFuture(HTTPClientError.invalidProxyResponse) } XCTAssertThrowsError(try httpClient.post(url: "http://localhost:\(httpBin.port)/post", body: body).wait()) - body = .stream(length: 50) { _ in + body = .stream(contentLength: 50) { _ in do { var request = try Request(url: "http://localhost:\(httpBin.port)/events/10/1") request.headers.add(name: "Accept", value: "text/event-stream") @@ -223,7 +223,7 @@ class HTTPClientInternalTests: XCTestCase { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) } - let body: HTTPClient.Body = .stream(length: 8) { writer in + let body: HTTPClient.Body = .stream(contentLength: 8) { writer in let buffer = ByteBuffer(string: "1234") return writer.write(.byteBuffer(buffer)).flatMap { let buffer = ByteBuffer(string: "4321") @@ -366,7 +366,7 @@ class HTTPClientInternalTests: XCTestCase { let el2 = group.next() XCTAssert(el1 !== el2) - let body: HTTPClient.Body = .stream(length: 8) { writer in + let body: HTTPClient.Body = .stream(contentLength: 8) { writer in XCTAssert(el1.inEventLoop) let buffer = ByteBuffer(string: "1234") return writer.write(.byteBuffer(buffer)).flatMap { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index b0b1be1d8..c93ab4cb5 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -312,7 +312,7 @@ class HTTPClientRequestTests: XCTestCase { request.method = .POST let sequence = AnySendableSequence(ByteBuffer(string: "post body").readableBytesView) - request.body = .bytes(sequence, length: .known(9)) + request.body = .bytes(sequence, length: .known(Int64(9))) var preparedRequest: PreparedRequest? XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } @@ -424,7 +424,7 @@ class HTTPClientRequestTests: XCTestCase { .async .map { ByteBuffer($0) } - request.body = .stream(asyncSequence, length: .known(9)) + request.body = .stream(asyncSequence, length: .known(Int64(9))) var preparedRequest: PreparedRequest? XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } @@ -476,7 +476,7 @@ class HTTPClientRequestTests: XCTestCase { String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) ).utf8, - length: .known(bagOfBytesToByteBufferConversionChunkSize * 3) + length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)) ).collect() let expectedChunks = [ @@ -495,7 +495,7 @@ class HTTPClientRequestTests: XCTestCase { Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) ), - length: .known(bagOfBytesToByteBufferConversionChunkSize * 3), + length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)), bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, byteBufferMaxSize: byteBufferMaxSize ).collect() @@ -516,7 +516,7 @@ class HTTPClientRequestTests: XCTestCase { } let body = try await HTTPClientRequest.Body.bytes( makeBytes(), - length: .known(bagOfBytesToByteBufferConversionChunkSize * 3) + length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)) ).collect() var firstChunk = ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) @@ -539,7 +539,7 @@ class HTTPClientRequestTests: XCTestCase { } let body = try await HTTPClientRequest.Body._bytes( makeBytes(), - length: .known(bagOfBytesToByteBufferConversionChunkSize * 3), + length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)), bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, byteBufferMaxSize: byteBufferMaxSize ).collect() @@ -614,8 +614,8 @@ extension HTTPClient.Body { } private struct LengthMismatch: Error { - var announcedLength: Int - var actualLength: Int + var announcedLength: Int64 + var actualLength: Int64 } @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -631,8 +631,8 @@ extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { case .sequence(let announcedLength, _, let generate): let buffer = generate(ByteBufferAllocator()) if case .known(let announcedLength) = announcedLength, - announcedLength != buffer.readableBytes { - throw LengthMismatch(announcedLength: announcedLength, actualLength: buffer.readableBytes) + announcedLength != Int64(buffer.readableBytes) { + throw LengthMismatch(announcedLength: announcedLength, actualLength: Int64(buffer.readableBytes)) } return buffer case .asyncSequence(length: let announcedLength, let generate): @@ -641,8 +641,8 @@ extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { accumulatedBuffer.writeBuffer(&buffer) } if case .known(let announcedLength) = announcedLength, - announcedLength != accumulatedBuffer.readableBytes { - throw LengthMismatch(announcedLength: announcedLength, actualLength: accumulatedBuffer.readableBytes) + announcedLength != Int64(accumulatedBuffer.readableBytes) { + throw LengthMismatch(announcedLength: announcedLength, actualLength: Int64(accumulatedBuffer.readableBytes)) } return accumulatedBuffer } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 1bfca1d30..51bc1f005 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -621,7 +621,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "post", method: .POST, headers: ["transfer-encoding": "chunked"], - body: .stream { streamWriter in + body: .stream(bodyStream: { streamWriter in _ = streamWriter.write(.byteBuffer(.init())) let promise = self.clientGroup.next().makePromise(of: Void.self) @@ -630,7 +630,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return promise.futureResult - }) + })) XCTAssertThrowsError(try localClient.execute(request: request).wait()) { XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) @@ -802,7 +802,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testUploadStreaming() throws { - let body: HTTPClient.Body = .stream(length: 8) { writer in + let body: HTTPClient.Body = .stream(contentLength: 8) { writer in let buffer = ByteBuffer(string: "1234") return writer.write(.byteBuffer(buffer)).flatMap { let buffer = ByteBuffer(string: "4321") @@ -1953,9 +1953,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testValidationErrorsAreSurfaced() throws { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .TRACE, body: .stream { _ in + let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .TRACE, body: .stream(bodyStream: { _ in self.defaultClient.eventLoopGroup.next().makeSucceededFuture(()) - }) + })) let runningRequest = self.defaultClient.execute(request: request) XCTAssertThrowsError(try runningRequest.wait()) { error in XCTAssertEqual(HTTPClientError.traceRequestWithBody, error as? HTTPClientError) @@ -2048,10 +2048,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { return try? HTTPClient.Request(url: "http://\(localAddress.ipAddress!):\(localAddress.port!)", method: .POST, headers: ["transfer-encoding": "chunked"], - body: .stream { streamWriter in + body: .stream(bodyStream: { streamWriter in streamWriterPromise.succeed(streamWriter) return sentOffAllBodyPartsPromise.futureResult - }) + })) } guard let server = makeServer(), let request = makeRequest(server: server) else { @@ -2083,7 +2083,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testUploadStreamingCallinToleratedFromOtsideEL() throws { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .POST, body: .stream(length: 4) { writer in + let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .POST, body: .stream(contentLength: 4) { writer in let promise = self.defaultClient.eventLoopGroup.next().makePromise(of: Void.self) // We have to toleare callins from any thread DispatchQueue(label: "upload-streaming").async { @@ -2602,9 +2602,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } var request = try HTTPClient.Request(url: "http://localhost:\(server.serverPort)/") - request.body = .stream { writer in + request.body = .stream(bodyStream: { writer in writer.write(.byteBuffer(ByteBuffer(string: "1234"))) - } + }) let future = client.execute(request: request) @@ -2703,7 +2703,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertThrowsError( try self.defaultClient.execute(request: Request(url: url, - body: .stream(length: 10) { streamWriter in + body: .stream(contentLength: 10) { streamWriter in let promise = self.defaultClient.eventLoopGroup.next().makePromise(of: Void.self) DispatchQueue(label: "content-length-test").async { streamWriter.write(.byteBuffer(ByteBuffer(string: "1"))).cascade(to: promise) @@ -2733,7 +2733,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertThrowsError( try self.defaultClient.execute(request: Request(url: url, - body: .stream(length: 1) { streamWriter in + body: .stream(contentLength: 1) { streamWriter in streamWriter.write(.byteBuffer(ByteBuffer(string: tooLong))) })).wait()) { error in XCTAssertEqual(error as! HTTPClientError, HTTPClientError.bodyLengthMismatch) @@ -2756,7 +2756,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testBodyUploadAfterEndFails() { let url = self.defaultHTTPBinURLPrefix + "post" - func uploader(_ streamWriter: HTTPClient.Body.StreamWriter) -> EventLoopFuture { + let uploader = { @Sendable (_ streamWriter: HTTPClient.Body.StreamWriter) -> EventLoopFuture in let done = streamWriter.write(.byteBuffer(ByteBuffer(string: "X"))) done.recover { error in XCTFail("unexpected error \(error)") @@ -2777,7 +2777,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } var request: HTTPClient.Request? - XCTAssertNoThrow(request = try Request(url: url, body: .stream(length: 1, uploader))) + XCTAssertNoThrow(request = try Request(url: url, body: .stream(contentLength: 1, bodyStream: uploader))) XCTAssertThrowsError(try self.defaultClient.execute(request: XCTUnwrap(request)).wait()) { XCTAssertEqual($0 as? HTTPClientError, .writeAfterRequestSent) } @@ -2793,7 +2793,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { _ = self.defaultClient.get(url: "http://localhost:\(self.defaultHTTPBin.port)/events/10/1") var request = try HTTPClient.Request(url: "http://localhost:\(self.defaultHTTPBin.port)/wait", method: .POST) - request.body = .stream { writer in + request.body = .stream(bodyStream: { writer in // Start writing chunks so tha we will try to write after read timeout is thrown for _ in 1...10 { _ = writer.write(.byteBuffer(ByteBuffer(string: "1234"))) @@ -2805,7 +2805,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return promise.futureResult - } + }) // We specify a deadline of 2 ms co that request will be timed out before all chunks are writtent, // we need to verify that second error on write after timeout does not lead to double-release. @@ -2968,10 +2968,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let delegate = ResponseStreamDelegate(eventLoop: delegateEL) - let body: HTTPClient.Body = .stream { writer in + let body: HTTPClient.Body = .stream(bodyStream: { writer in let finalPromise = writeEL.makePromise(of: Void.self) - func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { // always invoke from the wrong el to test thread safety writeEL.preconditionInEventLoop() @@ -3004,7 +3004,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - } + }) let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: delegateEL)) @@ -3068,9 +3068,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let body = ByteBuffer(bytes: 0..<11) var request = try Request(url: httpBin.baseURL) - request.body = .stream { writer in + request.body = .stream(bodyStream: { writer in writer.write(.byteBuffer(body)) - } + }) XCTAssertThrowsError(try self.defaultClient.execute( request: request, delegate: ResponseAccumulator(request: request, maxBodySize: 10) @@ -3086,9 +3086,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let body = ByteBuffer(bytes: 0..<10) var request = try Request(url: httpBin.baseURL) - request.body = .stream { writer in + request.body = .stream(bodyStream: { writer in writer.write(.byteBuffer(body)) - } + }) let response = try self.defaultClient.execute( request: request, delegate: ResponseAccumulator(request: request, maxBodySize: 10) @@ -3113,10 +3113,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let delegate = ResponseStreamDelegate(eventLoop: delegateEL) - let body: HTTPClient.Body = .stream { writer in + let body: HTTPClient.Body = .stream(bodyStream: { writer in let finalPromise = writeEL.makePromise(of: Void.self) - func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { // always invoke from the wrong el to test thread safety writeEL.preconditionInEventLoop() @@ -3143,7 +3143,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - } + }) let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: delegateEL)) @@ -3164,10 +3164,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } - let body: HTTPClient.Body = .stream { writer in + let body: HTTPClient.Body = .stream(bodyStream: { writer in let finalPromise = writeEL.makePromise(of: Void.self) - func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { // always invoke from the wrong el to test thread safety writeEL.preconditionInEventLoop() @@ -3194,7 +3194,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - } + }) let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request) @@ -3220,10 +3220,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } - let body: HTTPClient.Body = .stream { writer in + let body: HTTPClient.Body = .stream(bodyStream: { writer in let finalPromise = writeEL.makePromise(of: Void.self) - func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { + @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { // always invoke from the wrong el to test thread safety writeEL.preconditionInEventLoop() @@ -3250,7 +3250,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - } + }) let headers = HTTPHeaders([("Connection", "close")]) let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", headers: headers, body: body) diff --git a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift index 41285d5c5..756facb3f 100644 --- a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift +++ b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift @@ -40,7 +40,7 @@ final class NoBytesSentOverBodyLimitTests: XCTestCaseHTTPClientTestsBaseClass { let request = try Request( url: "http://localhost:\(server.serverPort)", - body: .stream(length: 1) { streamWriter in + body: .stream(contentLength: 1) { streamWriter in streamWriter.write(.byteBuffer(ByteBuffer(string: tooLong))) } ) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 610e429f5..fa094c1af 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -14,6 +14,7 @@ @testable import AsyncHTTPClient import Logging +import NIOConcurrencyHelpers import NIOCore import NIOEmbedded import NIOHTTP1 @@ -26,24 +27,36 @@ final class RequestBagTests: XCTestCase { defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } let logger = Logger(label: "test") - var writtenBytes = 0 - var writes = 0 + struct TestState { + var writtenBytes: Int = 0 + var writes: Int = 0 + var streamIsAllowedToWrite: Bool = false + } + + let testState = NIOLockedValueBox(TestState()) + let bytesToSent = (3000...10000).randomElement()! let expectedWrites = bytesToSent / 100 + ((bytesToSent % 100 > 0) ? 1 : 0) - var streamIsAllowedToWrite = false let writeDonePromise = embeddedEventLoop.makePromise(of: Void.self) - let requestBody: HTTPClient.Body = .stream(length: bytesToSent) { writer -> EventLoopFuture in - func write(donePromise: EventLoopPromise) { - XCTAssertTrue(streamIsAllowedToWrite) - guard writtenBytes < bytesToSent else { - return donePromise.succeed(()) + let requestBody: HTTPClient.Body = .stream(contentLength: Int64(bytesToSent)) { writer -> EventLoopFuture in + @Sendable func write(donePromise: EventLoopPromise) { + let futureWrite: EventLoopFuture? = testState.withLockedValue { state in + XCTAssertTrue(state.streamIsAllowedToWrite) + guard state.writtenBytes < bytesToSent else { + donePromise.succeed(()) + return nil + } + let byteCount = min(bytesToSent - state.writtenBytes, 100) + let buffer = ByteBuffer(bytes: [UInt8](repeating: 1, count: byteCount)) + state.writes += 1 + return writer.write(.byteBuffer(buffer)) } - let byteCount = min(bytesToSent - writtenBytes, 100) - let buffer = ByteBuffer(bytes: [UInt8](repeating: 1, count: byteCount)) - writes += 1 - writer.write(.byteBuffer(buffer)).whenSuccess { _ in - writtenBytes += 100 + + futureWrite?.whenSuccess { _ in + testState.withLockedValue { state in + state.writtenBytes += 100 + } write(donePromise: donePromise) } } @@ -81,9 +94,9 @@ final class RequestBagTests: XCTestCase { executor.runRequest(bag) XCTAssertEqual(delegate.hitDidSendRequestHead, 1) - streamIsAllowedToWrite = true + testState.withLockedValue { $0.streamIsAllowedToWrite = true } bag.resumeRequestBodyStream() - streamIsAllowedToWrite = false + testState.withLockedValue { $0.streamIsAllowedToWrite = false } // after starting the body stream we should have received two writes var receivedBytes = 0 @@ -91,14 +104,14 @@ final class RequestBagTests: XCTestCase { XCTAssertNoThrow(try executor.receiveRequestBody { receivedBytes += $0.readableBytes }) - XCTAssertEqual(delegate.hitDidSendRequestPart, writes) + XCTAssertEqual(delegate.hitDidSendRequestPart, testState.withLockedValue { $0.writes }) if i % 2 == 1 { - streamIsAllowedToWrite = true + testState.withLockedValue { $0.streamIsAllowedToWrite = true } executor.resumeRequestBodyStream() - streamIsAllowedToWrite = false + testState.withLockedValue { $0.streamIsAllowedToWrite = false } XCTAssertLessThanOrEqual(executor.requestBodyPartsCount, 2) - XCTAssertEqual(delegate.hitDidSendRequestPart, writes) + XCTAssertEqual(delegate.hitDidSendRequestPart, testState.withLockedValue { $0.writes }) } } @@ -153,7 +166,7 @@ final class RequestBagTests: XCTestCase { defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } let logger = Logger(label: "test") - let requestBody: HTTPClient.Body = .stream(length: 12) { writer -> EventLoopFuture in + let requestBody: HTTPClient.Body = .stream(contentLength: 12) { writer -> EventLoopFuture in writer.write(.byteBuffer(ByteBuffer(bytes: 0...3))).flatMap { _ -> EventLoopFuture in embeddedEventLoop.makeFailedFuture(TestError()) @@ -530,21 +543,21 @@ final class RequestBagTests: XCTestCase { var maybeRequest: HTTPClient.Request? let writeSecondPartPromise = embeddedEventLoop.makePromise(of: Void.self) + let firstWriteSuccess: NIOLockedValueBox = .init(false) XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( url: "https://swift.org", method: .POST, headers: ["content-length": "12"], - body: .stream(length: 12) { writer -> EventLoopFuture in - var firstWriteSuccess = false + body: .stream(contentLength: 12) { writer -> EventLoopFuture in return writer.write(.byteBuffer(.init(bytes: 0...3))).flatMap { _ in - firstWriteSuccess = true + firstWriteSuccess.withLockedValue { $0 = true } return writeSecondPartPromise.futureResult }.flatMap { return writer.write(.byteBuffer(.init(bytes: 4...7))) }.always { result in - XCTAssertTrue(firstWriteSuccess) + XCTAssertTrue(firstWriteSuccess.withLockedValue { $0 }) guard case .failure(let error) = result else { return XCTFail("Expected the second write to fail") @@ -859,11 +872,11 @@ final class RequestBagTests: XCTestCase { let writerPromise = group.any().makePromise(of: HTTPClient.Body.StreamWriter.self) let donePromise = group.any().makePromise(of: Void.self) - request.body = .stream { [leakDetector] writer in + request.body = .stream(bodyStream: { [leakDetector] writer in _ = leakDetector writerPromise.succeed(writer) return donePromise.futureResult - } + }) let resultFuture = httpClient.execute(request: request) request.body = nil diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index a8a2bb30e..40f71d010 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -517,7 +517,7 @@ final class TransactionTests: XCTestCase { var request = HTTPClientRequest(url: "https://localhost:\(httpBin.port)/") request.method = .POST request.headers = ["host": "localhost:\(httpBin.port)"] - request.body = .stream(streamWriter, length: .known(800)) + request.body = .stream(streamWriter, length: .known(Int64(800))) var maybePreparedRequest: PreparedRequest? XCTAssertNoThrow(maybePreparedRequest = try PreparedRequest(request)) From 07536f6a4ee1e0af0bf8f0e59ea758eb116f99d4 Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Wed, 3 Jul 2024 16:14:47 +0100 Subject: [PATCH 115/146] Resolve ambiguity issue for the `stream` function, remove @deprecated marking from the original implementation, and make argument labels consistent. (#749) --- Sources/AsyncHTTPClient/HTTPHandler.swift | 6 +-- .../HTTPClientTests.swift | 46 +++++++++---------- .../RequestBagTests.swift | 4 +- 3 files changed, 28 insertions(+), 28 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index c8a485023..b81a2c7a5 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -111,8 +111,8 @@ extension HTTPClient { /// - length: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `length`. /// - stream: Body chunk provider. + @_disfavoredOverload @preconcurrency - @available(*, deprecated, renamed: "stream(contentLength:bodyStream:)") public static func stream(length: Int? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { return Body(contentLength: length.flatMap { Int64($0) }, stream: stream) } @@ -123,8 +123,8 @@ extension HTTPClient { /// - contentLength: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `contentLength`. /// - bodyStream: Body chunk provider. - public static func stream(contentLength: Int64? = nil, bodyStream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { - return Body(contentLength: contentLength, stream: bodyStream) + public static func stream(contentLength: Int64? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { + return Body(contentLength: contentLength, stream: stream) } /// Create and stream body using a collection of bytes. diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 51bc1f005..c733d0497 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -621,7 +621,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "post", method: .POST, headers: ["transfer-encoding": "chunked"], - body: .stream(bodyStream: { streamWriter in + body: .stream { streamWriter in _ = streamWriter.write(.byteBuffer(.init())) let promise = self.clientGroup.next().makePromise(of: Void.self) @@ -630,7 +630,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return promise.futureResult - })) + }) XCTAssertThrowsError(try localClient.execute(request: request).wait()) { XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) @@ -1953,9 +1953,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testValidationErrorsAreSurfaced() throws { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .TRACE, body: .stream(bodyStream: { _ in + let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .TRACE, body: .stream { _ in self.defaultClient.eventLoopGroup.next().makeSucceededFuture(()) - })) + }) let runningRequest = self.defaultClient.execute(request: request) XCTAssertThrowsError(try runningRequest.wait()) { error in XCTAssertEqual(HTTPClientError.traceRequestWithBody, error as? HTTPClientError) @@ -2048,10 +2048,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { return try? HTTPClient.Request(url: "http://\(localAddress.ipAddress!):\(localAddress.port!)", method: .POST, headers: ["transfer-encoding": "chunked"], - body: .stream(bodyStream: { streamWriter in + body: .stream { streamWriter in streamWriterPromise.succeed(streamWriter) return sentOffAllBodyPartsPromise.futureResult - })) + }) } guard let server = makeServer(), let request = makeRequest(server: server) else { @@ -2602,9 +2602,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } var request = try HTTPClient.Request(url: "http://localhost:\(server.serverPort)/") - request.body = .stream(bodyStream: { writer in + request.body = .stream { writer in writer.write(.byteBuffer(ByteBuffer(string: "1234"))) - }) + } let future = client.execute(request: request) @@ -2777,7 +2777,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } var request: HTTPClient.Request? - XCTAssertNoThrow(request = try Request(url: url, body: .stream(contentLength: 1, bodyStream: uploader))) + XCTAssertNoThrow(request = try Request(url: url, body: .stream(contentLength: 1, uploader))) XCTAssertThrowsError(try self.defaultClient.execute(request: XCTUnwrap(request)).wait()) { XCTAssertEqual($0 as? HTTPClientError, .writeAfterRequestSent) } @@ -2793,7 +2793,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { _ = self.defaultClient.get(url: "http://localhost:\(self.defaultHTTPBin.port)/events/10/1") var request = try HTTPClient.Request(url: "http://localhost:\(self.defaultHTTPBin.port)/wait", method: .POST) - request.body = .stream(bodyStream: { writer in + request.body = .stream { writer in // Start writing chunks so tha we will try to write after read timeout is thrown for _ in 1...10 { _ = writer.write(.byteBuffer(ByteBuffer(string: "1234"))) @@ -2805,7 +2805,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return promise.futureResult - }) + } // We specify a deadline of 2 ms co that request will be timed out before all chunks are writtent, // we need to verify that second error on write after timeout does not lead to double-release. @@ -2968,7 +2968,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let delegate = ResponseStreamDelegate(eventLoop: delegateEL) - let body: HTTPClient.Body = .stream(bodyStream: { writer in + let body: HTTPClient.Body = .stream { writer in let finalPromise = writeEL.makePromise(of: Void.self) @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { @@ -3004,7 +3004,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - }) + } let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: delegateEL)) @@ -3068,9 +3068,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let body = ByteBuffer(bytes: 0..<11) var request = try Request(url: httpBin.baseURL) - request.body = .stream(bodyStream: { writer in + request.body = .stream { writer in writer.write(.byteBuffer(body)) - }) + } XCTAssertThrowsError(try self.defaultClient.execute( request: request, delegate: ResponseAccumulator(request: request, maxBodySize: 10) @@ -3086,9 +3086,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let body = ByteBuffer(bytes: 0..<10) var request = try Request(url: httpBin.baseURL) - request.body = .stream(bodyStream: { writer in + request.body = .stream { writer in writer.write(.byteBuffer(body)) - }) + } let response = try self.defaultClient.execute( request: request, delegate: ResponseAccumulator(request: request, maxBodySize: 10) @@ -3113,7 +3113,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let delegate = ResponseStreamDelegate(eventLoop: delegateEL) - let body: HTTPClient.Body = .stream(bodyStream: { writer in + let body: HTTPClient.Body = .stream { writer in let finalPromise = writeEL.makePromise(of: Void.self) @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { @@ -3143,7 +3143,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - }) + } let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: delegateEL)) @@ -3164,7 +3164,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } - let body: HTTPClient.Body = .stream(bodyStream: { writer in + let body: HTTPClient.Body = .stream { writer in let finalPromise = writeEL.makePromise(of: Void.self) @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { @@ -3194,7 +3194,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - }) + } let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", body: body) let future = httpClient.execute(request: request) @@ -3220,7 +3220,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let httpClient = HTTPClient(eventLoopGroupProvider: .shared(eventLoopGroup)) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } - let body: HTTPClient.Body = .stream(bodyStream: { writer in + let body: HTTPClient.Body = .stream { writer in let finalPromise = writeEL.makePromise(of: Void.self) @Sendable func writeLoop(_ writer: HTTPClient.Body.StreamWriter, index: Int) { @@ -3250,7 +3250,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } return finalPromise.futureResult - }) + } let headers = HTTPHeaders([("Connection", "close")]) let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)", headers: headers, body: body) diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index fa094c1af..a9b9bd0dd 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -872,11 +872,11 @@ final class RequestBagTests: XCTestCase { let writerPromise = group.any().makePromise(of: HTTPClient.Body.StreamWriter.self) let donePromise = group.any().makePromise(of: Void.self) - request.body = .stream(bodyStream: { [leakDetector] writer in + request.body = .stream { [leakDetector] writer in _ = leakDetector writerPromise.succeed(writer) return donePromise.futureResult - }) + } let resultFuture = httpClient.execute(request: request) request.body = nil From 54d1006dc90b9ee77b4d04d63ad8688d2215dc1e Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Tue, 9 Jul 2024 11:47:07 +0100 Subject: [PATCH 116/146] Add leading slash in relative URL requests if necessary (#747) --- Sources/AsyncHTTPClient/HTTPHandler.swift | 2 +- .../HTTPClientInternalTests.swift | 19 +++++++++++++++++++ .../HTTPClientTests.swift | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index b81a2c7a5..c13d1accd 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -662,7 +662,7 @@ extension URL { if self.path.isEmpty { return "/" } - return URLComponents(url: self, resolvingAgainstBaseURL: false)?.percentEncodedPath ?? self.path + return URLComponents(url: self, resolvingAgainstBaseURL: true)?.percentEncodedPath ?? self.path } var uri: String { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index 80446251c..a3246dde0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -142,6 +142,25 @@ class HTTPClientInternalTests: XCTestCase { XCTAssertEqual(request12.url.uri, "/some%2Fpathsegment1/pathsegment2") } + func testURIOfRelativeURLRequest() throws { + let requestNoLeadingSlash = try Request( + url: URL( + string: "percent%2Fencoded/hello", + relativeTo: URL(string: "http://127.0.0.1")! + )! + ) + + let requestWithLeadingSlash = try Request( + url: URL( + string: "/percent%2Fencoded/hello", + relativeTo: URL(string: "http://127.0.0.1")! + )! + ) + + XCTAssertEqual(requestNoLeadingSlash.url.uri, "/percent%2Fencoded/hello") + XCTAssertEqual(requestWithLeadingSlash.url.uri, "/percent%2Fencoded/hello") + } + func testChannelAndDelegateOnDifferentEventLoops() throws { class Delegate: HTTPClientResponseDelegate { typealias Response = ([Message], [Message]) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index c733d0497..2a30a07ca 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -495,6 +495,20 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(.ok, response.status) } + func testLeadingSlashRelativeURL() throws { + let noLeadingSlashURL = URL(string: "percent%2Fencoded/hello", relativeTo: URL(string: self.defaultHTTPBinURLPrefix)!)! + let withLeadingSlashURL = URL(string: "/percent%2Fencoded/hello", relativeTo: URL(string: self.defaultHTTPBinURLPrefix)!)! + + let noLeadingSlashURLRequest = try HTTPClient.Request(url: noLeadingSlashURL, method: .GET) + let withLeadingSlashURLRequest = try HTTPClient.Request(url: withLeadingSlashURL, method: .GET) + + let noLeadingSlashURLResponse = try self.defaultClient.execute(request: noLeadingSlashURLRequest).wait() + let withLeadingSlashURLResponse = try self.defaultClient.execute(request: withLeadingSlashURLRequest).wait() + + XCTAssertEqual(noLeadingSlashURLResponse.status, .ok) + XCTAssertEqual(withLeadingSlashURLResponse.status, .ok) + } + func testMultipleContentLengthHeaders() throws { let body = ByteBuffer(string: "hello world!") From 07f171bed7cd180518a6cb362ad4c37bfa4e53da Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Wed, 7 Aug 2024 15:49:48 +0100 Subject: [PATCH 117/146] Avoid using deprecated API in tests (#762) This gets CI working again --- Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift | 6 ++++-- Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 626f5b4ae..524fc6e07 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -527,9 +527,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let key = try NIOSSLPrivateKey(file: keyPath, format: .pem) let configuration = TLSConfiguration.makeServerConfiguration( certificateChain: try NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, - privateKey: .file(keyPath) + privateKey: .privateKey(key) ) let sslContext = try NIOSSLContext(configuration: configuration) let serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) @@ -597,10 +598,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { /// ``` let certPath = Bundle.module.path(forResource: "example.com.cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "example.com.private-key", ofType: "pem")! + let key = try NIOSSLPrivateKey(file: keyPath, format: .pem) let localhostCert = try NIOSSLCertificate.fromPEMFile(certPath) let configuration = TLSConfiguration.makeServerConfiguration( certificateChain: localhostCert.map { .certificate($0) }, - privateKey: .file(keyPath) + privateKey: .privateKey(key) ) let bin = HTTPBin(.http2(tlsConfiguration: configuration)) defer { XCTAssertNoThrow(try bin.shutdown()) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 2a30a07ca..ac0aee068 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1274,9 +1274,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let key = try NIOSSLPrivateKey(file: keyPath, format: .pem) let configuration = try TLSConfiguration.makeServerConfiguration( certificateChain: NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, - privateKey: .file(keyPath) + privateKey: .privateKey(key) ) let sslContext = try NIOSSLContext(configuration: configuration) @@ -1314,9 +1315,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { /// openssl req -x509 -newkey rsa:4096 -keyout self_signed_key.pem -out self_signed_cert.pem -sha256 -days 99999 -nodes -subj '/CN=localhost' let certPath = Bundle.module.path(forResource: "self_signed_cert", ofType: "pem")! let keyPath = Bundle.module.path(forResource: "self_signed_key", ofType: "pem")! + let key = try NIOSSLPrivateKey(file: keyPath, format: .pem) let configuration = try TLSConfiguration.makeServerConfiguration( certificateChain: NIOSSLCertificate.fromPEMFile(certPath).map { .certificate($0) }, - privateKey: .file(keyPath) + privateKey: .privateKey(key) ) let sslContext = try NIOSSLContext(configuration: configuration) From 0bd9111ca70755e5a7f934028869cd186b264187 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Wed, 7 Aug 2024 16:28:39 +0100 Subject: [PATCH 118/146] mark HTTPClient.Response Sendable (#759) `HTTPClient.Response` is trivially `Sendable`, let's mark it `Sendable`. --- Sources/AsyncHTTPClient/HTTPHandler.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index c13d1accd..bf452a85c 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -288,7 +288,7 @@ extension HTTPClient { } /// Represents an HTTP response. - public struct Response { + public struct Response: Sendable { /// Remote host of the request. public var host: String /// Response HTTP status. From 4b7a68e997a23af0a3e913b034f8cc430d7a84b4 Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Wed, 14 Aug 2024 10:56:26 +0100 Subject: [PATCH 119/146] Fix OOM issue when setting `concurrentHTTP1ConnectionsPerHostSoftLimit` to `Int.max` (#763) ### Motivation: When a user wishes to make the connection pool create as many concurrent connections as possible, a natural way to achieve this would be to set `.max` to the `concurrentHTTP1ConnectionsPerHostSoftLimit` property. ```swift HTTPClient.Configuration().connectionPool = .init( idleTimeout: .hours(1), concurrentHTTP1ConnectionsPerHostSoftLimit: .max ) ``` The `concurrentHTTP1ConnectionsPerHostSoftLimit` property is of type `Int`. Setting it to `Int.max` leads to `Int.max` being passed as an argument to `Array`s `.reserveCapacity(_:)` method, causing an OOM issue. Addresses Github Issue #751 ### Modifications: Capped the argument to `self.connections.reserveCapacity(_:)` to 1024 in `HTTPConnectionPool.HTTP1Connections` ### Result: Users can now set the `concurrentHTTP1ConnectionsPerHostSoftLimit` property to `.max` without causing an OOM issue. --- .../HTTPConnectionPool+HTTP1Connections.swift | 2 +- .../AsyncAwaitEndToEndTests.swift | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift index 935cdb2f6..f61413e1c 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift @@ -261,7 +261,7 @@ extension HTTPConnectionPool { init(maximumConcurrentConnections: Int, generator: Connection.ID.Generator, maximumConnectionUses: Int?) { self.connections = [] - self.connections.reserveCapacity(maximumConcurrentConnections) + self.connections.reserveCapacity(min(maximumConcurrentConnections, 1024)) self.overflowIndex = self.connections.endIndex self.maximumConcurrentConnections = maximumConcurrentConnections self.generator = generator diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 524fc6e07..c6311753b 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -642,6 +642,24 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } } + func testInsanelyHighConcurrentHTTP1ConnectionLimitDoesNotCrash() async throws { + let bin = HTTPBin(.http1_1(compress: false)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + + var httpClientConfig = HTTPClient.Configuration() + httpClientConfig.connectionPool = .init( + idleTimeout: .hours(1), + concurrentHTTP1ConnectionsPerHostSoftLimit: Int.max + ) + httpClientConfig.timeout = .init(connect: .seconds(10), read: .seconds(100), write: .seconds(100)) + + let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: httpClientConfig) + defer { XCTAssertNoThrow(try httpClient.syncShutdown()) } + + let request = HTTPClientRequest(url: "http://localhost:\(bin.port)") + _ = try await httpClient.execute(request, deadline: .now() + .seconds(2)) + } + func testRedirectChangesHostHeader() { XCTAsyncTest { let bin = HTTPBin(.http2(compress: false)) From 1290119b315a6fac14308e77509f4415a2729cd6 Mon Sep 17 00:00:00 2001 From: Ayush Garg Date: Thu, 15 Aug 2024 13:26:52 +0100 Subject: [PATCH 120/146] Assume http2 connection by default, instead of http1 (#758) Since most of the servers now conform to http2, the change here updates the behaviour of assuming the connection to be http2 and not http1 by default. It will migrate to http1 if the server only supports http1. One can set the `httpVersion` in `ClientConfiguration` to `.http1Only` which will start with http1 instead of http2. Additional Changes: - Fixed an off by one error in the maximum additional general purpose connection check - Updated tests --------- Co-authored-by: Ayush Garg Co-authored-by: David Nadoba Co-authored-by: Fabian Fett --- Package.swift | 2 +- .../ConnectionPool/HTTPConnectionPool.swift | 1 + .../HTTPConnectionPool+HTTP1Connections.swift | 6 +- .../HTTPConnectionPool+StateMachine.swift | 28 +++++--- ...PConnectionPool+HTTP1ConnectionsTest.swift | 28 ++++++++ .../HTTPConnectionPool+HTTP1StateTests.swift | 9 +++ ...onnectionPool+HTTP2StateMachineTests.swift | 44 ++++++------- .../HTTPConnectionPoolTests.swift | 65 ++++++++++++++++++- .../Mocks/MockConnectionPool.swift | 6 +- 9 files changed, 149 insertions(+), 40 deletions(-) diff --git a/Package.swift b/Package.swift index dae0d91c7..7e9498660 100644 --- a/Package.swift +++ b/Package.swift @@ -22,7 +22,7 @@ let package = Package( ], dependencies: [ .package(url: "https://github.com/apple/swift-nio.git", from: "2.62.0"), - .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.22.0"), + .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.27.1"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.19.0"), diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index eac4cc21f..093c1e328 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -72,6 +72,7 @@ final class HTTPConnectionPool { idGenerator: idGenerator, maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool.concurrentHTTP1ConnectionsPerHostSoftLimit, retryConnectionEstablishment: clientConfiguration.connectionPool.retryConnectionEstablishment, + preferHTTP1: clientConfiguration.httpVersion == .http1Only, maximumConnectionUses: clientConfiguration.maximumUsesPerConnection ) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift index f61413e1c..1428a918b 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift @@ -307,7 +307,7 @@ extension HTTPConnectionPool { } private var maximumAdditionalGeneralPurposeConnections: Int { - self.maximumConcurrentConnections - (self.overflowIndex - 1) + self.maximumConcurrentConnections - (self.overflowIndex) } /// Is there at least one connection that is able to run requests @@ -594,6 +594,7 @@ extension HTTPConnectionPool { eventLoop: eventLoop, maximumUses: self.maximumConnectionUses ) + self.connections.insert(newConnection, at: self.overflowIndex) /// If we can grow, we mark the connection as a general purpose connection. /// Otherwise, it will be an overflow connection which is only used once for requests with a required event loop @@ -610,6 +611,7 @@ extension HTTPConnectionPool { ) // TODO: Maybe we want to add a static init for backing off connections to HTTP1ConnectionState backingOffConnection.failedToConnect() + self.connections.insert(backingOffConnection, at: self.overflowIndex) /// If we can grow, we mark the connection as a general purpose connection. /// Otherwise, it will be an overflow connection which is only used once for requests with a required event loop @@ -637,7 +639,7 @@ extension HTTPConnectionPool { ) -> [(Connection.ID, EventLoop)] { // create new connections for requests with a required event loop - // we may already start connections for those requests and do not want to start to many + // we may already start connections for those requests and do not want to start too many let startingRequiredEventLoopConnectionCount = Dictionary( self.connections[self.overflowIndex.. Action { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift index f1a641216..dfeaf1d9c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift @@ -408,6 +408,34 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { XCTAssertTrue(context.eventLoop === el3) } + func testMigrationFromHTTP2WithPendingRequestsWithRequiredEventLoopSameAsStartingConnections() { + let elg = EmbeddedEventLoopGroup(loops: 4) + let generator = HTTPConnectionPool.Connection.ID.Generator() + var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + + let el1 = elg.next() + let el2 = elg.next() + + let conn1ID = generator.next() + let conn2ID = generator.next() + + connections.migrateFromHTTP2( + starting: [(conn1ID, el1)], + backingOff: [(conn2ID, el2)] + ) + + let stats = connections.stats + XCTAssertEqual(stats.idle, 0) + XCTAssertEqual(stats.leased, 0) + XCTAssertEqual(stats.connecting, 1) + XCTAssertEqual(stats.backingOff, 1) + + let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) + let (_, context) = connections.newHTTP1ConnectionEstablished(conn1) + XCTAssertEqual(context.use, .generalPurpose) + XCTAssertTrue(context.eventLoop === el1) + } + func testMigrationFromHTTP2WithPendingRequestsWithPreferredEventLoop() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift index 2df63a0f3..367fdaffb 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift @@ -29,6 +29,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -113,6 +114,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: false, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -181,6 +183,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 2, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -240,6 +243,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 2, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -278,6 +282,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 2, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -670,6 +675,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 6, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -710,6 +716,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 6, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -743,6 +750,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 6, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -768,6 +776,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 6, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index 30b49662a..046040266 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -720,6 +720,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -811,6 +812,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: false, maximumConnectionUses: nil ) @@ -858,6 +860,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) @@ -998,6 +1001,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: false, maximumConnectionUses: nil ) @@ -1014,11 +1018,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try queuer.queue(mockRequest, id: request1.id)) let http2Conn: HTTPConnectionPool.Connection = .__testOnly_connection(id: http2ConnID, eventLoop: el1) XCTAssertNoThrow(try connections.succeedConnectionCreationHTTP2(http2ConnID, maxConcurrentStreams: 10)) - let migrationAction1 = state.newHTTP2ConnectionCreated(http2Conn, maxConcurrentStreams: 10) - guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn) = migrationAction1.request else { - return XCTFail("unexpected request action \(migrationAction1.request)") + let executeAction1 = state.newHTTP2ConnectionCreated(http2Conn, maxConcurrentStreams: 10) + guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn) = executeAction1.request else { + return XCTFail("unexpected request action \(executeAction1.request)") } - XCTAssertEqual(migrationAction1.connection, .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil)) + XCTAssertEqual(requests.count, 1) for request in requests { XCTAssertNoThrow(try queuer.get(request.id, request: request.__testOnly_wrapped_request())) @@ -1069,6 +1073,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: false, maximumConnectionUses: nil ) @@ -1085,11 +1090,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try queuer.queue(mockRequest, id: request1.id)) let http2Conn: HTTPConnectionPool.Connection = .__testOnly_connection(id: http2ConnID, eventLoop: el1) XCTAssertNoThrow(try connections.succeedConnectionCreationHTTP2(http2ConnID, maxConcurrentStreams: 10)) - let migrationAction1 = state.newHTTP2ConnectionCreated(http2Conn, maxConcurrentStreams: 10) - guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn) = migrationAction1.request else { - return XCTFail("unexpected request action \(migrationAction1.request)") + let executeAction1 = state.newHTTP2ConnectionCreated(http2Conn, maxConcurrentStreams: 10) + guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn) = executeAction1.request else { + return XCTFail("unexpected request action \(executeAction1.request)") } - XCTAssertEqual(migrationAction1.connection, .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil)) + XCTAssertEqual(requests.count, 1) for request in requests { XCTAssertNoThrow(try queuer.get(request.id, request: request.__testOnly_wrapped_request())) @@ -1120,7 +1125,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } XCTAssertTrue(queuer.isEmpty) - // if we established a new http/1 connection we should migrate back to http/1, + // if we established a new http/1 connection we should migrate to http/1, // close the connection and shutdown the pool let http1Conn: HTTPConnectionPool.Connection = .__testOnly_connection(id: http1ConnId, eventLoop: el2) XCTAssertNoThrow(try connections.succeedConnectionCreationHTTP1(http1ConnId)) @@ -1146,11 +1151,12 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: false, maximumConnectionUses: nil ) var connectionIDs: [HTTPConnectionPool.Connection.ID] = [] - for el in [el1, el2, el2] { + for el in [el1, el2] { let mockRequest = MockHTTPScheduableRequest(eventLoop: el, requiresEventLoopForChannel: true) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -1164,7 +1170,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try queuer.queue(mockRequest, id: request.id)) } - // fail the two connections for el2 + // fail the connection for el2 for connectionID in connectionIDs.dropFirst() { struct SomeError: Error {} XCTAssertNoThrow(try connections.failConnectionCreation(connectionID)) @@ -1177,16 +1183,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } let http2ConnID1 = connectionIDs[0] let http2ConnID2 = connectionIDs[1] - let http2ConnID3 = connectionIDs[2] // let the first connection on el1 succeed as a http2 connection let http2Conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: http2ConnID1, eventLoop: el1) XCTAssertNoThrow(try connections.succeedConnectionCreationHTTP2(http2ConnID1, maxConcurrentStreams: 10)) - let migrationAction1 = state.newHTTP2ConnectionCreated(http2Conn1, maxConcurrentStreams: 10) - guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn1) = migrationAction1.request else { - return XCTFail("unexpected request action \(migrationAction1.request)") + let connectionAction = state.newHTTP2ConnectionCreated(http2Conn1, maxConcurrentStreams: 10) + guard case .executeRequestsAndCancelTimeouts(let requests, http2Conn1) = connectionAction.request else { + return XCTFail("unexpected request action \(connectionAction.request)") } - XCTAssertEqual(migrationAction1.connection, .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil)) XCTAssertEqual(requests.count, 1) for request in requests { XCTAssertNoThrow(try queuer.get(request.id, request: request.__testOnly_wrapped_request())) @@ -1205,14 +1209,6 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } XCTAssertTrue(eventLoop2 === el2) XCTAssertNoThrow(try connections.createConnection(newHttp2ConnID2, on: el2)) - - // we now have a starting connection for el2 and another one backing off - - // if the backoff timer fires now for a connection on el2, we should *not* start a new connection - XCTAssertNoThrow(try connections.connectionBackoffTimerDone(http2ConnID3)) - let action3 = state.connectionCreationBackoffDone(http2ConnID3) - XCTAssertEqual(action3.request, .none) - XCTAssertEqual(action3.connection, .none) } func testMaxConcurrentStreamsIsRespected() { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift index 2cf222afe..a75cfb63c 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift @@ -82,7 +82,6 @@ class HTTPConnectionPoolTests: XCTestCase { let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)") let poolDelegate = TestDelegate(eventLoop: eventLoop) - let pool = HTTPConnectionPool( eventLoopGroup: eventLoopGroup, sslContextCache: .init(), @@ -93,6 +92,70 @@ class HTTPConnectionPoolTests: XCTestCase { idGenerator: .init(), backgroundActivityLogger: .init(label: "test") ) + defer { + pool.shutdown() + XCTAssertNoThrow(try poolDelegate.future.wait()) + XCTAssertNoThrow(try eventLoop.scheduleTask(in: .milliseconds(100)) {}.futureResult.wait()) + XCTAssertEqual(httpBin.activeConnections, 0) + // Since we would migrate from h2 -> h1, which creates a general purpose connection + // for every connection in .starting state, after the first request which will + // be serviced by an overflow connection, the rest of requests will use the general + // purpose connection since they are all on the same event loop. + // Hence we will only create 1 overflow connection and 1 general purpose connection. + XCTAssertEqual(httpBin.createdConnections, 2) + } + + XCTAssertEqual(httpBin.createdConnections, 0) + + for _ in 0..<10 { + var maybeRequest: HTTPClient.Request? + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .init(.testOnly_exact(channelOn: eventLoopGroup.next(), delegateOn: eventLoopGroup.next())), + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + )) + + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } + + pool.executeRequest(requestBag) + XCTAssertNoThrow(try requestBag.task.futureResult.wait()) + + // Flakiness Alert: We check <= and >= instead of == + // While migration from h2 -> h1, one general purpose and one over flow connection + // will be created, there's no guarantee as to whether the request is executed + // after both are created. + XCTAssertGreaterThanOrEqual(httpBin.createdConnections, 1) + XCTAssertLessThanOrEqual(httpBin.createdConnections, 2) + } + } + + func testConnectionsForEventLoopRequirementsAreClosedH1Only() { + let httpBin = HTTPBin() + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) + let eventLoop = eventLoopGroup.next() + defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } + + let request = try! HTTPClient.Request(url: "http://localhost:\(httpBin.port)") + let poolDelegate = TestDelegate(eventLoop: eventLoop) + var configuration = HTTPClient.Configuration() + configuration.httpVersion = .http1Only + let pool = HTTPConnectionPool( + eventLoopGroup: eventLoopGroup, + sslContextCache: .init(), + tlsConfiguration: .none, + clientConfiguration: configuration, + key: .init(request), + delegate: poolDelegate, + idGenerator: .init(), + backgroundActivityLogger: .init(label: "test") + ) defer { pool.shutdown() XCTAssertNoThrow(try poolDelegate.future.wait()) diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift index ca41a1e39..7b4eb19d9 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift @@ -543,6 +543,7 @@ extension MockConnectionPool { idGenerator: .init(), maximumConcurrentHTTP1Connections: maxNumberOfConnections, retryConnectionEstablishment: true, + preferHTTP1: true, maximumConnectionUses: nil ) var connections = MockConnectionPool() @@ -608,6 +609,7 @@ extension MockConnectionPool { idGenerator: .init(), maximumConcurrentHTTP1Connections: 8, retryConnectionEstablishment: true, + preferHTTP1: false, maximumConnectionUses: nil ) var connections = MockConnectionPool() @@ -639,10 +641,6 @@ extension MockConnectionPool { throw SetupError.expectedPreviouslyQueuedRequestToBeRunNow } - guard case .migration(createConnections: let create, closeConnections: [], scheduleTimeout: nil) = action.connection, create.isEmpty else { - throw SetupError.expectedNoConnectionAction - } - guard try queuer.get(request.id, request: request.__testOnly_wrapped_request()) === mockRequest else { throw SetupError.expectedPreviouslyQueuedRequestToBeRunNow } From e8babad8226b9b3f956a252d5b80e36b0c9d62a9 Mon Sep 17 00:00:00 2001 From: Peter Adams <63288215+PeterAdams-A@users.noreply.github.com> Date: Fri, 16 Aug 2024 09:48:16 +0100 Subject: [PATCH 121/146] Add releases.yml config for github (#765) Motivation: Would like to use github to autogenerate release notes. Modifications: Bring over configuration consistent with NIO and other popular repos. Result: Once labels are synchronised release notes can be generated by github --- .github/release.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/release.yml diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 000000000..13c29b0e6 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,14 @@ +changelog: + categories: + - title: SemVer Major + labels: + - semver/major + - title: SemVer Minor + labels: + - semver/minor + - title: SemVer Patch + labels: + - semver/patch + - title: Other Changes + labels: + - semver/none From 776a1c230446bdadcb2cf7f853fe67abb701f3ed Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 3 Sep 2024 13:36:24 +0200 Subject: [PATCH 122/146] Fix NIO deprecations after update to `2.71.0` (#769) `NIOTooManyBytesError` now requires the `maxBytes` in its initializer. --- Package.swift | 2 +- Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift | 2 +- Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Package.swift b/Package.swift index 7e9498660..a482dc3d0 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.62.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.71.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.27.1"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index ee7f11592..1ca01f53f 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -108,7 +108,7 @@ extension HTTPClientResponse { case .transaction(_, let expectedContentLength): if let contentLength = expectedContentLength { if contentLength > maxBytes { - throw NIOTooManyBytesError() + throw NIOTooManyBytesError(maxBytes: maxBytes) } } case .anyAsyncSequence: diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index c6311753b..d44d047f6 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -907,7 +907,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { await XCTAssertThrowsError( try await response.body.collect(upTo: 3) ) { - XCTAssertEqualTypeAndValue($0, NIOTooManyBytesError()) + XCTAssertEqualTypeAndValue($0, NIOTooManyBytesError(maxBytes: 3)) } } } From 11205411bb60612f0a1a04f733fa71b4fb864ab9 Mon Sep 17 00:00:00 2001 From: Fabian Fett Date: Tue, 3 Sep 2024 13:54:44 +0200 Subject: [PATCH 123/146] Fix crash when writablity becomes false and races against finishing the http request (#768) ### Motivation If the channel's writability changed to false just before we finished a request, we currently run into a precondition. ### Changes - Remove the precondition and handle the case appropiatly ### Result A crash less. --- .../HTTP1/HTTP1ClientChannelHandler.swift | 3 +- .../HTTP1ClientChannelHandlerTests.swift | 50 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 04de8b352..86d2547ef 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -665,7 +665,8 @@ struct IdleWriteStateMachine { self.state = .requestEndSent return .clearIdleWriteTimeoutTimer case .waitingForWritabilityEnabled: - preconditionFailure("If the channel is not writable, we can't have sent the request end.") + self.state = .requestEndSent + return .none case .requestEndSent: return .none } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index f4f2d67f8..c91db94b3 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -376,6 +376,56 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { } } + func testIdleWriteTimeoutRaceToEnd() { + let embedded = EmbeddedChannel() + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream { _ in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + let scheduled = embedded.embeddedEventLoop.flatScheduleTask(in: .milliseconds(2)) { + embedded.embeddedEventLoop.makeSucceededVoidFuture() + } + return scheduled.futureResult + })) + + guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + let delegate = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow(maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(5)), + delegate: delegate + )) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + testUtils.connection.executeRequest(requestBag) + let expectedHeaders: HTTPHeaders = ["host": "localhost", "Transfer-Encoding": "chunked"] + XCTAssertEqual( + try embedded.readOutbound(as: HTTPClientRequestPart.self), + .head(HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: expectedHeaders)) + ) + + // change the writability to false. + embedded.isWritable = false + embedded.pipeline.fireChannelWritabilityChanged() + embedded.embeddedEventLoop.run() + + // let the writer, write an end (while writability is false) + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + + XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) + } + func testIdleWriteTimeoutWritabilityChanged() { let embedded = EmbeddedChannel() let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) From 6df8e1c17e68f0f93de2443b8c8cafca9ddcc89a Mon Sep 17 00:00:00 2001 From: Ian Anderson Date: Fri, 13 Sep 2024 23:55:55 -0700 Subject: [PATCH 124/146] Explicitly import locale modules (#771) The Darwin module is slowly being split up, and as it gets further along, it will stop importing some of the split-out modules like the one for locale.h that provides newlocale() and other locale API. However, there's a wrinkle that on platforms with xlocale, it's xlocale.h that provides most of the POSIX locale.h functions and not locale.h, so prefer the xlocale module when available. --- Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift | 6 +++++- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index b2e9d7b05..a46b4f759 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -13,7 +13,11 @@ //===----------------------------------------------------------------------===// import NIOHTTP1 -#if canImport(Darwin) +#if canImport(xlocale) +import xlocale +#elseif canImport(locale_h) +import locale_h +#elseif canImport(Darwin) import Darwin #elseif canImport(Musl) import Musl diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 7f28040c2..e8d6976c5 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -28,7 +28,11 @@ import NIOSSL import NIOTLS import NIOTransportServices import XCTest -#if canImport(Darwin) +#if canImport(xlocale) +import xlocale +#elseif canImport(locale_h) +import locale_h +#elseif canImport(Darwin) import Darwin #elseif canImport(Musl) import Musl From 10bd49c1b1d788753ae40c6914c753195c547730 Mon Sep 17 00:00:00 2001 From: Justin Date: Thu, 19 Sep 2024 14:46:11 +0530 Subject: [PATCH 125/146] add .git extensions to dependency URLs (#770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds .git extensions to the Swift Package Manager dependencies that didn't include them. For me, this resolves issues that I have had with an error produced by Xcode when updating to the latest package versions if I'm editing the project which depends on AHC in an Xcode Workspace. image The complete error is: `github.com: https://github.com/apple/swift-algorithms: The repository could not be found. Make sure a valid repository exists at the specified location and try again.` Based on conversations in the Vapor Discord server, adding these extensions "shouldn't" make a difference to the dependency resolution done by swift package manager, however adding them resolves the error. ๐Ÿคท Co-authored-by: Franz Busch --- Package.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Package.swift b/Package.swift index a482dc3d0..b645ee198 100644 --- a/Package.swift +++ b/Package.swift @@ -28,8 +28,8 @@ let package = Package( .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), - .package(url: "https://github.com/apple/swift-docc-plugin", from: "1.0.0"), - .package(url: "https://github.com/apple/swift-algorithms", from: "1.0.0"), + .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.0.0"), + .package(url: "https://github.com/apple/swift-algorithms.git", from: "1.0.0"), ], targets: [ .target( From 15dbe6dcee36563027b5a03a8d142d1488b9ba76 Mon Sep 17 00:00:00 2001 From: Anthony Doeraene <78789735+Aperence@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:22:17 +0200 Subject: [PATCH 126/146] Add an option to enable Multipath TCP on clients (#766) Multipath TCP (MPTCP) is a TCP extension allowing to enhance the reliability of the network by using multiple interfaces. This extension provides a seamless handover between interfaces in case of deterioration of the connection on the original one. In the context of iOS and Mac OS X, it could be really interesting to leverage the capabilities of MPTCP as they could benefit from their multiple interfaces (ethernet + Wi-fi for Mac OS X, Wi-fi + cellular for iOS). This contribution introduces patches to HTTPClient.Configuration and establishment of the Bootstraps. A supplementary field "enableMultipath" was added to the configuration, allowing to request the use of MPTCP. This flag is then used when creating the channels to configure the client. Note that in the future, it might also be potentially interesting to offer more precise configuration options for MPTCP on MacOS, as the Network framework allows also to select a type of service, instead of just offering the option to create MPTCP connections. Currently, when enabling MPTCP, only the Handover mode is used. --------- Co-authored-by: Cory Benfield --- .../HTTPConnectionPool+Factory.swift | 4 ++++ Sources/AsyncHTTPClient/HTTPClient.swift | 5 +++++ .../AsyncHTTPClientTests/HTTPClientTests.swift | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 1461a6620..3a0011d5e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -322,6 +322,7 @@ extension HTTPConnectionPool.ConnectionFactory { if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { return tsBootstrap .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) + .channelOption(NIOTSChannelOptions.multipathServiceType, value: self.clientConfiguration.enableMultipath ? .handover : .disabled) .connectTimeout(deadline - NIODeadline.now()) .channelInitializer { channel in do { @@ -338,6 +339,7 @@ extension HTTPConnectionPool.ConnectionFactory { if let nioBootstrap = ClientBootstrap(validatingGroup: eventLoop) { return nioBootstrap .connectTimeout(deadline - NIODeadline.now()) + .enableMPTCP(clientConfiguration.enableMultipath) } preconditionFailure("No matching bootstrap found") @@ -415,6 +417,7 @@ extension HTTPConnectionPool.ConnectionFactory { tsBootstrap .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) + .channelOption(NIOTSChannelOptions.multipathServiceType, value: self.clientConfiguration.enableMultipath ? .handover : .disabled) .connectTimeout(deadline - NIODeadline.now()) .tlsOptions(options) .channelInitializer { channel in @@ -443,6 +446,7 @@ extension HTTPConnectionPool.ConnectionFactory { let bootstrap = ClientBootstrap(group: eventLoop) .connectTimeout(deadline - NIODeadline.now()) + .enableMPTCP(clientConfiguration.enableMultipath) .channelInitializer { channel in sslContextFuture.flatMap { sslContext -> EventLoopFuture in do { diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index c52263318..096fb9387 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -738,6 +738,10 @@ public class HTTPClient { } } + /// Whether ``HTTPClient`` will use Multipath TCP or not + /// By default, don't use it + public var enableMultipath: Bool + public init( tlsConfiguration: TLSConfiguration? = nil, redirectConfiguration: RedirectConfiguration? = nil, @@ -755,6 +759,7 @@ public class HTTPClient { self.decompression = decompression self.httpVersion = .automatic self.networkFrameworkWaitForConnectivity = true + self.enableMultipath = false } public init(tlsConfiguration: TLSConfiguration? = nil, diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index ac0aee068..0c259c1bd 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3590,6 +3590,24 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(.ok, response.status) } + func testClientWithMultipath() throws { + do { + var conf = HTTPClient.Configuration() + conf.enableMultipath = true + let client = HTTPClient(configuration: conf) + defer { + XCTAssertNoThrow(try client.shutdown().wait()) + } + let response = try client.get(url: self.defaultHTTPBinURLPrefix + "get").wait() + XCTAssertEqual(.ok, response.status) + } catch let error as IOError where error.errnoCode == EINVAL || error.errnoCode == EPROTONOSUPPORT || error.errnoCode == ENOPROTOOPT { + // some old Linux kernels don't support MPTCP, skip this test in this case + // see https://www.mptcp.dev/implementation.html for details about each type + // of error + throw XCTSkip() + } + } + func testSingletonClientWorks() throws { let response = try HTTPClient.shared.get(url: self.defaultHTTPBinURLPrefix + "get").wait() XCTAssertEqual(.ok, response.status) From 38608db98544e3d65bb7859e59c91a031a7e5649 Mon Sep 17 00:00:00 2001 From: aryan-25 Date: Mon, 23 Sep 2024 12:31:45 +0100 Subject: [PATCH 127/146] Reduce time spent logging EventLoop description in HTTP1ClientChannelHandler (#772) ### Motivation: A performance test executing 100,000 sequential requests against a simple [`NIOHTTP1Server`](https://github.com/apple/swift-nio/blob/main/Sources/NIOHTTP1Server/README.md) revealed that 7% of total run time is spent in the setter of the `request` property in `HTTP1ClientChannelHandler` (GitHub Issue #754). The poor performance comes from [processing the string interpolation `"\(self.eventLoop)"`](https://github.com/swift-server/async-http-client/blob/6df8e1c17e68f0f93de2443b8c8cafca9ddcc89a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift#L39C17-L39C75) which under the hood calls a computed property. This problem can entirely be avoided by storing `eventLoop.description` when initializing `HTTP1ClientChannelHandler`, and using that stored value in `request`'s setter, rather than computing the property each time. ### Modifications: - Created a new property `let eventLoopDescription: Logger.MetadataValue` in `HTTP1ClientChannelHandler` that stores the description of the `eventLoop` argument that is passed into the initializer. - Replaced the string interpolation `"\(self.eventLoop)"` in `request`'s setter with `self.eventLoopDescription`. ### Result: `HTTP1ClientChannelHandler.eventLoop`'s `description` property is cached upon initialization rather than being computed each time in the `request` property's setter. --------- Co-authored-by: Cory Benfield --- .../ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 86d2547ef..41a56c91b 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -36,7 +36,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { if let newRequest = self.request { var requestLogger = newRequest.logger requestLogger[metadataKey: "ahc-connection-id"] = self.connectionIdLoggerMetadata - requestLogger[metadataKey: "ahc-el"] = "\(self.eventLoop)" + requestLogger[metadataKey: "ahc-el"] = self.eventLoopDescription self.logger = requestLogger if let idleReadTimeout = newRequest.requestOptions.idleReadTimeout { @@ -72,11 +72,13 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { private let backgroundLogger: Logger private var logger: Logger private let eventLoop: EventLoop + private let eventLoopDescription: Logger.MetadataValue private let connectionIdLoggerMetadata: Logger.MetadataValue var onConnectionIdle: () -> Void = {} init(eventLoop: EventLoop, backgroundLogger: Logger, connectionIdLoggerMetadata: Logger.MetadataValue) { self.eventLoop = eventLoop + self.eventLoopDescription = "\(eventLoop.description)" self.backgroundLogger = backgroundLogger self.logger = backgroundLogger self.connectionIdLoggerMetadata = connectionIdLoggerMetadata From 64abc77edf1ef81e69bd90a2ac386de615c8e8ea Mon Sep 17 00:00:00 2001 From: Alastair Houghton Date: Mon, 30 Sep 2024 18:17:01 +0100 Subject: [PATCH 128/146] Don't just import `locale_h`. (#775) On modularised platforms, #771 broke things because it changed from importing `Musl` or `Glibc` to importing just `locale_h`. The latter understandably doesn't define `errno` or `EOVERFLOW`, so we get a build failure. Fixes #773. --- Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index a46b4f759..9d9d6dfb7 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -17,13 +17,16 @@ import NIOHTTP1 import xlocale #elseif canImport(locale_h) import locale_h -#elseif canImport(Darwin) +#endif + +#if canImport(Darwin) import Darwin #elseif canImport(Musl) import Musl #elseif canImport(Glibc) import Glibc #endif + import CAsyncHTTPClient import NIOCore From 0a9b72369b9d87ab155ef585ef50700a34abf070 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 3 Oct 2024 11:02:00 +0100 Subject: [PATCH 129/146] workaround Foundation.URL behavior changes (#777) `Foundation.URL` has various behavior changes in Swift 6 to better match RFC 3986 which impact AHC. In particular it now no longer strips the square brackets in IPv6 hosts which are not tolerated by `inet_pton` so these must be manually stripped. https://github.com/swiftlang/swift-foundation/issues/957 https://github.com/swiftlang/swift-foundation/issues/958 https://github.com/swiftlang/swift-foundation/issues/962 --- .../HTTPClientRequest+Prepared.swift | 2 +- .../AsyncHTTPClient/DeconstructedURL.swift | 30 +++++++++++++++++++ .../HTTPClientTests.swift | 6 +++- 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index 2d5e3e2e0..0a3ec6442 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -45,7 +45,7 @@ extension HTTPClientRequest { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest.Prepared { init(_ request: HTTPClientRequest, dnsOverride: [String: String] = [:]) throws { - guard let url = URL(string: request.url) else { + guard !request.url.isEmpty, let url = URL(string: request.url) else { throw HTTPClientError.invalidURL } diff --git a/Sources/AsyncHTTPClient/DeconstructedURL.swift b/Sources/AsyncHTTPClient/DeconstructedURL.swift index 020c17455..52042bce3 100644 --- a/Sources/AsyncHTTPClient/DeconstructedURL.swift +++ b/Sources/AsyncHTTPClient/DeconstructedURL.swift @@ -48,9 +48,16 @@ extension DeconstructedURL { switch scheme { case .http, .https: + #if !canImport(Darwin) && compiler(>=6.0) + guard let urlHost = url.host, !urlHost.isEmpty else { + throw HTTPClientError.emptyHost + } + let host = urlHost.trimIPv6Brackets() + #else guard let host = url.host, !host.isEmpty else { throw HTTPClientError.emptyHost } + #endif self.init( scheme: scheme, connectionTarget: .init(remoteHost: host, port: url.port ?? scheme.defaultPort), @@ -81,3 +88,26 @@ extension DeconstructedURL { } } } + +#if !canImport(Darwin) && compiler(>=6.0) +extension String { + @inlinable internal func trimIPv6Brackets() -> String { + var utf8View = self.utf8[...] + + var modified = false + if utf8View.first == UInt8(ascii: "[") { + utf8View = utf8View.dropFirst() + modified = true + } + if utf8View.last == UInt8(ascii: "]") { + utf8View = utf8View.dropLast() + modified = true + } + + if modified { + return String(Substring(utf8View)) + } + return self + } +} +#endif diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 0c259c1bd..9b0ad587e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -43,8 +43,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(request2.url.path, "") let request3 = try Request(url: "unix:///tmp/file") - XCTAssertNil(request3.url.host) XCTAssertEqual(request3.host, "") + #if os(Linux) && compiler(>=6.0) + XCTAssertEqual(request3.url.host, "") + #else + XCTAssertNil(request3.url.host) + #endif XCTAssertEqual(request3.url.path, "/tmp/file") XCTAssertEqual(request3.port, 80) XCTAssertFalse(request3.useTLS) From acaca2d50d6736df99da37775612c56f4ec0a18a Mon Sep 17 00:00:00 2001 From: Agam Dua Date: Mon, 21 Oct 2024 08:27:28 -0700 Subject: [PATCH 130/146] Added: ability to set basic authentication on requests (#778) Motivation: As an HTTP library, async-http-client should have authentication support. Modifications: This adds a `setBasicAuth()` method to both HTTPClientRequest and `HTTPClient.Request` and their related unit tests. Result: Library users will be able to leverage this method to use basic authentication on their requests without implementing this in their own projects. Note: I also ran the tests (`swift test`) with the `docker.io/library/swift:6.0-focal` and `docker.io/library/swift:5.10.1-focal` to ensure linux compatibility. Signed-off-by: Agam Dua --- .../AsyncAwait/HTTPClientRequest+auth.swift | 27 +++++++++++++ Sources/AsyncHTTPClient/BasicAuth.swift | 39 +++++++++++++++++++ Sources/AsyncHTTPClient/HTTPHandler.swift | 9 +++++ .../HTTPClientRequestTests.swift | 11 ++++++ .../HTTPClientTests.swift | 7 ++++ 5 files changed, 93 insertions(+) create mode 100644 Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+auth.swift create mode 100644 Sources/AsyncHTTPClient/BasicAuth.swift diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+auth.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+auth.swift new file mode 100644 index 000000000..106a8f76b --- /dev/null +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+auth.swift @@ -0,0 +1,27 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2024 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClientRequest { + /// Set basic auth for a request. + /// + /// - parameters: + /// - username: the username to authenticate with + /// - password: authentication password associated with the username + public mutating func setBasicAuth(username: String, password: String) { + self.headers.setBasicAuth(username: username, password: password) + } +} diff --git a/Sources/AsyncHTTPClient/BasicAuth.swift b/Sources/AsyncHTTPClient/BasicAuth.swift new file mode 100644 index 000000000..3e69f8277 --- /dev/null +++ b/Sources/AsyncHTTPClient/BasicAuth.swift @@ -0,0 +1,39 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2024 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Foundation +import NIOHTTP1 + +/// Generates base64 encoded username + password for http basic auth. +/// +/// - Parameters: +/// - username: the username to authenticate with +/// - password: authentication password associated with the username +/// - Returns: encoded credentials to use the Authorization: Basic http header. +func encodeBasicAuthCredentials(username: String, password: String) -> String { + var value = Data() + value.reserveCapacity(username.utf8.count + password.utf8.count + 1) + value.append(contentsOf: username.utf8) + value.append(UInt8(ascii: ":")) + value.append(contentsOf: password.utf8) + return value.base64EncodedString() +} + +extension HTTPHeaders { + /// Sets the basic auth header + mutating func setBasicAuth(username: String, password: String) { + let encoded = encodeBasicAuthCredentials(username: username, password: password) + self.replaceOrAdd(name: "Authorization", value: "Basic \(encoded)") + } +} diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index bf452a85c..d989b8a6c 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -285,6 +285,15 @@ extension HTTPClient { return (head, metadata) } + + /// Set basic auth for a request. + /// + /// - parameters: + /// - username: the username to authenticate with + /// - password: authentication password associated with the username + public mutating func setBasicAuth(username: String, password: String) { + self.headers.setBasicAuth(username: username, password: password) + } } /// Represents an HTTP response. diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index c93ab4cb5..05e22f2d2 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -57,6 +57,17 @@ class HTTPClientRequestTests: XCTestCase { } } + func testBasicAuth() { + XCTAsyncTest { + var request = Request(url: "https://example.com/get") + request.setBasicAuth(username: "foo", password: "bar") + var preparedRequest: PreparedRequest? + XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) + guard let preparedRequest = preparedRequest else { return } + XCTAssertEqual(preparedRequest.head.headers.first(name: "Authorization")!, "Basic Zm9vOmJhcg==") + } + } + func testUnixScheme() { XCTAsyncTest { var request = Request(url: "unix://%2Fexample%2Ffolder.sock/some_path") diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 9b0ad587e..cf3578ed1 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -3668,4 +3668,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let response3 = try await client.execute(request, timeout: /* infinity */ .hours(99)) XCTAssertEqual(.ok, response3.status) } + + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) + func testRequestBasicAuth() async throws { + var request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix) + request.setBasicAuth(username: "foo", password: "bar") + XCTAssertEqual(request.headers.first(name: "Authorization"), "Basic Zm9vOmJhcg==") + } } From c62114232734a99f7718fffc306d182086f9d744 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Tue, 29 Oct 2024 15:01:46 +0000 Subject: [PATCH 131/146] Adopt GitHub actions (#780) Migrate CI to use GitHub Actions. ### Motivation: To migrate to GitHub actions and centralised infrastructure. ### Modifications: Changes of note: * Adopt swift-format using rules from SwiftNIO. * Remove scripts and docker files which are no longer needed. * Disabled warnings-as-errors on Swift 6.0 CI pipelines for now. ### Result: Feature parity with old CI. --- .github/workflows/main.yml | 18 + .github/workflows/pull_request.yml | 30 + .github/workflows/pull_request_label.yml | 18 + .licenseignore | 37 + .swift-format | 68 + .swiftformat | 24 - CONTRIBUTING.md | 6 +- Examples/GetHTML/GetHTML.swift | 2 +- Examples/GetJSON/GetJSON.swift | 2 +- Examples/Package.swift | 9 +- NOTICE.txt | 4 +- Package.swift | 5 +- .../AsyncAwait/HTTPClient+execute.swift | 67 +- .../HTTPClientRequest+Prepared.swift | 9 +- .../AsyncAwait/HTTPClientRequest.swift | 134 +- .../AsyncAwait/HTTPClientResponse.swift | 25 +- .../AsyncAwait/Transaction+StateMachine.swift | 138 +- .../AsyncAwait/Transaction.swift | 8 +- Sources/AsyncHTTPClient/Base64.swift | 252 +- .../BestEffortHashableTLSConfiguration.swift | 2 +- .../Configuration+BrowserLike.swift | 3 +- Sources/AsyncHTTPClient/ConnectionPool.swift | 8 +- .../HTTP1ProxyConnectHandler.swift | 16 +- .../ChannelHandler/SOCKSEventsHandler.swift | 2 +- .../ChannelHandler/TLSEventsHandler.swift | 2 +- .../HTTP1/HTTP1ClientChannelHandler.swift | 43 +- .../HTTP1/HTTP1Connection.swift | 10 +- .../HTTP1/HTTP1ConnectionStateMachine.swift | 20 +- .../HTTP2/HTTP2ClientRequestHandler.swift | 22 +- .../HTTP2/HTTP2Connection.swift | 29 +- .../HTTP2/HTTP2IdleHandler.swift | 12 +- .../HTTPConnectionPool+Factory.swift | 123 +- .../HTTPConnectionPool+Manager.swift | 18 +- .../ConnectionPool/HTTPConnectionPool.swift | 208 +- .../HTTPRequestStateMachine+Demand.swift | 8 +- .../HTTPRequestStateMachine.swift | 185 +- .../HTTPConnectionPool+Backoff.swift | 1 + .../HTTPConnectionPool+HTTP1Connections.swift | 28 +- ...HTTPConnectionPool+HTTP1StateMachine.swift | 23 +- .../HTTPConnectionPool+HTTP2Connections.swift | 79 +- ...HTTPConnectionPool+HTTP2StateMachine.swift | 62 +- .../HTTPConnectionPool+StateMachine.swift | 193 +- .../FileDownloadDelegate.swift | 11 +- .../FoundationExtensions.swift | 15 +- .../HTTPClient+HTTPCookie.swift | 27 +- .../AsyncHTTPClient/HTTPClient+Proxy.swift | 11 +- Sources/AsyncHTTPClient/HTTPClient.swift | 449 ++-- Sources/AsyncHTTPClient/HTTPHandler.swift | 144 +- Sources/AsyncHTTPClient/LRUCache.swift | 8 +- .../NIOTransportServices/NWErrorHandler.swift | 11 +- .../NWWaitingHandler.swift | 5 +- .../TLSConfiguration.swift | 39 +- Sources/AsyncHTTPClient/RedirectState.swift | 3 +- .../RequestBag+StateMachine.swift | 59 +- Sources/AsyncHTTPClient/RequestBag.swift | 20 +- .../AsyncHTTPClient/RequestValidation.swift | 39 +- Sources/AsyncHTTPClient/SSLContextCache.swift | 26 +- Sources/AsyncHTTPClient/Singleton.swift | 2 +- .../StringConvertibleInstances.swift | 2 +- Sources/AsyncHTTPClient/Utils.swift | 15 +- .../AsyncAwaitEndToEndTests.swift | 335 ++- .../AsyncTestHelpers.swift | 6 +- ...nPoolSizeConfigValueIsRespectedTests.swift | 7 +- .../EmbeddedChannel+HTTPConvenience.swift | 5 +- .../HTTP1ClientChannelHandlerTests.swift | 521 ++-- .../HTTP1ConnectionStateMachineTests.swift | 148 +- .../HTTP1ConnectionTests.swift | 540 +++-- .../HTTP1ProxyConnectHandlerTests.swift | 3 +- .../HTTP2ClientRequestHandlerTests.swift | 328 ++- .../HTTP2ClientTests.swift | 79 +- .../HTTP2ConnectionTests.swift | 155 +- .../HTTP2IdleHandlerTests.swift | 108 +- .../HTTPClient+SOCKSTests.swift | 53 +- .../AsyncHTTPClientTests/HTTPClientBase.swift | 24 +- .../HTTPClientCookieTests.swift | 39 +- ...TTPClientInformationalResponsesTests.swift | 53 +- .../HTTPClientInternalTests.swift | 151 +- .../HTTPClientNIOTSTests.swift | 40 +- .../HTTPClientRequestTests.swift | 647 +++-- .../HTTPClientResponseTests.swift | 27 +- .../HTTPClientTestUtils.swift | 279 ++- .../HTTPClientTests.swift | 2116 +++++++++++------ ...entUncleanSSLConnectionShutdownTests.swift | 79 +- .../HTTPConnectionPool+FactoryTests.swift | 119 +- ...PConnectionPool+HTTP1ConnectionsTest.swift | 129 +- .../HTTPConnectionPool+HTTP1StateTests.swift | 118 +- ...PConnectionPool+HTTP2ConnectionsTest.swift | 90 +- ...onnectionPool+HTTP2StateMachineTests.swift | 344 ++- .../HTTPConnectionPool+ManagerTests.swift | 43 +- ...HTTPConnectionPool+RequestQueueTests.swift | 3 +- .../HTTPConnectionPool+StateTestUtils.swift | 60 +- .../HTTPConnectionPoolTests.swift | 195 +- .../HTTPRequestStateMachineTests.swift | 537 ++++- .../IdleTimeoutNoReuseTests.swift | 7 +- .../AsyncHTTPClientTests/LRUCacheTests.swift | 3 +- .../Mocks/MockConnectionPool.swift | 28 +- .../Mocks/MockHTTPExecutableRequest.swift | 3 +- .../Mocks/MockRequestExecutor.swift | 34 +- .../Mocks/MockRequestQueuer.swift | 3 +- .../NWWaitingHandlerTests.swift | 14 +- .../NoBytesSentOverBodyLimitTests.swift | 7 +- .../RacePoolIdleConnectionsAndGetTests.swift | 13 +- .../RequestBagTests.swift | 517 ++-- .../RequestValidationTests.swift | 83 +- .../ResponseDelayGetTests.swift | 19 +- .../SOCKSEventsHandlerTests.swift | 3 +- .../AsyncHTTPClientTests/SOCKSTestUtils.swift | 49 +- .../SSLContextCacheTests.swift | 53 +- .../StressGetHttpsTests.swift | 19 +- .../TLSEventsHandlerTests.swift | 3 +- .../Transaction+StateMachineTests.swift | 25 +- .../TransactionTests.swift | 67 +- .../XCTest+AsyncAwait.swift | 32 +- docker/Dockerfile | 34 - docker/docker-compose.2204.510.yaml | 22 - docker/docker-compose.2204.58.yaml | 22 - docker/docker-compose.2204.59.yaml | 22 - docker/docker-compose.2204.main.yaml | 21 - docker/docker-compose.yaml | 45 - scripts/check-docs.sh | 23 - scripts/check_no_api_breakages.sh | 68 - scripts/generate_docs.sh | 114 - scripts/soundness.sh | 152 -- 123 files changed, 7280 insertions(+), 4445 deletions(-) create mode 100644 .github/workflows/main.yml create mode 100644 .github/workflows/pull_request.yml create mode 100644 .github/workflows/pull_request_label.yml create mode 100644 .licenseignore create mode 100644 .swift-format delete mode 100644 .swiftformat delete mode 100644 docker/Dockerfile delete mode 100644 docker/docker-compose.2204.510.yaml delete mode 100644 docker/docker-compose.2204.58.yaml delete mode 100644 docker/docker-compose.2204.59.yaml delete mode 100644 docker/docker-compose.2204.main.yaml delete mode 100644 docker/docker-compose.yaml delete mode 100755 scripts/check-docs.sh delete mode 100755 scripts/check_no_api_breakages.sh delete mode 100755 scripts/generate_docs.sh delete mode 100755 scripts/soundness.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 000000000..6e5453369 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,18 @@ +name: Main + +on: + push: + branches: [main] + schedule: + - cron: "0 8,20 * * *" + +jobs: + unit-tests: + name: Unit tests + uses: apple/swift-nio/.github/workflows/unit_tests.yml@main + with: + linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml new file mode 100644 index 000000000..9d7185505 --- /dev/null +++ b/.github/workflows/pull_request.yml @@ -0,0 +1,30 @@ +name: PR + +on: + pull_request: + types: [opened, reopened, synchronize] + +jobs: + soundness: + name: Soundness + uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main + with: + license_header_check_project_name: "AsyncHTTPClient" + unit-tests: + name: Unit tests + uses: apple/swift-nio/.github/workflows/unit_tests.yml@main + with: + linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" + linux_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + + cxx-interop: + name: Cxx interop + uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main + + swift-6-language-mode: + name: Swift 6 Language Mode + uses: apple/swift-nio/.github/workflows/swift_6_language_mode.yml@main + if: false # Disabled for now. diff --git a/.github/workflows/pull_request_label.yml b/.github/workflows/pull_request_label.yml new file mode 100644 index 000000000..86f199f32 --- /dev/null +++ b/.github/workflows/pull_request_label.yml @@ -0,0 +1,18 @@ +name: PR label + +on: + pull_request: + types: [labeled, unlabeled, opened, reopened, synchronize] + +jobs: + semver-label-check: + name: Semantic Version label check + runs-on: ubuntu-latest + timeout-minutes: 1 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Check for Semantic Version label + uses: apple/swift-nio/.github/actions/pull_request_semver_label_checker@main diff --git a/.licenseignore b/.licenseignore new file mode 100644 index 000000000..edceaab62 --- /dev/null +++ b/.licenseignore @@ -0,0 +1,37 @@ +.gitignore +**/.gitignore +.licenseignore +.gitattributes +.git-blame-ignore-revs +.mailfilter +.mailmap +.spi.yml +.swift-format +.editorconfig +.github/* +*.md +*.txt +*.yml +*.yaml +*.json +Package.swift +**/Package.swift +Package@-*.swift +**/Package@-*.swift +Package.resolved +**/Package.resolved +Makefile +*.modulemap +**/*.modulemap +**/*.docc/* +*.xcprivacy +**/*.xcprivacy +*.symlink +**/*.symlink +Dockerfile +**/Dockerfile +.dockerignore +Snippets/* +dev/git.commit.template +.unacceptablelanguageignore +Tests/AsyncHTTPClientTests/Resources/*.pem diff --git a/.swift-format b/.swift-format new file mode 100644 index 000000000..7e8ae7391 --- /dev/null +++ b/.swift-format @@ -0,0 +1,68 @@ +{ + "version" : 1, + "indentation" : { + "spaces" : 4 + }, + "tabWidth" : 4, + "fileScopedDeclarationPrivacy" : { + "accessLevel" : "private" + }, + "spacesAroundRangeFormationOperators" : false, + "indentConditionalCompilationBlocks" : false, + "indentSwitchCaseLabels" : false, + "lineBreakAroundMultilineExpressionChainComponents" : false, + "lineBreakBeforeControlFlowKeywords" : false, + "lineBreakBeforeEachArgument" : true, + "lineBreakBeforeEachGenericRequirement" : true, + "lineLength" : 120, + "maximumBlankLines" : 1, + "respectsExistingLineBreaks" : true, + "prioritizeKeepingFunctionOutputTogether" : true, + "noAssignmentInExpressions" : { + "allowedFunctions" : [ + "XCTAssertNoThrow", + "XCTAssertThrowsError" + ] + }, + "rules" : { + "AllPublicDeclarationsHaveDocumentation" : false, + "AlwaysUseLiteralForEmptyCollectionInit" : false, + "AlwaysUseLowerCamelCase" : false, + "AmbiguousTrailingClosureOverload" : true, + "BeginDocumentationCommentWithOneLineSummary" : false, + "DoNotUseSemicolons" : true, + "DontRepeatTypeInStaticProperties" : true, + "FileScopedDeclarationPrivacy" : true, + "FullyIndirectEnum" : true, + "GroupNumericLiterals" : true, + "IdentifiersMustBeASCII" : true, + "NeverForceUnwrap" : false, + "NeverUseForceTry" : false, + "NeverUseImplicitlyUnwrappedOptionals" : false, + "NoAccessLevelOnExtensionDeclaration" : true, + "NoAssignmentInExpressions" : true, + "NoBlockComments" : true, + "NoCasesWithOnlyFallthrough" : true, + "NoEmptyTrailingClosureParentheses" : true, + "NoLabelsInCasePatterns" : true, + "NoLeadingUnderscores" : false, + "NoParensAroundConditions" : true, + "NoVoidReturnOnFunctionSignature" : true, + "OmitExplicitReturns" : true, + "OneCasePerLine" : true, + "OneVariableDeclarationPerLine" : true, + "OnlyOneTrailingClosureArgument" : true, + "OrderedImports" : true, + "ReplaceForEachWithForLoop" : true, + "ReturnVoidInsteadOfEmptyTuple" : true, + "UseEarlyExits" : false, + "UseExplicitNilCheckInConditions" : false, + "UseLetInEveryBoundCaseVariable" : false, + "UseShorthandTypeNames" : true, + "UseSingleLinePropertyGetter" : false, + "UseSynthesizedInitializer" : false, + "UseTripleSlashForDocumentationComments" : true, + "UseWhereClausesInForLoops" : false, + "ValidateDocumentationComments" : false + } +} diff --git a/.swiftformat b/.swiftformat deleted file mode 100644 index 7b7c486ea..000000000 --- a/.swiftformat +++ /dev/null @@ -1,24 +0,0 @@ -# file options - ---swiftversion 5.4 ---exclude .build - -# format options - ---self insert ---patternlet inline ---ranges nospace ---stripunusedargs unnamed-only ---ifdef no-indent ---extensionacl on-declarations ---disable typeSugar # https://github.com/nicklockwood/SwiftFormat/issues/636 ---disable andOperator ---disable wrapMultilineStatementBraces ---disable enumNamespaces ---disable redundantExtensionACL ---disable redundantReturn ---disable preferKeyPath ---disable sortedSwitchCases ---disable numberFormatting - -# rules diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3803bb618..dddcb3ba4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,10 +65,10 @@ We require that your commit messages match our template. The easiest way to do t git config commit.template dev/git.commit.template -### Make sure Tests work on Linux -AsyncHTTPClient uses XCTest to run tests on both macOS and Linux. While the macOS version of XCTest is able to use the Objective-C runtime to discover tests at execution time, the Linux version is not. -For this reason, whenever you add new tests **you have to run a script** that generates the hooks needed to run those tests on Linux, or our CI will complain that the tests are not all present on Linux. To do this, merely execute `ruby ./scripts/generate_linux_tests.rb` at the root of the package and check the changes it made. +### Run CI checks locally + +You can run the Github Actions workflows locally using [act](https://github.com/nektos/act). For detailed steps on how to do this please see [https://github.com/swiftlang/github-workflows?tab=readme-ov-file#running-workflows-locally](https://github.com/swiftlang/github-workflows?tab=readme-ov-file#running-workflows-locally). ## How to contribute your work diff --git a/Examples/GetHTML/GetHTML.swift b/Examples/GetHTML/GetHTML.swift index 98d6eb3c6..ca3bacbea 100644 --- a/Examples/GetHTML/GetHTML.swift +++ b/Examples/GetHTML/GetHTML.swift @@ -23,7 +23,7 @@ struct GetHTML { let request = HTTPClientRequest(url: "https://apple.com") let response = try await httpClient.execute(request, timeout: .seconds(30)) print("HTTP head", response) - let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB print(String(buffer: body)) } catch { print("request failed:", error) diff --git a/Examples/GetJSON/GetJSON.swift b/Examples/GetJSON/GetJSON.swift index 8f77c4a89..1af7a5144 100644 --- a/Examples/GetJSON/GetJSON.swift +++ b/Examples/GetJSON/GetJSON.swift @@ -38,7 +38,7 @@ struct GetJSON { let request = HTTPClientRequest(url: "https://xkcd.com/info.0.json") let response = try await httpClient.execute(request, timeout: .seconds(30)) print("HTTP head", response) - let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB + let body = try await response.body.collect(upTo: 1024 * 1024) // 1 MB // we use an overload defined in `NIOFoundationCompat` for `decode(_:from:)` to // efficiently decode from a `ByteBuffer` let comic = try JSONDecoder().decode(Comic.self, from: body) diff --git a/Examples/Package.swift b/Examples/Package.swift index 696092cba..9986b17b5 100644 --- a/Examples/Package.swift +++ b/Examples/Package.swift @@ -43,7 +43,8 @@ let package = Package( dependencies: [ .product(name: "AsyncHTTPClient", package: "async-http-client"), .product(name: "NIOCore", package: "swift-nio"), - ], path: "GetHTML" + ], + path: "GetHTML" ), .executableTarget( name: "GetJSON", @@ -51,14 +52,16 @@ let package = Package( .product(name: "AsyncHTTPClient", package: "async-http-client"), .product(name: "NIOCore", package: "swift-nio"), .product(name: "NIOFoundationCompat", package: "swift-nio"), - ], path: "GetJSON" + ], + path: "GetJSON" ), .executableTarget( name: "StreamingByteCounter", dependencies: [ .product(name: "AsyncHTTPClient", package: "async-http-client"), .product(name: "NIOCore", package: "swift-nio"), - ], path: "StreamingByteCounter" + ], + path: "StreamingByteCounter" ), ] ) diff --git a/NOTICE.txt b/NOTICE.txt index 095a11740..86a969171 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -50,13 +50,13 @@ This product contains a derivation of the Tony Stone's 'process_test_files.rb'. * https://www.apache.org/licenses/LICENSE-2.0 * HOMEPAGE: * https://github.com/tonystone/build-tools/commit/6c417b7569df24597a48a9aa7b505b636e8f73a1 - * https://github.com/tonystone/build-tools/blob/master/source/xctest_tool.rb + * https://github.com/tonystone/build-tools/blob/cf3440f43bde2053430285b4ed0709c865892eb5/source/xctest_tool.rb --- This product contains a derivation of Fabian Fett's 'Base64.swift'. * LICENSE (Apache License 2.0): - * https://github.com/fabianfett/swift-base64-kit/blob/master/LICENSE + * https://github.com/swift-extras/swift-extras-base64/blob/b8af49699d59ad065b801715a5009619100245ca/LICENSE * HOMEPAGE: * https://github.com/fabianfett/swift-base64-kit diff --git a/Package.swift b/Package.swift index b645ee198..bec6c9114 100644 --- a/Package.swift +++ b/Package.swift @@ -18,7 +18,7 @@ import PackageDescription let package = Package( name: "async-http-client", products: [ - .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]), + .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]) ], dependencies: [ .package(url: "https://github.com/apple/swift-nio.git", from: "2.71.0"), @@ -28,14 +28,13 @@ let package = Package( .package(url: "https://github.com/apple/swift-nio-transport-services.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-log.git", from: "1.4.4"), .package(url: "https://github.com/apple/swift-atomics.git", from: "1.0.2"), - .package(url: "https://github.com/apple/swift-docc-plugin.git", from: "1.0.0"), .package(url: "https://github.com/apple/swift-algorithms.git", from: "1.0.0"), ], targets: [ .target( name: "CAsyncHTTPClient", cSettings: [ - .define("_GNU_SOURCE"), + .define("_GNU_SOURCE") ] ), .target( diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index ef858443e..fc1dbc209 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -12,11 +12,12 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import Logging import NIOCore import NIOHTTP1 +import struct Foundation.URL + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClient { /// Execute arbitrary HTTP requests. @@ -85,11 +86,13 @@ extension HTTPClient { return response } - guard let redirectURL = response.headers.extractRedirectTarget( - status: response.status, - originalURL: preparedRequest.url, - originalScheme: preparedRequest.poolKey.scheme - ) else { + guard + let redirectURL = response.headers.extractRedirectTarget( + status: response.status, + originalURL: preparedRequest.url, + originalScheme: preparedRequest.poolKey.scheme + ) + else { // response does not want a redirect return response } @@ -120,31 +123,35 @@ extension HTTPClient { ) async throws -> HTTPClientResponse { let cancelHandler = TransactionCancelHandler() - return try await withTaskCancellationHandler(operation: { () async throws -> HTTPClientResponse in - let eventLoop = self.eventLoopGroup.any() - let deadlineTask = eventLoop.scheduleTask(deadline: deadline) { - cancelHandler.cancel(reason: .deadlineExceeded) - } - defer { - deadlineTask.cancel() - } - return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) -> Void in - let transaction = Transaction( - request: request, - requestOptions: .fromClientConfiguration(self.configuration), - logger: logger, - connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), - preferredEventLoop: eventLoop, - responseContinuation: continuation - ) - - cancelHandler.registerTransaction(transaction) - - self.poolManager.executeRequest(transaction) + return try await withTaskCancellationHandler( + operation: { () async throws -> HTTPClientResponse in + let eventLoop = self.eventLoopGroup.any() + let deadlineTask = eventLoop.scheduleTask(deadline: deadline) { + cancelHandler.cancel(reason: .deadlineExceeded) + } + defer { + deadlineTask.cancel() + } + return try await withCheckedThrowingContinuation { + (continuation: CheckedContinuation) -> Void in + let transaction = Transaction( + request: request, + requestOptions: .fromClientConfiguration(self.configuration), + logger: logger, + connectionDeadline: .now() + (self.configuration.timeout.connectionCreationTimeout), + preferredEventLoop: eventLoop, + responseContinuation: continuation + ) + + cancelHandler.registerTransaction(transaction) + + self.poolManager.executeRequest(transaction) + } + }, + onCancel: { + cancelHandler.cancel(reason: .taskCanceled) } - }, onCancel: { - cancelHandler.cancel(reason: .taskCanceled) - }) + ) } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift index 0a3ec6442..d4eeae03e 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest+Prepared.swift @@ -12,11 +12,12 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import NIOCore import NIOHTTP1 import NIOSSL +import struct Foundation.URL + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientRequest { struct Prepared { @@ -81,7 +82,11 @@ extension HTTPClientRequest.Prepared.Body { case .asyncSequence(let length, let makeAsyncIterator): self = .asyncSequence(length: length, nextBodyPart: makeAsyncIterator()) case .sequence(let length, let canBeConsumedMultipleTimes, let makeCompleteBody): - self = .sequence(length: length, canBeConsumedMultipleTimes: canBeConsumedMultipleTimes, makeCompleteBody: makeCompleteBody) + self = .sequence( + length: length, + canBeConsumedMultipleTimes: canBeConsumedMultipleTimes, + makeCompleteBody: makeCompleteBody + ) case .byteBuffer(let byteBuffer): self = .byteBuffer(byteBuffer) } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift index ad81bfa32..f07a2ed41 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientRequest.swift @@ -176,23 +176,29 @@ extension HTTPClientRequest.Body { // the maximum size of a ByteBuffer. if bufferPointer.count <= byteBufferMaxSize { let buffer = ByteBuffer(bytes: bufferPointer) - return Self(.sequence( - length: length.storage, - canBeConsumedMultipleTimes: true, - makeCompleteBody: { _ in buffer } - )) + return Self( + .sequence( + length: length.storage, + canBeConsumedMultipleTimes: true, + makeCompleteBody: { _ in buffer } + ) + ) } else { // we need to copy `bufferPointer` eagerly as the pointer is only valid during the call to `withContiguousStorageIfAvailable` - let buffers: Array = bufferPointer.chunks(ofCount: byteBufferMaxSize).map { ByteBuffer(bytes: $0) } - return Self(.asyncSequence( - length: length.storage, - makeAsyncIterator: { - var iterator = buffers.makeIterator() - return { _ in - iterator.next() + let buffers: [ByteBuffer] = bufferPointer.chunks(ofCount: byteBufferMaxSize).map { + ByteBuffer(bytes: $0) + } + return Self( + .asyncSequence( + length: length.storage, + makeAsyncIterator: { + var iterator = buffers.makeIterator() + return { _ in + iterator.next() + } } - } - )) + ) + ) } } if let body = body { @@ -200,21 +206,23 @@ extension HTTPClientRequest.Body { } // slow path - return Self(.asyncSequence( - length: length.storage - ) { - var iterator = bytes.makeIterator() - return { allocator in - var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) - while buffer.writableBytes > 0, let byte = iterator.next() { - buffer.writeInteger(byte) - } - if buffer.readableBytes > 0 { - return buffer + return Self( + .asyncSequence( + length: length.storage + ) { + var iterator = bytes.makeIterator() + return { allocator in + var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) + while buffer.writableBytes > 0, let byte = iterator.next() { + buffer.writeInteger(byte) + } + if buffer.readableBytes > 0 { + return buffer + } + return nil } - return nil } - }) + ) } /// Create an ``HTTPClientRequest/Body-swift.struct`` from a `Collection` of bytes. @@ -237,25 +245,29 @@ extension HTTPClientRequest.Body { length: Length ) -> Self where Bytes.Element == UInt8 { if bytes.count <= bagOfBytesToByteBufferConversionChunkSize { - return self.init(.sequence( - length: length.storage, - canBeConsumedMultipleTimes: true - ) { allocator in - allocator.buffer(bytes: bytes) - }) + return self.init( + .sequence( + length: length.storage, + canBeConsumedMultipleTimes: true + ) { allocator in + allocator.buffer(bytes: bytes) + } + ) } else { - return self.init(.asyncSequence( - length: length.storage, - makeAsyncIterator: { - var iterator = bytes.chunks(ofCount: bagOfBytesToByteBufferConversionChunkSize).makeIterator() - return { allocator in - guard let chunk = iterator.next() else { - return nil + return self.init( + .asyncSequence( + length: length.storage, + makeAsyncIterator: { + var iterator = bytes.chunks(ofCount: bagOfBytesToByteBufferConversionChunkSize).makeIterator() + return { allocator in + guard let chunk = iterator.next() else { + return nil + } + return allocator.buffer(bytes: chunk) } - return allocator.buffer(bytes: chunk) } - } - )) + ) + ) } } @@ -276,12 +288,14 @@ extension HTTPClientRequest.Body { _ sequenceOfBytes: SequenceOfBytes, length: Length ) -> Self where SequenceOfBytes.Element == ByteBuffer { - let body = self.init(.asyncSequence(length: length.storage) { - var iterator = sequenceOfBytes.makeAsyncIterator() - return { _ -> ByteBuffer? in - try await iterator.next() + let body = self.init( + .asyncSequence(length: length.storage) { + var iterator = sequenceOfBytes.makeAsyncIterator() + return { _ -> ByteBuffer? in + try await iterator.next() + } } - }) + ) return body } @@ -304,19 +318,21 @@ extension HTTPClientRequest.Body { _ bytes: Bytes, length: Length ) -> Self where Bytes.Element == UInt8 { - let body = self.init(.asyncSequence(length: length.storage) { - var iterator = bytes.makeAsyncIterator() - return { allocator -> ByteBuffer? in - var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) - while buffer.writableBytes > 0, let byte = try await iterator.next() { - buffer.writeInteger(byte) - } - if buffer.readableBytes > 0 { - return buffer + let body = self.init( + .asyncSequence(length: length.storage) { + var iterator = bytes.makeAsyncIterator() + return { allocator -> ByteBuffer? in + var buffer = allocator.buffer(capacity: bagOfBytesToByteBufferConversionChunkSize) + while buffer.writableBytes > 0, let byte = try await iterator.next() { + buffer.writeInteger(byte) + } + if buffer.readableBytes > 0 { + return buffer + } + return nil } - return nil } - }) + ) return body } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift index 1ca01f53f..832eb7b41 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClientResponse.swift @@ -55,14 +55,16 @@ public struct HTTPClientResponse: Sendable { version: version, status: status, headers: headers, - body: .init(.transaction( - body, - expectedContentLength: HTTPClientResponse.expectedContentLength( - requestMethod: requestMethod, - headers: headers, - status: status + body: .init( + .transaction( + body, + expectedContentLength: HTTPClientResponse.expectedContentLength( + requestMethod: requestMethod, + headers: headers, + status: status + ) ) - )) + ) ) } } @@ -116,7 +118,8 @@ extension HTTPClientResponse { } /// calling collect function within here in order to ensure the correct nested type - func collect(_ body: Body, maxBytes: Int) async throws -> ByteBuffer where Body.Element == ByteBuffer { + func collect(_ body: Body, maxBytes: Int) async throws -> ByteBuffer + where Body.Element == ByteBuffer { try await body.collect(upTo: maxBytes) } return try await collect(self, maxBytes: maxBytes) @@ -126,7 +129,11 @@ extension HTTPClientResponse { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClientResponse { - static func expectedContentLength(requestMethod: HTTPMethod, headers: HTTPHeaders, status: HTTPResponseStatus) -> Int? { + static func expectedContentLength( + requestMethod: HTTPMethod, + headers: HTTPHeaders, + status: HTTPResponseStatus + ) -> Int? { if status == .notModified { return 0 } else if requestMethod == .HEAD { diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index ad49332c0..47b424f04 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -82,9 +82,20 @@ extension Transaction { enum FailAction { case none /// fail response before head received. scheduler and executor are exclusive here. - case failResponseHead(CheckedContinuation, Error, HTTPRequestScheduler?, HTTPRequestExecutor?, bodyStreamContinuation: CheckedContinuation?) + case failResponseHead( + CheckedContinuation, + Error, + HTTPRequestScheduler?, + HTTPRequestExecutor?, + bodyStreamContinuation: CheckedContinuation? + ) /// fail response after response head received. fail the response stream (aka call to `next()`) - case failResponseStream(TransactionBody.Source, Error, HTTPRequestExecutor, bodyStreamContinuation: CheckedContinuation?) + case failResponseStream( + TransactionBody.Source, + Error, + HTTPRequestExecutor, + bodyStreamContinuation: CheckedContinuation? + ) case failRequestStreamContinuation(CheckedContinuation, Error) } @@ -116,24 +127,41 @@ extension Transaction { switch requestStreamState { case .paused(continuation: .some(let continuation)): self.state = .finished(error: error) - return .failResponseHead(context.continuation, error, nil, context.executor, bodyStreamContinuation: continuation) + return .failResponseHead( + context.continuation, + error, + nil, + context.executor, + bodyStreamContinuation: continuation + ) case .requestHeadSent, .finished, .producing, .paused(continuation: .none): self.state = .finished(error: error) - return .failResponseHead(context.continuation, error, nil, context.executor, bodyStreamContinuation: nil) + return .failResponseHead( + context.continuation, + error, + nil, + context.executor, + bodyStreamContinuation: nil + ) } case .executing(let context, let requestStreamState, .streamingBody(let source)): self.state = .finished(error: error) switch requestStreamState { case .paused(let bodyStreamContinuation): - return .failResponseStream(source, error, context.executor, bodyStreamContinuation: bodyStreamContinuation) + return .failResponseStream( + source, + error, + context.executor, + bodyStreamContinuation: bodyStreamContinuation + ) case .finished, .producing, .requestHeadSent: return .failResponseStream(source, error, context.executor, bodyStreamContinuation: nil) } case .finished(error: _), - .executing(_, _, .finished): + .executing(_, _, .finished): return .none } } @@ -165,7 +193,7 @@ extension Transaction { return .cancel(executor) case .executing, - .finished(error: .none): + .finished(error: .none): preconditionFailure("Invalid state: \(self.state)") } } @@ -179,7 +207,9 @@ extension Transaction { mutating func resumeRequestBodyStream() -> ResumeProducingAction { switch self.state { case .initialized, .queued, .deadlineExceededWhileQueued: - preconditionFailure("Received a resumeBodyRequest on a request, that isn't executing. Invalid state: \(self.state)") + preconditionFailure( + "Received a resumeBodyRequest on a request, that isn't executing. Invalid state: \(self.state)" + ) case .executing(let context, .requestHeadSent, let responseState): // the request can start to send its body. @@ -187,7 +217,9 @@ extension Transaction { return .startStream(context.allocator) case .executing(_, .producing, _): - preconditionFailure("Received a resumeBodyRequest on a request, that is producing. Invalid state: \(self.state)") + preconditionFailure( + "Received a resumeBodyRequest on a request, that is producing. Invalid state: \(self.state)" + ) case .executing(let context, .paused(.none), let responseState): // request stream is currently paused, but there is no write waiting. We don't need @@ -213,17 +245,17 @@ extension Transaction { mutating func pauseRequestBodyStream() { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, .requestHeadSent, _): + .queued, + .deadlineExceededWhileQueued, + .executing(_, .requestHeadSent, _): preconditionFailure("A request stream can only be resumed, if the request was started") case .executing(let context, .producing, let responseSteam): self.state = .executing(context, .paused(continuation: nil), responseSteam) case .executing(_, .paused, _), - .executing(_, .finished, _), - .finished: + .executing(_, .finished, _), + .finished: // the channels writability changed to paused after we have already forwarded all // request bytes. Can be ignored. break @@ -239,10 +271,12 @@ extension Transaction { func writeNextRequestPart() -> NextWriteAction { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, .requestHeadSent, _): - preconditionFailure("A request stream can only produce, if the request was started. Invalid state: \(self.state)") + .queued, + .deadlineExceededWhileQueued, + .executing(_, .requestHeadSent, _): + preconditionFailure( + "A request stream can only produce, if the request was started. Invalid state: \(self.state)" + ) case .executing(let context, .producing, _): // We are currently producing the request body. The executors channel is writable. @@ -260,7 +294,9 @@ extension Transaction { return .writeAndWait(context.executor) case .executing(_, .paused(continuation: .some), _): - preconditionFailure("A write continuation already exists, but we tried to set another one. Invalid state: \(self.state)") + preconditionFailure( + "A write continuation already exists, but we tried to set another one. Invalid state: \(self.state)" + ) case .finished, .executing(_, .finished, _): return .fail @@ -270,11 +306,13 @@ extension Transaction { mutating func waitForRequestBodyDemand(continuation: CheckedContinuation) { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, .requestHeadSent, _), - .executing(_, .finished, _): - preconditionFailure("A request stream can only produce, if the request was started. Invalid state: \(self.state)") + .queued, + .deadlineExceededWhileQueued, + .executing(_, .requestHeadSent, _), + .executing(_, .finished, _): + preconditionFailure( + "A request stream can only produce, if the request was started. Invalid state: \(self.state)" + ) case .executing(_, .producing, _): preconditionFailure() @@ -303,17 +341,19 @@ extension Transaction { mutating func finishRequestBodyStream() -> FinishAction { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, .finished, _): + .queued, + .deadlineExceededWhileQueued, + .executing(_, .finished, _): preconditionFailure("Invalid state: \(self.state)") case .executing(_, .paused(continuation: .some), _): - preconditionFailure("Received a request body end, while having a registered back-pressure continuation. Invalid state: \(self.state)") + preconditionFailure( + "Received a request body end, while having a registered back-pressure continuation. Invalid state: \(self.state)" + ) case .executing(let context, .producing, let responseState), - .executing(let context, .paused(continuation: .none), let responseState), - .executing(let context, .requestHeadSent, let responseState): + .executing(let context, .paused(continuation: .none), let responseState), + .executing(let context, .requestHeadSent, let responseState): switch responseState { case .finished: @@ -345,10 +385,10 @@ extension Transaction { ) -> ReceiveResponseHeadAction { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, _, .streamingBody), - .executing(_, _, .finished): + .queued, + .deadlineExceededWhileQueued, + .executing(_, _, .streamingBody), + .executing(_, _, .finished): preconditionFailure("invalid state \(self.state)") case .executing(let context, let requestState, .waitingForResponseHead): @@ -381,15 +421,15 @@ extension Transaction { mutating func produceMore() -> ProduceMoreAction { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, _, .waitingForResponseHead): + .queued, + .deadlineExceededWhileQueued, + .executing(_, _, .waitingForResponseHead): preconditionFailure("invalid state \(self.state)") case .executing(let context, _, .streamingBody): return .requestMoreResponseBodyParts(context.executor) case .finished, - .executing(_, _, .finished): + .executing(_, _, .finished): return .none } } @@ -402,7 +442,9 @@ extension Transaction { mutating func receiveResponseBodyParts(_ buffer: CircularBuffer) -> ReceiveResponsePartAction { switch self.state { case .initialized, .queued, .deadlineExceededWhileQueued: - preconditionFailure("Received a response body part, but request hasn't started yet. Invalid state: \(self.state)") + preconditionFailure( + "Received a response body part, but request hasn't started yet. Invalid state: \(self.state)" + ) case .executing(_, _, .waitingForResponseHead): preconditionFailure("If we receive a response body, we must have received a head before") @@ -415,7 +457,9 @@ extension Transaction { return .none case .executing(_, _, .finished): - preconditionFailure("Received response end. Must not receive further body parts after that. Invalid state: \(self.state)") + preconditionFailure( + "Received response end. Must not receive further body parts after that. Invalid state: \(self.state)" + ) } } @@ -427,10 +471,12 @@ extension Transaction { mutating func succeedRequest(_ newChunks: CircularBuffer?) -> ReceiveResponseEndAction { switch self.state { case .initialized, - .queued, - .deadlineExceededWhileQueued, - .executing(_, _, .waitingForResponseHead): - preconditionFailure("Received no response head, but received a response end. Invalid state: \(self.state)") + .queued, + .deadlineExceededWhileQueued, + .executing(_, _, .waitingForResponseHead): + preconditionFailure( + "Received no response head, but received a response end. Invalid state: \(self.state)" + ) case .executing(let context, let requestState, .streamingBody(let source)): self.state = .executing(context, requestState, .finished) @@ -439,7 +485,9 @@ extension Transaction { // the request failed or was cancelled before, we can ignore all events return .none case .executing(_, _, .finished): - preconditionFailure("Already received an eof or error before. Must not receive further events. Invalid state: \(self.state)") + preconditionFailure( + "Already received an eof or error before. Must not receive further events. Invalid state: \(self.state)" + ) } } diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index 6d9192642..e420935f1 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -146,8 +146,8 @@ import NIOSSL @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension Transaction: HTTPSchedulableRequest { var poolKey: ConnectionPool.Key { self.request.poolKey } - var tlsConfiguration: TLSConfiguration? { return self.request.tlsConfiguration } - var requiredEventLoop: EventLoop? { return nil } + var tlsConfiguration: TLSConfiguration? { self.request.tlsConfiguration } + var requiredEventLoop: EventLoop? { nil } func requestWasQueued(_ scheduler: HTTPRequestScheduler) { self.stateLock.withLock { @@ -290,7 +290,7 @@ extension Transaction: HTTPExecutableRequest { case .failResponseHead(let continuation, let error, let scheduler, let executor, let bodyStreamContinuation): continuation.resume(throwing: error) bodyStreamContinuation?.resume(throwing: error) - scheduler?.cancelRequest(self) // NOTE: scheduler and executor are exclusive here + scheduler?.cancelRequest(self) // NOTE: scheduler and executor are exclusive here executor?.cancelRequest(self) case .failResponseStream(let source, let error, let executor, let requestBodyStreamContinuation): @@ -317,7 +317,7 @@ extension Transaction: HTTPExecutableRequest { scheduler?.cancelRequest(self) executor?.cancelRequest(self) bodyStreamContinuation?.resume(throwing: HTTPClientError.deadlineExceeded) - case .cancelSchedulerOnly(scheduler: let scheduler): + case .cancelSchedulerOnly(let scheduler): scheduler.cancelRequest(self) case .none: break diff --git a/Sources/AsyncHTTPClient/Base64.swift b/Sources/AsyncHTTPClient/Base64.swift index eed511a8c..3162e7251 100644 --- a/Sources/AsyncHTTPClient/Base64.swift +++ b/Sources/AsyncHTTPClient/Base64.swift @@ -19,142 +19,156 @@ extension String { - /// Base64 encode a collection of UInt8 to a string, without the use of Foundation. - @inlinable - init(base64Encoding bytes: Buffer) - where Buffer.Element == UInt8 - { - self = Base64.encode(bytes: bytes) - } + /// Base64 encode a collection of UInt8 to a string, without the use of Foundation. + @inlinable + init(base64Encoding bytes: Buffer) + where Buffer.Element == UInt8 { + self = Base64.encode(bytes: bytes) + } } +// swift-format-ignore: DontRepeatTypeInStaticProperties @usableFromInline internal struct Base64 { - @inlinable - static func encode(bytes: Buffer) - -> String where Buffer.Element == UInt8 - { - guard !bytes.isEmpty else { - return "" - } - // In Base64, 3 bytes become 4 output characters, and we pad to the - // nearest multiple of four. - let base64StringLength = ((bytes.count + 2) / 3) * 4 - let alphabet = Base64.encodeBase64 - - return String(customUnsafeUninitializedCapacity: base64StringLength) { backingStorage in - var input = bytes.makeIterator() - var offset = 0 - while let firstByte = input.next() { - let secondByte = input.next() - let thirdByte = input.next() - - backingStorage[offset] = Base64.encode(alphabet: alphabet, firstByte: firstByte) - backingStorage[offset + 1] = Base64.encode(alphabet: alphabet, firstByte: firstByte, secondByte: secondByte) - backingStorage[offset + 2] = Base64.encode(alphabet: alphabet, secondByte: secondByte, thirdByte: thirdByte) - backingStorage[offset + 3] = Base64.encode(alphabet: alphabet, thirdByte: thirdByte) - offset += 4 - } - return offset + @inlinable + static func encode( + bytes: Buffer + ) + -> String where Buffer.Element == UInt8 + { + guard !bytes.isEmpty else { + return "" + } + // In Base64, 3 bytes become 4 output characters, and we pad to the + // nearest multiple of four. + let base64StringLength = ((bytes.count + 2) / 3) * 4 + let alphabet = Base64.encodeBase64 + + return String(customUnsafeUninitializedCapacity: base64StringLength) { backingStorage in + var input = bytes.makeIterator() + var offset = 0 + while let firstByte = input.next() { + let secondByte = input.next() + let thirdByte = input.next() + + backingStorage[offset] = Base64.encode(alphabet: alphabet, firstByte: firstByte) + backingStorage[offset + 1] = Base64.encode( + alphabet: alphabet, + firstByte: firstByte, + secondByte: secondByte + ) + backingStorage[offset + 2] = Base64.encode( + alphabet: alphabet, + secondByte: secondByte, + thirdByte: thirdByte + ) + backingStorage[offset + 3] = Base64.encode(alphabet: alphabet, thirdByte: thirdByte) + offset += 4 + } + return offset + } } - } - - // MARK: Internal - - // The base64 unicode table. - @usableFromInline - static let encodeBase64: [UInt8] = [ - UInt8(ascii: "A"), UInt8(ascii: "B"), UInt8(ascii: "C"), UInt8(ascii: "D"), - UInt8(ascii: "E"), UInt8(ascii: "F"), UInt8(ascii: "G"), UInt8(ascii: "H"), - UInt8(ascii: "I"), UInt8(ascii: "J"), UInt8(ascii: "K"), UInt8(ascii: "L"), - UInt8(ascii: "M"), UInt8(ascii: "N"), UInt8(ascii: "O"), UInt8(ascii: "P"), - UInt8(ascii: "Q"), UInt8(ascii: "R"), UInt8(ascii: "S"), UInt8(ascii: "T"), - UInt8(ascii: "U"), UInt8(ascii: "V"), UInt8(ascii: "W"), UInt8(ascii: "X"), - UInt8(ascii: "Y"), UInt8(ascii: "Z"), UInt8(ascii: "a"), UInt8(ascii: "b"), - UInt8(ascii: "c"), UInt8(ascii: "d"), UInt8(ascii: "e"), UInt8(ascii: "f"), - UInt8(ascii: "g"), UInt8(ascii: "h"), UInt8(ascii: "i"), UInt8(ascii: "j"), - UInt8(ascii: "k"), UInt8(ascii: "l"), UInt8(ascii: "m"), UInt8(ascii: "n"), - UInt8(ascii: "o"), UInt8(ascii: "p"), UInt8(ascii: "q"), UInt8(ascii: "r"), - UInt8(ascii: "s"), UInt8(ascii: "t"), UInt8(ascii: "u"), UInt8(ascii: "v"), - UInt8(ascii: "w"), UInt8(ascii: "x"), UInt8(ascii: "y"), UInt8(ascii: "z"), - UInt8(ascii: "0"), UInt8(ascii: "1"), UInt8(ascii: "2"), UInt8(ascii: "3"), - UInt8(ascii: "4"), UInt8(ascii: "5"), UInt8(ascii: "6"), UInt8(ascii: "7"), - UInt8(ascii: "8"), UInt8(ascii: "9"), UInt8(ascii: "+"), UInt8(ascii: "/"), - ] - - static let encodePaddingCharacter: UInt8 = UInt8(ascii: "=") - - @usableFromInline - static func encode(alphabet: [UInt8], firstByte: UInt8) -> UInt8 { - let index = firstByte >> 2 - return alphabet[Int(index)] - } - - @usableFromInline - static func encode(alphabet: [UInt8], firstByte: UInt8, secondByte: UInt8?) -> UInt8 { - var index = (firstByte & 0b00000011) << 4 - if let secondByte = secondByte { - index += (secondByte & 0b11110000) >> 4 + + // MARK: Internal + + // The base64 unicode table. + @usableFromInline + static let encodeBase64: [UInt8] = [ + UInt8(ascii: "A"), UInt8(ascii: "B"), UInt8(ascii: "C"), UInt8(ascii: "D"), + UInt8(ascii: "E"), UInt8(ascii: "F"), UInt8(ascii: "G"), UInt8(ascii: "H"), + UInt8(ascii: "I"), UInt8(ascii: "J"), UInt8(ascii: "K"), UInt8(ascii: "L"), + UInt8(ascii: "M"), UInt8(ascii: "N"), UInt8(ascii: "O"), UInt8(ascii: "P"), + UInt8(ascii: "Q"), UInt8(ascii: "R"), UInt8(ascii: "S"), UInt8(ascii: "T"), + UInt8(ascii: "U"), UInt8(ascii: "V"), UInt8(ascii: "W"), UInt8(ascii: "X"), + UInt8(ascii: "Y"), UInt8(ascii: "Z"), UInt8(ascii: "a"), UInt8(ascii: "b"), + UInt8(ascii: "c"), UInt8(ascii: "d"), UInt8(ascii: "e"), UInt8(ascii: "f"), + UInt8(ascii: "g"), UInt8(ascii: "h"), UInt8(ascii: "i"), UInt8(ascii: "j"), + UInt8(ascii: "k"), UInt8(ascii: "l"), UInt8(ascii: "m"), UInt8(ascii: "n"), + UInt8(ascii: "o"), UInt8(ascii: "p"), UInt8(ascii: "q"), UInt8(ascii: "r"), + UInt8(ascii: "s"), UInt8(ascii: "t"), UInt8(ascii: "u"), UInt8(ascii: "v"), + UInt8(ascii: "w"), UInt8(ascii: "x"), UInt8(ascii: "y"), UInt8(ascii: "z"), + UInt8(ascii: "0"), UInt8(ascii: "1"), UInt8(ascii: "2"), UInt8(ascii: "3"), + UInt8(ascii: "4"), UInt8(ascii: "5"), UInt8(ascii: "6"), UInt8(ascii: "7"), + UInt8(ascii: "8"), UInt8(ascii: "9"), UInt8(ascii: "+"), UInt8(ascii: "/"), + ] + + static let encodePaddingCharacter: UInt8 = UInt8(ascii: "=") + + @usableFromInline + static func encode(alphabet: [UInt8], firstByte: UInt8) -> UInt8 { + let index = firstByte >> 2 + return alphabet[Int(index)] } - return alphabet[Int(index)] - } - - @usableFromInline - static func encode(alphabet: [UInt8], secondByte: UInt8?, thirdByte: UInt8?) -> UInt8 { - guard let secondByte = secondByte else { - // No second byte means we are just emitting padding. - return Base64.encodePaddingCharacter + + @usableFromInline + static func encode(alphabet: [UInt8], firstByte: UInt8, secondByte: UInt8?) -> UInt8 { + var index = (firstByte & 0b00000011) << 4 + if let secondByte = secondByte { + index += (secondByte & 0b11110000) >> 4 + } + return alphabet[Int(index)] } - var index = (secondByte & 0b00001111) << 2 - if let thirdByte = thirdByte { - index += (thirdByte & 0b11000000) >> 6 + + @usableFromInline + static func encode(alphabet: [UInt8], secondByte: UInt8?, thirdByte: UInt8?) -> UInt8 { + guard let secondByte = secondByte else { + // No second byte means we are just emitting padding. + return Base64.encodePaddingCharacter + } + var index = (secondByte & 0b00001111) << 2 + if let thirdByte = thirdByte { + index += (thirdByte & 0b11000000) >> 6 + } + return alphabet[Int(index)] } - return alphabet[Int(index)] - } - - @usableFromInline - static func encode(alphabet: [UInt8], thirdByte: UInt8?) -> UInt8 { - guard let thirdByte = thirdByte else { - // No third byte means just padding. - return Base64.encodePaddingCharacter + + @usableFromInline + static func encode(alphabet: [UInt8], thirdByte: UInt8?) -> UInt8 { + guard let thirdByte = thirdByte else { + // No third byte means just padding. + return Base64.encodePaddingCharacter + } + let index = thirdByte & 0b00111111 + return alphabet[Int(index)] } - let index = thirdByte & 0b00111111 - return alphabet[Int(index)] - } } extension String { - /// This is a backport of a proposed String initializer that will allow writing directly into an uninitialized String's backing memory. - /// - /// As this API does not exist prior to 5.3 on Linux, or on older Apple platforms, we fake it out with a pointer and accept the extra copy. - @inlinable - init(backportUnsafeUninitializedCapacity capacity: Int, - initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int) rethrows { - // The buffer will store zero terminated C string - let buffer = UnsafeMutableBufferPointer.allocate(capacity: capacity + 1) - defer { - buffer.deallocate() + /// This is a backport of a proposed String initializer that will allow writing directly into an uninitialized String's backing memory. + /// + /// As this API does not exist prior to 5.3 on Linux, or on older Apple platforms, we fake it out with a pointer and accept the extra copy. + @inlinable + init( + backportUnsafeUninitializedCapacity capacity: Int, + initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int + ) rethrows { + // The buffer will store zero terminated C string + let buffer = UnsafeMutableBufferPointer.allocate(capacity: capacity + 1) + defer { + buffer.deallocate() + } + + let initializedCount = try initializer(buffer) + precondition(initializedCount <= capacity, "Overran buffer in initializer!") + // add zero termination + buffer[initializedCount] = 0 + + self = String(cString: buffer.baseAddress!) } - - let initializedCount = try initializer(buffer) - precondition(initializedCount <= capacity, "Overran buffer in initializer!") - // add zero termination - buffer[initializedCount] = 0 - - self = String(cString: buffer.baseAddress!) - } } extension String { - @inlinable - init(customUnsafeUninitializedCapacity capacity: Int, - initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int) rethrows { - if #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) { - try self.init(unsafeUninitializedCapacity: capacity, initializingUTF8With: initializer) - } else { - try self.init(backportUnsafeUninitializedCapacity: capacity, initializingUTF8With: initializer) + @inlinable + init( + customUnsafeUninitializedCapacity capacity: Int, + initializingUTF8With initializer: (_ buffer: UnsafeMutableBufferPointer) throws -> Int + ) rethrows { + if #available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) { + try self.init(unsafeUninitializedCapacity: capacity, initializingUTF8With: initializer) + } else { + try self.init(backportUnsafeUninitializedCapacity: capacity, initializingUTF8With: initializer) + } } - } } diff --git a/Sources/AsyncHTTPClient/BestEffortHashableTLSConfiguration.swift b/Sources/AsyncHTTPClient/BestEffortHashableTLSConfiguration.swift index 58169f645..aca0ce235 100644 --- a/Sources/AsyncHTTPClient/BestEffortHashableTLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/BestEffortHashableTLSConfiguration.swift @@ -27,6 +27,6 @@ struct BestEffortHashableTLSConfiguration: Hashable { } static func == (lhs: BestEffortHashableTLSConfiguration, rhs: BestEffortHashableTLSConfiguration) -> Bool { - return lhs.base.bestEffortEquals(rhs.base) + lhs.base.bestEffortEquals(rhs.base) } } diff --git a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift index b1ee8d5a9..39aefe975 100644 --- a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift +++ b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +// swift-format-ignore: DontRepeatTypeInStaticProperties extension HTTPClient.Configuration { /// The ``HTTPClient/Configuration`` for ``HTTPClient/shared`` which tries to mimic the platform's default or prevalent browser as closely as possible. /// @@ -27,7 +28,7 @@ extension HTTPClient.Configuration { /// - Linux (non-Android): Google Chrome public static var singletonConfiguration: HTTPClient.Configuration { // To start with, let's go with these values. Obtained from Firefox's config. - return HTTPClient.Configuration( + HTTPClient.Configuration( certificateVerification: .fullVerification, redirectConfiguration: .follow(max: 20, allowCycles: false), timeout: Timeout(connect: .seconds(90), read: .seconds(90)), diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index 8cca70750..776d1f6df 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -29,8 +29,7 @@ extension String { var ipv4Address = in_addr() var ipv6Address = in6_addr() return self.withCString { host in - inet_pton(AF_INET, host, &ipv4Address) == 1 || - inet_pton(AF_INET6, host, &ipv6Address) == 1 + inet_pton(AF_INET, host, &ipv4Address) == 1 || inet_pton(AF_INET6, host, &ipv6Address) == 1 } } } @@ -67,12 +66,13 @@ enum ConnectionPool { switch self.connectionTarget { case .ipAddress(let serialization, let addr): hostDescription = "\(serialization):\(addr.port!)" - case .domain(let domain, port: let port): + case .domain(let domain, let port): hostDescription = "\(domain):\(port)" case .unixSocket(let socketPath): hostDescription = socketPath } - return "\(self.scheme)://\(hostDescription)\(self.serverNameIndicatorOverride.map { " SNI: \($0)" } ?? "") TLS-hash: \(hash) " + return + "\(self.scheme)://\(hostDescription)\(self.serverNameIndicatorOverride.map { " SNI: \($0)" } ?? "") TLS-hash: \(hash) " } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift index fbcd4f9c0..db7b7b7ef 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/HTTP1ProxyConnectHandler.swift @@ -42,7 +42,7 @@ final class HTTP1ProxyConnectHandler: ChannelDuplexHandler, RemovableChannelHand private var proxyEstablishedPromise: EventLoopPromise? var proxyEstablishedFuture: EventLoopFuture? { - return self.proxyEstablishedPromise?.futureResult + self.proxyEstablishedPromise?.futureResult } convenience init( @@ -53,10 +53,10 @@ final class HTTP1ProxyConnectHandler: ChannelDuplexHandler, RemovableChannelHand let targetHost: String let targetPort: Int switch target { - case .ipAddress(serialization: let serialization, address: let address): + case .ipAddress(let serialization, let address): targetHost = serialization targetPort = address.port! - case .domain(name: let domain, port: let port): + case .domain(name: let domain, let port): targetHost = domain targetPort = port case .unixSocket: @@ -70,10 +70,12 @@ final class HTTP1ProxyConnectHandler: ChannelDuplexHandler, RemovableChannelHand ) } - init(targetHost: String, - targetPort: Int, - proxyAuthorization: HTTPClient.Authorization?, - deadline: NIODeadline) { + init( + targetHost: String, + targetPort: Int, + proxyAuthorization: HTTPClient.Authorization?, + deadline: NIODeadline + ) { self.targetHost = targetHost self.targetPort = targetPort self.proxyAuthorization = proxyAuthorization diff --git a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/SOCKSEventsHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/SOCKSEventsHandler.swift index 5a46f44a7..a98f97d4d 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/SOCKSEventsHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/SOCKSEventsHandler.swift @@ -31,7 +31,7 @@ final class SOCKSEventsHandler: ChannelInboundHandler, RemovableChannelHandler { private var socksEstablishedPromise: EventLoopPromise? var socksEstablishedFuture: EventLoopFuture? { - return self.socksEstablishedPromise?.futureResult + self.socksEstablishedPromise?.futureResult } private let deadline: NIODeadline diff --git a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/TLSEventsHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/TLSEventsHandler.swift index aab26fda8..bebd0bcc7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/TLSEventsHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/ChannelHandler/TLSEventsHandler.swift @@ -31,7 +31,7 @@ final class TLSEventsHandler: ChannelInboundHandler, RemovableChannelHandler { private var tlsEstablishedPromise: EventLoopPromise? var tlsEstablishedFuture: EventLoopFuture? { - return self.tlsEstablishedPromise?.futureResult + self.tlsEstablishedPromise?.futureResult } private let deadline: NIODeadline? diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 41a56c91b..74a0c72d7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -100,9 +100,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { // MARK: Channel Inbound Handler func channelActive(context: ChannelHandlerContext) { - self.logger.trace("Channel active", metadata: [ - "ahc-channel-writable": "\(context.channel.isWritable)", - ]) + self.logger.trace( + "Channel active", + metadata: [ + "ahc-channel-writable": "\(context.channel.isWritable)" + ] + ) let action = self.state.channelActive(isWritable: context.channel.isWritable) self.run(action, context: context) @@ -116,9 +119,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } func channelWritabilityChanged(context: ChannelHandlerContext) { - self.logger.trace("Channel writability changed", metadata: [ - "ahc-channel-writable": "\(context.channel.isWritable)", - ]) + self.logger.trace( + "Channel writability changed", + metadata: [ + "ahc-channel-writable": "\(context.channel.isWritable)" + ] + ) if let timeoutAction = self.idleWriteTimeoutStateMachine?.channelWritabilityChanged(context: context) { self.runTimeoutAction(timeoutAction, context: context) @@ -132,9 +138,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { func channelRead(context: ChannelHandlerContext, data: NIOAny) { let httpPart = self.unwrapInboundIn(data) - self.logger.trace("HTTP response part received", metadata: [ - "ahc-http-part": "\(httpPart)", - ]) + self.logger.trace( + "HTTP response part received", + metadata: [ + "ahc-http-part": "\(httpPart)" + ] + ) if let timeoutAction = self.idleReadTimeoutStateMachine?.channelRead(httpPart) { self.runTimeoutAction(timeoutAction, context: context) @@ -152,9 +161,12 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { } func errorCaught(context: ChannelHandlerContext, error: Error) { - self.logger.trace("Channel error caught", metadata: [ - "ahc-error": "\(error)", - ]) + self.logger.trace( + "Channel error caught", + metadata: [ + "ahc-error": "\(error)" + ] + ) let action = self.state.errorHappened(error) self.run(action, context: context) @@ -447,7 +459,8 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { // MARK: Private HTTPRequestExecutor - private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { + private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) + { guard self.request === request, let context = self.channelContext else { // Because the HTTPExecutableRequest may run in a different thread to our eventLoop, // calls from the HTTPExecutableRequest to our ChannelHandler may arrive here after @@ -691,7 +704,9 @@ struct IdleWriteStateMachine { self.state = .waitingForWritabilityEnabled return .clearIdleWriteTimeoutTimer case .waitingForWritabilityEnabled: - preconditionFailure("If the channel was writable before, then we should have been waiting for more data.") + preconditionFailure( + "If the channel was writable before, then we should have been waiting for more data." + ) case .requestEndSent: return .none } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift index ee0a78498..e0496f2e3 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1Connection.swift @@ -39,9 +39,11 @@ final class HTTP1Connection { let id: HTTPConnectionPool.Connection.ID - init(channel: Channel, - connectionID: HTTPConnectionPool.Connection.ID, - delegate: HTTP1ConnectionDelegate) { + init( + channel: Channel, + connectionID: HTTPConnectionPool.Connection.ID, + delegate: HTTP1ConnectionDelegate + ) { self.channel = channel self.id = connectionID self.delegate = delegate @@ -80,7 +82,7 @@ final class HTTP1Connection { } func close(promise: EventLoopPromise?) { - return self.channel.close(mode: .all, promise: promise) + self.channel.close(mode: .all, promise: promise) } func close() -> EventLoopFuture { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index ed4594183..aee0736ff 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -140,7 +140,7 @@ struct HTTP1ConnectionStateMachine { self.state = .closed return .fireChannelError(error, closeConnection: false) - case .inRequest(var requestStateMachine, close: let close): + case .inRequest(var requestStateMachine, let close): return self.avoidingStateMachineCoW { state -> Action in let action = requestStateMachine.errorHappened(error) state = .inRequest(requestStateMachine, close: close) @@ -239,7 +239,9 @@ struct HTTP1ConnectionStateMachine { mutating func requestCancelled(closeConnection: Bool) -> Action { switch self.state { case .initialized: - fatalError("This event must only happen, if the connection is leased. During startup this is impossible. Invalid state: \(self.state)") + fatalError( + "This event must only happen, if the connection is leased. During startup this is impossible. Invalid state: \(self.state)" + ) case .idle: if closeConnection { @@ -249,7 +251,7 @@ struct HTTP1ConnectionStateMachine { return .wait } - case .inRequest(var requestStateMachine, close: let close): + case .inRequest(var requestStateMachine, let close): return self.avoidingStateMachineCoW { state -> Action in let action = requestStateMachine.requestCancelled() state = .inRequest(requestStateMachine, close: close || closeConnection) @@ -415,12 +417,16 @@ extension HTTP1ConnectionStateMachine { } extension HTTP1ConnectionStateMachine.State { - fileprivate mutating func modify(with action: HTTPRequestStateMachine.Action) -> HTTP1ConnectionStateMachine.Action { + fileprivate mutating func modify(with action: HTTPRequestStateMachine.Action) -> HTTP1ConnectionStateMachine.Action + { switch action { case .sendRequestHead(let head, let sendEnd): return .sendRequestHead(head, sendEnd: sendEnd) case .notifyRequestHeadSendSuccessfully(let resumeRequestBodyStream, let startIdleTimer): - return .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: resumeRequestBodyStream, startIdleTimer: startIdleTimer) + return .notifyRequestHeadSendSuccessfully( + resumeRequestBodyStream: resumeRequestBodyStream, + startIdleTimer: startIdleTimer + ) case .pauseRequestBodyStream: return .pauseRequestBodyStream case .resumeRequestBodyStream: @@ -458,7 +464,7 @@ extension HTTP1ConnectionStateMachine.State { fatalError("Invalid state: \(self)") case .idle: fatalError("How can we fail a task, if we are idle") - case .inRequest(_, close: let close): + case .inRequest(_, let close): if case .close(let promise) = finalAction { self = .closing return .failRequest(error, .close(promise)) @@ -502,7 +508,7 @@ extension HTTP1ConnectionStateMachine: CustomStringConvertible { return ".initialized" case .idle: return ".idle" - case .inRequest(let request, close: let close): + case .inRequest(let request, let close): return ".inRequest(\(request), closeAfterRequest: \(close))" case .closing: return ".closing" diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 1520ff414..01a248d72 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -68,8 +68,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } func handlerAdded(context: ChannelHandlerContext) { - assert(context.eventLoop === self.eventLoop, - "The handler must be added to a channel that runs on the eventLoop it was initialized with.") + assert( + context.eventLoop === self.eventLoop, + "The handler must be added to a channel that runs on the eventLoop it was initialized with." + ) self.channelContext = context let isWritable = context.channel.isActive && context.channel.isWritable @@ -216,7 +218,7 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // that the request is neither failed nor finished yet self.request!.resumeRequestBodyStream() - case .forwardResponseHead(let head, pauseRequestBodyStream: let pauseRequestBodyStream): + case .forwardResponseHead(let head, let pauseRequestBodyStream): // We can force unwrap the request here, as we have just validated in the state machine, // that the request is neither failed nor finished yet self.request!.receiveResponseHead(head) @@ -268,7 +270,10 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { self.run(self.state.headSent(), context: context) } - private func runSuccessfulFinalAction(_ action: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, context: ChannelHandlerContext) { + private func runSuccessfulFinalAction( + _ action: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, + context: ChannelHandlerContext + ) { switch action { case .close, .none: // The actions returned here come from an `HTTPRequestStateMachine` that assumes http/1.1 @@ -281,7 +286,11 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } } - private func runFailedFinalAction(_ action: HTTPRequestStateMachine.Action.FinalFailedRequestAction, context: ChannelHandlerContext, error: Error) { + private func runFailedFinalAction( + _ action: HTTPRequestStateMachine.Action.FinalFailedRequestAction, + context: ChannelHandlerContext, + error: Error + ) { // We must close the http2 stream after the request has finished. Since the request failed, // we have no idea what the h2 streams state was. To be on the save side, we explicitly close // the h2 stream. This will break a reference cycle in HTTP2Connection. @@ -368,7 +377,8 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { // MARK: Private HTTPRequestExecutor - private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { + private func writeRequestBodyPart0(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) + { guard self.request === request, let context = self.channelContext else { // Because the HTTPExecutableRequest may run in a different thread to our eventLoop, // calls from the HTTPExecutableRequest to our ChannelHandler may arrive here after diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift index ab43558c0..5e4ae6e01 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2Connection.swift @@ -89,12 +89,14 @@ final class HTTP2Connection { self.channel.closeFuture } - init(channel: Channel, - connectionID: HTTPConnectionPool.Connection.ID, - decompression: HTTPClient.Decompression, - maximumConnectionUses: Int?, - delegate: HTTP2ConnectionDelegate, - logger: Logger) { + init( + channel: Channel, + connectionID: HTTPConnectionPool.Connection.ID, + decompression: HTTPClient.Decompression, + maximumConnectionUses: Int?, + delegate: HTTP2ConnectionDelegate, + logger: Logger + ) { self.channel = channel self.id = connectionID self.decompression = decompression @@ -103,7 +105,7 @@ final class HTTP2Connection { self.multiplexer = HTTP2StreamMultiplexer( mode: .client, channel: channel, - targetWindowSize: 8 * 1024 * 1024, // 8mb + targetWindowSize: 8 * 1024 * 1024, // 8mb outboundBufferSizeHighWatermark: 8196, outboundBufferSizeLowWatermark: 4092, inboundStreamInitializer: { channel -> EventLoopFuture in @@ -162,7 +164,7 @@ final class HTTP2Connection { } func close(promise: EventLoopPromise?) { - return self.channel.close(mode: .all, promise: promise) + self.channel.close(mode: .all, promise: promise) } func close() -> EventLoopFuture { @@ -199,7 +201,11 @@ final class HTTP2Connection { let sync = self.channel.pipeline.syncOperations let http2Handler = NIOHTTP2Handler(mode: .client, initialSettings: Self.defaultSettings) - let idleHandler = HTTP2IdleHandler(delegate: self, logger: self.logger, maximumConnectionUses: self.maximumConnectionUses) + let idleHandler = HTTP2IdleHandler( + delegate: self, + logger: self.logger, + maximumConnectionUses: self.maximumConnectionUses + ) try sync.addHandler(http2Handler, position: .last) try sync.addHandler(idleHandler, position: .last) @@ -221,7 +227,8 @@ final class HTTP2Connection { case .active: let createStreamChannelPromise = self.channel.eventLoop.makePromise(of: Channel.self) - self.multiplexer.createStreamChannel(promise: createStreamChannelPromise) { channel -> EventLoopFuture in + self.multiplexer.createStreamChannel(promise: createStreamChannelPromise) { + channel -> EventLoopFuture in do { // the connection may have been asked to shutdown while we created the child. in // this @@ -278,7 +285,7 @@ final class HTTP2Connection { self.state = .closing // inform all open streams, that the currently running request should be cancelled. - self.openStreams.forEach { box in + for box in self.openStreams { box.channel.triggerUserOutboundEvent(HTTPConnectionEvent.shutdownRequested, promise: nil) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift index 06458cb7e..64a151489 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2IdleHandler.swift @@ -184,9 +184,15 @@ extension HTTP2IdleHandler { self.state = .active(openStreams: 0, maxStreams: maxStreams, remainingUses: remainingUses) return .notifyConnectionNewMaxStreamsSettings(maxStreams) - case .active(openStreams: let openStreams, maxStreams: let maxStreams, remainingUses: let remainingUses): - if let newMaxStreams = settings.last(where: { $0.parameter == .maxConcurrentStreams })?.value, newMaxStreams != maxStreams { - self.state = .active(openStreams: openStreams, maxStreams: newMaxStreams, remainingUses: remainingUses) + case .active(let openStreams, let maxStreams, let remainingUses): + if let newMaxStreams = settings.last(where: { $0.parameter == .maxConcurrentStreams })?.value, + newMaxStreams != maxStreams + { + self.state = .active( + openStreams: openStreams, + maxStreams: newMaxStreams, + remainingUses: remainingUses + ) return .notifyConnectionNewMaxStreamsSettings(newMaxStreams) } return .nothing diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 3a0011d5e..0aad0c8dd 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -20,6 +20,7 @@ import NIOPosix import NIOSOCKS import NIOSSL import NIOTLS + #if canImport(Network) import NIOTransportServices #endif @@ -31,14 +32,17 @@ extension HTTPConnectionPool { let tlsConfiguration: TLSConfiguration let sslContextCache: SSLContextCache - init(key: ConnectionPool.Key, - tlsConfiguration: TLSConfiguration?, - clientConfiguration: HTTPClient.Configuration, - sslContextCache: SSLContextCache) { + init( + key: ConnectionPool.Key, + tlsConfiguration: TLSConfiguration?, + clientConfiguration: HTTPClient.Configuration, + sslContextCache: SSLContextCache + ) { self.key = key self.clientConfiguration = clientConfiguration self.sslContextCache = sslContextCache - self.tlsConfiguration = tlsConfiguration ?? clientConfiguration.tlsConfiguration ?? .makeClientConfiguration() + self.tlsConfiguration = + tlsConfiguration ?? clientConfiguration.tlsConfiguration ?? .makeClientConfiguration() } } } @@ -63,7 +67,13 @@ extension HTTPConnectionPool.ConnectionFactory { var logger = logger logger[metadataKey: "ahc-connection-id"] = "\(connectionID)" - self.makeChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger).whenComplete { result in + self.makeChannel( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop, + logger: logger + ).whenComplete { result in switch result { case .success(.http1_1(let channel)): do { @@ -137,7 +147,13 @@ extension HTTPConnectionPool.ConnectionFactory { ) } } else { - channelFuture = self.makeNonProxiedChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger) + channelFuture = self.makeNonProxiedChannel( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop, + logger: logger + ) } // let's map `ChannelError.connectTimeout` into a `HTTPClientError.connectTimeout` @@ -160,10 +176,22 @@ extension HTTPConnectionPool.ConnectionFactory { ) -> EventLoopFuture { switch self.key.scheme { case .http, .httpUnix, .unix: - return self.makePlainChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop).map { .http1_1($0) } + return self.makePlainChannel( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop + ).map { .http1_1($0) } case .https, .httpsUnix: - return self.makeTLSChannel(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop, logger: logger).flatMapThrowing { - channel, negotiated in + return self.makeTLSChannel( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop, + logger: logger + ).flatMapThrowing { + channel, + negotiated in try self.matchALPNToHTTPVersion(negotiated, channel: channel) } @@ -177,7 +205,12 @@ extension HTTPConnectionPool.ConnectionFactory { eventLoop: EventLoop ) -> EventLoopFuture { precondition(!self.key.scheme.usesTLS, "Unexpected scheme") - return self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop).connect(target: self.key.connectionTarget) + return self.makePlainBootstrap( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop + ).connect(target: self.key.connectionTarget) } private func makeHTTPProxyChannel( @@ -191,7 +224,12 @@ extension HTTPConnectionPool.ConnectionFactory { // A proxy connection starts with a plain text connection to the proxy server. After // the connection has been established with the proxy server, the connection might be // upgraded to TLS before we send our first request. - let bootstrap = self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop) + let bootstrap = self.makePlainBootstrap( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop + ) return bootstrap.connect(host: proxy.host, port: proxy.port).flatMap { channel in let encoder = HTTPRequestEncoder() let decoder = ByteToMessageHandler(HTTPResponseDecoder(leftOverBytesStrategy: .dropBytes)) @@ -234,7 +272,12 @@ extension HTTPConnectionPool.ConnectionFactory { // A proxy connection starts with a plain text connection to the proxy server. After // the connection has been established with the proxy server, the connection might be // upgraded to TLS before we send our first request. - let bootstrap = self.makePlainBootstrap(requester: requester, connectionID: connectionID, deadline: deadline, eventLoop: eventLoop) + let bootstrap = self.makePlainBootstrap( + requester: requester, + connectionID: connectionID, + deadline: deadline, + eventLoop: eventLoop + ) return bootstrap.connect(host: proxy.host, port: proxy.port).flatMap { channel in let socksConnectHandler = SOCKSClientHandler(targetAddress: SOCKSAddress(self.key.connectionTarget)) let socksEventHandler = SOCKSEventsHandler(deadline: deadline) @@ -319,15 +362,26 @@ extension HTTPConnectionPool.ConnectionFactory { eventLoop: EventLoop ) -> NIOClientTCPBootstrapProtocol { #if canImport(Network) - if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { - return tsBootstrap - .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) - .channelOption(NIOTSChannelOptions.multipathServiceType, value: self.clientConfiguration.enableMultipath ? .handover : .disabled) + if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), + let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) + { + return + tsBootstrap + .channelOption( + NIOTSChannelOptions.waitForActivity, + value: self.clientConfiguration.networkFrameworkWaitForConnectivity + ) + .channelOption( + NIOTSChannelOptions.multipathServiceType, + value: self.clientConfiguration.enableMultipath ? .handover : .disabled + ) .connectTimeout(deadline - NIODeadline.now()) .channelInitializer { channel in do { try channel.pipeline.syncOperations.addHandler(HTTPClient.NWErrorHandler()) - try channel.pipeline.syncOperations.addHandler(NWWaitingHandler(requester: requester, connectionID: connectionID)) + try channel.pipeline.syncOperations.addHandler( + NWWaitingHandler(requester: requester, connectionID: connectionID) + ) return channel.eventLoop.makeSucceededVoidFuture() } catch { return channel.eventLoop.makeFailedFuture(error) @@ -337,7 +391,8 @@ extension HTTPConnectionPool.ConnectionFactory { #endif if let nioBootstrap = ClientBootstrap(validatingGroup: eventLoop) { - return nioBootstrap + return + nioBootstrap .connectTimeout(deadline - NIODeadline.now()) .enableMPTCP(clientConfiguration.enableMultipath) } @@ -362,7 +417,7 @@ extension HTTPConnectionPool.ConnectionFactory { ) var channelFuture = bootstrapFuture.flatMap { bootstrap -> EventLoopFuture in - return bootstrap.connect(target: self.key.connectionTarget) + bootstrap.connect(target: self.key.connectionTarget) }.flatMap { channel -> EventLoopFuture<(Channel, String?)> in do { // if the channel is closed before flatMap is executed, all ChannelHandler are removed @@ -375,7 +430,10 @@ extension HTTPConnectionPool.ConnectionFactory { channel.pipeline.removeHandler(tlsEventHandler).map { (channel, negotiated) } } } catch { - assert(channel.isActive == false, "if the channel is still active then TLSEventsHandler must be present but got error \(error)") + assert( + channel.isActive == false, + "if the channel is still active then TLSEventsHandler must be present but got error \(error)" + ) return channel.eventLoop.makeFailedFuture(HTTPClientError.remoteConnectionClosed) } } @@ -410,20 +468,33 @@ extension HTTPConnectionPool.ConnectionFactory { } #if canImport(Network) - if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) { + if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), + let tsBootstrap = NIOTSConnectionBootstrap(validatingGroup: eventLoop) + { // create NIOClientTCPBootstrap with NIOTS TLS provider - let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions(on: eventLoop, serverNameIndicatorOverride: key.serverNameIndicatorOverride).map { + let bootstrapFuture = tlsConfig.getNWProtocolTLSOptions( + on: eventLoop, + serverNameIndicatorOverride: key.serverNameIndicatorOverride + ).map { options -> NIOClientTCPBootstrapProtocol in tsBootstrap - .channelOption(NIOTSChannelOptions.waitForActivity, value: self.clientConfiguration.networkFrameworkWaitForConnectivity) - .channelOption(NIOTSChannelOptions.multipathServiceType, value: self.clientConfiguration.enableMultipath ? .handover : .disabled) + .channelOption( + NIOTSChannelOptions.waitForActivity, + value: self.clientConfiguration.networkFrameworkWaitForConnectivity + ) + .channelOption( + NIOTSChannelOptions.multipathServiceType, + value: self.clientConfiguration.enableMultipath ? .handover : .disabled + ) .connectTimeout(deadline - NIODeadline.now()) .tlsOptions(options) .channelInitializer { channel in do { try channel.pipeline.syncOperations.addHandler(HTTPClient.NWErrorHandler()) - try channel.pipeline.syncOperations.addHandler(NWWaitingHandler(requester: requester, connectionID: connectionID)) + try channel.pipeline.syncOperations.addHandler( + NWWaitingHandler(requester: requester, connectionID: connectionID) + ) // we don't need to set a TLS deadline for NIOTS connections, since the // TLS handshake is part of the TS connection bootstrap. If the TLS // handshake times out the complete connection creation will be failed. diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift index f5a0540cf..3fdf93752 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Manager.swift @@ -39,9 +39,11 @@ extension HTTPConnectionPool { private let sslContextCache = SSLContextCache() - init(eventLoopGroup: EventLoopGroup, - configuration: HTTPClient.Configuration, - backgroundActivityLogger logger: Logger) { + init( + eventLoopGroup: EventLoopGroup, + configuration: HTTPClient.Configuration, + backgroundActivityLogger logger: Logger + ) { self.eventLoopGroup = eventLoopGroup self.configuration = configuration self.logger = logger @@ -118,7 +120,7 @@ extension HTTPConnectionPool { promise?.succeed(false) case .shutdown(let pools): - pools.values.forEach { pool in + for pool in pools.values { pool.shutdown() } } @@ -140,7 +142,9 @@ extension HTTPConnectionPool.Manager: HTTPConnectionPoolDelegate { case .shuttingDown(let promise, let soFarUnclean): guard self._pools.removeValue(forKey: pool.key) === pool else { - preconditionFailure("Expected that the pool was created by this manager and is known for this reason.") + preconditionFailure( + "Expected that the pool was created by this manager and is known for this reason." + ) } if self._pools.isEmpty { @@ -154,7 +158,7 @@ extension HTTPConnectionPool.Manager: HTTPConnectionPoolDelegate { } switch closeAction { - case .close(let promise, unclean: let unclean): + case .close(let promise, let unclean): promise?.succeed(unclean) case .wait: break @@ -173,7 +177,7 @@ extension HTTPConnectionPool.Connection.ID { } func next() -> Int { - return self.atomic.loadThenWrappingIncrement(ordering: .relaxed) + self.atomic.loadThenWrappingIncrement(ordering: .relaxed) } } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index 093c1e328..e7f1d8ce5 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -44,14 +44,16 @@ final class HTTPConnectionPool { let delegate: HTTPConnectionPoolDelegate - init(eventLoopGroup: EventLoopGroup, - sslContextCache: SSLContextCache, - tlsConfiguration: TLSConfiguration?, - clientConfiguration: HTTPClient.Configuration, - key: ConnectionPool.Key, - delegate: HTTPConnectionPoolDelegate, - idGenerator: Connection.ID.Generator, - backgroundActivityLogger logger: Logger) { + init( + eventLoopGroup: EventLoopGroup, + sslContextCache: SSLContextCache, + tlsConfiguration: TLSConfiguration?, + clientConfiguration: HTTPClient.Configuration, + key: ConnectionPool.Key, + delegate: HTTPConnectionPoolDelegate, + idGenerator: Connection.ID.Generator, + backgroundActivityLogger logger: Logger + ) { self.eventLoopGroup = eventLoopGroup self.connectionFactory = ConnectionFactory( key: key, @@ -70,7 +72,8 @@ final class HTTPConnectionPool { self._state = StateMachine( idGenerator: idGenerator, - maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool.concurrentHTTP1ConnectionsPerHostSoftLimit, + maximumConcurrentHTTP1Connections: clientConfiguration.connectionPool + .concurrentHTTP1ConnectionsPerHostSoftLimit, retryConnectionEstablishment: clientConfiguration.connectionPool.retryConnectionEstablishment, preferHTTP1: clientConfiguration.httpVersion == .http1Only, maximumConnectionUses: clientConfiguration.maximumUsesPerConnection @@ -150,7 +153,7 @@ final class HTTPConnectionPool { self.unlocked = Unlocked(connection: .none, request: .none) switch stateMachineAction.request { - case .executeRequest(let request, let connection, cancelTimeout: let cancelTimeout): + case .executeRequest(let request, let connection, let cancelTimeout): if cancelTimeout { self.locked.request = .cancelRequestTimeout(request.id) } @@ -158,7 +161,7 @@ final class HTTPConnectionPool { case .executeRequestsAndCancelTimeouts(let requests, let connection): self.locked.request = .cancelRequestTimeouts(requests) self.unlocked.request = .executeRequests(requests, connection) - case .failRequest(let request, let error, cancelTimeout: let cancelTimeout): + case .failRequest(let request, let error, let cancelTimeout): if cancelTimeout { self.locked.request = .cancelRequestTimeout(request.id) } @@ -175,15 +178,15 @@ final class HTTPConnectionPool { switch stateMachineAction.connection { case .createConnection(let connectionID, on: let eventLoop): self.unlocked.connection = .createConnection(connectionID, on: eventLoop) - case .scheduleBackoffTimer(let connectionID, backoff: let backoff, on: let eventLoop): + case .scheduleBackoffTimer(let connectionID, let backoff, on: let eventLoop): self.locked.connection = .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop) case .scheduleTimeoutTimer(let connectionID, on: let eventLoop): self.locked.connection = .scheduleTimeoutTimer(connectionID, on: eventLoop) case .cancelTimeoutTimer(let connectionID): self.locked.connection = .cancelTimeoutTimer(connectionID) - case .closeConnection(let connection, isShutdown: let isShutdown): + case .closeConnection(let connection, let isShutdown): self.unlocked.connection = .closeConnection(connection, isShutdown: isShutdown) - case .cleanupConnections(var cleanupContext, isShutdown: let isShutdown): + case .cleanupConnections(var cleanupContext, let isShutdown): // self.locked.connection = .cancelBackoffTimers(cleanupContext.connectBackoff) cleanupContext.connectBackoff = [] @@ -221,7 +224,7 @@ final class HTTPConnectionPool { private func runLockedConnectionAction(_ action: Actions.ConnectionAction.Locked) { switch action { - case .scheduleBackoffTimer(let connectionID, backoff: let backoff, on: let eventLoop): + case .scheduleBackoffTimer(let connectionID, let backoff, on: let eventLoop): self.scheduleConnectionStartBackoffTimer(connectionID, backoff, on: eventLoop) case .scheduleTimeoutTimer(let connectionID, on: let eventLoop): @@ -249,7 +252,7 @@ final class HTTPConnectionPool { self.cancelRequestTimeout(requestID) case .cancelRequestTimeouts(let requests): - requests.forEach { self.cancelRequestTimeout($0.id) } + for request in requests { self.cancelRequestTimeout(request.id) } case .none: break @@ -266,10 +269,13 @@ final class HTTPConnectionPool { case .createConnection(let connectionID, let eventLoop): self.createConnection(connectionID, on: eventLoop) - case .closeConnection(let connection, isShutdown: let isShutdown): - self.logger.trace("close connection", metadata: [ - "ahc-connection-id": "\(connection.id)", - ]) + case .closeConnection(let connection, let isShutdown): + self.logger.trace( + "close connection", + metadata: [ + "ahc-connection-id": "\(connection.id)" + ] + ) // we are not interested in the close promise... connection.close(promise: nil) @@ -278,7 +284,7 @@ final class HTTPConnectionPool { self.delegate.connectionPoolDidShutdown(self, unclean: unclean) } - case .cleanupConnections(let cleanupContext, isShutdown: let isShutdown): + case .cleanupConnections(let cleanupContext, let isShutdown): for connection in cleanupContext.close { connection.close(promise: nil) } @@ -315,13 +321,13 @@ final class HTTPConnectionPool { connection.executeRequest(request.req) case .executeRequests(let requests, let connection): - requests.forEach { connection.executeRequest($0.req) } + for request in requests { connection.executeRequest(request.req) } case .failRequest(let request, let error): request.req.fail(error) case .failRequests(let requests, let error): - requests.forEach { $0.req.fail(error) } + for request in requests { request.req.fail(error) } case .none: break @@ -329,9 +335,12 @@ final class HTTPConnectionPool { } private func createConnection(_ connectionID: Connection.ID, on eventLoop: EventLoop) { - self.logger.trace("Opening fresh connection", metadata: [ - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.trace( + "Opening fresh connection", + metadata: [ + "ahc-connection-id": "\(connectionID)" + ] + ) // Even though this function is called make it actually creates/establishes a connection. // TBD: Should we rename it? To what? self.connectionFactory.makeConnection( @@ -374,9 +383,12 @@ final class HTTPConnectionPool { } private func scheduleIdleTimerForConnection(_ connectionID: Connection.ID, on eventLoop: EventLoop) { - self.logger.trace("Schedule idle connection timeout timer", metadata: [ - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.trace( + "Schedule idle connection timeout timer", + metadata: [ + "ahc-connection-id": "\(connectionID)" + ] + ) let scheduled = eventLoop.scheduleTask(in: self.idleConnectionTimeout) { // there might be a race between a cancelTimer call and the triggering // of this scheduled task. both want to acquire the lock @@ -394,9 +406,12 @@ final class HTTPConnectionPool { } private func cancelIdleTimerForConnection(_ connectionID: Connection.ID) { - self.logger.trace("Cancel idle connection timeout timer", metadata: [ - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.trace( + "Cancel idle connection timeout timer", + metadata: [ + "ahc-connection-id": "\(connectionID)" + ] + ) guard let cancelTimer = self._idleTimer.removeValue(forKey: connectionID) else { preconditionFailure("Expected to have an idle timer for connection \(connectionID) at this point.") } @@ -408,9 +423,12 @@ final class HTTPConnectionPool { _ timeAmount: TimeAmount, on eventLoop: EventLoop ) { - self.logger.trace("Schedule connection creation backoff timer", metadata: [ - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.trace( + "Schedule connection creation backoff timer", + metadata: [ + "ahc-connection-id": "\(connectionID)" + ] + ) let scheduled = eventLoop.scheduleTask(in: timeAmount) { // there might be a race between a backoffTimer and the pool shutting down. @@ -439,41 +457,53 @@ final class HTTPConnectionPool { extension HTTPConnectionPool: HTTPConnectionRequester { func http1ConnectionCreated(_ connection: HTTP1Connection) { - self.logger.trace("successfully created connection", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/1.1", - ]) + self.logger.trace( + "successfully created connection", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/1.1", + ] + ) self.modifyStateAndRunActions { $0.newHTTP1ConnectionCreated(.http1_1(connection)) } } func http2ConnectionCreated(_ connection: HTTP2Connection, maximumStreams: Int) { - self.logger.trace("successfully created connection", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/2", - "ahc-max-streams": "\(maximumStreams)", - ]) + self.logger.trace( + "successfully created connection", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/2", + "ahc-max-streams": "\(maximumStreams)", + ] + ) self.modifyStateAndRunActions { $0.newHTTP2ConnectionCreated(.http2(connection), maxConcurrentStreams: maximumStreams) } } func failedToCreateHTTPConnection(_ connectionID: HTTPConnectionPool.Connection.ID, error: Error) { - self.logger.debug("connection attempt failed", metadata: [ - "ahc-error": "\(error)", - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.debug( + "connection attempt failed", + metadata: [ + "ahc-error": "\(error)", + "ahc-connection-id": "\(connectionID)", + ] + ) self.modifyStateAndRunActions { $0.failedToCreateNewConnection(error, connectionID: connectionID) } } func waitingForConnectivity(_ connectionID: HTTPConnectionPool.Connection.ID, error: Error) { - self.logger.debug("waiting for connectivity", metadata: [ - "ahc-error": "\(error)", - "ahc-connection-id": "\(connectionID)", - ]) + self.logger.debug( + "waiting for connectivity", + metadata: [ + "ahc-error": "\(error)", + "ahc-connection-id": "\(connectionID)", + ] + ) self.modifyStateAndRunActions { $0.waitingForConnectivity(error, connectionID: connectionID) } @@ -482,20 +512,26 @@ extension HTTPConnectionPool: HTTPConnectionRequester { extension HTTPConnectionPool: HTTP1ConnectionDelegate { func http1ConnectionClosed(_ connection: HTTP1Connection) { - self.logger.debug("connection closed", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/1.1", - ]) + self.logger.debug( + "connection closed", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/1.1", + ] + ) self.modifyStateAndRunActions { $0.http1ConnectionClosed(connection.id) } } func http1ConnectionReleased(_ connection: HTTP1Connection) { - self.logger.trace("releasing connection", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/1.1", - ]) + self.logger.trace( + "releasing connection", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/1.1", + ] + ) self.modifyStateAndRunActions { $0.http1ConnectionReleased(connection.id) } @@ -504,41 +540,53 @@ extension HTTPConnectionPool: HTTP1ConnectionDelegate { extension HTTPConnectionPool: HTTP2ConnectionDelegate { func http2Connection(_ connection: HTTP2Connection, newMaxStreamSetting: Int) { - self.logger.debug("new max stream setting", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/2", - "ahc-max-streams": "\(newMaxStreamSetting)", - ]) + self.logger.debug( + "new max stream setting", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/2", + "ahc-max-streams": "\(newMaxStreamSetting)", + ] + ) self.modifyStateAndRunActions { $0.newHTTP2MaxConcurrentStreamsReceived(connection.id, newMaxStreams: newMaxStreamSetting) } } func http2ConnectionGoAwayReceived(_ connection: HTTP2Connection) { - self.logger.debug("connection go away received", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/2", - ]) + self.logger.debug( + "connection go away received", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/2", + ] + ) self.modifyStateAndRunActions { $0.http2ConnectionGoAwayReceived(connection.id) } } func http2ConnectionClosed(_ connection: HTTP2Connection) { - self.logger.debug("connection closed", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/2", - ]) + self.logger.debug( + "connection closed", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/2", + ] + ) self.modifyStateAndRunActions { $0.http2ConnectionClosed(connection.id) } } func http2ConnectionStreamClosed(_ connection: HTTP2Connection, availableStreams: Int) { - self.logger.trace("stream closed", metadata: [ - "ahc-connection-id": "\(connection.id)", - "ahc-http-version": "http/2", - ]) + self.logger.trace( + "stream closed", + metadata: [ + "ahc-connection-id": "\(connection.id)", + "ahc-http-version": "http/2", + ] + ) self.modifyStateAndRunActions { $0.http2ConnectionStreamClosed(connection.id) } @@ -642,7 +690,9 @@ extension HTTPConnectionPool { return lhsConn.id == rhsConn.id case (.http2(let lhsConn), .http2(let rhsConn)): return lhsConn.id == rhsConn.id - case (.__testOnly_connection(let lhsID, let lhsEventLoop), .__testOnly_connection(let rhsID, let rhsEventLoop)): + case ( + .__testOnly_connection(let lhsID, let lhsEventLoop), .__testOnly_connection(let rhsID, let rhsEventLoop) + ): return lhsID == rhsID && lhsEventLoop === rhsEventLoop default: return false @@ -723,7 +773,7 @@ struct EventLoopID: Hashable { } static func __testOnly_fakeID(_ id: Int) -> EventLoopID { - return EventLoopID(.__testOnly_fakeID(id)) + EventLoopID(.__testOnly_fakeID(id)) } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine+Demand.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine+Demand.swift index 90578bc87..5c5b893e0 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine+Demand.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine+Demand.swift @@ -104,8 +104,8 @@ extension HTTPRequestStateMachine { // forwarded to the user. case .waitingForRead, - .waitingForDemand, - .waitingForReadOrDemand: + .waitingForDemand, + .waitingForReadOrDemand: return nil case .modifying: @@ -174,8 +174,8 @@ extension HTTPRequestStateMachine { return (buffer, .none) case .waitingForReadOrDemand(let buffer), - .waitingForRead(let buffer), - .waitingForDemand(let buffer): + .waitingForRead(let buffer), + .waitingForDemand(let buffer): // Normally this code path should never be hit. However there is one way to trigger // this: // diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift index 533062036..e06389360 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPRequestStateMachine.swift @@ -161,10 +161,10 @@ struct HTTPRequestStateMachine { switch self.state { case .initialized, - .running(.streaming(_, _, producer: .producing), _), - .running(.endSent, _), - .finished, - .failed: + .running(.streaming(_, _, producer: .producing), _), + .running(.endSent, _), + .finished, + .failed: return .wait case .waitForChannelToBecomeWritable(let head, let metadata): @@ -196,11 +196,11 @@ struct HTTPRequestStateMachine { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(.streaming(_, _, producer: .paused), _), - .running(.endSent, _), - .finished, - .failed: + .waitForChannelToBecomeWritable, + .running(.streaming(_, _, producer: .paused), _), + .running(.endSent, _), + .finished, + .failed: return .wait case .running(.streaming(let expectedBodyLength, let sentBodyBytes, producer: .producing), let responseState): @@ -219,13 +219,16 @@ struct HTTPRequestStateMachine { mutating func errorHappened(_ error: Error) -> Action { if let error = error as? NIOSSLError, - error == .uncleanShutdown, - let action = self.handleNIOSSLUncleanShutdownError() { + error == .uncleanShutdown, + let action = self.handleNIOSSLUncleanShutdownError() + { return action } switch self.state { case .initialized: - preconditionFailure("After the state machine has been initialized, start must be called immediately. Thus this state is unreachable") + preconditionFailure( + "After the state machine has been initialized, start must be called immediately. Thus this state is unreachable" + ) case .waitForChannelToBecomeWritable: // the request failed, before it was sent onto the wire. self.state = .failed(error) @@ -247,14 +250,14 @@ struct HTTPRequestStateMachine { private mutating func handleNIOSSLUncleanShutdownError() -> Action? { switch self.state { case .running(.streaming, .waitingForHead), - .running(.endSent, .waitingForHead): + .running(.endSent, .waitingForHead): // if we received a NIOSSL.uncleanShutdown before we got an answer we should handle // this like a normal connection close. We will receive a call to channelInactive after // this error. return .wait case .running(.streaming, .receivingBody(let responseHead, _)), - .running(.endSent, .receivingBody(let responseHead, _)): + .running(.endSent, .receivingBody(let responseHead, _)): // This code is only reachable for request and responses, which we expect to have a body. // We depend on logic from the HTTPResponseDecoder here. The decoder will emit an // HTTPResponsePart.end right after the HTTPResponsePart.head, for every request with a @@ -263,7 +266,9 @@ struct HTTPRequestStateMachine { // For this reason we only need to check the "content-length" or "transfer-encoding" // headers here to determine if we are potentially in an EOF terminated response. - if responseHead.headers.contains(name: "content-length") || responseHead.headers.contains(name: "transfer-encoding") { + if responseHead.headers.contains(name: "content-length") + || responseHead.headers.contains(name: "transfer-encoding") + { // If we have already received the response head, the parser will ensure that we // receive a complete response, if the content-length or transfer-encoding header // was set. In this case we can ignore the NIOSSLError.uncleanShutdown. We will see @@ -285,9 +290,11 @@ struct HTTPRequestStateMachine { mutating func requestStreamPartReceived(_ part: IOData, promise: EventLoopPromise?) -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(.endSent, _): - preconditionFailure("We must be in the request streaming phase, if we receive further body parts. Invalid state: \(self.state)") + .waitForChannelToBecomeWritable, + .running(.endSent, _): + preconditionFailure( + "We must be in the request streaming phase, if we receive further body parts. Invalid state: \(self.state)" + ) case .running(.streaming(_, _, let producerState), .receivingBody(let head, _)) where head.status.code >= 300: // If we have already received a response head with status >= 300, we won't send out any @@ -349,9 +356,11 @@ struct HTTPRequestStateMachine { mutating func requestStreamFinished(promise: EventLoopPromise?) -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(.endSent, _): - preconditionFailure("A request body stream end is only expected if we are in state request streaming. Invalid state: \(self.state)") + .waitForChannelToBecomeWritable, + .running(.endSent, _): + preconditionFailure( + "A request body stream end is only expected if we are in state request streaming. Invalid state: \(self.state)" + ) case .running(.streaming(let expectedBodyLength, let sentBodyBytes, _), .waitingForHead): if let expected = expectedBodyLength, expected != sentBodyBytes { @@ -363,7 +372,10 @@ struct HTTPRequestStateMachine { self.state = .running(.endSent, .waitingForHead) return .sendRequestEnd(promise) - case .running(.streaming(let expectedBodyLength, let sentBodyBytes, _), .receivingBody(let head, let streamState)): + case .running( + .streaming(let expectedBodyLength, let sentBodyBytes, _), + .receivingBody(let head, let streamState) + ): assert(head.status.code < 300) if let expected = expectedBodyLength, expected != sentBodyBytes { @@ -456,11 +468,11 @@ struct HTTPRequestStateMachine { mutating func read() -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(_, .waitingForHead), - .running(_, .endReceived), - .finished, - .failed: + .waitForChannelToBecomeWritable, + .running(_, .waitingForHead), + .running(_, .endReceived), + .finished, + .failed: // If we are not in the middle of streaming the response body, we always want to get // more data... return .read @@ -493,11 +505,11 @@ struct HTTPRequestStateMachine { mutating func channelReadComplete() -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(_, .waitingForHead), - .running(_, .endReceived), - .finished, - .failed: + .waitForChannelToBecomeWritable, + .running(_, .waitingForHead), + .running(_, .endReceived), + .finished, + .failed: return .wait case .running(let requestState, .receivingBody(let head, var streamState)): @@ -528,7 +540,9 @@ struct HTTPRequestStateMachine { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response head before sending a request head ourselves \(self.state)") + preconditionFailure( + "How can we receive a response head before sending a request head ourselves \(self.state)" + ) case .running(.streaming(let expectedBodyLength, let sentBodyBytes, producer: .paused), .waitingForHead): self.state = .running( @@ -546,7 +560,11 @@ struct HTTPRequestStateMachine { return .forwardResponseHead(head, pauseRequestBodyStream: true) } else { self.state = .running( - .streaming(expectedBodyLength: expectedBodyLength, sentBodyBytes: sentBodyBytes, producer: .producing), + .streaming( + expectedBodyLength: expectedBodyLength, + sentBodyBytes: sentBodyBytes, + producer: .producing + ), .receivingBody(head, .init()) ) return .forwardResponseHead(head, pauseRequestBodyStream: false) @@ -557,7 +575,9 @@ struct HTTPRequestStateMachine { return .forwardResponseHead(head, pauseRequestBodyStream: false) case .running(_, .receivingBody), .running(_, .endReceived), .finished: - preconditionFailure("How can we successfully finish the request, before having received a head. Invalid state: \(self.state)") + preconditionFailure( + "How can we successfully finish the request, before having received a head. Invalid state: \(self.state)" + ) case .failed: return .wait @@ -569,10 +589,14 @@ struct HTTPRequestStateMachine { mutating func receivedHTTPResponseBodyPart(_ body: ByteBuffer) -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response head before completely sending a request head ourselves. Invalid state: \(self.state)") + preconditionFailure( + "How can we receive a response head before completely sending a request head ourselves. Invalid state: \(self.state)" + ) case .running(_, .waitingForHead): - preconditionFailure("How can we receive a response body, if we haven't received a head. Invalid state: \(self.state)") + preconditionFailure( + "How can we receive a response body, if we haven't received a head. Invalid state: \(self.state)" + ) case .running(let requestState, .receivingBody(let head, var responseStreamState)): return self.avoidingStateMachineCoW { state -> Action in @@ -582,7 +606,9 @@ struct HTTPRequestStateMachine { } case .running(_, .endReceived), .finished: - preconditionFailure("How can we successfully finish the request, before having received a head. Invalid state: \(self.state)") + preconditionFailure( + "How can we successfully finish the request, before having received a head. Invalid state: \(self.state)" + ) case .failed: return .wait @@ -595,20 +621,31 @@ struct HTTPRequestStateMachine { private mutating func receivedHTTPResponseEnd() -> Action { switch self.state { case .initialized, .waitForChannelToBecomeWritable: - preconditionFailure("How can we receive a response end before completely sending a request head ourselves. Invalid state: \(self.state)") + preconditionFailure( + "How can we receive a response end before completely sending a request head ourselves. Invalid state: \(self.state)" + ) case .running(_, .waitingForHead): - preconditionFailure("How can we receive a response end, if we haven't a received a head. Invalid state: \(self.state)") + preconditionFailure( + "How can we receive a response end, if we haven't a received a head. Invalid state: \(self.state)" + ) - case .running(.streaming(let expectedBodyLength, let sentBodyBytes, let producerState), .receivingBody(let head, var responseStreamState)) - where head.status.code < 300: + case .running( + .streaming(let expectedBodyLength, let sentBodyBytes, let producerState), + .receivingBody(let head, var responseStreamState) + ) + where head.status.code < 300: return self.avoidingStateMachineCoW { state -> Action in let (remainingBuffer, connectionAction) = responseStreamState.end() switch connectionAction { case .none: state = .running( - .streaming(expectedBodyLength: expectedBodyLength, sentBodyBytes: sentBodyBytes, producer: producerState), + .streaming( + expectedBodyLength: expectedBodyLength, + sentBodyBytes: sentBodyBytes, + producer: producerState + ), .endReceived ) return .forwardResponseBodyParts(remainingBuffer) @@ -624,7 +661,10 @@ struct HTTPRequestStateMachine { case .running(.streaming(_, _, let producerState), .receivingBody(let head, var responseStreamState)): assert(head.status.code >= 300) - assert(producerState == .paused, "Expected to have paused the request body stream, when the head was received. Invalid state: \(self.state)") + assert( + producerState == .paused, + "Expected to have paused the request body stream, when the head was received. Invalid state: \(self.state)" + ) return self.avoidingStateMachineCoW { state -> Action in // We can ignore the connectionAction from the responseStreamState, since the @@ -647,7 +687,9 @@ struct HTTPRequestStateMachine { } case .running(_, .endReceived), .finished: - preconditionFailure("How can we receive a response end, if another one was already received. Invalid state: \(self.state)") + preconditionFailure( + "How can we receive a response end, if another one was already received. Invalid state: \(self.state)" + ) case .failed: return .wait @@ -660,9 +702,11 @@ struct HTTPRequestStateMachine { mutating func demandMoreResponseBodyParts() -> Action { switch self.state { case .initialized, - .running(_, .waitingForHead), - .waitForChannelToBecomeWritable: - preconditionFailure("The response is expected to only ask for more data after the response head was forwarded \(self.state)") + .running(_, .waitingForHead), + .waitForChannelToBecomeWritable: + preconditionFailure( + "The response is expected to only ask for more data after the response head was forwarded \(self.state)" + ) case .running(let requestState, .receivingBody(let head, var responseStreamState)): return self.avoidingStateMachineCoW { state -> Action in @@ -672,8 +716,8 @@ struct HTTPRequestStateMachine { } case .running(_, .endReceived), - .finished, - .failed: + .finished, + .failed: return .wait case .modifying: @@ -684,9 +728,11 @@ struct HTTPRequestStateMachine { mutating func idleReadTimeoutTriggered() -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable, - .running(.streaming, _): - preconditionFailure("We only schedule idle read timeouts after we have sent the complete request. Invalid state: \(self.state)") + .waitForChannelToBecomeWritable, + .running(.streaming, _): + preconditionFailure( + "We only schedule idle read timeouts after we have sent the complete request. Invalid state: \(self.state)" + ) case .running(.endSent, .waitingForHead), .running(.endSent, .receivingBody): let error = HTTPClientError.readTimeout @@ -707,8 +753,10 @@ struct HTTPRequestStateMachine { mutating func idleWriteTimeoutTriggered() -> Action { switch self.state { case .initialized, - .waitForChannelToBecomeWritable: - preconditionFailure("We only schedule idle write timeouts while the request is being sent. Invalid state: \(self.state)") + .waitForChannelToBecomeWritable: + preconditionFailure( + "We only schedule idle write timeouts while the request is being sent. Invalid state: \(self.state)" + ) case .running(.streaming, _): let error = HTTPClientError.writeTimeout @@ -733,7 +781,10 @@ struct HTTPRequestStateMachine { self.state = .running(.endSent, .waitingForHead) return .sendRequestHead(head, sendEnd: true) } else { - self.state = .running(.streaming(expectedBodyLength: length, sentBodyBytes: 0, producer: .paused), .waitingForHead) + self.state = .running( + .streaming(expectedBodyLength: length, sentBodyBytes: 0, producer: .paused), + .waitingForHead + ) return .sendRequestHead(head, sendEnd: false) } } @@ -745,11 +796,14 @@ struct HTTPRequestStateMachine { case .running(.streaming(let expectedBodyLength, let sentBodyBytes, producer: .paused), let responseState): let startProducing = self.isChannelWritable && expectedBodyLength != sentBodyBytes - self.state = .running(.streaming( - expectedBodyLength: expectedBodyLength, - sentBodyBytes: sentBodyBytes, - producer: startProducing ? .producing : .paused - ), responseState) + self.state = .running( + .streaming( + expectedBodyLength: expectedBodyLength, + sentBodyBytes: sentBodyBytes, + producer: startProducing ? .producing : .paused + ), + responseState + ) return .notifyRequestHeadSendSuccessfully( resumeRequestBodyStream: startProducing, startIdleTimer: false @@ -757,7 +811,9 @@ struct HTTPRequestStateMachine { case .running(.endSent, _): return .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) case .running(.streaming(_, _, producer: .producing), _): - preconditionFailure("request body producing can not start before we have successfully send the header \(self.state)") + preconditionFailure( + "request body producing can not start before we have successfully send the header \(self.state)" + ) case .failed: return .wait @@ -830,7 +886,8 @@ extension HTTPRequestStateMachine: CustomStringConvertible { case .waitForChannelToBecomeWritable: return "HTTPRequestStateMachine(.waitForChannelToBecomeWritable, isWritable: \(self.isChannelWritable))" case .running(let requestState, let responseState): - return "HTTPRequestStateMachine(.running(request: \(requestState), response: \(responseState)), isWritable: \(self.isChannelWritable))" + return + "HTTPRequestStateMachine(.running(request: \(requestState), response: \(responseState)), isWritable: \(self.isChannelWritable))" case .finished: return "HTTPRequestStateMachine(.finished, isWritable: \(self.isChannelWritable))" case .failed(let error): @@ -844,7 +901,7 @@ extension HTTPRequestStateMachine: CustomStringConvertible { extension HTTPRequestStateMachine.RequestState: CustomStringConvertible { var description: String { switch self { - case .streaming(expectedBodyLength: let expected, let sent, producer: let producer): + case .streaming(expectedBodyLength: let expected, let sent, let producer): return ".streaming(sent: \(expected != nil ? String(expected!) : "-"), sent: \(sent), producer: \(producer)" case .endSent: return ".endSent" diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift index cc7c7cfa1..86a54273d 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift @@ -13,6 +13,7 @@ //===----------------------------------------------------------------------===// import NIOCore + #if canImport(Darwin) import func Darwin.pow #elseif canImport(Musl) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift index 1428a918b..15138a141 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1Connections.swift @@ -71,7 +71,7 @@ extension HTTPConnectionPool { var idleAndNoRemainingUses: Bool { switch self.state { - case .idle(_, since: _, remainingUses: let remainingUses): + case .idle(_, since: _, let remainingUses): if let remainingUses = remainingUses { return remainingUses <= 0 } else { @@ -139,7 +139,7 @@ extension HTTPConnectionPool { mutating func lease() -> Connection { switch self.state { - case .idle(let connection, since: _, remainingUses: let remainingUses): + case .idle(let connection, since: _, let remainingUses): self.state = .leased(connection, remainingUses: remainingUses.map { $0 - 1 }) return connection case .backingOff, .starting, .leased, .closed: @@ -208,7 +208,9 @@ extension HTTPConnectionPool { context.cancel.append(connection) return .keepConnection case .closed: - preconditionFailure("Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)") + preconditionFailure( + "Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)" + ) } } @@ -232,7 +234,9 @@ extension HTTPConnectionPool { case .leased: return .keepConnection case .closed: - preconditionFailure("Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)") + preconditionFailure( + "Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)" + ) } } } @@ -316,7 +320,7 @@ extension HTTPConnectionPool { } func startingEventLoopConnections(on eventLoop: EventLoop) -> Int { - return self.connections[self.overflowIndex.. [(Connection.ID, EventLoop)] in // We need a connection for each queued request with a required event loop. // Therefore, we look how many request we have queued for a given `eventLoop` and // how many connections we are already starting on the given `eventLoop`. // If we have not enough, we will create additional connections to have at least // on connection per request. - let connectionsToStart = requestCount - startingRequiredEventLoopConnectionCount[eventLoop.id, default: 0] + let connectionsToStart = + requestCount - startingRequiredEventLoopConnectionCount[eventLoop.id, default: 0] return stride(from: 0, to: connectionsToStart, by: 1).lazy.map { _ in (self.createNewOverflowConnection(on: eventLoop), eventLoop) } @@ -668,7 +677,8 @@ extension HTTPConnectionPool { // event loop we will continue with the event loop with the second most queued requests // and so on and so forth. The `generalPurposeRequestCountGroupedByPreferredEventLoop` // array is already ordered so we can just iterate over it without sorting by request count. - let newGeneralPurposeConnections: [(Connection.ID, EventLoop)] = generalPurposeRequestCountGroupedByPreferredEventLoop + let newGeneralPurposeConnections: [(Connection.ID, EventLoop)] = + generalPurposeRequestCountGroupedByPreferredEventLoop // we do not want to allocated intermediate arrays. .lazy // we flatten the grouped list of event loops by lazily repeating the event loop diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift index 2629b0ea2..09b1dc85e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP1StateMachine.swift @@ -80,7 +80,10 @@ extension HTTPConnectionPool { requests: RequestQueue ) -> ConnectionMigrationAction { precondition(self.connections.isEmpty, "expected an empty state machine but connections are not empty") - precondition(self.http2Connections == nil, "expected an empty state machine but http2Connections are not nil") + precondition( + self.http2Connections == nil, + "expected an empty state machine but http2Connections are not nil" + ) precondition(self.requests.isEmpty, "expected an empty state machine but requests are not empty") self.requests = requests @@ -100,7 +103,8 @@ extension HTTPConnectionPool { let createConnections = self.connections.createConnectionsAfterMigrationIfNeeded( requiredEventLoopOfPendingRequests: requests.requestCountGroupedByRequiredEventLoop(), - generalPurposeRequestCountGroupedByPreferredEventLoop: requests.generalPurposeRequestCountGroupedByPreferredEventLoop() + generalPurposeRequestCountGroupedByPreferredEventLoop: + requests.generalPurposeRequestCountGroupedByPreferredEventLoop() ) if !http2Connections.isEmpty { @@ -229,7 +233,9 @@ extension HTTPConnectionPool { case .running: guard self.retryConnectionEstablishment else { guard let (index, _) = self.connections.failConnection(connectionID) else { - preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + preconditionFailure( + "A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost." + ) } self.connections.removeConnection(at: index) @@ -295,7 +301,10 @@ extension HTTPConnectionPool { return .none } - precondition(self.lifecycleState == .running, "If we are shutting down, we must not have any idle connections") + precondition( + self.lifecycleState == .running, + "If we are shutting down, we must not have any idle connections" + ) return .init( request: .none, @@ -561,7 +570,8 @@ extension HTTPConnectionPool { // MARK: HTTP2 - mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action { + mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action + { // The `http2Connections` are optional here: // Connections report events back to us, if they are in a shutdown that was // initiated by the state machine. For this reason this callback might be invoked @@ -663,6 +673,7 @@ extension HTTPConnectionPool.HTTP1StateMachine: CustomStringConvertible { let stats = self.connections.stats let queued = self.requests.count - return "connections: [connecting: \(stats.connecting) | backoff: \(stats.backingOff) | leased: \(stats.leased) | idle: \(stats.idle)], queued: \(queued)" + return + "connections: [connecting: \(stats.connecting) | backoff: \(stats.backingOff) | leased: \(stats.leased) | idle: \(stats.idle)], queued: \(queued)" } } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift index 01d68b8e4..dbb6b2d30 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2Connections.swift @@ -117,7 +117,13 @@ extension HTTPConnectionPool { preconditionFailure("Invalid state: \(self.state)") case .starting(let maxUses): - self.state = .active(conn, maxStreams: maxStreams, usedStreams: 0, lastIdle: .now(), remainingUses: maxUses) + self.state = .active( + conn, + maxStreams: maxStreams, + usedStreams: 0, + lastIdle: .now(), + remainingUses: maxUses + ) if let maxUses = maxUses { return min(maxStreams, maxUses) } else { @@ -136,7 +142,13 @@ extension HTTPConnectionPool { preconditionFailure("Invalid state for updating max concurrent streams: \(self.state)") case .active(let conn, _, let usedStreams, let lastIdle, let remainingUses): - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses) + self.state = .active( + conn, + maxStreams: maxStreams, + usedStreams: usedStreams, + lastIdle: lastIdle, + remainingUses: remainingUses + ) let availableStreams = max(maxStreams - usedStreams, 0) if let remainingUses = remainingUses { return min(remainingUses, availableStreams) @@ -192,8 +204,17 @@ extension HTTPConnectionPool { case .active(let conn, let maxStreams, var usedStreams, let lastIdle, let remainingUses): usedStreams += count precondition(usedStreams <= maxStreams, "tried to lease a connection which is not available") - precondition(remainingUses.map { $0 >= count } ?? true, "tried to lease streams from a connection which does not have enough remaining streams") - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses.map { $0 - count }) + precondition( + remainingUses.map { $0 >= count } ?? true, + "tried to lease streams from a connection which does not have enough remaining streams" + ) + self.state = .active( + conn, + maxStreams: maxStreams, + usedStreams: usedStreams, + lastIdle: lastIdle, + remainingUses: remainingUses.map { $0 - count } + ) return conn } } @@ -212,7 +233,13 @@ extension HTTPConnectionPool { lastIdle = .now() } - self.state = .active(conn, maxStreams: maxStreams, usedStreams: usedStreams, lastIdle: lastIdle, remainingUses: remainingUses) + self.state = .active( + conn, + maxStreams: maxStreams, + usedStreams: usedStreams, + lastIdle: lastIdle, + remainingUses: remainingUses + ) let availableStreams = max(maxStreams &- usedStreams, 0) if let remainingUses = remainingUses { return min(availableStreams, remainingUses) @@ -282,7 +309,9 @@ extension HTTPConnectionPool { return .keepConnection case .closed: - preconditionFailure("Unexpected state for cleanup: Did not expect to have closed connections in the state machine.") + preconditionFailure( + "Unexpected state for cleanup: Did not expect to have closed connections in the state machine." + ) } } @@ -341,7 +370,9 @@ extension HTTPConnectionPool { return .removeConnection case .closed: - preconditionFailure("Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)") + preconditionFailure( + "Unexpected state: Did not expect to have connections with this state in the state machine: \(self.state)" + ) } } @@ -388,16 +419,20 @@ extension HTTPConnectionPool { backingOff: [(Connection.ID, EventLoop)] ) { for (connectionID, eventLoop) in starting { - let newConnection = HTTP2ConnectionState(connectionID: connectionID, - eventLoop: eventLoop, - maximumUses: self.maximumConnectionUses) + let newConnection = HTTP2ConnectionState( + connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) self.connections.append(newConnection) } for (connectionID, eventLoop) in backingOff { - var backingOffConnection = HTTP2ConnectionState(connectionID: connectionID, - eventLoop: eventLoop, - maximumUses: self.maximumConnectionUses) + var backingOffConnection = HTTP2ConnectionState( + connectionID: connectionID, + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) // TODO: Maybe we want to add a static init for backing off connections to HTTP2ConnectionState backingOffConnection.failedToConnect() self.connections.append(backingOffConnection) @@ -503,9 +538,11 @@ extension HTTPConnectionPool { "we should not create more than one connection per event loop" ) - let connection = HTTP2ConnectionState(connectionID: self.generator.next(), - eventLoop: eventLoop, - maximumUses: self.maximumConnectionUses) + let connection = HTTP2ConnectionState( + connectionID: self.generator.next(), + eventLoop: eventLoop, + maximumUses: self.maximumConnectionUses + ) self.connections.append(connection) return connection.connectionID } @@ -518,11 +555,17 @@ extension HTTPConnectionPool { /// - Returns: An index and an ``EstablishedConnectionContext`` to determine the next action for the now idle connection. /// Call ``leaseStreams(at:count:)`` or ``closeConnection(at:)`` with the supplied index after /// this. - mutating func newHTTP2ConnectionEstablished(_ connection: Connection, maxConcurrentStreams: Int) -> (Int, EstablishedConnectionContext) { + mutating func newHTTP2ConnectionEstablished( + _ connection: Connection, + maxConcurrentStreams: Int + ) -> (Int, EstablishedConnectionContext) { guard let index = self.connections.firstIndex(where: { $0.connectionID == connection.id }) else { preconditionFailure("There is a new connection that we didn't request!") } - precondition(connection.eventLoop === self.connections[index].eventLoop, "Expected the new connection to be on EL") + precondition( + connection.eventLoop === self.connections[index].eventLoop, + "Expected the new connection to be on EL" + ) let availableStreams = self.connections[index].connected(connection, maxStreams: maxConcurrentStreams) let context = EstablishedConnectionContext( availableStreams: availableStreams, diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift index 83a7647f4..2372cab4b 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+HTTP2StateMachine.swift @@ -47,8 +47,10 @@ extension HTTPConnectionPool { self.idGenerator = idGenerator self.requests = RequestQueue() - self.connections = HTTP2Connections(generator: idGenerator, - maximumConnectionUses: maximumConnectionUses) + self.connections = HTTP2Connections( + generator: idGenerator, + maximumConnectionUses: maximumConnectionUses + ) self.lifecycleState = lifecycleState self.retryConnectionEstablishment = retryConnectionEstablishment } @@ -83,7 +85,10 @@ extension HTTPConnectionPool { requests: RequestQueue ) -> ConnectionMigrationAction { precondition(self.connections.isEmpty, "expected an empty state machine but connections are not empty") - precondition(self.http1Connections == nil, "expected an empty state machine but http1Connections are not nil") + precondition( + self.http1Connections == nil, + "expected an empty state machine but http1Connections are not nil" + ) precondition(self.requests.isEmpty, "expected an empty state machine but requests are not empty") self.requests = requests @@ -93,7 +98,7 @@ extension HTTPConnectionPool { self.connections = http2Connections } - var http1Connections = http1Connections // make http1Connections mutable + var http1Connections = http1Connections // make http1Connections mutable let context = http1Connections.migrateToHTTP2() self.connections.migrateFromHTTP1( starting: context.starting, @@ -215,7 +220,10 @@ extension HTTPConnectionPool { .init(self._newHTTP2ConnectionEstablished(connection, maxConcurrentStreams: maxConcurrentStreams)) } - private mutating func _newHTTP2ConnectionEstablished(_ connection: Connection, maxConcurrentStreams: Int) -> EstablishedAction { + private mutating func _newHTTP2ConnectionEstablished( + _ connection: Connection, + maxConcurrentStreams: Int + ) -> EstablishedAction { self.failedConsecutiveConnectionAttempts = 0 self.lastConnectFailure = nil if self.connections.hasActiveConnection(for: connection.eventLoop) { @@ -296,8 +304,14 @@ extension HTTPConnectionPool { } } - mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action { - guard let (index, context) = self.connections.newHTTP2MaxConcurrentStreamsReceived(connectionID, newMaxStreams: newMaxStreams) else { + mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action + { + guard + let (index, context) = self.connections.newHTTP2MaxConcurrentStreamsReceived( + connectionID, + newMaxStreams: newMaxStreams + ) + else { // When a connection close is initiated by the connection pool, the connection will // still report further events (like newMaxConcurrentStreamsReceived) to the state // machine. In those cases we must ignore the event. @@ -341,15 +355,15 @@ extension HTTPConnectionPool { // we need to start a new on connection in two cases: let needGeneralPurposeConnection = // 1. if we have general purpose requests - !self.requests.isEmpty(for: nil) && + !self.requests.isEmpty(for: nil) // and no connection starting or active - !context.hasGeneralPurposeConnection + && !context.hasGeneralPurposeConnection let needRequiredEventLoopConnection = // 2. or if we have requests for a required event loop - !self.requests.isEmpty(for: eventLoop) && + !self.requests.isEmpty(for: eventLoop) // and no connection starting or active for the given event loop - !context.hasConnectionOnSpecifiedEventLoop + && !context.hasConnectionOnSpecifiedEventLoop guard needGeneralPurposeConnection || needRequiredEventLoopConnection else { // otherwise we can remove the connection @@ -357,7 +371,8 @@ extension HTTPConnectionPool { return .none } - let (newConnectionID, previousEventLoop) = self.connections.createNewConnectionByReplacingClosedConnection(at: index) + let (newConnectionID, previousEventLoop) = self.connections + .createNewConnectionByReplacingClosedConnection(at: index) precondition(previousEventLoop === eventLoop) return .init( @@ -413,7 +428,9 @@ extension HTTPConnectionPool { case .running: guard self.retryConnectionEstablishment else { guard let (index, _) = self.connections.failConnection(connectionID) else { - preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + preconditionFailure( + "A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost." + ) } self.connections.removeConnection(at: index) @@ -425,10 +442,15 @@ extension HTTPConnectionPool { let eventLoop = self.connections.backoffNextConnectionAttempt(connectionID) let backoff = calculateBackoff(failedAttempt: self.failedConsecutiveConnectionAttempts) - return .init(request: .none, connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop)) + return .init( + request: .none, + connection: .scheduleBackoffTimer(connectionID, backoff: backoff, on: eventLoop) + ) case .shuttingDown: guard let (index, context) = self.connections.failConnection(connectionID) else { - preconditionFailure("A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost.") + preconditionFailure( + "A connection attempt failed, that the state machine knows nothing about. Somewhere state was lost." + ) } return self.nextActionForFailedConnection(at: index, on: context.eventLoop) case .shutDown: @@ -505,7 +527,10 @@ extension HTTPConnectionPool { return .none } - precondition(self.lifecycleState == .running, "If we are shutting down, we must not have any idle connections") + precondition( + self.lifecycleState == .running, + "If we are shutting down, we must not have any idle connections" + ) return .init( request: .none, @@ -558,7 +583,10 @@ extension HTTPConnectionPool { case .shuttingDown(let unclean): if self.connections.isEmpty { // if the http2connections are empty as well, there are no more connections. Shutdown completed. - return .init(request: .none, connection: .closeConnection(connection, isShutdown: .yes(unclean: unclean))) + return .init( + request: .none, + connection: .closeConnection(connection, isShutdown: .yes(unclean: unclean)) + ) } else { return .init(request: .none, connection: .closeConnection(connection, isShutdown: .no)) } diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift index a86bbe8a3..6dfd4223e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+StateMachine.swift @@ -134,11 +134,14 @@ extension HTTPConnectionPool { } mutating func executeRequest(_ request: Request) -> Action { - self.state.modify(http1: { http1 in - http1.executeRequest(request) - }, http2: { http2 in - http2.executeRequest(request) - }) + self.state.modify( + http1: { http1 in + http1.executeRequest(request) + }, + http2: { http2 in + http2.executeRequest(request) + } + ) } mutating func newHTTP1ConnectionCreated(_ connection: Connection) -> Action { @@ -199,60 +202,82 @@ extension HTTPConnectionPool { } } - mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action { - self.state.modify(http1: { http1 in - http1.newHTTP2MaxConcurrentStreamsReceived(connectionID, newMaxStreams: newMaxStreams) - }, http2: { http2 in - http2.newHTTP2MaxConcurrentStreamsReceived(connectionID, newMaxStreams: newMaxStreams) - }) + mutating func newHTTP2MaxConcurrentStreamsReceived(_ connectionID: Connection.ID, newMaxStreams: Int) -> Action + { + self.state.modify( + http1: { http1 in + http1.newHTTP2MaxConcurrentStreamsReceived(connectionID, newMaxStreams: newMaxStreams) + }, + http2: { http2 in + http2.newHTTP2MaxConcurrentStreamsReceived(connectionID, newMaxStreams: newMaxStreams) + } + ) } mutating func http2ConnectionGoAwayReceived(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.http2ConnectionGoAwayReceived(connectionID) - }, http2: { http2 in - http2.http2ConnectionGoAwayReceived(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.http2ConnectionGoAwayReceived(connectionID) + }, + http2: { http2 in + http2.http2ConnectionGoAwayReceived(connectionID) + } + ) } mutating func http2ConnectionClosed(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.http2ConnectionClosed(connectionID) - }, http2: { http2 in - http2.http2ConnectionClosed(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.http2ConnectionClosed(connectionID) + }, + http2: { http2 in + http2.http2ConnectionClosed(connectionID) + } + ) } mutating func http2ConnectionStreamClosed(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.http2ConnectionStreamClosed(connectionID) - }, http2: { http2 in - http2.http2ConnectionStreamClosed(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.http2ConnectionStreamClosed(connectionID) + }, + http2: { http2 in + http2.http2ConnectionStreamClosed(connectionID) + } + ) } mutating func failedToCreateNewConnection(_ error: Error, connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.failedToCreateNewConnection(error, connectionID: connectionID) - }, http2: { http2 in - http2.failedToCreateNewConnection(error, connectionID: connectionID) - }) + self.state.modify( + http1: { http1 in + http1.failedToCreateNewConnection(error, connectionID: connectionID) + }, + http2: { http2 in + http2.failedToCreateNewConnection(error, connectionID: connectionID) + } + ) } mutating func waitingForConnectivity(_ error: Error, connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.waitingForConnectivity(error, connectionID: connectionID) - }, http2: { http2 in - http2.waitingForConnectivity(error, connectionID: connectionID) - }) + self.state.modify( + http1: { http1 in + http1.waitingForConnectivity(error, connectionID: connectionID) + }, + http2: { http2 in + http2.waitingForConnectivity(error, connectionID: connectionID) + } + ) } mutating func connectionCreationBackoffDone(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.connectionCreationBackoffDone(connectionID) - }, http2: { http2 in - http2.connectionCreationBackoffDone(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.connectionCreationBackoffDone(connectionID) + }, + http2: { http2 in + http2.connectionCreationBackoffDone(connectionID) + } + ) } /// A request has timed out. @@ -261,11 +286,14 @@ extension HTTPConnectionPool { /// request, but don't need to cancel the timer (it already triggered). If a request is cancelled /// we don't need to fail it but we need to cancel its timeout timer. mutating func timeoutRequest(_ requestID: Request.ID) -> Action { - self.state.modify(http1: { http1 in - http1.timeoutRequest(requestID) - }, http2: { http2 in - http2.timeoutRequest(requestID) - }) + self.state.modify( + http1: { http1 in + http1.timeoutRequest(requestID) + }, + http2: { http2 in + http2.timeoutRequest(requestID) + } + ) } /// A request was cancelled. @@ -274,44 +302,59 @@ extension HTTPConnectionPool { /// need to cancel its timeout timer. If a request times out, we need to fail the request, but don't /// need to cancel the timer (it already triggered). mutating func cancelRequest(_ requestID: Request.ID) -> Action { - self.state.modify(http1: { http1 in - http1.cancelRequest(requestID) - }, http2: { http2 in - http2.cancelRequest(requestID) - }) + self.state.modify( + http1: { http1 in + http1.cancelRequest(requestID) + }, + http2: { http2 in + http2.cancelRequest(requestID) + } + ) } mutating func connectionIdleTimeout(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.connectionIdleTimeout(connectionID) - }, http2: { http2 in - http2.connectionIdleTimeout(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.connectionIdleTimeout(connectionID) + }, + http2: { http2 in + http2.connectionIdleTimeout(connectionID) + } + ) } /// A connection has been closed mutating func http1ConnectionClosed(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.http1ConnectionClosed(connectionID) - }, http2: { http2 in - http2.http1ConnectionClosed(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.http1ConnectionClosed(connectionID) + }, + http2: { http2 in + http2.http1ConnectionClosed(connectionID) + } + ) } mutating func http1ConnectionReleased(_ connectionID: Connection.ID) -> Action { - self.state.modify(http1: { http1 in - http1.http1ConnectionReleased(connectionID) - }, http2: { http2 in - http2.http1ConnectionReleased(connectionID) - }) + self.state.modify( + http1: { http1 in + http1.http1ConnectionReleased(connectionID) + }, + http2: { http2 in + http2.http1ConnectionReleased(connectionID) + } + ) } mutating func shutdown() -> Action { - return self.state.modify(http1: { http1 in - http1.shutdown() - }, http2: { http2 in - http2.shutdown() - }) + self.state.modify( + http1: { http1 in + http1.shutdown() + }, + http2: { http2 in + http2.shutdown() + } + ) } } } @@ -362,7 +405,10 @@ extension HTTPConnectionPool.StateMachine { enum EstablishedConnectionAction { case none case scheduleTimeoutTimer(HTTPConnectionPool.Connection.ID, on: EventLoop) - case closeConnection(HTTPConnectionPool.Connection, isShutdown: HTTPConnectionPool.StateMachine.ConnectionAction.IsShutdown) + case closeConnection( + HTTPConnectionPool.Connection, + isShutdown: HTTPConnectionPool.StateMachine.ConnectionAction.IsShutdown + ) } } @@ -403,8 +449,7 @@ extension HTTPConnectionPool.StateMachine.ConnectionAction { case .closeConnection(let connection, let isShutdown): guard isShutdown == .no else { precondition( - migrationAction.closeConnections.isEmpty && - migrationAction.createConnections.isEmpty, + migrationAction.closeConnections.isEmpty && migrationAction.createConnections.isEmpty, "migration actions are not supported during shutdown" ) return .closeConnection(connection, isShutdown: isShutdown) diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index 9a351f3c1..1f869506a 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -87,12 +87,12 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { path: path, pool: .some(pool), reportHead: reportHead.map { reportHead in - return { _, head in + { _, head in reportHead(head) } }, reportProgress: reportProgress.map { reportProgress in - return { _, head in + { _, head in reportProgress(head) } } @@ -117,12 +117,12 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { path: path, pool: nil, reportHead: reportHead.map { reportHead in - return { _, head in + { _, head in reportHead(head) } }, reportProgress: reportProgress.map { reportProgress in - return { _, head in + { _, head in reportProgress(head) } } @@ -136,7 +136,8 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { self.reportHead?(task, head) if let totalBytesString = head.headers.first(name: "Content-Length"), - let totalBytes = Int(totalBytesString) { + let totalBytes = Int(totalBytesString) + { self.progress.totalBytes = totalBytes } diff --git a/Sources/AsyncHTTPClient/FoundationExtensions.swift b/Sources/AsyncHTTPClient/FoundationExtensions.swift index 545da756b..452cb7b13 100644 --- a/Sources/AsyncHTTPClient/FoundationExtensions.swift +++ b/Sources/AsyncHTTPClient/FoundationExtensions.swift @@ -39,7 +39,16 @@ extension HTTPClient.Cookie { /// - maxAge: The cookie's age in seconds, defaults to nil. /// - httpOnly: Whether this cookie should be used by HTTP servers only, defaults to false. /// - secure: Whether this cookie should only be sent using secure channels, defaults to false. - public init(name: String, value: String, path: String = "/", domain: String? = nil, expires: Date? = nil, maxAge: Int? = nil, httpOnly: Bool = false, secure: Bool = false) { + public init( + name: String, + value: String, + path: String = "/", + domain: String? = nil, + expires: Date? = nil, + maxAge: Int? = nil, + httpOnly: Bool = false, + secure: Bool = false + ) { // FIXME: This should be failable and validate the inputs // (for example, checking that the strings are ASCII, path begins with "/", domain is not empty, etc). self.init( @@ -59,8 +68,8 @@ extension HTTPClient.Body { /// Create and stream body using `Data`. /// /// - parameters: - /// - bytes: Body `Data` representation. + /// - data: Body `Data` representation. public static func data(_ data: Data) -> HTTPClient.Body { - return self.bytes(data) + self.bytes(data) } } diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index 9d9d6dfb7..847a99af2 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -12,7 +12,10 @@ // //===----------------------------------------------------------------------===// +import CAsyncHTTPClient +import NIOCore import NIOHTTP1 + #if canImport(xlocale) import xlocale #elseif canImport(locale_h) @@ -27,9 +30,6 @@ import Musl import Glibc #endif -import CAsyncHTTPClient -import NIOCore - extension HTTPClient { /// A representation of an HTTP cookie. public struct Cookie: Sendable { @@ -55,7 +55,6 @@ extension HTTPClient { /// - parameters: /// - header: String representation of the `Set-Cookie` response header. /// - defaultDomain: Default domain to use if cookie was sent without one. - /// - returns: nil if the header is invalid. public init?(header: String, defaultDomain: String) { // The parsing of "Set-Cookie" headers is defined by Section 5.2, RFC-6265: // https://datatracker.ietf.org/doc/html/rfc6265#section-5.2 @@ -136,7 +135,16 @@ extension HTTPClient { /// - maxAge: The cookie's age in seconds, defaults to nil. /// - httpOnly: Whether this cookie should be used by HTTP servers only, defaults to false. /// - secure: Whether this cookie should only be sent using secure channels, defaults to false. - internal init(name: String, value: String, path: String = "/", domain: String? = nil, expires_timestamp: Int64? = nil, maxAge: Int? = nil, httpOnly: Bool = false, secure: Bool = false) { + internal init( + name: String, + value: String, + path: String = "/", + domain: String? = nil, + expires_timestamp: Int64? = nil, + maxAge: Int? = nil, + httpOnly: Bool = false, + secure: Bool = false + ) { self.name = name self.value = value self.path = path @@ -152,7 +160,7 @@ extension HTTPClient { extension HTTPClient.Response { /// List of HTTP cookies returned by the server. public var cookies: [HTTPClient.Cookie] { - return self.headers["set-cookie"].compactMap { HTTPClient.Cookie(header: $0, defaultDomain: self.host) } + self.headers["set-cookie"].compactMap { HTTPClient.Cookie(header: $0, defaultDomain: self.host) } } } @@ -222,7 +230,8 @@ private func parseTimestamp(_ utf8: String.UTF8View.SubSequence, format: String) } private func parseCookieTime(_ timestampUTF8: String.UTF8View.SubSequence) -> Int64? { - if timestampUTF8.contains(where: { $0 < 0x20 /* Control characters */ || $0 == 0x7F /* DEL */ }) { + // 0x20: Control characters or 0x7F: DEL + if timestampUTF8.contains(where: { $0 < 0x20 || $0 == 0x7F }) { return nil } var timestampUTF8 = timestampUTF8 @@ -235,8 +244,8 @@ private func parseCookieTime(_ timestampUTF8: String.UTF8View.SubSequence) -> In } guard var timeComponents = parseTimestamp(timestampUTF8, format: "%a, %d %b %Y %H:%M:%S") - ?? parseTimestamp(timestampUTF8, format: "%a, %d-%b-%y %H:%M:%S") - ?? parseTimestamp(timestampUTF8, format: "%a %b %d %H:%M:%S %Y") + ?? parseTimestamp(timestampUTF8, format: "%a, %d-%b-%y %H:%M:%S") + ?? parseTimestamp(timestampUTF8, format: "%a %b %d %H:%M:%S %Y") else { return nil } diff --git a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift index 25b4b4555..e95c828ce 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+Proxy.swift @@ -38,7 +38,10 @@ extension HTTPClient.Configuration { /// Specifies Proxy server authorization. public var authorization: HTTPClient.Authorization? { set { - precondition(self.type == .http(self.authorization), "SOCKS authorization support is not yet implemented.") + precondition( + self.type == .http(self.authorization), + "SOCKS authorization support is not yet implemented." + ) self.type = .http(newValue) } @@ -60,7 +63,7 @@ extension HTTPClient.Configuration { /// - host: proxy server host. /// - port: proxy server port. public static func server(host: String, port: Int) -> Proxy { - return .init(host: host, port: port, type: .http(nil)) + .init(host: host, port: port, type: .http(nil)) } /// Create a HTTP proxy. @@ -70,7 +73,7 @@ extension HTTPClient.Configuration { /// - port: proxy server port. /// - authorization: proxy server authorization. public static func server(host: String, port: Int, authorization: HTTPClient.Authorization? = nil) -> Proxy { - return .init(host: host, port: port, type: .http(authorization)) + .init(host: host, port: port, type: .http(authorization)) } /// Create a SOCKSv5 proxy. @@ -78,7 +81,7 @@ extension HTTPClient.Configuration { /// - parameter port: The SOCKSv5 proxy port, defaults to 1080. /// - returns: A new instance of `Proxy` configured to connect to a `SOCKSv5` server. public static func socksServer(host: String, port: Int = 1080) -> Proxy { - return .init(host: host, port: port, type: .socks) + .init(host: host, port: port, type: .socks) } } } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 096fb9387..130a59f99 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -26,7 +26,7 @@ import NIOTransportServices extension Logger { private func requestInfo(_ request: HTTPClient.Request) -> Logger.Metadata.Value { - return "\(request.method) \(request.url)" + "\(request.method) \(request.url)" } func attachingRequestInformation(_ request: HTTPClient.Request, requestID: Int) -> Logger { @@ -80,23 +80,31 @@ public class HTTPClient { /// - parameters: /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. /// - configuration: Client configuration. - public convenience init(eventLoopGroupProvider: EventLoopGroupProvider, - configuration: Configuration = Configuration()) { - self.init(eventLoopGroupProvider: eventLoopGroupProvider, - configuration: configuration, - backgroundActivityLogger: HTTPClient.loggingDisabled) + public convenience init( + eventLoopGroupProvider: EventLoopGroupProvider, + configuration: Configuration = Configuration() + ) { + self.init( + eventLoopGroupProvider: eventLoopGroupProvider, + configuration: configuration, + backgroundActivityLogger: HTTPClient.loggingDisabled + ) } /// Create an ``HTTPClient`` with specified `EventLoopGroup` and configuration. /// /// - parameters: - /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. + /// - eventLoopGroup: Specify how `EventLoopGroup` will be created. /// - configuration: Client configuration. - public convenience init(eventLoopGroup: EventLoopGroup = HTTPClient.defaultEventLoopGroup, - configuration: Configuration = Configuration()) { - self.init(eventLoopGroupProvider: .shared(eventLoopGroup), - configuration: configuration, - backgroundActivityLogger: HTTPClient.loggingDisabled) + public convenience init( + eventLoopGroup: EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration() + ) { + self.init( + eventLoopGroupProvider: .shared(eventLoopGroup), + configuration: configuration, + backgroundActivityLogger: HTTPClient.loggingDisabled + ) } /// Create an ``HTTPClient`` with specified `EventLoopGroup` provider and configuration. @@ -104,21 +112,26 @@ public class HTTPClient { /// - parameters: /// - eventLoopGroupProvider: Specify how `EventLoopGroup` will be created. /// - configuration: Client configuration. - public convenience init(eventLoopGroupProvider: EventLoopGroupProvider, - configuration: Configuration = Configuration(), - backgroundActivityLogger: Logger) { + /// - backgroundActivityLogger: The logger to use for background activity logs. + public convenience init( + eventLoopGroupProvider: EventLoopGroupProvider, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger + ) { let eventLoopGroup: any EventLoopGroup switch eventLoopGroupProvider { case .shared(let group): eventLoopGroup = group - default: // handle `.createNew` without a deprecation warning + default: // handle `.createNew` without a deprecation warning eventLoopGroup = HTTPClient.defaultEventLoopGroup } - self.init(eventLoopGroup: eventLoopGroup, - configuration: configuration, - backgroundActivityLogger: backgroundActivityLogger) + self.init( + eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: backgroundActivityLogger + ) } /// Create an ``HTTPClient`` with specified `EventLoopGroup` and configuration. @@ -127,19 +140,25 @@ public class HTTPClient { /// - eventLoopGroup: The `EventLoopGroup` that the ``HTTPClient`` will use. /// - configuration: Client configuration. /// - backgroundActivityLogger: The `Logger` that will be used to log background any activity that's not associated with a request. - public convenience init(eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, - configuration: Configuration = Configuration(), - backgroundActivityLogger: Logger) { - self.init(eventLoopGroup: eventLoopGroup, - configuration: configuration, - backgroundActivityLogger: backgroundActivityLogger, - canBeShutDown: true) + public convenience init( + eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger + ) { + self.init( + eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: backgroundActivityLogger, + canBeShutDown: true + ) } - internal required init(eventLoopGroup: EventLoopGroup, - configuration: Configuration = Configuration(), - backgroundActivityLogger: Logger, - canBeShutDown: Bool) { + internal required init( + eventLoopGroup: EventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger, + canBeShutDown: Bool + ) { self.canBeShutDown = canBeShutDown self.eventLoopGroup = eventLoopGroup self.configuration = configuration @@ -158,15 +177,19 @@ public class HTTPClient { case .shutDown: break case .shuttingDown: - preconditionFailure(""" - This state should be totally unreachable. While the HTTPClient is shutting down a \ - reference cycle should exist, that prevents it from deinit. - """) + preconditionFailure( + """ + This state should be totally unreachable. While the HTTPClient is shutting down a \ + reference cycle should exist, that prevents it from deinit. + """ + ) case .upAndRunning: - preconditionFailure(""" - Client not shut down before the deinit. Please call client.shutdown() when no \ - longer needed. Otherwise memory will leak. - """) + preconditionFailure( + """ + Client not shut down before the deinit. Please call client.shutdown() when no \ + longer needed. Otherwise memory will leak. + """ + ) } } } @@ -191,16 +214,19 @@ public class HTTPClient { /// In general, setting this parameter to `true` should make it easier and faster to catch related programming errors. func syncShutdown(requiresCleanClose: Bool) throws { if let eventLoop = MultiThreadedEventLoopGroup.currentEventLoop { - preconditionFailure(""" - BUG DETECTED: syncShutdown() must not be called when on an EventLoop. - Calling syncShutdown() on any EventLoop can lead to deadlocks. - Current eventLoop: \(eventLoop) - """) + preconditionFailure( + """ + BUG DETECTED: syncShutdown() must not be called when on an EventLoop. + Calling syncShutdown() on any EventLoop can lead to deadlocks. + Current eventLoop: \(eventLoop) + """ + ) } let errorStorageLock = NIOLock() let errorStorage: UnsafeMutableTransferBox = .init(nil) let continuation = DispatchWorkItem {} - self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown")) { error in + self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown")) + { error in if let error = error { errorStorageLock.withLock { errorStorage.wrappedValue = error @@ -301,7 +327,7 @@ public class HTTPClient { /// - url: Remote URL. /// - deadline: Point in time by which the request must complete. public func get(url: String, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.get(url: url, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.get(url: url, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute `GET` request using specified URL. @@ -311,7 +337,7 @@ public class HTTPClient { /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. public func get(url: String, deadline: NIODeadline? = nil, logger: Logger) -> EventLoopFuture { - return self.execute(.GET, url: url, deadline: deadline, logger: logger) + self.execute(.GET, url: url, deadline: deadline, logger: logger) } /// Execute `POST` request using specified URL. @@ -321,7 +347,7 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. public func post(url: String, body: Body? = nil, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.post(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.post(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute `POST` request using specified URL. @@ -331,8 +357,13 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func post(url: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger) -> EventLoopFuture { - return self.execute(.POST, url: url, body: body, deadline: deadline, logger: logger) + public func post( + url: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger + ) -> EventLoopFuture { + self.execute(.POST, url: url, body: body, deadline: deadline, logger: logger) } /// Execute `PATCH` request using specified URL. @@ -342,7 +373,7 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. public func patch(url: String, body: Body? = nil, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.patch(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.patch(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute `PATCH` request using specified URL. @@ -352,8 +383,13 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func patch(url: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger) -> EventLoopFuture { - return self.execute(.PATCH, url: url, body: body, deadline: deadline, logger: logger) + public func patch( + url: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger + ) -> EventLoopFuture { + self.execute(.PATCH, url: url, body: body, deadline: deadline, logger: logger) } /// Execute `PUT` request using specified URL. @@ -363,7 +399,7 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. public func put(url: String, body: Body? = nil, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.put(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.put(url: url, body: body, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute `PUT` request using specified URL. @@ -373,8 +409,13 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func put(url: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger) -> EventLoopFuture { - return self.execute(.PUT, url: url, body: body, deadline: deadline, logger: logger) + public func put( + url: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger + ) -> EventLoopFuture { + self.execute(.PUT, url: url, body: body, deadline: deadline, logger: logger) } /// Execute `DELETE` request using specified URL. @@ -383,7 +424,7 @@ public class HTTPClient { /// - url: Remote URL. /// - deadline: The time when the request must have been completed by. public func delete(url: String, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.delete(url: url, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.delete(url: url, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute `DELETE` request using specified URL. @@ -393,7 +434,7 @@ public class HTTPClient { /// - deadline: The time when the request must have been completed by. /// - logger: The logger to use for this request. public func delete(url: String, deadline: NIODeadline? = nil, logger: Logger) -> EventLoopFuture { - return self.execute(.DELETE, url: url, deadline: deadline, logger: logger) + self.execute(.DELETE, url: url, deadline: deadline, logger: logger) } /// Execute arbitrary HTTP request using specified URL. @@ -404,7 +445,13 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func execute(_ method: HTTPMethod = .GET, url: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger? = nil) -> EventLoopFuture { + public func execute( + _ method: HTTPMethod = .GET, + url: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger? = nil + ) -> EventLoopFuture { do { let request = try Request(url: url, method: method, body: body) return self.execute(request: request, deadline: deadline, logger: logger ?? HTTPClient.loggingDisabled) @@ -422,7 +469,14 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func execute(_ method: HTTPMethod = .GET, socketPath: String, urlPath: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger? = nil) -> EventLoopFuture { + public func execute( + _ method: HTTPMethod = .GET, + socketPath: String, + urlPath: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger? = nil + ) -> EventLoopFuture { do { guard let url = URL(httpURLWithSocketPath: socketPath, uri: urlPath) else { throw HTTPClientError.invalidURL @@ -443,7 +497,14 @@ public class HTTPClient { /// - body: Request body. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func execute(_ method: HTTPMethod = .GET, secureSocketPath: String, urlPath: String, body: Body? = nil, deadline: NIODeadline? = nil, logger: Logger? = nil) -> EventLoopFuture { + public func execute( + _ method: HTTPMethod = .GET, + secureSocketPath: String, + urlPath: String, + body: Body? = nil, + deadline: NIODeadline? = nil, + logger: Logger? = nil + ) -> EventLoopFuture { do { guard let url = URL(httpsURLWithSocketPath: secureSocketPath, uri: urlPath) else { throw HTTPClientError.invalidURL @@ -461,7 +522,7 @@ public class HTTPClient { /// - request: HTTP request to execute. /// - deadline: Point in time by which the request must complete. public func execute(request: Request, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.execute(request: request, deadline: deadline, logger: HTTPClient.loggingDisabled) + self.execute(request: request, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute arbitrary HTTP request using specified URL. @@ -481,26 +542,40 @@ public class HTTPClient { /// - request: HTTP request to execute. /// - eventLoop: NIO Event Loop preference. /// - deadline: Point in time by which the request must complete. - public func execute(request: Request, eventLoop: EventLoopPreference, deadline: NIODeadline? = nil) -> EventLoopFuture { - return self.execute(request: request, - eventLoop: eventLoop, - deadline: deadline, - logger: HTTPClient.loggingDisabled) + public func execute( + request: Request, + eventLoop: EventLoopPreference, + deadline: NIODeadline? = nil + ) -> EventLoopFuture { + self.execute( + request: request, + eventLoop: eventLoop, + deadline: deadline, + logger: HTTPClient.loggingDisabled + ) } /// Execute arbitrary HTTP request and handle response processing using provided delegate. /// /// - parameters: /// - request: HTTP request to execute. - /// - eventLoop: NIO Event Loop preference. + /// - eventLoopPreference: NIO Event Loop preference. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func execute(request: Request, - eventLoop eventLoopPreference: EventLoopPreference, - deadline: NIODeadline? = nil, - logger: Logger?) -> EventLoopFuture { + public func execute( + request: Request, + eventLoop eventLoopPreference: EventLoopPreference, + deadline: NIODeadline? = nil, + logger: Logger? + ) -> EventLoopFuture { let accumulator = ResponseAccumulator(request: request) - return self.execute(request: request, delegate: accumulator, eventLoop: eventLoopPreference, deadline: deadline, logger: logger).futureResult + return self.execute( + request: request, + delegate: accumulator, + eventLoop: eventLoopPreference, + deadline: deadline, + logger: logger + ).futureResult } /// Execute arbitrary HTTP request and handle response processing using provided delegate. @@ -509,10 +584,12 @@ public class HTTPClient { /// - request: HTTP request to execute. /// - delegate: Delegate to process response parts. /// - deadline: Point in time by which the request must complete. - public func execute(request: Request, - delegate: Delegate, - deadline: NIODeadline? = nil) -> Task { - return self.execute(request: request, delegate: delegate, deadline: deadline, logger: HTTPClient.loggingDisabled) + public func execute( + request: Request, + delegate: Delegate, + deadline: NIODeadline? = nil + ) -> Task { + self.execute(request: request, delegate: delegate, deadline: deadline, logger: HTTPClient.loggingDisabled) } /// Execute arbitrary HTTP request and handle response processing using provided delegate. @@ -522,11 +599,13 @@ public class HTTPClient { /// - delegate: Delegate to process response parts. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. - public func execute(request: Request, - delegate: Delegate, - deadline: NIODeadline? = nil, - logger: Logger) -> Task { - return self.execute(request: request, delegate: delegate, eventLoop: .indifferent, deadline: deadline, logger: logger) + public func execute( + request: Request, + delegate: Delegate, + deadline: NIODeadline? = nil, + logger: Logger + ) -> Task { + self.execute(request: request, delegate: delegate, eventLoop: .indifferent, deadline: deadline, logger: logger) } /// Execute arbitrary HTTP request and handle response processing using provided delegate. @@ -534,18 +613,21 @@ public class HTTPClient { /// - parameters: /// - request: HTTP request to execute. /// - delegate: Delegate to process response parts. - /// - eventLoop: NIO Event Loop preference. + /// - eventLoopPreference: NIO Event Loop preference. /// - deadline: Point in time by which the request must complete. - /// - logger: The logger to use for this request. - public func execute(request: Request, - delegate: Delegate, - eventLoop eventLoopPreference: EventLoopPreference, - deadline: NIODeadline? = nil) -> Task { - return self.execute(request: request, - delegate: delegate, - eventLoop: eventLoopPreference, - deadline: deadline, - logger: HTTPClient.loggingDisabled) + public func execute( + request: Request, + delegate: Delegate, + eventLoop eventLoopPreference: EventLoopPreference, + deadline: NIODeadline? = nil + ) -> Task { + self.execute( + request: request, + delegate: delegate, + eventLoop: eventLoopPreference, + deadline: deadline, + logger: HTTPClient.loggingDisabled + ) } /// Execute arbitrary HTTP request and handle response processing using provided delegate. @@ -553,7 +635,7 @@ public class HTTPClient { /// - parameters: /// - request: HTTP request to execute. /// - delegate: Delegate to process response parts. - /// - eventLoop: NIO Event Loop preference. + /// - eventLoopPreference: NIO Event Loop preference. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. public func execute( @@ -561,14 +643,14 @@ public class HTTPClient { delegate: Delegate, eventLoop eventLoopPreference: EventLoopPreference, deadline: NIODeadline? = nil, - logger originalLogger: Logger? + logger: Logger? ) -> Task { self._execute( request: request, delegate: delegate, eventLoop: eventLoopPreference, deadline: deadline, - logger: originalLogger, + logger: logger, redirectState: RedirectState( self.configuration.redirectConfiguration.mode, initialURL: request.url.absoluteString @@ -592,25 +674,38 @@ public class HTTPClient { logger originalLogger: Logger?, redirectState: RedirectState? ) -> Task { - let logger = (originalLogger ?? HTTPClient.loggingDisabled).attachingRequestInformation(request, requestID: globalRequestID.wrappingIncrementThenLoad(ordering: .relaxed)) + let logger = (originalLogger ?? HTTPClient.loggingDisabled).attachingRequestInformation( + request, + requestID: globalRequestID.wrappingIncrementThenLoad(ordering: .relaxed) + ) let taskEL: EventLoop switch eventLoopPreference.preference { case .indifferent: // if possible we want a connection on the current `EventLoop` taskEL = self.eventLoopGroup.any() case .delegate(on: let eventLoop): - precondition(self.eventLoopGroup.makeIterator().contains { $0 === eventLoop }, "Provided EventLoop must be part of clients EventLoopGroup.") + precondition( + self.eventLoopGroup.makeIterator().contains { $0 === eventLoop }, + "Provided EventLoop must be part of clients EventLoopGroup." + ) taskEL = eventLoop case .delegateAndChannel(on: let eventLoop): - precondition(self.eventLoopGroup.makeIterator().contains { $0 === eventLoop }, "Provided EventLoop must be part of clients EventLoopGroup.") + precondition( + self.eventLoopGroup.makeIterator().contains { $0 === eventLoop }, + "Provided EventLoop must be part of clients EventLoopGroup." + ) taskEL = eventLoop case .testOnly_exact(_, delegateOn: let delegateEL): taskEL = delegateEL } - logger.trace("selected EventLoop for task given the preference", - metadata: ["ahc-eventloop": "\(taskEL)", - "ahc-el-preference": "\(eventLoopPreference)"]) + logger.trace( + "selected EventLoop for task given the preference", + metadata: [ + "ahc-eventloop": "\(taskEL)", + "ahc-el-preference": "\(eventLoopPreference)", + ] + ) let failedTask: Task? = self.stateLock.withLock { switch self.state { @@ -618,10 +713,12 @@ public class HTTPClient { return nil case .shuttingDown, .shutDown: logger.debug("client is shutting down, failing request") - return Task.failedTask(eventLoop: taskEL, - error: HTTPClientError.alreadyShutdown, - logger: logger, - makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool) + return Task.failedTask( + eventLoop: taskEL, + error: HTTPClientError.alreadyShutdown, + logger: logger, + makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool + ) } } @@ -644,7 +741,11 @@ public class HTTPClient { } }() - let task = Task(eventLoop: taskEL, logger: logger, makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool) + let task = Task( + eventLoop: taskEL, + logger: logger, + makeOrGetFileIOThreadPool: self.makeOrGetFileIOThreadPool + ) do { let requestBag = try RequestBag( request: request, @@ -711,7 +812,12 @@ public class HTTPClient { /// Enables automatic body decompression. Supported algorithms are gzip and deflate. public var decompression: Decompression /// Ignore TLS unclean shutdown error, defaults to `false`. - @available(*, deprecated, message: "AsyncHTTPClient now correctly supports handling unexpected SSL connection drops. This property is ignored") + @available( + *, + deprecated, + message: + "AsyncHTTPClient now correctly supports handling unexpected SSL connection drops. This property is ignored" + ) public var ignoreUncleanSSLShutdown: Bool { get { false } set {} @@ -762,12 +868,14 @@ public class HTTPClient { self.enableMultipath = false } - public init(tlsConfiguration: TLSConfiguration? = nil, - redirectConfiguration: RedirectConfiguration? = nil, - timeout: Timeout = Timeout(), - proxy: Proxy? = nil, - ignoreUncleanSSLShutdown: Bool = false, - decompression: Decompression = .disabled) { + public init( + tlsConfiguration: TLSConfiguration? = nil, + redirectConfiguration: RedirectConfiguration? = nil, + timeout: Timeout = Timeout(), + proxy: Proxy? = nil, + ignoreUncleanSSLShutdown: Bool = false, + decompression: Decompression = .disabled + ) { self.init( tlsConfiguration: tlsConfiguration, redirectConfiguration: redirectConfiguration, @@ -779,49 +887,59 @@ public class HTTPClient { ) } - public init(certificateVerification: CertificateVerification, - redirectConfiguration: RedirectConfiguration? = nil, - timeout: Timeout = Timeout(), - maximumAllowedIdleTimeInConnectionPool: TimeAmount = .seconds(60), - proxy: Proxy? = nil, - ignoreUncleanSSLShutdown: Bool = false, - decompression: Decompression = .disabled) { + public init( + certificateVerification: CertificateVerification, + redirectConfiguration: RedirectConfiguration? = nil, + timeout: Timeout = Timeout(), + maximumAllowedIdleTimeInConnectionPool: TimeAmount = .seconds(60), + proxy: Proxy? = nil, + ignoreUncleanSSLShutdown: Bool = false, + decompression: Decompression = .disabled + ) { var tlsConfig = TLSConfiguration.makeClientConfiguration() tlsConfig.certificateVerification = certificateVerification - self.init(tlsConfiguration: tlsConfig, - redirectConfiguration: redirectConfiguration, - timeout: timeout, - connectionPool: ConnectionPool(idleTimeout: maximumAllowedIdleTimeInConnectionPool), - proxy: proxy, - ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, - decompression: decompression) + self.init( + tlsConfiguration: tlsConfig, + redirectConfiguration: redirectConfiguration, + timeout: timeout, + connectionPool: ConnectionPool(idleTimeout: maximumAllowedIdleTimeInConnectionPool), + proxy: proxy, + ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, + decompression: decompression + ) } - public init(certificateVerification: CertificateVerification, - redirectConfiguration: RedirectConfiguration? = nil, - timeout: Timeout = Timeout(), - connectionPool: TimeAmount = .seconds(60), - proxy: Proxy? = nil, - ignoreUncleanSSLShutdown: Bool = false, - decompression: Decompression = .disabled, - backgroundActivityLogger: Logger?) { + public init( + certificateVerification: CertificateVerification, + redirectConfiguration: RedirectConfiguration? = nil, + timeout: Timeout = Timeout(), + connectionPool: TimeAmount = .seconds(60), + proxy: Proxy? = nil, + ignoreUncleanSSLShutdown: Bool = false, + decompression: Decompression = .disabled, + backgroundActivityLogger: Logger? + ) { var tlsConfig = TLSConfiguration.makeClientConfiguration() tlsConfig.certificateVerification = certificateVerification - self.init(tlsConfiguration: tlsConfig, - redirectConfiguration: redirectConfiguration, - timeout: timeout, - connectionPool: ConnectionPool(idleTimeout: connectionPool), - proxy: proxy, - ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, - decompression: decompression) + self.init( + tlsConfiguration: tlsConfig, + redirectConfiguration: redirectConfiguration, + timeout: timeout, + connectionPool: ConnectionPool(idleTimeout: connectionPool), + proxy: proxy, + ignoreUncleanSSLShutdown: ignoreUncleanSSLShutdown, + decompression: decompression + ) } - public init(certificateVerification: CertificateVerification, - redirectConfiguration: RedirectConfiguration? = nil, - timeout: Timeout = Timeout(), - proxy: Proxy? = nil, - ignoreUncleanSSLShutdown: Bool = false, - decompression: Decompression = .disabled) { + public init( + certificateVerification: CertificateVerification, + redirectConfiguration: RedirectConfiguration? = nil, + timeout: Timeout = Timeout(), + proxy: Proxy? = nil, + ignoreUncleanSSLShutdown: Bool = false, + decompression: Decompression = .disabled + ) { self.init( certificateVerification: certificateVerification, redirectConfiguration: redirectConfiguration, @@ -875,7 +993,7 @@ public class HTTPClient { /// `EventLoop` but will not establish a new network connection just to satisfy the `EventLoop` preference if /// another existing connection on a different `EventLoop` is readily available from a connection pool. public static func delegate(on eventLoop: EventLoop) -> EventLoopPreference { - return EventLoopPreference(.delegate(on: eventLoop)) + EventLoopPreference(.delegate(on: eventLoop)) } /// The delegate and the `Channel` will be run on the specified EventLoop. @@ -883,7 +1001,7 @@ public class HTTPClient { /// Use this for use-cases where you prefer a new connection to be established over re-using an existing /// connection that might be on a different `EventLoop`. public static func delegateAndChannel(on eventLoop: EventLoop) -> EventLoopPreference { - return EventLoopPreference(.delegateAndChannel(on: eventLoop)) + EventLoopPreference(.delegateAndChannel(on: eventLoop)) } } @@ -907,7 +1025,7 @@ public class HTTPClient { extension HTTPClient.EventLoopGroupProvider { /// Shares ``HTTPClient/defaultEventLoopGroup`` which is a singleton `EventLoopGroup` suitable for the platform. public static var singleton: Self { - return .shared(HTTPClient.defaultEventLoopGroup) + .shared(HTTPClient.defaultEventLoopGroup) } } @@ -1010,7 +1128,9 @@ extension HTTPClient.Configuration { /// - allowCycles: Whether cycles are allowed. /// /// - warning: Cycle detection will keep all visited URLs in memory which means a malicious server could use this as a denial-of-service vector. - public static func follow(max: Int, allowCycles: Bool) -> RedirectConfiguration { return .init(configuration: .follow(max: max, allowCycles: allowCycles)) } + public static func follow(max: Int, allowCycles: Bool) -> RedirectConfiguration { + .init(configuration: .follow(max: max, allowCycles: allowCycles)) + } } /// Connection pool configuration. @@ -1108,7 +1228,7 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { } public var description: String { - return "HTTPClientError.\(String(describing: self.code))" + "HTTPClientError.\(String(describing: self.code))" } /// Short description of the error that can be used in case a bounded set of error descriptions is expected, e.g. to @@ -1198,7 +1318,9 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { /// URL does not contain scheme. public static let emptyScheme = HTTPClientError(code: .emptyScheme) /// Provided URL scheme is not supported, supported schemes are: `http` and `https` - public static func unsupportedScheme(_ scheme: String) -> HTTPClientError { return HTTPClientError(code: .unsupportedScheme(scheme)) } + public static func unsupportedScheme(_ scheme: String) -> HTTPClientError { + HTTPClientError(code: .unsupportedScheme(scheme)) + } /// Request timed out while waiting for response. public static let readTimeout = HTTPClientError(code: .readTimeout) /// Request timed out. @@ -1227,9 +1349,13 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { /// A body was sent in a request with method TRACE. public static let traceRequestWithBody = HTTPClientError(code: .traceRequestWithBody) /// Header field names contain invalid characters. - public static func invalidHeaderFieldNames(_ names: [String]) -> HTTPClientError { return HTTPClientError(code: .invalidHeaderFieldNames(names)) } + public static func invalidHeaderFieldNames(_ names: [String]) -> HTTPClientError { + HTTPClientError(code: .invalidHeaderFieldNames(names)) + } /// Header field values contain invalid characters. - public static func invalidHeaderFieldValues(_ values: [String]) -> HTTPClientError { return HTTPClientError(code: .invalidHeaderFieldValues(values)) } + public static func invalidHeaderFieldValues(_ values: [String]) -> HTTPClientError { + HTTPClientError(code: .invalidHeaderFieldValues(values)) + } /// Body length is not equal to `Content-Length`. public static let bodyLengthMismatch = HTTPClientError(code: .bodyLengthMismatch) /// Body part was written after request was fully sent. @@ -1247,12 +1373,12 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { public static let tlsHandshakeTimeout = HTTPClientError(code: .tlsHandshakeTimeout) /// The remote server only offered an unsupported application protocol public static func serverOfferedUnsupportedApplicationProtocol(_ proto: String) -> HTTPClientError { - return HTTPClientError(code: .serverOfferedUnsupportedApplicationProtocol(proto)) + HTTPClientError(code: .serverOfferedUnsupportedApplicationProtocol(proto)) } /// The globally shared singleton ``HTTPClient`` cannot be shut down. public static var shutdownUnsupported: HTTPClientError { - return HTTPClientError(code: .shutdownUnsupported) + HTTPClientError(code: .shutdownUnsupported) } /// The request deadline was exceeded. The request was cancelled because of this. @@ -1269,6 +1395,11 @@ public struct HTTPClientError: Error, Equatable, CustomStringConvertible { /// - Tasks are not processed fast enough on the existing connections, to process all waiters in time public static let getConnectionFromPoolTimeout = HTTPClientError(code: .getConnectionFromPoolTimeout) - @available(*, deprecated, message: "AsyncHTTPClient now correctly supports informational headers. For this reason `httpEndReceivedAfterHeadWith1xx` will not be thrown anymore.") + @available( + *, + deprecated, + message: + "AsyncHTTPClient now correctly supports informational headers. For this reason `httpEndReceivedAfterHeadWith1xx` will not be thrown anymore." + ) public static let httpEndReceivedAfterHeadWith1xx = HTTPClientError(code: .httpEndReceivedAfterHeadWith1xx) } diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index d989b8a6c..7db1ce33c 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -43,17 +43,18 @@ extension HTTPClient { /// - parameters: /// - data: `IOData` to write. public func write(_ data: IOData) -> EventLoopFuture { - return self.closure(data) + self.closure(data) } @inlinable - func writeChunks(of bytes: Bytes, maxChunkSize: Int) -> EventLoopFuture where Bytes.Element == UInt8 { + func writeChunks(of bytes: Bytes, maxChunkSize: Int) -> EventLoopFuture + where Bytes.Element == UInt8 { let iterator = UnsafeMutableTransferBox(bytes.chunks(ofCount: maxChunkSize).makeIterator()) guard let chunk = iterator.wrappedValue.next() else { return self.write(IOData.byteBuffer(.init())) } - @Sendable // can't use closure here as we recursively call ourselves which closures can't do + @Sendable // can't use closure here as we recursively call ourselves which closures can't do func writeNextChunk(_ chunk: Bytes.SubSequence) -> EventLoopFuture { if let nextChunk = iterator.wrappedValue.next() { return self.write(.byteBuffer(ByteBuffer(bytes: chunk))).flatMap { @@ -100,7 +101,7 @@ extension HTTPClient { /// - parameters: /// - buffer: Body `ByteBuffer` representation. public static func byteBuffer(_ buffer: ByteBuffer) -> Body { - return Body(contentLength: Int64(buffer.readableBytes)) { writer in + Body(contentLength: Int64(buffer.readableBytes)) { writer in writer.write(.byteBuffer(buffer)) } } @@ -113,8 +114,11 @@ extension HTTPClient { /// - stream: Body chunk provider. @_disfavoredOverload @preconcurrency - public static func stream(length: Int? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { - return Body(contentLength: length.flatMap { Int64($0) }, stream: stream) + public static func stream( + length: Int? = nil, + _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture + ) -> Body { + Body(contentLength: length.flatMap { Int64($0) }, stream: stream) } /// Create and stream body using ``StreamWriter``. @@ -122,19 +126,23 @@ extension HTTPClient { /// - parameters: /// - contentLength: Body size. If nil, `Transfer-Encoding` will automatically be set to `chunked`. Otherwise a `Content-Length` /// header is set with the given `contentLength`. - /// - bodyStream: Body chunk provider. - public static func stream(contentLength: Int64? = nil, _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture) -> Body { - return Body(contentLength: contentLength, stream: stream) + /// - stream: Body chunk provider. + public static func stream( + contentLength: Int64? = nil, + _ stream: @Sendable @escaping (StreamWriter) -> EventLoopFuture + ) -> Body { + Body(contentLength: contentLength, stream: stream) } /// Create and stream body using a collection of bytes. /// /// - parameters: - /// - data: Body binary representation. + /// - bytes: Body binary representation. @preconcurrency @inlinable - public static func bytes(_ bytes: Bytes) -> Body where Bytes: RandomAccessCollection, Bytes: Sendable, Bytes.Element == UInt8 { - return Body(contentLength: Int64(bytes.count)) { writer in + public static func bytes(_ bytes: Bytes) -> Body + where Bytes: RandomAccessCollection, Bytes: Sendable, Bytes.Element == UInt8 { + Body(contentLength: Int64(bytes.count)) { writer in if bytes.count <= bagOfBytesToByteBufferConversionChunkSize { return writer.write(.byteBuffer(ByteBuffer(bytes: bytes))) } else { @@ -148,7 +156,7 @@ extension HTTPClient { /// - parameters: /// - string: Body `String` representation. public static func string(_ string: String) -> Body { - return Body(contentLength: Int64(string.utf8.count)) { writer in + Body(contentLength: Int64(string.utf8.count)) { writer in if string.utf8.count <= bagOfBytesToByteBufferConversionChunkSize { return writer.write(.byteBuffer(ByteBuffer(string: string))) } else { @@ -184,7 +192,6 @@ extension HTTPClient { /// /// - parameters: /// - url: Remote `URL`. - /// - version: HTTP version. /// - method: HTTP method. /// - headers: Custom HTTP headers. /// - body: Request body. @@ -193,7 +200,12 @@ extension HTTPClient { /// - `emptyScheme` if URL does not contain HTTP scheme. /// - `unsupportedScheme` if URL does contains unsupported HTTP scheme. /// - `emptyHost` if URL does not contains a host. - public init(url: String, method: HTTPMethod = .GET, headers: HTTPHeaders = HTTPHeaders(), body: Body? = nil) throws { + public init( + url: String, + method: HTTPMethod = .GET, + headers: HTTPHeaders = HTTPHeaders(), + body: Body? = nil + ) throws { try self.init(url: url, method: method, headers: headers, body: body, tlsConfiguration: nil) } @@ -201,7 +213,6 @@ extension HTTPClient { /// /// - parameters: /// - url: Remote `URL`. - /// - version: HTTP version. /// - method: HTTP method. /// - headers: Custom HTTP headers. /// - body: Request body. @@ -211,7 +222,13 @@ extension HTTPClient { /// - `emptyScheme` if URL does not contain HTTP scheme. /// - `unsupportedScheme` if URL does contains unsupported HTTP scheme. /// - `emptyHost` if URL does not contains a host. - public init(url: String, method: HTTPMethod = .GET, headers: HTTPHeaders = HTTPHeaders(), body: Body? = nil, tlsConfiguration: TLSConfiguration?) throws { + public init( + url: String, + method: HTTPMethod = .GET, + headers: HTTPHeaders = HTTPHeaders(), + body: Body? = nil, + tlsConfiguration: TLSConfiguration? + ) throws { guard let url = URL(string: url) else { throw HTTPClientError.invalidURL } @@ -231,7 +248,8 @@ extension HTTPClient { /// - `unsupportedScheme` if URL does contains unsupported HTTP scheme. /// - `emptyHost` if URL does not contains a host. /// - `missingSocketPath` if URL does not contains a socketPath as an encoded host. - public init(url: URL, method: HTTPMethod = .GET, headers: HTTPHeaders = HTTPHeaders(), body: Body? = nil) throws { + public init(url: URL, method: HTTPMethod = .GET, headers: HTTPHeaders = HTTPHeaders(), body: Body? = nil) throws + { try self.init(url: url, method: method, headers: headers, body: body, tlsConfiguration: nil) } @@ -248,7 +266,13 @@ extension HTTPClient { /// - `unsupportedScheme` if URL does contains unsupported HTTP scheme. /// - `emptyHost` if URL does not contains a host. /// - `missingSocketPath` if URL does not contains a socketPath as an encoded host. - public init(url: URL, method: HTTPMethod = .GET, headers: HTTPHeaders = HTTPHeaders(), body: Body? = nil, tlsConfiguration: TLSConfiguration?) throws { + public init( + url: URL, + method: HTTPMethod = .GET, + headers: HTTPHeaders = HTTPHeaders(), + body: Body? = nil, + tlsConfiguration: TLSConfiguration? + ) throws { self.deconstructedURL = try DeconstructedURL(url: url) self.url = url @@ -281,7 +305,10 @@ extension HTTPClient { head.headers.addHostIfNeeded(for: self.deconstructedURL) - let metadata = try head.headers.validateAndSetTransportFraming(method: self.method, bodyLength: .init(self.body)) + let metadata = try head.headers.validateAndSetTransportFraming( + method: self.method, + bodyLength: .init(self.body) + ) return (head, metadata) } @@ -333,7 +360,13 @@ extension HTTPClient { /// - version: Response HTTP version. /// - headers: Reponse HTTP headers. /// - body: Response body. - public init(host: String, status: HTTPResponseStatus, version: HTTPVersion, headers: HTTPHeaders, body: ByteBuffer?) { + public init( + host: String, + status: HTTPResponseStatus, + version: HTTPVersion, + headers: HTTPHeaders, + body: ByteBuffer? + ) { self.host = host self.status = status self.version = version @@ -357,19 +390,19 @@ extension HTTPClient { /// HTTP basic auth. public static func basic(username: String, password: String) -> HTTPClient.Authorization { - return .basic(credentials: Base64.encode(bytes: "\(username):\(password)".utf8)) + .basic(credentials: Base64.encode(bytes: "\(username):\(password)".utf8)) } /// HTTP basic auth. /// /// This version uses the raw string directly. public static func basic(credentials: String) -> HTTPClient.Authorization { - return .init(scheme: .Basic(credentials)) + .init(scheme: .Basic(credentials)) } /// HTTP bearer auth public static func bearer(tokens: String) -> HTTPClient.Authorization { - return .init(scheme: .Bearer(tokens)) + .init(scheme: .Bearer(tokens)) } /// The header string for this auth field. @@ -406,7 +439,7 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { } public var description: String { - return "ResponseTooBigError: received response body exceeds maximum accepted size of \(self.maxBodySize) bytes" + "ResponseTooBigError: received response body exceeds maximum accepted size of \(self.maxBodySize) bytes" } } @@ -450,9 +483,10 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { switch self.state { case .idle: if self.requestMethod != .HEAD, - let contentLength = head.headers.first(name: "Content-Length"), - let announcedBodySize = Int(contentLength), - announcedBodySize > self.maxBodySize { + let contentLength = head.headers.first(name: "Content-Length"), + let announcedBodySize = Int(contentLength), + announcedBodySize > self.maxBodySize + { let error = ResponseTooBigError(maxBodySize: maxBodySize) self.state = .error(error) return task.eventLoop.makeFailedFuture(error) @@ -515,9 +549,21 @@ public final class ResponseAccumulator: HTTPClientResponseDelegate { case .idle: preconditionFailure("no head received before end") case .head(let head): - return Response(host: self.requestHost, status: head.status, version: head.version, headers: head.headers, body: nil) + return Response( + host: self.requestHost, + status: head.status, + version: head.version, + headers: head.headers, + body: nil + ) case .body(let head, let body): - return Response(host: self.requestHost, status: head.status, version: head.version, headers: head.headers, body: body) + return Response( + host: self.requestHost, + status: head.status, + version: head.version, + headers: head.headers, + body: body + ) case .end: preconditionFailure("request already processed") case .error(let error): @@ -650,14 +696,14 @@ extension HTTPClientResponseDelegate { /// /// By default, this does nothing. public func didReceiveHead(task: HTTPClient.Task, _: HTTPResponseHead) -> EventLoopFuture { - return task.eventLoop.makeSucceededVoidFuture() + task.eventLoop.makeSucceededVoidFuture() } /// Default implementation of ``HTTPClientResponseDelegate/didReceiveBodyPart(task:_:)-4fd4v``. /// /// By default, this does nothing. public func didReceiveBodyPart(task: HTTPClient.Task, _: ByteBuffer) -> EventLoopFuture { - return task.eventLoop.makeSucceededVoidFuture() + task.eventLoop.makeSucceededVoidFuture() } /// Default implementation of ``HTTPClientResponseDelegate/didReceiveError(task:_:)-fhsg``. @@ -685,7 +731,7 @@ extension URL { } func hasTheSameOrigin(as other: URL) -> Bool { - return self.host == other.host && self.scheme == other.scheme && self.port == other.port + self.host == other.host && self.scheme == other.scheme && self.port == other.port } /// Initializes a newly created HTTP URL connecting to a unix domain socket path. The socket path is encoded as the URL's host, replacing percent encoding invalid path characters, and will use the "http+unix" scheme. @@ -732,7 +778,7 @@ extension HTTPClient { /// The `EventLoop` the delegate will be executed on. public let eventLoop: EventLoop /// The `Logger` used by the `Task` for logging. - public let logger: Logger // We are okay to store the logger here because a Task is for only one request. + public let logger: Logger // We are okay to store the logger here because a Task is for only one request. let promise: EventLoopPromise @@ -772,14 +818,18 @@ extension HTTPClient { logger: Logger, makeOrGetFileIOThreadPool: @escaping () -> NIOThreadPool ) -> Task { - let task = self.init(eventLoop: eventLoop, logger: logger, makeOrGetFileIOThreadPool: makeOrGetFileIOThreadPool) + let task = self.init( + eventLoop: eventLoop, + logger: logger, + makeOrGetFileIOThreadPool: makeOrGetFileIOThreadPool + ) task.promise.fail(error) return task } /// `EventLoopFuture` for the response returned by this request. public var futureResult: EventLoopFuture { - return self.promise.futureResult + self.promise.futureResult } /// Waits for execution of this request to complete. @@ -788,7 +838,7 @@ extension HTTPClient { /// - throws: The error value of ``futureResult`` if it errors. @available(*, noasync, message: "wait() can block indefinitely, prefer get()", renamed: "get()") public func wait() throws -> Response { - return try self.promise.futureResult.wait() + try self.promise.futureResult.wait() } /// Provides the result of this request. @@ -797,7 +847,7 @@ extension HTTPClient { /// - throws: The error value of ``futureResult`` if it errors. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) public func get() async throws -> Response { - return try await self.promise.futureResult.get() + try await self.promise.futureResult.get() } /// Cancels the request execution. @@ -806,7 +856,7 @@ extension HTTPClient { } /// Cancels the request execution with a custom `Error`. - /// - Parameter reason: the error that is used to fail the promise + /// - Parameter error: the error that is used to fail the promise public func fail(reason error: Error) { let taskDelegate = self.lock.withLock { () -> HTTPClientTaskDelegate? in self._isCancelled = true @@ -816,15 +866,19 @@ extension HTTPClient { taskDelegate?.fail(error) } - func succeed(promise: EventLoopPromise?, - with value: Response, - delegateType: Delegate.Type, - closing: Bool) { + func succeed( + promise: EventLoopPromise?, + with value: Response, + delegateType: Delegate.Type, + closing: Bool + ) { promise?.succeed(value) } - func fail(with error: Error, - delegateType: Delegate.Type) { + func fail( + with error: Error, + delegateType: Delegate.Type + ) { self.promise.fail(error) } } diff --git a/Sources/AsyncHTTPClient/LRUCache.swift b/Sources/AsyncHTTPClient/LRUCache.swift index 0a01da0d2..f8b58c36a 100644 --- a/Sources/AsyncHTTPClient/LRUCache.swift +++ b/Sources/AsyncHTTPClient/LRUCache.swift @@ -52,9 +52,11 @@ struct LRUCache { @discardableResult mutating func append(key: Key, value: Value) -> Value { - let newElement = Element(generation: self.generation, - key: key, - value: value) + let newElement = Element( + generation: self.generation, + key: key, + value: value + ) if let found = self.bumpGenerationAndFindIndex(key: key) { self.elements[found] = newElement return value diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift index 9796bc2af..148b4a4c4 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/NWErrorHandler.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -#if canImport(Network) -import Network -#endif import NIOCore import NIOHTTP1 import NIOTransportServices +#if canImport(Network) +import Network +#endif + extension HTTPClient { #if canImport(Network) /// A wrapper for `POSIX` errors thrown by `Network.framework`. @@ -38,7 +39,7 @@ extension HTTPClient { self.reason = reason } - public var description: String { return self.reason } + public var description: String { self.reason } } /// A wrapper for TLS errors thrown by `Network.framework`. @@ -58,7 +59,7 @@ extension HTTPClient { self.reason = reason } - public var description: String { return self.reason } + public var description: String { self.reason } } #endif diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift b/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift index 3474a8821..d7c6055ec 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/NWWaitingHandler.swift @@ -33,7 +33,10 @@ final class NWWaitingHandler: ChannelInbound func userInboundEventTriggered(context: ChannelHandlerContext, event: Any) { if let waitingEvent = event as? NIOTSNetworkEvents.WaitingForConnectivity { - self.requester.waitingForConnectivity(self.connectionID, error: HTTPClient.NWErrorHandler.translateError(waitingEvent.transientError)) + self.requester.waitingForConnectivity( + self.connectionID, + error: HTTPClient.NWErrorHandler.translateError(waitingEvent.transientError) + ) } context.fireUserInboundEventTriggered(event) } diff --git a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift index cb6bd43bd..ef505e3b7 100644 --- a/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift +++ b/Sources/AsyncHTTPClient/NIOTransportServices/TLSConfiguration.swift @@ -66,7 +66,10 @@ extension TLSConfiguration { /// /// - Parameter eventLoop: EventLoop to wait for creation of options on /// - Returns: Future holding NWProtocolTLS Options - func getNWProtocolTLSOptions(on eventLoop: EventLoop, serverNameIndicatorOverride: String?) -> EventLoopFuture { + func getNWProtocolTLSOptions( + on eventLoop: EventLoop, + serverNameIndicatorOverride: String? + ) -> EventLoopFuture { let promise = eventLoop.makePromise(of: NWProtocolTLS.Options.self) Self.tlsDispatchQueue.async { do { @@ -86,11 +89,11 @@ extension TLSConfiguration { let options = NWProtocolTLS.Options() let useMTELGExplainer = """ - You can still use this configuration option on macOS if you initialize HTTPClient \ - with a MultiThreadedEventLoopGroup. Please note that using MultiThreadedEventLoopGroup \ - will make AsyncHTTPClient use NIO on BSD Sockets and not Network.framework (which is the preferred \ - platform networking stack). - """ + You can still use this configuration option on macOS if you initialize HTTPClient \ + with a MultiThreadedEventLoopGroup. Please note that using MultiThreadedEventLoopGroup \ + will make AsyncHTTPClient use NIO on BSD Sockets and not Network.framework (which is the preferred \ + platform networking stack). + """ if let serverNameIndicatorOverride = serverNameIndicatorOverride { serverNameIndicatorOverride.withCString { serverNameIndicatorOverride in @@ -100,15 +103,24 @@ extension TLSConfiguration { // minimum TLS protocol if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) { - sec_protocol_options_set_min_tls_protocol_version(options.securityProtocolOptions, self.minimumTLSVersion.nwTLSProtocolVersion) + sec_protocol_options_set_min_tls_protocol_version( + options.securityProtocolOptions, + self.minimumTLSVersion.nwTLSProtocolVersion + ) } else { - sec_protocol_options_set_tls_min_version(options.securityProtocolOptions, self.minimumTLSVersion.sslProtocol) + sec_protocol_options_set_tls_min_version( + options.securityProtocolOptions, + self.minimumTLSVersion.sslProtocol + ) } // maximum TLS protocol if let maximumTLSVersion = self.maximumTLSVersion { if #available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) { - sec_protocol_options_set_max_tls_protocol_version(options.securityProtocolOptions, maximumTLSVersion.nwTLSProtocolVersion) + sec_protocol_options_set_max_tls_protocol_version( + options.securityProtocolOptions, + maximumTLSVersion.nwTLSProtocolVersion + ) } else { sec_protocol_options_set_tls_max_version(options.securityProtocolOptions, maximumTLSVersion.sslProtocol) } @@ -161,8 +173,10 @@ extension TLSConfiguration { break } - precondition(self.certificateVerification != .noHostnameVerification, - "TLSConfiguration.certificateVerification = .noHostnameVerification is not supported. \(useMTELGExplainer)") + precondition( + self.certificateVerification != .noHostnameVerification, + "TLSConfiguration.certificateVerification = .noHostnameVerification is not supported. \(useMTELGExplainer)" + ) if certificateVerification != .fullVerification || trustRoots != nil { // add verify block to control certificate verification @@ -196,7 +210,8 @@ extension TLSConfiguration { } } } - }, Self.tlsDispatchQueue + }, + Self.tlsDispatchQueue ) } return options diff --git a/Sources/AsyncHTTPClient/RedirectState.swift b/Sources/AsyncHTTPClient/RedirectState.swift index c4e427ef1..95de2d508 100644 --- a/Sources/AsyncHTTPClient/RedirectState.swift +++ b/Sources/AsyncHTTPClient/RedirectState.swift @@ -12,9 +12,10 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import NIOHTTP1 +import struct Foundation.URL + typealias RedirectMode = HTTPClient.Configuration.RedirectConfiguration.Mode struct RedirectState { diff --git a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift index e7fad6850..37b2a42f0 100644 --- a/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift +++ b/Sources/AsyncHTTPClient/RequestBag+StateMachine.swift @@ -12,10 +12,11 @@ // //===----------------------------------------------------------------------===// -import struct Foundation.URL import NIOCore import NIOHTTP1 +import struct Foundation.URL + extension HTTPClient { /// The maximum body size allowed, before a redirect response is cancelled. 3KB. /// @@ -302,10 +303,12 @@ extension RequestBag.StateMachine { preconditionFailure("If we receive a response, we must not have received something else before") } - if let redirectHandler = redirectHandler, let redirectURL = redirectHandler.redirectTarget( - status: head.status, - responseHeaders: head.headers - ) { + if let redirectHandler = redirectHandler, + let redirectURL = redirectHandler.redirectTarget( + status: head.status, + responseHeaders: head.headers + ) + { // If we will redirect, we need to consume the response's body ASAP, to be able to // reuse the existing connection. We will consume a response body, if the body is // smaller than 3kb. @@ -348,7 +351,9 @@ extension RequestBag.StateMachine { case .executing(let executor, let requestState, .buffering(var currentBuffer, next: let next)): guard case .askExecutorForMore = next else { - preconditionFailure("If we have received an error or eof before, why did we get another body part? Next: \(next)") + preconditionFailure( + "If we have received an error or eof before, why did we get another body part? Next: \(next)" + ) } self.state = .modifying @@ -405,7 +410,9 @@ extension RequestBag.StateMachine { case .executing(let executor, let requestState, .buffering(var buffer, next: let next)): guard case .askExecutorForMore = next else { - preconditionFailure("If we have received an error or eof before, why did we get another body part? Next: \(next)") + preconditionFailure( + "If we have received an error or eof before, why did we get another body part? Next: \(next)" + ) } if buffer.isEmpty, let newChunks = newChunks, !newChunks.isEmpty { @@ -463,7 +470,9 @@ extension RequestBag.StateMachine { case .initialized, .queued, .deadlineExceededWhileQueued: preconditionFailure("Invalid state: \(self.state)") case .executing(_, _, .initialized): - preconditionFailure("Invalid state: Must have received response head, before this method is called for the first time") + preconditionFailure( + "Invalid state: Must have received response head, before this method is called for the first time" + ) case .executing(_, _, .buffering(_, next: .error(let connectionError))): // if an error was received from the connection, we fail the task with the one @@ -476,17 +485,23 @@ extension RequestBag.StateMachine { return .failTask(error, executorToCancel: executor) case .executing(_, _, .waitingForRemote): - preconditionFailure("Invalid state... We just returned from a consumption function. We can't already be waiting") + preconditionFailure( + "Invalid state... We just returned from a consumption function. We can't already be waiting" + ) case .redirected: - preconditionFailure("Invalid state... Redirect don't call out to delegate functions. Thus we should never land here.") + preconditionFailure( + "Invalid state... Redirect don't call out to delegate functions. Thus we should never land here." + ) case .finished(error: .some): // don't overwrite existing errors return .doNothing case .finished(error: .none): - preconditionFailure("Invalid state... If no error occured, this must not be called, after the request was finished") + preconditionFailure( + "Invalid state... If no error occured, this must not be called, after the request was finished" + ) case .modifying: preconditionFailure() @@ -499,7 +514,9 @@ extension RequestBag.StateMachine { preconditionFailure("Invalid state: \(self.state)") case .executing(_, _, .initialized): - preconditionFailure("Invalid state: Must have received response head, before this method is called for the first time") + preconditionFailure( + "Invalid state: Must have received response head, before this method is called for the first time" + ) case .executing(let executor, let requestState, .buffering(var buffer, next: .askExecutorForMore)): self.state = .modifying @@ -529,7 +546,9 @@ extension RequestBag.StateMachine { return .failTask(error, executorToCancel: nil) case .executing(_, _, .waitingForRemote): - preconditionFailure("Invalid state... We just returned from a consumption function. We can't already be waiting") + preconditionFailure( + "Invalid state... We just returned from a consumption function. We can't already be waiting" + ) case .redirected: return .doNothing @@ -538,7 +557,9 @@ extension RequestBag.StateMachine { return .doNothing case .finished(error: .none): - preconditionFailure("Invalid state... If no error occurred, this must not be called, after the request was finished") + preconditionFailure( + "Invalid state... If no error occurred, this must not be called, after the request was finished" + ) case .modifying: preconditionFailure() @@ -559,11 +580,11 @@ extension RequestBag.StateMachine { return .cancelScheduler(queuer) case .initialized, - .deadlineExceededWhileQueued, - .executing, - .finished, - .redirected, - .modifying: + .deadlineExceededWhileQueued, + .executing, + .finished, + .redirected, + .modifying: /// if we are not in the queued state, we can fail early by just calling down to `self.fail(_:)` /// which does the appropriate state transition for us. return .fail(self.fail(HTTPClientError.deadlineExceeded)) diff --git a/Sources/AsyncHTTPClient/RequestBag.swift b/Sources/AsyncHTTPClient/RequestBag.swift index c5472fc6f..f2720d9ef 100644 --- a/Sources/AsyncHTTPClient/RequestBag.swift +++ b/Sources/AsyncHTTPClient/RequestBag.swift @@ -58,13 +58,15 @@ final class RequestBag { let eventLoopPreference: HTTPClient.EventLoopPreference - init(request: HTTPClient.Request, - eventLoopPreference: HTTPClient.EventLoopPreference, - task: HTTPClient.Task, - redirectHandler: RedirectHandler?, - connectionDeadline: NIODeadline, - requestOptions: RequestOptions, - delegate: Delegate) throws { + init( + request: HTTPClient.Request, + eventLoopPreference: HTTPClient.EventLoopPreference, + task: HTTPClient.Task, + redirectHandler: RedirectHandler?, + connectionDeadline: NIODeadline, + requestOptions: RequestOptions, + delegate: Delegate + ) throws { self.poolKey = .init(request, dnsOverride: requestOptions.dnsOverride) self.eventLoopPreference = eventLoopPreference self.task = task @@ -435,8 +437,8 @@ extension RequestBag: HTTPExecutableRequest { case .indifferent: return self.task.eventLoop case .delegate(let eventLoop), - .delegateAndChannel(on: let eventLoop), - .testOnly_exact(channelOn: let eventLoop, delegateOn: _): + .delegateAndChannel(on: let eventLoop), + .testOnly_exact(channelOn: let eventLoop, delegateOn: _): return eventLoop } } diff --git a/Sources/AsyncHTTPClient/RequestValidation.swift b/Sources/AsyncHTTPClient/RequestValidation.swift index 87224a3b2..f338e06a9 100644 --- a/Sources/AsyncHTTPClient/RequestValidation.swift +++ b/Sources/AsyncHTTPClient/RequestValidation.swift @@ -50,23 +50,23 @@ extension HTTPHeaders { let satisfy = name.utf8.allSatisfy { char -> Bool in switch char { case UInt8(ascii: "a")...UInt8(ascii: "z"), - UInt8(ascii: "A")...UInt8(ascii: "Z"), - UInt8(ascii: "0")...UInt8(ascii: "9"), - UInt8(ascii: "!"), - UInt8(ascii: "#"), - UInt8(ascii: "$"), - UInt8(ascii: "%"), - UInt8(ascii: "&"), - UInt8(ascii: "'"), - UInt8(ascii: "*"), - UInt8(ascii: "+"), - UInt8(ascii: "-"), - UInt8(ascii: "."), - UInt8(ascii: "^"), - UInt8(ascii: "_"), - UInt8(ascii: "`"), - UInt8(ascii: "|"), - UInt8(ascii: "~"): + UInt8(ascii: "A")...UInt8(ascii: "Z"), + UInt8(ascii: "0")...UInt8(ascii: "9"), + UInt8(ascii: "!"), + UInt8(ascii: "#"), + UInt8(ascii: "$"), + UInt8(ascii: "%"), + UInt8(ascii: "&"), + UInt8(ascii: "'"), + UInt8(ascii: "*"), + UInt8(ascii: "+"), + UInt8(ascii: "-"), + UInt8(ascii: "."), + UInt8(ascii: "^"), + UInt8(ascii: "_"), + UInt8(ascii: "`"), + UInt8(ascii: "|"), + UInt8(ascii: "~"): return true default: return false @@ -166,13 +166,14 @@ extension HTTPHeaders { mutating func addHostIfNeeded(for url: DeconstructedURL) { // if no host header was set, let's use the url host guard !self.contains(name: "host"), - var host = url.connectionTarget.host + var host = url.connectionTarget.host else { return } // if the request uses a non-default port, we need to add it after the host if let port = url.connectionTarget.port, - port != url.scheme.defaultPort { + port != url.scheme.defaultPort + { host += ":\(port)" } self.add(name: "host", value: host) diff --git a/Sources/AsyncHTTPClient/SSLContextCache.swift b/Sources/AsyncHTTPClient/SSLContextCache.swift index 660a04942..599003e56 100644 --- a/Sources/AsyncHTTPClient/SSLContextCache.swift +++ b/Sources/AsyncHTTPClient/SSLContextCache.swift @@ -25,30 +25,38 @@ final class SSLContextCache { } extension SSLContextCache { - func sslContext(tlsConfiguration: TLSConfiguration, - eventLoop: EventLoop, - logger: Logger) -> EventLoopFuture { + func sslContext( + tlsConfiguration: TLSConfiguration, + eventLoop: EventLoop, + logger: Logger + ) -> EventLoopFuture { let eqTLSConfiguration = BestEffortHashableTLSConfiguration(wrapping: tlsConfiguration) let sslContext = self.lock.withLock { self.sslContextCache.find(key: eqTLSConfiguration) } if let sslContext = sslContext { - logger.trace("found SSL context in cache", - metadata: ["ahc-tls-config": "\(tlsConfiguration)"]) + logger.trace( + "found SSL context in cache", + metadata: ["ahc-tls-config": "\(tlsConfiguration)"] + ) return eventLoop.makeSucceededFuture(sslContext) } - logger.trace("creating new SSL context", - metadata: ["ahc-tls-config": "\(tlsConfiguration)"]) + logger.trace( + "creating new SSL context", + metadata: ["ahc-tls-config": "\(tlsConfiguration)"] + ) let newSSLContext = self.offloadQueue.asyncWithFuture(eventLoop: eventLoop) { try NIOSSLContext(configuration: tlsConfiguration) } newSSLContext.whenSuccess { (newSSLContext: NIOSSLContext) -> Void in self.lock.withLock { () -> Void in - self.sslContextCache.append(key: eqTLSConfiguration, - value: newSSLContext) + self.sslContextCache.append( + key: eqTLSConfiguration, + value: newSSLContext + ) } } diff --git a/Sources/AsyncHTTPClient/Singleton.swift b/Sources/AsyncHTTPClient/Singleton.swift index 149f7586f..0ddf1bc40 100644 --- a/Sources/AsyncHTTPClient/Singleton.swift +++ b/Sources/AsyncHTTPClient/Singleton.swift @@ -20,7 +20,7 @@ extension HTTPClient { /// - `EventLoopGroup` is ``HTTPClient/defaultEventLoopGroup`` (matching the platform default) /// - logging is disabled public static var shared: HTTPClient { - return globallySharedHTTPClient + globallySharedHTTPClient } } diff --git a/Sources/AsyncHTTPClient/StringConvertibleInstances.swift b/Sources/AsyncHTTPClient/StringConvertibleInstances.swift index f75fb0d87..61d4b067a 100644 --- a/Sources/AsyncHTTPClient/StringConvertibleInstances.swift +++ b/Sources/AsyncHTTPClient/StringConvertibleInstances.swift @@ -14,6 +14,6 @@ extension HTTPClient.EventLoopPreference: CustomStringConvertible { public var description: String { - return "\(self.preference)" + "\(self.preference)" } } diff --git a/Sources/AsyncHTTPClient/Utils.swift b/Sources/AsyncHTTPClient/Utils.swift index f8618ea17..abdd5bbc2 100644 --- a/Sources/AsyncHTTPClient/Utils.swift +++ b/Sources/AsyncHTTPClient/Utils.swift @@ -29,11 +29,11 @@ public final class HTTPClientCopyingDelegate: HTTPClientResponseDelegate { } public func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { - return self.chunkHandler(buffer) + self.chunkHandler(buffer) } public func didFinishRequest(task: HTTPClient.Task) throws { - return () + () } } @@ -44,7 +44,12 @@ public final class HTTPClientCopyingDelegate: HTTPClientResponseDelegate { /// https://forums.swift.org/t/support-debug-only-code/11037 for a discussion. @inlinable internal func debugOnly(_ body: () -> Void) { - assert({ body(); return true }()) + assert( + { + body() + return true + }() + ) } extension BidirectionalCollection where Element: Equatable { @@ -61,8 +66,8 @@ extension BidirectionalCollection where Element: Equatable { guard self[ourIdx] == suffix[suffixIdx] else { return false } } guard suffixIdx == suffix.startIndex else { - return false // Exhausted self, but 'suffix' has elements remaining. + return false // Exhausted self, but 'suffix' has elements remaining. } - return true // Exhausted 'other' without finding a mismatch. + return true // Exhausted 'other' without finding a mismatch. } } diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index d44d047f6..f58e07730 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOPosix import NIOSSL import XCTest +@testable import AsyncHTTPClient + private func makeDefaultHTTPClient( eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .singleton ) -> HTTPClient { @@ -65,9 +66,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) let request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } @@ -85,9 +88,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) let request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } @@ -107,13 +112,17 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST request.body = .bytes(ByteBuffer(string: "1234")) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], ["4"]) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -129,13 +138,17 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST request.body = .bytes(AnySendableSequence("1234".utf8), length: .unknown) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -151,13 +164,17 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST request.body = .bytes(AnySendableCollection("1234".utf8), length: .unknown) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -173,13 +190,17 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST request.body = .bytes(ByteBuffer(string: "1234").readableBytesView) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], ["4"]) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -206,7 +227,7 @@ final class AsyncAwaitEndToEndTests: XCTestCase { } func makeAsyncIterator() -> AsyncSequenceByteBufferGenerator { - return self + self } } @@ -225,19 +246,23 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST let sequence = AsyncSequenceByteBufferGenerator( - chunkSize: 4_194_304, // 4MB chunk - totalChunks: 768 // Total = 3GB + chunkSize: 4_194_304, // 4MB chunk + totalChunks: 768 // Total = 3GB ) request.body = .stream(sequence, length: .unknown) - let response: HTTPClientResponse = try await client.execute(request, deadline: .now() + .seconds(30), logger: logger) + let response: HTTPClientResponse = try await client.execute( + request, + deadline: .now() + .seconds(30), + logger: logger + ) XCTAssertEqual(response.headers["content-length"], []) var receivedBytes: Int64 = 0 for try await part in response.body { receivedBytes += Int64(part.readableBytes) } - XCTAssertEqual(receivedBytes, 3_221_225_472) // 3GB + XCTAssertEqual(receivedBytes, 3_221_225_472) // 3GB } func testPostWithAsyncSequenceOfByteBuffers() { @@ -249,19 +274,26 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") request.method = .POST - request.body = .stream([ - ByteBuffer(string: "1"), - ByteBuffer(string: "2"), - ByteBuffer(string: "34"), - ].async, length: .unknown) + request.body = .stream( + [ + ByteBuffer(string: "1"), + ByteBuffer(string: "2"), + ByteBuffer(string: "34"), + ].async, + length: .unknown + ) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -277,13 +309,17 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.method = .POST request.body = .stream("1234".utf8.async, length: .unknown) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -300,9 +336,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let streamWriter = AsyncSequenceWriter() request.body = .stream(streamWriter, length: .unknown) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) let fragments = [ @@ -313,16 +351,20 @@ final class AsyncAwaitEndToEndTests: XCTestCase { var bodyIterator = response.body.makeAsyncIterator() for expectedFragment in fragments { streamWriter.write(expectedFragment) - guard let actualFragment = await XCTAssertNoThrowWithResult( - try await bodyIterator.next() - ) else { return } + guard + let actualFragment = await XCTAssertNoThrowWithResult( + try await bodyIterator.next() + ) + else { return } XCTAssertEqual(expectedFragment, actualFragment) } streamWriter.end() - guard let lastResult = await XCTAssertNoThrowWithResult( - try await bodyIterator.next() - ) else { return } + guard + let lastResult = await XCTAssertNoThrowWithResult( + try await bodyIterator.next() + ) + else { return } XCTAssertEqual(lastResult, nil) } } @@ -339,9 +381,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let streamWriter = AsyncSequenceWriter() request.body = .stream(streamWriter, length: .unknown) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response.headers["content-length"], []) let fragments = [ @@ -353,16 +397,20 @@ final class AsyncAwaitEndToEndTests: XCTestCase { var bodyIterator = response.body.makeAsyncIterator() for expectedFragment in fragments { streamWriter.write(expectedFragment) - guard let actualFragment = await XCTAssertNoThrowWithResult( - try await bodyIterator.next() - ) else { return } + guard + let actualFragment = await XCTAssertNoThrowWithResult( + try await bodyIterator.next() + ) + else { return } XCTAssertEqual(expectedFragment, actualFragment) } streamWriter.end() - guard let lastResult = await XCTAssertNoThrowWithResult( - try await bodyIterator.next() - ) else { return } + guard + let lastResult = await XCTAssertNoThrowWithResult( + try await bodyIterator.next() + ) + else { return } XCTAssertEqual(lastResult, nil) } } @@ -436,7 +484,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { // a race between deadline and connect timer can result in either error. // If closing happens really fast we might shutdown the pipeline before we fail the request. // If the pipeline is closed we may receive a `.remoteConnectionClosed`. - XCTAssertTrue([.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), "unexpected error \(error)") + XCTAssertTrue( + [.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), + "unexpected error \(error)" + ) } } } @@ -460,7 +511,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { // a race between deadline and connect timer can result in either error. // If closing happens really fast we might shutdown the pipeline before we fail the request. // If the pipeline is closed we may receive a `.remoteConnectionClosed`. - XCTAssertTrue([.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), "unexpected error \(error)") + XCTAssertTrue( + [.deadlineExceeded, .connectTimeout, .remoteConnectionClosed].contains(error), + "unexpected error \(error)" + ) } } } @@ -500,8 +554,10 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let url = "http://localhost:\(port)/get" #endif - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150)))) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150))) + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) @@ -549,7 +605,8 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let localClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://localhost:\(port)") - await XCTAssertThrowsError(try await localClient.execute(request, deadline: .now() + .seconds(2))) { error in + await XCTAssertThrowsError(try await localClient.execute(request, deadline: .now() + .seconds(2))) { + error in #if canImport(Network) guard let nwTLSError = error as? HTTPClient.NWTLSError else { XCTFail("could not cast \(error) of type \(type(of: error)) to \(HTTPClient.NWTLSError.self)") @@ -558,7 +615,8 @@ final class AsyncAwaitEndToEndTests: XCTestCase { XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") #else guard let sslError = error as? NIOSSLError, - case .handshakeFailed(.sslError) = sslError else { + case .handshakeFailed(.sslError) = sslError + else { XCTFail("unexpected error \(error)") return } @@ -619,7 +677,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let localClient = HTTPClient(eventLoopGroupProvider: .singleton, configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } let request = HTTPClientRequest(url: "https://example.com:\(bin.port)/echohostheader") - let response = await XCTAssertNoThrowWithResult(try await localClient.execute(request, deadline: .now() + .seconds(2))) + let response = await XCTAssertNoThrowWithResult( + try await localClient.execute(request, deadline: .now() + .seconds(2)) + ) XCTAssertEqual(response?.status, .ok) XCTAssertEqual(response?.version, .http2) var body = try await response?.body.collect(upTo: 1024) @@ -634,9 +694,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let client = makeDefaultHTTPClient() defer { XCTAssertNoThrow(try client.syncShutdown()) } let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) - let request = HTTPClientRequest(url: "") // invalid URL + let request = HTTPClientRequest(url: "") // invalid URL - await XCTAssertThrowsError(try await client.execute(request, deadline: .now() + .seconds(2), logger: logger)) { + await XCTAssertThrowsError( + try await client.execute(request, deadline: .now() + .seconds(2), logger: logger) + ) { XCTAssertEqual($0 as? HTTPClientError, .invalidURL) } } @@ -668,14 +730,21 @@ final class AsyncAwaitEndToEndTests: XCTestCase { defer { XCTAssertNoThrow(try client.syncShutdown()) } let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://127.0.0.1:\(bin.port)/redirect/target") - request.headers.replaceOrAdd(name: "X-Target-Redirect-URL", value: "https://localhost:\(bin.port)/echohostheader") + request.headers.replaceOrAdd( + name: "X-Target-Redirect-URL", + value: "https://localhost:\(bin.port)/echohostheader" + ) - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { + return + } + guard let body = await XCTAssertNoThrowWithResult(try await response.body.collect(upTo: 1024)) else { return } - guard let body = await XCTAssertNoThrowWithResult(try await response.body.collect(upTo: 1024)) else { return } var maybeRequestInfo: RequestInfo? XCTAssertNoThrow(maybeRequestInfo = try JSONDecoder().decode(RequestInfo.self, from: body)) guard let requestInfo = maybeRequestInfo else { return } @@ -722,28 +791,39 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/") request.method = .POST - request.body = .stream([ - ByteBuffer(string: "1"), - ByteBuffer(string: "2"), - ByteBuffer(string: "34"), - ].async, length: .unknown) + request.body = .stream( + [ + ByteBuffer(string: "1"), + ByteBuffer(string: "2"), + ByteBuffer(string: "34"), + ].async, + length: .unknown + ) - guard let response1 = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response1 = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response1.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response1.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response1.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) - guard let response2 = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response2 = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } XCTAssertEqual(response2.headers["content-length"], []) - guard let body = await XCTAssertNoThrowWithResult( - try await response2.body.collect(upTo: 1024) - ) else { return } + guard + let body = await XCTAssertNoThrowWithResult( + try await response2.body.collect(upTo: 1024) + ) + else { return } XCTAssertEqual(body, ByteBuffer(string: "1234")) } } @@ -782,9 +862,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.headers.add(name: weirdAllowedFieldName, value: "present") // This should work fine. - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } @@ -801,7 +883,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") request.headers.add(name: forbiddenFieldName, value: "present") - await XCTAssertThrowsError(try await client.execute(request, deadline: .now() + .seconds(10), logger: logger)) { error in + await XCTAssertThrowsError( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) { error in XCTAssertEqual(error as? HTTPClientError, .invalidHeaderFieldNames([forbiddenFieldName])) } } @@ -825,15 +909,18 @@ final class AsyncAwaitEndToEndTests: XCTestCase { let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) // We reject all ASCII control characters except HTAB and tolerate everything else. - let weirdAllowedFieldValue = "!\" \t#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + let weirdAllowedFieldValue = + "!\" \t#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") request.headers.add(name: "Weird-Value", value: weirdAllowedFieldValue) // This should work fine. - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } @@ -850,7 +937,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { var request = HTTPClientRequest(url: "https://localhost:\(bin.port)/get") request.headers.add(name: "Weird-Value", value: forbiddenFieldValue) - await XCTAssertThrowsError(try await client.execute(request, deadline: .now() + .seconds(10), logger: logger)) { error in + await XCTAssertThrowsError( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) { error in XCTAssertEqual(error as? HTTPClientError, .invalidHeaderFieldValues([forbiddenFieldValue])) } } @@ -863,9 +952,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { request.headers.add(name: "Weird-Value", value: evenWeirderAllowedValue) // This should work fine. - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } @@ -882,9 +973,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { defer { XCTAssertNoThrow(try client.syncShutdown()) } let request = try HTTPClient.Request(url: "https://localhost:\(bin.port)/get") - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request: request).get() - ) else { + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request: request).get() + ) + else { return } @@ -901,9 +994,11 @@ final class AsyncAwaitEndToEndTests: XCTestCase { defer { XCTAssertNoThrow(try client.syncShutdown()) } let logger = Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) let request = HTTPClientRequest(url: "https://localhost:\(bin.port)/content-length-without-body") - guard let response = await XCTAssertNoThrowWithResult( - try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) - ) else { return } + guard + let response = await XCTAssertNoThrowWithResult( + try await client.execute(request, deadline: .now() + .seconds(10), logger: logger) + ) + else { return } await XCTAssertThrowsError( try await response.body.collect(upTo: 3) ) { diff --git a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift index 147b24dca..cbab922a4 100644 --- a/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift +++ b/Tests/AsyncHTTPClientTests/AsyncTestHelpers.swift @@ -33,7 +33,7 @@ final class AsyncSequenceWriter: AsyncSequence, @unchecked Sendable { } func makeAsyncIterator() -> Iterator { - return Iterator(self) + Iterator(self) } private enum State { @@ -117,7 +117,9 @@ final class AsyncSequenceWriter: AsyncSequence, @unchecked Sendable { case .waiting: let state = self._state self.lock.unlock() - preconditionFailure("Expected that there is always only one concurrent call to next. Invalid state: \(state)") + preconditionFailure( + "Expected that there is always only one concurrent call to next. Invalid state: \(state)" + ) } } diff --git a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift index 79c304fc2..962791334 100644 --- a/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift +++ b/Tests/AsyncHTTPClientTests/ConnectionPoolSizeConfigValueIsRespectedTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,6 +26,10 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class ConnectionPoolSizeConfigValueIsRespectedTests: XCTestCaseHTTPClientTestsBaseClass { func testConnectionPoolSizeConfigValueIsRespected() { let numberOfRequestsPerThread = 1000 diff --git a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift index 5e7a1a9bc..914d03612 100644 --- a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift +++ b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOEmbedded import NIOHTTP1 import NIOHTTP2 +@testable import AsyncHTTPClient + extension EmbeddedChannel { public func receiveHeadAndVerify(_ verify: (HTTPRequestHead) throws -> Void = { _ in }) throws { let part = try self.readOutbound(as: HTTPClientRequestPart.self) @@ -111,6 +112,6 @@ public struct HTTP1EmbeddedChannelError: Error, Hashable, CustomStringConvertibl } public var description: String { - return self.reason + self.reason } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index c91db94b3..53af0823d 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOEmbedded import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + class HTTP1ClientChannelHandlerTests: XCTestCase { func testResponseBackpressure() { let embedded = EmbeddedChannel() @@ -32,27 +33,35 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(testUtils.readEventHandler.readHitCounter, 0) embedded.read() @@ -113,22 +122,30 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 100) { writer in - testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 100) { writer in + testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } // the handler only writes once the channel is writable @@ -143,12 +160,14 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { testWriter.writabilityChanged(true) embedded.pipeline.fireChannelWritabilityChanged() - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .POST) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - XCTAssertEqual($0.headers.first(name: "content-length"), "100") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .POST) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + XCTAssertEqual($0.headers.first(name: "content-length"), "100") + } + ) // the next body write will be executed once we tick the el. before we make the channel // unwritable @@ -162,9 +181,11 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { embedded.embeddedEventLoop.run() - XCTAssertNoThrow(try embedded.receiveBodyAndVerify { - XCTAssertEqual($0.readableBytes, 2) - }) + XCTAssertNoThrow( + try embedded.receiveBodyAndVerify { + XCTAssertEqual($0.readableBytes, 2) + } + ) XCTAssertEqual(testWriter.written, index + 1) @@ -201,24 +222,28 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) XCTAssertTrue(embedded.isActive) @@ -247,27 +272,35 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(testUtils.readEventHandler.readHitCounter, 0) embedded.read() @@ -299,27 +332,35 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(testUtils.readEventHandler.readHitCounter, 0) embedded.read() @@ -345,25 +386,33 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in - // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) - return testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 10) { writer in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + return testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -383,27 +432,35 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream { _ in - // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. - let scheduled = embedded.embeddedEventLoop.flatScheduleTask(in: .milliseconds(2)) { - embedded.embeddedEventLoop.makeSucceededVoidFuture() - } - return scheduled.futureResult - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream { _ in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + let scheduled = embedded.embeddedEventLoop.flatScheduleTask(in: .milliseconds(2)) { + embedded.embeddedEventLoop.makeSucceededVoidFuture() + } + return scheduled.futureResult + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(5)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(5)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -434,34 +491,42 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in - embedded.isWritable = false - embedded.pipeline.fireChannelWritabilityChanged() - // This should not trigger any errors or timeouts, because the timer isn't running - // as the channel is not writable. - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) - - // Now that the channel will become writable, this should trigger a timeout. - embedded.isWritable = true - embedded.pipeline.fireChannelWritabilityChanged() - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) - - return testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 10) { writer in + embedded.isWritable = false + embedded.pipeline.fireChannelWritabilityChanged() + // This should not trigger any errors or timeouts, because the timer isn't running + // as the channel is not writable. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) + + // Now that the channel will become writable, this should trigger a timeout. + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + + return testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -482,22 +547,30 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 2) { writer in - return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 2) { writer in + testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -528,27 +601,35 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "50")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "50")]) + ) XCTAssertEqual(testUtils.readEventHandler.readHitCounter, 0) embedded.read() @@ -599,7 +680,12 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } - XCTAssertNoThrow(try embedded.pipeline.syncOperations.addHandler(FailWriteHandler(), position: .after(testUtils.readEventHandler))) + XCTAssertNoThrow( + try embedded.pipeline.syncOperations.addHandler( + FailWriteHandler(), + position: .after(testUtils.readEventHandler) + ) + ) let logger = Logger(label: "test") @@ -609,16 +695,20 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) - guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) + guard let requestBag = maybeRequestBag else { + return XCTFail("Expected to be able to create a request bag") + } embedded.isWritable = false XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) @@ -645,22 +735,30 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in - testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 10) { writer in + testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } XCTAssertNoThrow(try embedded.pipeline.addHandler(FailEndHandler(), position: .first).wait()) @@ -668,12 +766,14 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { // Execute the request and we'll receive the head. testWriter.writabilityChanged(true) testUtils.connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .POST) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - XCTAssertEqual($0.headers.first(name: "content-length"), "10") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .POST) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + XCTAssertEqual($0.headers.first(name: "content-length"), "10") + } + ) // We're going to immediately send the response head and end. let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) XCTAssertNoThrow(try embedded.writeInbound(HTTPClientResponsePart.head(responseHead))) @@ -689,9 +789,11 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { embedded.embeddedEventLoop.run() XCTAssertEqual(testWriter.written, 5) for _ in 0..<5 { - XCTAssertNoThrow(try embedded.receiveBodyAndVerify { - XCTAssertEqual($0.readableBytes, 2) - }) + XCTAssertNoThrow( + try embedded.receiveBodyAndVerify { + XCTAssertEqual($0.readableBytes, 2) + } + ) } embedded.embeddedEventLoop.run() @@ -722,10 +824,13 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { backgroundLogger: Logger(label: "no-op", factory: SwiftLogNoOpLogHandler.init), connectionIdLoggerMetadata: "test connection" ) - let channel = EmbeddedChannel(handlers: [ - ChangeWritabilityOnFlush(), - handler, - ], loop: eventLoop) + let channel = EmbeddedChannel( + handlers: [ + ChangeWritabilityOnFlush(), + handler, + ], + loop: eventLoop + ) try channel.connect(to: .init(ipAddress: "127.0.0.1", port: 80)).wait() let request = MockHTTPExecutableRequest() @@ -825,7 +930,10 @@ class ResponseBackpressureDelegate: HTTPClientResponseDelegate { return newPromise.futureResult case .waitingForRemote(var promiseBuffer): - assert(!promiseBuffer.isEmpty, "assert expected to be waiting if we have at least one promise in the buffer") + assert( + !promiseBuffer.isEmpty, + "assert expected to be waiting if we have at least one promise in the buffer" + ) let promise = self.eventLoop.makePromise(of: ByteBuffer?.self) promiseBuffer.append(promise) self.state = .waitingForRemote(promiseBuffer) @@ -864,7 +972,10 @@ class ResponseBackpressureDelegate: HTTPClientResponseDelegate { func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { switch self.state { case .waitingForRemote(var promiseBuffer): - assert(!promiseBuffer.isEmpty, "assert expected to be waiting if we have at least one promise in the buffer") + assert( + !promiseBuffer.isEmpty, + "assert expected to be waiting if we have at least one promise in the buffer" + ) let promise = promiseBuffer.removeFirst() if promiseBuffer.isEmpty { let newBackpressurePromise = self.eventLoop.makePromise(of: Void.self) @@ -883,7 +994,9 @@ class ResponseBackpressureDelegate: HTTPClientResponseDelegate { return promise.futureResult case .buffering(.some): - preconditionFailure("Did receive response part should not be called, before the previous promise was succeeded.") + preconditionFailure( + "Did receive response part should not be called, before the previous promise was succeeded." + ) case .done, .consuming: preconditionFailure("Invalid state: \(self.state)") @@ -893,8 +1006,8 @@ class ResponseBackpressureDelegate: HTTPClientResponseDelegate { func didFinishRequest(task: HTTPClient.Task) throws { switch self.state { case .waitingForRemote(let promiseBuffer): - promiseBuffer.forEach { - $0.succeed(.none) + for promise in promiseBuffer { + promise.succeed(.none) } self.state = .done @@ -905,7 +1018,9 @@ class ResponseBackpressureDelegate: HTTPClientResponseDelegate { preconditionFailure("Invalid state: \(self.state)") case .buffering(.some): - preconditionFailure("Did receive response part should not be called, before the previous promise was succeeded.") + preconditionFailure( + "Did receive response part should not be called, before the previous promise was succeeded." + ) } } } diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index e256aa49e..18831d32f 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOHTTP1 import NIOHTTPCompression import XCTest +@testable import AsyncHTTPClient + class HTTP1ConnectionStateMachineTests: XCTestCase { func testPOSTRequestWithWriteAndReadBackpressure() { var state = HTTP1ConnectionStateMachine() @@ -27,7 +28,10 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.runNewRequest(head: requestHead, metadata: metadata), .wait) XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: false)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) @@ -51,7 +55,10 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { XCTAssertEqual(state.requestStreamFinished(promise: nil), .sendRequestEnd(nil)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.informConnectionIsIdle, .init([responseBody]))) @@ -66,10 +73,16 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "12"]) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part0 = ByteBuffer(bytes: 0...3) let part1 = ByteBuffer(bytes: 4...7) let part2 = ByteBuffer(bytes: 8...11) @@ -95,10 +108,16 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: true, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init([responseBody]))) @@ -112,10 +131,16 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_0, status: .ok, headers: ["content-length": "4"]) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init([responseBody]))) @@ -129,10 +154,20 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) - - let responseHead = HTTPResponseHead(version: .http1_0, status: .ok, headers: ["content-length": "4", "connection": "keep-alive"]) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) + + let responseHead = HTTPResponseHead( + version: .http1_0, + status: .ok, + headers: ["content-length": "4", "connection": "keep-alive"] + ) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.informConnectionIsIdle, .init([responseBody]))) @@ -147,10 +182,16 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["connection": "close"]) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init([responseBody]))) @@ -191,13 +232,19 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.runNewRequest(head: requestHead, metadata: metadata), .wait) XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: false)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) - XCTAssertEqual(state.requestCancelled(closeConnection: false), .failRequest(HTTPClientError.cancelled, .close(nil))) + XCTAssertEqual( + state.requestCancelled(closeConnection: false), + .failRequest(HTTPClientError.cancelled, .close(nil)) + ) } func testNewRequestAfterErrorHappened() { @@ -218,9 +265,17 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { XCTAssertEqual(state.channelActive(isWritable: true), .fireChannelActive) XCTAssertEqual(state.requestCancelled(closeConnection: false), .wait, "Should be ignored.") XCTAssertEqual(state.requestCancelled(closeConnection: true), .close, "Should lead to connection closure.") - XCTAssertEqual(state.requestCancelled(closeConnection: true), .wait, "Should be ignored. Connection is already closing") + XCTAssertEqual( + state.requestCancelled(closeConnection: true), + .wait, + "Should be ignored. Connection is already closing" + ) XCTAssertEqual(state.channelInactive(), .fireChannelInactive) - XCTAssertEqual(state.requestCancelled(closeConnection: true), .wait, "Should be ignored. Connection is already closed") + XCTAssertEqual( + state.requestCancelled(closeConnection: true), + .wait, + "Should be ignored. Connection is already closed" + ) } func testReadsAreForwardedIfConnectionIsClosing() { @@ -248,7 +303,10 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: ["content-length": "4"]) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) XCTAssertEqual(state.runNewRequest(head: requestHead, metadata: metadata), .wait) - XCTAssertEqual(state.requestCancelled(closeConnection: false), .failRequest(HTTPClientError.cancelled, .informConnectionIsIdle)) + XCTAssertEqual( + state.requestCancelled(closeConnection: false), + .failRequest(HTTPClientError.cancelled, .informConnectionIsIdle) + ) } func testConnectionIsClosedIfErrorHappensWhileInRequest() { @@ -258,9 +316,15 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.body(ByteBuffer(string: "Hello world!\n"))), .wait) XCTAssertEqual(state.channelRead(.body(ByteBuffer(string: "Foo Bar!\n"))), .wait) let decompressionError = NIOHTTPDecompression.DecompressionError.limit @@ -274,9 +338,15 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .switchingProtocols) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, [])) } @@ -287,8 +357,14 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true)) - let responseHead = HTTPResponseHead(version: .http1_1, status: .init(statusCode: 103, reasonPhrase: "Early Hints")) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .init(statusCode: 103, reasonPhrase: "Early Hints") + ) XCTAssertEqual(state.channelRead(.head(responseHead)), .wait) XCTAssertEqual(state.channelInactive(), .failRequest(HTTPClientError.remoteConnectionClosed, .none)) } @@ -339,13 +415,19 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { case (.resumeRequestBodyStream, .resumeRequestBodyStream): return true - case (.forwardResponseHead(let lhsHead, let lhsPauseRequestBodyStream), .forwardResponseHead(let rhsHead, let rhsPauseRequestBodyStream)): + case ( + .forwardResponseHead(let lhsHead, let lhsPauseRequestBodyStream), + .forwardResponseHead(let rhsHead, let rhsPauseRequestBodyStream) + ): return lhsHead == rhsHead && lhsPauseRequestBodyStream == rhsPauseRequestBodyStream case (.forwardResponseBodyParts(let lhsData), .forwardResponseBodyParts(let rhsData)): return lhsData == rhsData - case (.succeedRequest(let lhsFinalAction, let lhsFinalBuffer), .succeedRequest(let rhsFinalAction, let rhsFinalBuffer)): + case ( + .succeedRequest(let lhsFinalAction, let lhsFinalBuffer), + .succeedRequest(let rhsFinalAction, let rhsFinalBuffer) + ): return lhsFinalAction == rhsFinalAction && lhsFinalBuffer == rhsFinalBuffer case (.failRequest(_, let lhsFinalAction), .failRequest(_, let rhsFinalAction)): @@ -367,7 +449,10 @@ extension HTTP1ConnectionStateMachine.Action: Equatable { } extension HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction: Equatable { - public static func == (lhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction, rhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction) -> Bool { + public static func == ( + lhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction, + rhs: HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction + ) -> Bool { switch (lhs, rhs) { case (.close, .close): return true @@ -382,7 +467,10 @@ extension HTTP1ConnectionStateMachine.Action.FinalSuccessfulStreamAction: Equata } extension HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction: Equatable { - public static func == (lhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction, rhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction) -> Bool { + public static func == ( + lhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction, + rhs: HTTP1ConnectionStateMachine.Action.FinalFailedStreamAction + ) -> Bool { switch (lhs, rhs) { case (.close(let lhsPromise), .close(let rhsPromise)): return lhsPromise?.futureResult == rhsPromise?.futureResult diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift index 5ea8bb77c..5f980bccb 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOConcurrencyHelpers import NIOCore @@ -23,6 +22,8 @@ import NIOPosix import NIOTestUtils import XCTest +@testable import AsyncHTTPClient + class HTTP1ConnectionTests: XCTestCase { func testCreateNewConnectionWithDecompression() { let embedded = EmbeddedChannel() @@ -31,16 +32,20 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertNoThrow(try embedded.connect(to: SocketAddress(ipAddress: "127.0.0.1", port: 3000)).wait()) var connection: HTTP1Connection? - XCTAssertNoThrow(connection = try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: MockHTTP1ConnectionDelegate(), - decompression: .enabled(limit: .ratio(4)), - logger: logger - )) + XCTAssertNoThrow( + connection = try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: MockHTTP1ConnectionDelegate(), + decompression: .enabled(limit: .ratio(4)), + logger: logger + ) + ) XCTAssertNotNil(try embedded.pipeline.syncOperations.handler(type: HTTPRequestEncoder.self)) - XCTAssertNotNil(try embedded.pipeline.syncOperations.handler(type: ByteToMessageHandler.self)) + XCTAssertNotNil( + try embedded.pipeline.syncOperations.handler(type: ByteToMessageHandler.self) + ) XCTAssertNotNil(try embedded.pipeline.syncOperations.handler(type: NIOHTTPResponseDecompressor.self)) XCTAssertNoThrow(try connection?.close().wait()) @@ -54,17 +59,22 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertNoThrow(try embedded.connect(to: SocketAddress(ipAddress: "127.0.0.1", port: 3000)).wait()) - XCTAssertNoThrow(try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: MockHTTP1ConnectionDelegate(), - decompression: .disabled, - logger: logger - )) + XCTAssertNoThrow( + try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: MockHTTP1ConnectionDelegate(), + decompression: .disabled, + logger: logger + ) + ) XCTAssertNotNil(try embedded.pipeline.syncOperations.handler(type: HTTPRequestEncoder.self)) - XCTAssertNotNil(try embedded.pipeline.syncOperations.handler(type: ByteToMessageHandler.self)) - XCTAssertThrowsError(try embedded.pipeline.syncOperations.handler(type: NIOHTTPResponseDecompressor.self)) { error in + XCTAssertNotNil( + try embedded.pipeline.syncOperations.handler(type: ByteToMessageHandler.self) + ) + XCTAssertThrowsError(try embedded.pipeline.syncOperations.handler(type: NIOHTTPResponseDecompressor.self)) { + error in XCTAssertEqual(error as? ChannelPipelineError, .notFound) } } @@ -78,13 +88,15 @@ class HTTP1ConnectionTests: XCTestCase { embedded.embeddedEventLoop.run() let logger = Logger(label: "test.http1.connection") - XCTAssertThrowsError(try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: MockHTTP1ConnectionDelegate(), - decompression: .disabled, - logger: logger - )) + XCTAssertThrowsError( + try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: MockHTTP1ConnectionDelegate(), + decompression: .disabled, + logger: logger + ) + ) } func testGETRequest() { @@ -113,30 +125,32 @@ class HTTP1ConnectionTests: XCTestCase { .wait() var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "http://localhost/hello/swift", - method: .POST, - body: .stream(contentLength: 4) { writer -> EventLoopFuture in - @Sendable func recursive(count: UInt8, promise: EventLoopPromise) { - guard count < 4 else { - return promise.succeed(()) - } + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/hello/swift", + method: .POST, + body: .stream(contentLength: 4) { writer -> EventLoopFuture in + @Sendable func recursive(count: UInt8, promise: EventLoopPromise) { + guard count < 4 else { + return promise.succeed(()) + } - writer.write(.byteBuffer(ByteBuffer(bytes: [count]))).whenComplete { result in - switch result { - case .failure(let error): - XCTFail("Unexpected error: \(error)") - case .success: - recursive(count: count + 1, promise: promise) + writer.write(.byteBuffer(ByteBuffer(bytes: [count]))).whenComplete { result in + switch result { + case .failure(let error): + XCTFail("Unexpected error: \(error)") + case .success: + recursive(count: count + 1, promise: promise) + } } } - } - let promise = clientEL.makePromise(of: Void.self) - recursive(count: 0, promise: promise) - return promise.futureResult - } - )) + let promise = clientEL.makePromise(of: Void.self) + recursive(count: 0, promise: promise) + return promise.futureResult + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to have a connection and a request") @@ -145,33 +159,39 @@ class HTTP1ConnectionTests: XCTestCase { let task = HTTPClient.Task(eventLoop: clientEL, logger: logger) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: clientEL), - task: task, - redirectHandler: nil, - connectionDeadline: .now() + .seconds(60), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: request) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: clientEL), + task: task, + redirectHandler: nil, + connectionDeadline: .now() + .seconds(60), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: request) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } connection.executeRequest(requestBag) - XCTAssertNoThrow(try server.receiveHeadAndVerify { head in - XCTAssertEqual(head.method, .POST) - XCTAssertEqual(head.uri, "/hello/swift") - XCTAssertEqual(head.headers["content-length"].first, "4") - }) + XCTAssertNoThrow( + try server.receiveHeadAndVerify { head in + XCTAssertEqual(head.method, .POST) + XCTAssertEqual(head.uri, "/hello/swift") + XCTAssertEqual(head.headers["content-length"].first, "4") + } + ) var received: UInt8 = 0 while received < 4 { - XCTAssertNoThrow(try server.receiveBodyAndVerify { body in - var body = body - while let read = body.readInteger(as: UInt8.self) { - XCTAssertEqual(received, read) - received += 1 + XCTAssertNoThrow( + try server.receiveBodyAndVerify { body in + var body = body + while let read = body.readInteger(as: UInt8.self) { + XCTAssertEqual(received, read) + received += 1 + } } - }) + ) } XCTAssertEqual(received, 4) XCTAssertNoThrow(try server.receiveEnd()) @@ -198,17 +218,23 @@ class HTTP1ConnectionTests: XCTestCase { var maybeChannel: Channel? - XCTAssertNoThrow(maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait()) + XCTAssertNoThrow( + maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait() + ) let connectionDelegate = MockConnectionDelegate() let logger = Logger(label: "test") var maybeConnection: HTTP1Connection? - XCTAssertNoThrow(maybeConnection = try eventLoop.submit { try HTTP1Connection.start( - channel: XCTUnwrap(maybeChannel), - connectionID: 0, - delegate: connectionDelegate, - decompression: .disabled, - logger: logger - ) }.wait()) + XCTAssertNoThrow( + maybeConnection = try eventLoop.submit { + try HTTP1Connection.start( + channel: XCTUnwrap(maybeChannel), + connectionID: 0, + delegate: connectionDelegate, + decompression: .disabled, + logger: logger + ) + }.wait() + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } var maybeRequest: HTTPClient.Request? @@ -217,15 +243,17 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: eventLoopGroup.next()), - task: .init(eventLoop: eventLoopGroup.next(), logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: eventLoopGroup.next()), + task: .init(eventLoop: eventLoopGroup.next(), logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } connection.executeRequest(requestBag) @@ -248,21 +276,29 @@ class HTTP1ConnectionTests: XCTestCase { defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } let closeOnRequest = (30...100).randomElement()! - let httpBin = HTTPBin(handlerFactory: { _ in SuddenlySendsCloseHeaderChannelHandler(closeOnRequest: closeOnRequest) }) + let httpBin = HTTPBin(handlerFactory: { _ in + SuddenlySendsCloseHeaderChannelHandler(closeOnRequest: closeOnRequest) + }) var maybeChannel: Channel? - XCTAssertNoThrow(maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait()) + XCTAssertNoThrow( + maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait() + ) let connectionDelegate = MockConnectionDelegate() let logger = Logger(label: "test") var maybeConnection: HTTP1Connection? - XCTAssertNoThrow(maybeConnection = try eventLoop.submit { try HTTP1Connection.start( - channel: XCTUnwrap(maybeChannel), - connectionID: 0, - delegate: connectionDelegate, - decompression: .disabled, - logger: logger - ) }.wait()) + XCTAssertNoThrow( + maybeConnection = try eventLoop.submit { + try HTTP1Connection.start( + channel: XCTUnwrap(maybeChannel), + connectionID: 0, + delegate: connectionDelegate, + decompression: .disabled, + logger: logger + ) + }.wait() + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } var counter = 0 @@ -275,16 +311,20 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: eventLoopGroup.next()), - task: .init(eventLoop: eventLoopGroup.next(), logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) - guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: eventLoopGroup.next()), + task: .init(eventLoop: eventLoopGroup.next(), logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) + guard let requestBag = maybeRequestBag else { + return XCTFail("Expected to be able to create a request bag") + } connection.executeRequest(requestBag) @@ -293,7 +333,7 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertEqual(response?.status, .ok) if response?.headers.first(name: "connection") == "close" { - break // the loop + break // the loop } else { XCTAssertEqual(httpBin.activeConnections, 1) XCTAssertEqual(connectionDelegate.hitConnectionReleased, counter) @@ -306,8 +346,11 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertEqual(counter, closeOnRequest) XCTAssertEqual(connectionDelegate.hitConnectionClosed, 1) - XCTAssertEqual(connectionDelegate.hitConnectionReleased, counter - 1, - "If a close header is received connection release is not triggered.") + XCTAssertEqual( + connectionDelegate.hitConnectionReleased, + counter - 1, + "If a close header is received connection release is not triggered." + ) // we need to wait a small amount of time to see the connection close on the server try! eventLoop.scheduleTask(in: .milliseconds(200)) {}.futureResult.wait() @@ -324,17 +367,23 @@ class HTTP1ConnectionTests: XCTestCase { var maybeChannel: Channel? - XCTAssertNoThrow(maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait()) + XCTAssertNoThrow( + maybeChannel = try ClientBootstrap(group: eventLoop).connect(host: "localhost", port: httpBin.port).wait() + ) let connectionDelegate = MockConnectionDelegate() let logger = Logger(label: "test") var maybeConnection: HTTP1Connection? - XCTAssertNoThrow(maybeConnection = try eventLoop.submit { try HTTP1Connection.start( - channel: XCTUnwrap(maybeChannel), - connectionID: 0, - delegate: connectionDelegate, - decompression: .disabled, - logger: logger - ) }.wait()) + XCTAssertNoThrow( + maybeConnection = try eventLoop.submit { + try HTTP1Connection.start( + channel: XCTUnwrap(maybeChannel), + connectionID: 0, + delegate: connectionDelegate, + decompression: .disabled, + logger: logger + ) + }.wait() + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection here") } var maybeRequest: HTTPClient.Request? @@ -343,15 +392,17 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: eventLoopGroup.next()), - task: .init(eventLoop: eventLoopGroup.next(), logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: eventLoopGroup.next()), + task: .init(eventLoop: eventLoopGroup.next(), logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } connection.executeRequest(requestBag) @@ -373,13 +424,15 @@ class HTTP1ConnectionTests: XCTestCase { var maybeConnection: HTTP1Connection? let connectionDelegate = MockConnectionDelegate() - XCTAssertNoThrow(maybeConnection = try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: connectionDelegate, - decompression: .enabled(limit: .ratio(4)), - logger: logger - )) + XCTAssertNoThrow( + maybeConnection = try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: connectionDelegate, + decompression: .enabled(limit: .ratio(4)), + logger: logger + ) + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } var maybeRequest: HTTPClient.Request? @@ -388,38 +441,40 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end let responseString = """ - HTTP/1.1 101 Switching Protocols\r\n\ - Upgrade: websocket\r\n\ - Sec-WebSocket-Accept: xAMUK7/Il9bLRFJrikq6mm8CNZI=\r\n\ - Connection: upgrade\r\n\ - date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ - \r\n\ - \r\nfoo bar baz - """ + HTTP/1.1 101 Switching Protocols\r\n\ + Upgrade: websocket\r\n\ + Sec-WebSocket-Accept: xAMUK7/Il9bLRFJrikq6mm8CNZI=\r\n\ + Connection: upgrade\r\n\ + date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ + \r\n\ + \r\nfoo bar baz + """ XCTAssertTrue(embedded.isActive) XCTAssertEqual(connectionDelegate.hitConnectionClosed, 0) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) XCTAssertNoThrow(try embedded.writeInbound(ByteBuffer(string: responseString))) XCTAssertFalse(embedded.isActive) - (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. + (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. XCTAssertEqual(connectionDelegate.hitConnectionClosed, 1) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) @@ -438,13 +493,15 @@ class HTTP1ConnectionTests: XCTestCase { var maybeConnection: HTTP1Connection? let connectionDelegate = MockConnectionDelegate() - XCTAssertNoThrow(maybeConnection = try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: connectionDelegate, - decompression: .enabled(limit: .ratio(4)), - logger: logger - )) + XCTAssertNoThrow( + maybeConnection = try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: connectionDelegate, + decompression: .enabled(limit: .ratio(4)), + logger: logger + ) + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } var maybeRequest: HTTPClient.Request? @@ -453,28 +510,30 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } connection.executeRequest(requestBag) - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end let responseString = """ - HTTP/1.1 103 Early Hints\r\n\ - date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ - \r\n\ - \r\n - """ + HTTP/1.1 103 Early Hints\r\n\ + date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ + \r\n\ + \r\n + """ XCTAssertTrue(embedded.isActive) XCTAssertEqual(connectionDelegate.hitConnectionClosed, 0) @@ -484,7 +543,7 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertTrue(embedded.isActive, "The connection remains active after the informational response head") XCTAssertNoThrow(try embedded.close().wait(), "the connection was closed") - embedded.embeddedEventLoop.run() // tick once to run futures. + embedded.embeddedEventLoop.run() // tick once to run futures. XCTAssertEqual(connectionDelegate.hitConnectionClosed, 1) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) @@ -500,20 +559,22 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertNoThrow(try embedded.connect(to: SocketAddress(ipAddress: "127.0.0.1", port: 0)).wait()) let connectionDelegate = MockConnectionDelegate() - XCTAssertNoThrow(try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: connectionDelegate, - decompression: .enabled(limit: .ratio(4)), - logger: logger - )) + XCTAssertNoThrow( + try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: connectionDelegate, + decompression: .enabled(limit: .ratio(4)), + logger: logger + ) + ) let responseString = """ - HTTP/1.1 200 OK\r\n\ - date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ - \r\n\ - \r\n - """ + HTTP/1.1 200 OK\r\n\ + date: Mon, 27 Sep 2021 17:53:14 GMT\r\n\ + \r\n\ + \r\n + """ XCTAssertEqual(connectionDelegate.hitConnectionClosed, 0) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) @@ -522,7 +583,7 @@ class HTTP1ConnectionTests: XCTestCase { XCTAssertEqual($0 as? NIOHTTPDecoderError, .unsolicitedResponse) } XCTAssertFalse(embedded.isActive) - (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. + (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. XCTAssertEqual(connectionDelegate.hitConnectionClosed, 1) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) } @@ -535,13 +596,15 @@ class HTTP1ConnectionTests: XCTestCase { var maybeConnection: HTTP1Connection? let connectionDelegate = MockConnectionDelegate() - XCTAssertNoThrow(maybeConnection = try HTTP1Connection.start( - channel: embedded, - connectionID: 0, - delegate: connectionDelegate, - decompression: .enabled(limit: .ratio(4)), - logger: logger - )) + XCTAssertNoThrow( + maybeConnection = try HTTP1Connection.start( + channel: embedded, + connectionID: 0, + delegate: connectionDelegate, + decompression: .enabled(limit: .ratio(4)), + logger: logger + ) + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point.") } var maybeRequest: HTTPClient.Request? @@ -550,32 +613,34 @@ class HTTP1ConnectionTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } connection.executeRequest(requestBag) let responseString = """ - HTTP/1.0 200 OK\r\n\ - HTTP/1.0 200 OK\r\n\r\n - """ + HTTP/1.0 200 OK\r\n\ + HTTP/1.0 200 OK\r\n\r\n + """ - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head - XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // head + XCTAssertNoThrow(try embedded.readOutbound(as: ByteBuffer.self)) // end XCTAssertEqual(connectionDelegate.hitConnectionClosed, 0) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) XCTAssertNoThrow(try embedded.writeInbound(ByteBuffer(string: responseString))) XCTAssertFalse(embedded.isActive) - (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. + (embedded.eventLoop as! EmbeddedEventLoop).run() // tick once to run futures. XCTAssertEqual(connectionDelegate.hitConnectionClosed, 1) XCTAssertEqual(connectionDelegate.hitConnectionReleased, 0) } @@ -606,7 +671,7 @@ class HTTP1ConnectionTests: XCTestCase { } var reads: Int { - return self.lock.withLock { + self.lock.withLock { self._reads } } @@ -618,7 +683,7 @@ class HTTP1ConnectionTests: XCTestCase { } func didReceiveHead(task: HTTPClient.Task, _ head: HTTPResponseHead) -> EventLoopFuture { - return task.futureResult.eventLoop.makeSucceededVoidFuture() + task.futureResult.eventLoop.makeSucceededVoidFuture() } func didReceiveBodyPart(task: HTTPClient.Task, _ buffer: ByteBuffer) -> EventLoopFuture { @@ -679,34 +744,42 @@ class HTTP1ConnectionTests: XCTestCase { defer { XCTAssertNoThrow(try httpBin.shutdown()) } var maybeChannel: Channel? - XCTAssertNoThrow(maybeChannel = try ClientBootstrap(group: eventLoopGroup) - .channelOption(ChannelOptions.maxMessagesPerRead, value: 1) - .channelOption(ChannelOptions.recvAllocator, value: FixedSizeRecvByteBufferAllocator(capacity: 1)) - .connect(host: "localhost", port: httpBin.port) - .wait()) + XCTAssertNoThrow( + maybeChannel = try ClientBootstrap(group: eventLoopGroup) + .channelOption(ChannelOptions.maxMessagesPerRead, value: 1) + .channelOption(ChannelOptions.recvAllocator, value: FixedSizeRecvByteBufferAllocator(capacity: 1)) + .connect(host: "localhost", port: httpBin.port) + .wait() + ) guard let channel = maybeChannel else { return XCTFail("Expected to have a channel at this point") } let connectionDelegate = MockConnectionDelegate() var maybeConnection: HTTP1Connection? - XCTAssertNoThrow(maybeConnection = try channel.eventLoop.submit { try HTTP1Connection.start( - channel: channel, - connectionID: 0, - delegate: connectionDelegate, - decompression: .disabled, - logger: logger - ) }.wait()) + XCTAssertNoThrow( + maybeConnection = try channel.eventLoop.submit { + try HTTP1Connection.start( + channel: channel, + connectionID: 0, + delegate: connectionDelegate, + decompression: .disabled, + logger: logger + ) + }.wait() + ) guard let connection = maybeConnection else { return XCTFail("Expected to have a connection at this point") } var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: HTTPClient.Request(url: "http://localhost:\(httpBin.port)/custom"), - eventLoopPreference: .delegate(on: requestEventLoop), - task: .init(eventLoop: requestEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: backpressureDelegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: HTTPClient.Request(url: "http://localhost:\(httpBin.port)/custom"), + eventLoopPreference: .delegate(on: requestEventLoop), + task: .init(eventLoop: requestEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: backpressureDelegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } backpressureDelegate.willExecuteOnChannel(connection.channel) @@ -764,7 +837,12 @@ class SuddenlySendsCloseHeaderChannelHandler: ChannelInboundHandler { break case .end: if self.closeOnRequest == self.counter { - context.write(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: ["connection": "close"]))), promise: nil) + context.write( + self.wrapOutboundOut( + .head(.init(version: .http1_1, status: .ok, headers: ["connection": "close"])) + ), + promise: nil + ) context.write(self.wrapOutboundOut(.end(nil)), promise: nil) context.flush() self.counter += 1 diff --git a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift index b3917173f..d75865da2 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ProxyConnectHandlerTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + class HTTP1ProxyConnectHandlerTests: XCTestCase { func testProxyConnectWithoutAuthorizationSuccess() { let embedded = EmbeddedChannel() diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift index 2428199a4..1f5f1b4c0 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientRequestHandlerTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOEmbedded import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + class HTTP2ClientRequestHandlerTests: XCTestCase { func testResponseBackpressure() { let embedded = EmbeddedChannel() @@ -34,28 +35,36 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.write(requestBag, promise: nil) XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertEqual(try embedded.readOutbound(as: HTTPClientRequestPart.self), .end(nil)) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(readEventHandler.readHitCounter, 0) embedded.read() @@ -115,22 +124,30 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 50) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 100) { writer in - testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 100) { writer in + testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = false @@ -143,12 +160,14 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { testWriter.writabilityChanged(true) embedded.pipeline.fireChannelWritabilityChanged() - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .POST) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - XCTAssertEqual($0.headers.first(name: "content-length"), "100") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .POST) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + XCTAssertEqual($0.headers.first(name: "content-length"), "100") + } + ) // the next body write will be executed once we tick the el. before we make the channel // unwritable @@ -162,9 +181,11 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { embedded.embeddedEventLoop.run() - XCTAssertNoThrow(try embedded.receiveBodyAndVerify { - XCTAssertEqual($0.readableBytes, 2) - }) + XCTAssertNoThrow( + try embedded.receiveBodyAndVerify { + XCTAssertEqual($0.readableBytes, 2) + } + ) XCTAssertEqual(testWriter.written, index + 1) @@ -198,27 +219,35 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.write(requestBag, promise: nil) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(readEventHandler.readHitCounter, 0) embedded.read() @@ -248,27 +277,35 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.write(requestBag, promise: nil) - XCTAssertNoThrow(try embedded.receiveHeadAndVerify { - XCTAssertEqual($0.method, .GET) - XCTAssertEqual($0.uri, "/") - XCTAssertEqual($0.headers.first(name: "host"), "localhost") - }) + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) XCTAssertNoThrow(try embedded.receiveEnd()) - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) XCTAssertEqual(readEventHandler.readHitCounter, 0) embedded.read() @@ -295,24 +332,32 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in - // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) - return testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 10) { writer in + // Advance time by more than the idle write timeout (that's 1 millisecond) to trigger the timeout. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + return testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -335,34 +380,42 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 10) { writer in - embedded.isWritable = false - embedded.pipeline.fireChannelWritabilityChanged() - // This should not trigger any errors or timeouts, because the timer isn't running - // as the channel is not writable. - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) - - // Now that the channel will become writable, this should trigger a timeout. - embedded.isWritable = true - embedded.pipeline.fireChannelWritabilityChanged() - embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) - - return testWriter.start(writer: writer) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 10) { writer in + embedded.isWritable = false + embedded.pipeline.fireChannelWritabilityChanged() + // This should not trigger any errors or timeouts, because the timer isn't running + // as the channel is not writable. + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(20)) + + // Now that the channel will become writable, this should trigger a timeout. + embedded.isWritable = true + embedded.pipeline.fireChannelWritabilityChanged() + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(2)) + + return testWriter.start(writer: writer) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -385,22 +438,30 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let testWriter = TestBackpressureWriter(eventLoop: embedded.eventLoop, parts: 5) var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/", method: .POST, body: .stream(contentLength: 2) { writer in - return testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) - })) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost/", + method: .POST, + body: .stream(contentLength: 2) { writer in + testWriter.start(writer: writer, expectedErrors: [HTTPClientError.cancelled]) + } + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to be able to create a request") } let delegate = ResponseBackpressureDelegate(eventLoop: embedded.eventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleWriteTimeout: .milliseconds(1)), + delegate: delegate + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } embedded.isWritable = true @@ -451,16 +512,20 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let delegate = ResponseAccumulator(request: request) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embedded.eventLoop), - task: .init(eventLoop: embedded.eventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), - delegate: delegate - )) - guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(idleReadTimeout: .milliseconds(200)), + delegate: delegate + ) + ) + guard let requestBag = maybeRequestBag else { + return XCTFail("Expected to be able to create a request bag") + } embedded.isWritable = false XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) @@ -494,10 +559,13 @@ class HTTP2ClientRequestHandlerTests: XCTestCase { let handler = HTTP2ClientRequestHandler( eventLoop: eventLoop ) - let channel = EmbeddedChannel(handlers: [ - ChangeWritabilityOnFlush(), - handler, - ], loop: eventLoop) + let channel = EmbeddedChannel( + handlers: [ + ChangeWritabilityOnFlush(), + handler, + ], + loop: eventLoop + ) try channel.connect(to: .init(ipAddress: "127.0.0.1", port: 80)).wait() let request = MockHTTPExecutableRequest() diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 889cd38b9..1d6c0c8f8 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -12,10 +12,7 @@ // //===----------------------------------------------------------------------===// -/* NOT @testable */ import AsyncHTTPClient // Tests that really need @testable go into HTTP2ClientInternalTests.swift -#if canImport(Network) -import Network -#endif +import AsyncHTTPClient // NOT @testable - tests that really need @testable go into HTTP2ClientInternalTests.swift import Logging import NIOCore import NIOHTTP1 @@ -23,6 +20,10 @@ import NIOPosix import NIOSSL import XCTest +#if canImport(Network) +import Network +#endif + class HTTP2ClientTests: XCTestCase { func makeDefaultHTTPClient( eventLoopGroupProvider: HTTPClient.EventLoopGroupProvider = .singleton @@ -132,8 +133,8 @@ class HTTP2ClientTests: XCTestCase { let q = DispatchQueue(label: "worker \(w)") q.async(group: allDone) { func go() { - allWorkersReady.signal() // tell the driver we're ready - allWorkersGo.wait() // wait for the driver to let us go + allWorkersReady.signal() // tell the driver we're ready + allWorkersGo.wait() // wait for the driver to let us go for _ in 0..] = [] - XCTAssertNoThrow(results = try EventLoopFuture - .whenAllComplete(responses, on: clientGroup.next()) - .timeout(after: .seconds(2)) - .wait()) + XCTAssertNoThrow( + results = + try EventLoopFuture + .whenAllComplete(responses, on: clientGroup.next()) + .timeout(after: .seconds(2)) + .wait() + ) for result in results { switch result { @@ -397,7 +402,11 @@ class HTTP2ClientTests: XCTestCase { XCTAssertNoThrow(maybeRequest1 = try HTTPClient.Request(url: "https://localhost:\(bin.port)/get")) guard let request1 = maybeRequest1 else { return } - let task1 = client.execute(request: request1, delegate: ResponseAccumulator(request: request1), eventLoop: .delegateAndChannel(on: el1)) + let task1 = client.execute( + request: request1, + delegate: ResponseAccumulator(request: request1), + eventLoop: .delegateAndChannel(on: el1) + ) var response1: ResponseAccumulator.Response? XCTAssertNoThrow(response1 = try task1.wait()) @@ -408,15 +417,17 @@ class HTTP2ClientTests: XCTestCase { let serverGroup = MultiThreadedEventLoopGroup(numberOfThreads: 1) defer { XCTAssertNoThrow(try serverGroup.syncShutdownGracefully()) } var maybeServer: Channel? - XCTAssertNoThrow(maybeServer = try ServerBootstrap(group: serverGroup) - .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) - .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), value: 1) - .childChannelInitializer { channel in - channel.close() - } - .childChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) - .bind(host: "127.0.0.1", port: serverPort) - .wait()) + XCTAssertNoThrow( + maybeServer = try ServerBootstrap(group: serverGroup) + .serverChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) + .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), value: 1) + .childChannelInitializer { channel in + channel.close() + } + .childChannelOption(ChannelOptions.socketOption(.so_reuseaddr), value: 1) + .bind(host: "127.0.0.1", port: serverPort) + .wait() + ) // shutting down the old server closes all connections immediately XCTAssertNoThrow(try bin.shutdown()) // client is now in HTTP/2 state and the HTTPBin is closed @@ -427,7 +438,11 @@ class HTTP2ClientTests: XCTestCase { XCTAssertNoThrow(maybeRequest2 = try HTTPClient.Request(url: "https://localhost:\(serverPort)/")) guard let request2 = maybeRequest2 else { return } - let task2 = client.execute(request: request2, delegate: ResponseAccumulator(request: request2), eventLoop: .delegateAndChannel(on: el2)) + let task2 = client.execute( + request: request2, + delegate: ResponseAccumulator(request: request2), + eventLoop: .delegateAndChannel(on: el2) + ) XCTAssertThrowsError(try task2.wait()) { error in XCTAssertNil( error as? HTTPClientError, @@ -474,11 +489,17 @@ private final class SendHeaderAndWaitChannelHandler: ChannelInboundHandler { let requestPart = self.unwrapInboundIn(data) switch requestPart { case .head: - context.writeAndFlush(self.wrapOutboundOut(.head(HTTPResponseHead( - version: HTTPVersion(major: 1, minor: 1), - status: .ok - )) - ), promise: nil) + context.writeAndFlush( + self.wrapOutboundOut( + .head( + HTTPResponseHead( + version: HTTPVersion(major: 1, minor: 1), + status: .ok + ) + ) + ), + promise: nil + ) case .body, .end: return } diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index 2e82fafba..acf81beac 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOConcurrencyHelpers import NIOCore @@ -24,6 +23,8 @@ import NIOSSL import NIOTestUtils import XCTest +@testable import AsyncHTTPClient + class HTTP2ConnectionTests: XCTestCase { func testCreateNewConnectionFailureClosedIO() { let embedded = EmbeddedChannel() @@ -34,14 +35,16 @@ class HTTP2ConnectionTests: XCTestCase { embedded.embeddedEventLoop.run() let logger = Logger(label: "test.http2.connection") - XCTAssertThrowsError(try HTTP2Connection.start( - channel: embedded, - connectionID: 0, - delegate: TestHTTP2ConnectionDelegate(), - decompression: .disabled, - maximumConnectionUses: nil, - logger: logger - ).wait()) + XCTAssertThrowsError( + try HTTP2Connection.start( + channel: embedded, + connectionID: 0, + delegate: TestHTTP2ConnectionDelegate(), + decompression: .disabled, + maximumConnectionUses: nil, + logger: logger + ).wait() + ) } func testConnectionToleratesShutdownEventsAfterAlreadyClosed() { @@ -80,11 +83,12 @@ class HTTP2ConnectionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - ) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") @@ -93,15 +97,17 @@ class HTTP2ConnectionTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to have a request bag at this point") } @@ -136,11 +142,13 @@ class HTTP2ConnectionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - )) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) + ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") } @@ -154,15 +162,17 @@ class HTTP2ConnectionTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to have a request bag at this point") } @@ -200,11 +210,12 @@ class HTTP2ConnectionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - ) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") @@ -216,15 +227,17 @@ class HTTP2ConnectionTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to have a request bag at this point") } @@ -292,11 +305,13 @@ class HTTP2ConnectionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - )) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) + ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") } @@ -304,15 +319,17 @@ class HTTP2ConnectionTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to have a request bag at this point") } @@ -321,7 +338,9 @@ class HTTP2ConnectionTests: XCTestCase { XCTAssertNoThrow(try serverReceivedRequestPromise.futureResult.wait()) var channelCount: Int? - XCTAssertNoThrow(channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait()) + XCTAssertNoThrow( + channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait() + ) XCTAssertEqual(channelCount, 1) triggerResponsePromise.succeed(()) @@ -331,7 +350,9 @@ class HTTP2ConnectionTests: XCTestCase { var retryCount = 0 let maxRetries = 1000 while retryCount < maxRetries { - XCTAssertNoThrow(channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait()) + XCTAssertNoThrow( + channelCount = try eventLoop.submit { http2Connection.__forTesting_getStreamChannels().count }.wait() + ) if channelCount == 0 { break } diff --git a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift index 611e31457..f2b56daa0 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2IdleHandlerTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOEmbedded import NIOHTTP2 import XCTest +@testable import AsyncHTTPClient + class HTTP2IdleHandlerTests: XCTestCase { func testReceiveSettingsWithMaxConcurrentStreamSetting() { let delegate = MockHTTP2IdleHandlerDelegate() @@ -26,7 +27,10 @@ class HTTP2IdleHandlerTests: XCTestCase { let embedded = EmbeddedChannel(handlers: [idleHandler]) XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) - let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)]))) + let settingsFrame = HTTP2Frame( + streamID: 0, + payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)])) + ) XCTAssertEqual(delegate.maxStreams, nil) XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) XCTAssertEqual(delegate.maxStreams, 10) @@ -41,7 +45,11 @@ class HTTP2IdleHandlerTests: XCTestCase { let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([]))) XCTAssertEqual(delegate.maxStreams, nil) XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) - XCTAssertEqual(delegate.maxStreams, 100, "Expected to assume 100 maxConcurrentConnection, if no setting was present") + XCTAssertEqual( + delegate.maxStreams, + 100, + "Expected to assume 100 maxConcurrentConnection, if no setting was present" + ) } func testEmptySettingsDontOverwriteMaxConcurrentStreamSetting() { @@ -50,7 +58,10 @@ class HTTP2IdleHandlerTests: XCTestCase { let embedded = EmbeddedChannel(handlers: [idleHandler]) XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) - let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)]))) + let settingsFrame = HTTP2Frame( + streamID: 0, + payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)])) + ) XCTAssertEqual(delegate.maxStreams, nil) XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) XCTAssertEqual(delegate.maxStreams, 10) @@ -66,12 +77,18 @@ class HTTP2IdleHandlerTests: XCTestCase { let embedded = EmbeddedChannel(handlers: [idleHandler]) XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) - let settingsFrame = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)]))) + let settingsFrame = HTTP2Frame( + streamID: 0, + payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 10)])) + ) XCTAssertEqual(delegate.maxStreams, nil) XCTAssertNoThrow(try embedded.writeInbound(settingsFrame)) XCTAssertEqual(delegate.maxStreams, 10) - let emptySettings = HTTP2Frame(streamID: 0, payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 20)]))) + let emptySettings = HTTP2Frame( + streamID: 0, + payload: .settings(.settings([.init(parameter: .maxConcurrentStreams, value: 20)])) + ) XCTAssertNoThrow(try embedded.writeInbound(emptySettings)) XCTAssertEqual(delegate.maxStreams, 20) } @@ -83,7 +100,10 @@ class HTTP2IdleHandlerTests: XCTestCase { XCTAssertNoThrow(try embedded.connect(to: .makeAddressResolvingHost("localhost", port: 0)).wait()) let randomStreamID = HTTP2StreamID((0.., - _ head: HTTPResponseHead) -> EventLoopFuture { + public func didReceiveHead( + task: HTTPClient.Task, + _ head: HTTPResponseHead + ) -> EventLoopFuture { self.eventLoop.assertInEventLoop() self.receivedMessages.append(.head(head)) return self.randoEL.makeSucceededFuture(()) } - func didReceiveBodyPart(task: HTTPClient.Task, - _ buffer: ByteBuffer) -> EventLoopFuture { + func didReceiveBodyPart( + task: HTTPClient.Task, + _ buffer: ByteBuffer + ) -> EventLoopFuture { self.eventLoop.assertInEventLoop() self.receivedMessages.append(.bodyPart(buffer)) return self.randoEL.makeSucceededFuture(()) @@ -250,22 +255,38 @@ class HTTPClientInternalTests: XCTestCase { } } - let request = try Request(url: "http://127.0.0.1:\(server.serverPort)/custom", - body: body) + let request = try Request( + url: "http://127.0.0.1:\(server.serverPort)/custom", + body: body + ) let delegate = Delegate(expectedEventLoop: delegateEL, randomOtherEventLoop: randoEL) - let future = httpClient.execute(request: request, - delegate: delegate, - eventLoop: .init(.testOnly_exact(channelOn: channelEL, - delegateOn: delegateEL))).futureResult - - XCTAssertNoThrow(try server.readInbound()) // .head - XCTAssertNoThrow(try server.readInbound()) // .body - XCTAssertNoThrow(try server.readInbound()) // .end + let future = httpClient.execute( + request: request, + delegate: delegate, + eventLoop: .init( + .testOnly_exact( + channelOn: channelEL, + delegateOn: delegateEL + ) + ) + ).futureResult + + XCTAssertNoThrow(try server.readInbound()) // .head + XCTAssertNoThrow(try server.readInbound()) // .body + XCTAssertNoThrow(try server.readInbound()) // .end // Send 3 parts, but only one should be received until the future is complete - XCTAssertNoThrow(try server.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), - status: .ok, - headers: HTTPHeaders([("Transfer-Encoding", "chunked")]))))) + XCTAssertNoThrow( + try server.writeOutbound( + .head( + .init( + version: .init(major: 1, minor: 1), + status: .ok, + headers: HTTPHeaders([("Transfer-Encoding", "chunked")]) + ) + ) + ) + ) let buffer = ByteBuffer(string: "1234") XCTAssertNoThrow(try server.writeOutbound(.body(.byteBuffer(buffer)))) XCTAssertNoThrow(try server.writeOutbound(.end(nil))) @@ -297,7 +318,7 @@ class HTTPClientInternalTests: XCTestCase { switch sentMessages.dropFirst(3).first { case .some(.sentRequest): - () // OK + () // OK default: XCTFail("wrong message") } @@ -335,7 +356,10 @@ class HTTPClientInternalTests: XCTestCase { let el = group.next() let req1 = client.execute(request: request, eventLoop: .delegate(on: el)) let req2 = client.execute(request: request, eventLoop: .delegateAndChannel(on: el)) - let req3 = client.execute(request: request, eventLoop: .init(.testOnly_exact(channelOn: el, delegateOn: el))) + let req3 = client.execute( + request: request, + eventLoop: .init(.testOnly_exact(channelOn: el, delegateOn: el)) + ) XCTAssert(req1.eventLoop === el) XCTAssert(req2.eventLoop === el) XCTAssert(req3.eventLoop === el) @@ -354,8 +378,8 @@ class HTTPClientInternalTests: XCTestCase { _ = httpClient.get(url: "http://localhost:\(server.serverPort)/wait") - XCTAssertNoThrow(try server.readInbound()) // .head - XCTAssertNoThrow(try server.readInbound()) // .end + XCTAssertNoThrow(try server.readInbound()) // .head + XCTAssertNoThrow(try server.readInbound()) // .end do { try httpClient.syncShutdown(requiresCleanClose: true) @@ -395,10 +419,16 @@ class HTTPClientInternalTests: XCTestCase { } } let request = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/post", method: .POST, body: body) - let response = httpClient.execute(request: request, - delegate: ResponseAccumulator(request: request), - eventLoop: HTTPClient.EventLoopPreference(.testOnly_exact(channelOn: el2, - delegateOn: el1))) + let response = httpClient.execute( + request: request, + delegate: ResponseAccumulator(request: request), + eventLoop: HTTPClient.EventLoopPreference( + .testOnly_exact( + channelOn: el2, + delegateOn: el1 + ) + ) + ) XCTAssert(el1 === response.eventLoop) XCTAssertNoThrow(try response.wait()) } @@ -419,7 +449,11 @@ class HTTPClientInternalTests: XCTestCase { let request = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)//get") let delegate = ResponseAccumulator(request: request) - let task = client.execute(request: request, delegate: delegate, eventLoop: .init(.testOnly_exact(channelOn: el1, delegateOn: el2))) + let task = client.execute( + request: request, + delegate: delegate, + eventLoop: .init(.testOnly_exact(channelOn: el1, delegateOn: el2)) + ) XCTAssertTrue(task.futureResult.eventLoop === el2) XCTAssertNoThrow(try task.wait()) } @@ -460,7 +494,11 @@ class HTTPClientInternalTests: XCTestCase { let request = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/get") let delegate = TestDelegate(expectedEL: el1) XCTAssertNoThrow(try httpBin.shutdown()) - let task = client.execute(request: request, delegate: delegate, eventLoop: .init(.testOnly_exact(channelOn: el2, delegateOn: el1))) + let task = client.execute( + request: request, + delegate: delegate, + eventLoop: .init(.testOnly_exact(channelOn: el2, delegateOn: el1)) + ) XCTAssertThrowsError(try task.wait()) XCTAssertTrue(delegate.receivedError) } @@ -493,10 +531,13 @@ class HTTPClientInternalTests: XCTestCase { let request6 = try Request(url: "https://127.0.0.1") XCTAssertEqual(request6.deconstructedURL.scheme, .https) - XCTAssertEqual(request6.deconstructedURL.connectionTarget, .ipAddress( - serialization: "127.0.0.1", - address: try! SocketAddress(ipAddress: "127.0.0.1", port: 443) - )) + XCTAssertEqual( + request6.deconstructedURL.connectionTarget, + .ipAddress( + serialization: "127.0.0.1", + address: try! SocketAddress(ipAddress: "127.0.0.1", port: 443) + ) + ) XCTAssertEqual(request6.deconstructedURL.uri, "/") let request7 = try Request(url: "https://0x7F.1:9999") @@ -506,18 +547,24 @@ class HTTPClientInternalTests: XCTestCase { let request8 = try Request(url: "http://[::1]") XCTAssertEqual(request8.deconstructedURL.scheme, .http) - XCTAssertEqual(request8.deconstructedURL.connectionTarget, .ipAddress( - serialization: "[::1]", - address: try! SocketAddress(ipAddress: "::1", port: 80) - )) + XCTAssertEqual( + request8.deconstructedURL.connectionTarget, + .ipAddress( + serialization: "[::1]", + address: try! SocketAddress(ipAddress: "::1", port: 80) + ) + ) XCTAssertEqual(request8.deconstructedURL.uri, "/") let request9 = try Request(url: "http://[763e:61d9::6ACA:3100:6274]:4242/foo/bar?baz") XCTAssertEqual(request9.deconstructedURL.scheme, .http) - XCTAssertEqual(request9.deconstructedURL.connectionTarget, .ipAddress( - serialization: "[763e:61d9::6ACA:3100:6274]", - address: try! SocketAddress(ipAddress: "763e:61d9::6aca:3100:6274", port: 4242) - )) + XCTAssertEqual( + request9.deconstructedURL.connectionTarget, + .ipAddress( + serialization: "[763e:61d9::6ACA:3100:6274]", + address: try! SocketAddress(ipAddress: "763e:61d9::6aca:3100:6274", port: 4242) + ) + ) XCTAssertEqual(request9.deconstructedURL.uri, "/foo/bar?baz") // Some systems have quirks in their implementations of 'ntop' which cause them to write @@ -526,18 +573,24 @@ class HTTPClientInternalTests: XCTestCase { // so the serialization must be kept verbatim as it was given in the request. let request10 = try Request(url: "http://[::c0a8:1]:4242/foo/bar?baz") XCTAssertEqual(request10.deconstructedURL.scheme, .http) - XCTAssertEqual(request10.deconstructedURL.connectionTarget, .ipAddress( - serialization: "[::c0a8:1]", - address: try! SocketAddress(ipAddress: "::c0a8:1", port: 4242) - )) + XCTAssertEqual( + request10.deconstructedURL.connectionTarget, + .ipAddress( + serialization: "[::c0a8:1]", + address: try! SocketAddress(ipAddress: "::c0a8:1", port: 4242) + ) + ) XCTAssertEqual(request10.deconstructedURL.uri, "/foo/bar?baz") let request11 = try Request(url: "http://[::192.168.0.1]:4242/foo/bar?baz") XCTAssertEqual(request11.deconstructedURL.scheme, .http) - XCTAssertEqual(request11.deconstructedURL.connectionTarget, .ipAddress( - serialization: "[::192.168.0.1]", - address: try! SocketAddress(ipAddress: "::192.168.0.1", port: 4242) - )) + XCTAssertEqual( + request11.deconstructedURL.connectionTarget, + .ipAddress( + serialization: "[::192.168.0.1]", + address: try! SocketAddress(ipAddress: "::192.168.0.1", port: 4242) + ) + ) XCTAssertEqual(request11.deconstructedURL.uri, "/foo/bar?baz") } @@ -566,7 +619,7 @@ class HTTPClientInternalTests: XCTestCase { } // Empty collection. do { - let elements: Array = [] + let elements: [Int] = [] XCTAssertTrue(elements.hasSuffix([])) XCTAssertFalse(elements.hasSuffix([0])) XCTAssertFalse(elements.hasSuffix([42])) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift index 3bbac632b..4c2d24dc4 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientNIOTSTests.swift @@ -12,10 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient -#if canImport(Network) -import Network -#endif import NIOConcurrencyHelpers import NIOCore import NIOPosix @@ -23,6 +19,12 @@ import NIOSSL import NIOTransportServices import XCTest +@testable import AsyncHTTPClient + +#if canImport(Network) +import Network +#endif + class HTTPClientNIOTSTests: XCTestCase { var clientGroup: EventLoopGroup! @@ -57,8 +59,10 @@ class HTTPClientNIOTSTests: XCTestCase { let httpBin = HTTPBin(.http1_1(ssl: true)) let config = HTTPClient.Configuration() .enableFastFailureModeForTesting() - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: config) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: config + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) XCTAssertNoThrow(try httpBin.shutdown()) @@ -69,8 +73,10 @@ class HTTPClientNIOTSTests: XCTestCase { _ = try httpClient.get(url: "https://localhost:\(httpBin.port)/get").wait() XCTFail("This should have failed") } catch let error as HTTPClient.NWTLSError { - XCTAssert(error.status == errSSLHandshakeFail || error.status == errSSLBadCert, - "unexpected NWTLSError with status \(error.status)") + XCTAssert( + error.status == errSSLHandshakeFail || error.status == errSSLBadCert, + "unexpected NWTLSError with status \(error.status)" + ) } catch { XCTFail("Error should have been NWTLSError not \(type(of: error))") } @@ -86,8 +92,10 @@ class HTTPClientNIOTSTests: XCTestCase { let config = HTTPClient.Configuration() .enableFastFailureModeForTesting() - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: config) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: config + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) @@ -106,9 +114,15 @@ class HTTPClientNIOTSTests: XCTestCase { guard isTestingNIOTS() else { return } #if canImport(Network) let httpBin = HTTPBin(.http1_1(ssl: false)) - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(timeout: .init(connect: .milliseconds(100), - read: .milliseconds(100)))) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init( + timeout: .init( + connect: .milliseconds(100), + read: .milliseconds(100) + ) + ) + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown(requiresCleanClose: true)) diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 05e22f2d2..a92d129a4 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -13,10 +13,11 @@ //===----------------------------------------------------------------------===// import Algorithms -@testable import AsyncHTTPClient import NIOCore import XCTest +@testable import AsyncHTTPClient + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) class HTTPClientRequestTests: XCTestCase { private typealias Request = HTTPClientRequest @@ -27,31 +28,40 @@ class HTTPClientRequestTests: XCTestCase { XCTAsyncTest { var request = Request(url: "https://example.com/get") request.headers = [ - "custom-header": "custom-header-value", + "custom-header": "custom-header-value" ] var preparedRequest: PreparedRequest? XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .https, - connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .GET, - uri: "/get", - headers: [ - "host": "example.com", - "custom-header": "custom-header-value", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .https, + connectionTarget: .domain(name: "example.com", port: 443), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .GET, + uri: "/get", + headers: [ + "host": "example.com", + "custom-header": "custom-header-value", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } @@ -76,22 +86,31 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .unix, - connectionTarget: .unixSocket(path: "/some_path"), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .GET, - uri: "/", - headers: ["custom-header": "custom-value"] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .unix, + connectionTarget: .unixSocket(path: "/some_path"), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .GET, + uri: "/", + headers: ["custom-header": "custom-value"] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } @@ -105,22 +124,31 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .httpUnix, - connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .GET, - uri: "/some_path", - headers: ["custom-header": "custom-value"] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .httpUnix, + connectionTarget: .unixSocket(path: "/example/folder.sock"), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .GET, + uri: "/some_path", + headers: ["custom-header": "custom-value"] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } @@ -134,22 +162,31 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .httpsUnix, - connectionTarget: .unixSocket(path: "/example/folder.sock"), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .GET, - uri: "/some_path", - headers: ["custom-header": "custom-value"] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .httpsUnix, + connectionTarget: .unixSocket(path: "/example/folder.sock"), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .GET, + uri: "/some_path", + headers: ["custom-header": "custom-value"] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } @@ -162,22 +199,31 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .https, - connectionTarget: .domain(name: "example.com", port: 443), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .GET, - uri: "/get", - headers: ["host": "example.com"] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .https, + connectionTarget: .domain(name: "example.com", port: 443), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .GET, + uri: "/get", + headers: ["host": "example.com"] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) } @@ -191,25 +237,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "0", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "0", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) @@ -225,25 +280,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "0", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(0) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "0", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(0) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, ByteBuffer()) @@ -259,25 +323,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "9", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(9) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "9", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(9) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -293,25 +366,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "transfer-encoding": "chunked", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .stream - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "transfer-encoding": "chunked", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .stream + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -328,25 +410,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "9", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(9) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "9", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(9) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -362,25 +453,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "9", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(9) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "9", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(9) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -401,25 +501,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "transfer-encoding": "chunked", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .stream - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "transfer-encoding": "chunked", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .stream + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -440,25 +549,34 @@ class HTTPClientRequestTests: XCTestCase { XCTAssertNoThrow(preparedRequest = try PreparedRequest(request)) guard let preparedRequest = preparedRequest else { return } - XCTAssertEqual(preparedRequest.poolKey, .init( - scheme: .http, - connectionTarget: .domain(name: "example.com", port: 80), - tlsConfiguration: nil, - serverNameIndicatorOverride: nil - )) - XCTAssertEqual(preparedRequest.head, .init( - version: .http1_1, - method: .POST, - uri: "/post", - headers: [ - "host": "example.com", - "content-length": "9", - ] - )) - XCTAssertEqual(preparedRequest.requestFramingMetadata, .init( - connectionClose: false, - body: .fixedSize(9) - )) + XCTAssertEqual( + preparedRequest.poolKey, + .init( + scheme: .http, + connectionTarget: .domain(name: "example.com", port: 80), + tlsConfiguration: nil, + serverNameIndicatorOverride: nil + ) + ) + XCTAssertEqual( + preparedRequest.head, + .init( + version: .http1_1, + method: .POST, + uri: "/post", + headers: [ + "host": "example.com", + "content-length": "9", + ] + ) + ) + XCTAssertEqual( + preparedRequest.requestFramingMetadata, + .init( + connectionClose: false, + body: .fixedSize(9) + ) + ) guard let buffer = await XCTAssertNoThrowWithResult(try await preparedRequest.body.read()) else { return } XCTAssertEqual(buffer, .init(string: "post body")) } @@ -466,9 +584,9 @@ class HTTPClientRequestTests: XCTestCase { func testChunkingRandomAccessCollection() async throws { let body = try await HTTPClientRequest.Body.bytes( - Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) ).collect() let expectedChunks = [ @@ -482,11 +600,9 @@ class HTTPClientRequestTests: XCTestCase { func testChunkingCollection() async throws { let body = try await HTTPClientRequest.Body.bytes( - ( - String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + - String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + - String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) - ).utf8, + (String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize)).utf8, length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)) ).collect() @@ -503,8 +619,8 @@ class HTTPClientRequestTests: XCTestCase { let bagOfBytesToByteBufferConversionChunkSize = 8 let body = try await HTTPClientRequest.Body._bytes( AnySequence( - Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) ), length: .known(Int64(bagOfBytesToByteBufferConversionChunkSize * 3)), bagOfBytesToByteBufferConversionChunkSize: bagOfBytesToByteBufferConversionChunkSize, @@ -521,9 +637,9 @@ class HTTPClientRequestTests: XCTestCase { func testChunkingSequenceFastPath() async throws { func makeBytes() -> some Sequence & Sendable { - Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) } let body = try await HTTPClientRequest.Body.bytes( makeBytes(), @@ -534,7 +650,7 @@ class HTTPClientRequestTests: XCTestCase { firstChunk.writeImmutableBuffer(ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize)) firstChunk.writeImmutableBuffer(ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize)) let expectedChunks = [ - firstChunk, + firstChunk ] XCTAssertEqual(body, expectedChunks) @@ -544,9 +660,9 @@ class HTTPClientRequestTests: XCTestCase { let bagOfBytesToByteBufferConversionChunkSize = 8 let byteBufferMaxSize = 16 func makeBytes() -> some Sequence & Sendable { - Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) } let body = try await HTTPClientRequest.Body._bytes( makeBytes(), @@ -568,9 +684,9 @@ class HTTPClientRequestTests: XCTestCase { func testBodyStringChunking() throws { let body = try HTTPClient.Body.string( - String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + - String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + - String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) + String(repeating: "0", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "1", count: bagOfBytesToByteBufferConversionChunkSize) + + String(repeating: "2", count: bagOfBytesToByteBufferConversionChunkSize) ).collect().wait() let expectedChunks = [ @@ -584,9 +700,9 @@ class HTTPClientRequestTests: XCTestCase { func testBodyChunkingRandomAccessCollection() throws { let body = try HTTPClient.Body.bytes( - Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + - Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) + Array(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize) + + Array(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize) ).collect().wait() let expectedChunks = [ @@ -642,7 +758,8 @@ extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { case .sequence(let announcedLength, _, let generate): let buffer = generate(ByteBufferAllocator()) if case .known(let announcedLength) = announcedLength, - announcedLength != Int64(buffer.readableBytes) { + announcedLength != Int64(buffer.readableBytes) + { throw LengthMismatch(announcedLength: announcedLength, actualLength: Int64(buffer.readableBytes)) } return buffer @@ -652,8 +769,12 @@ extension Optional where Wrapped == HTTPClientRequest.Prepared.Body { accumulatedBuffer.writeBuffer(&buffer) } if case .known(let announcedLength) = announcedLength, - announcedLength != Int64(accumulatedBuffer.readableBytes) { - throw LengthMismatch(announcedLength: announcedLength, actualLength: Int64(accumulatedBuffer.readableBytes)) + announcedLength != Int64(accumulatedBuffer.readableBytes) + { + throw LengthMismatch( + announcedLength: announcedLength, + actualLength: Int64(accumulatedBuffer.readableBytes) + ) } return accumulatedBuffer } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift index 2c6c9afac..fd2b7ee4e 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift @@ -12,25 +12,38 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import XCTest +@testable import AsyncHTTPClient + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) final class HTTPClientResponseTests: XCTestCase { func testSimpleResponse() { - let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "1025"], status: .ok) + let response = HTTPClientResponse.expectedContentLength( + requestMethod: .GET, + headers: ["content-length": "1025"], + status: .ok + ) XCTAssertEqual(response, 1025) } func testSimpleResponseNotModified() { - let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "1025"], status: .notModified) + let response = HTTPClientResponse.expectedContentLength( + requestMethod: .GET, + headers: ["content-length": "1025"], + status: .notModified + ) XCTAssertEqual(response, 0) } func testSimpleResponseHeadRequestMethod() { - let response = HTTPClientResponse.expectedContentLength(requestMethod: .HEAD, headers: ["content-length": "1025"], status: .ok) + let response = HTTPClientResponse.expectedContentLength( + requestMethod: .HEAD, + headers: ["content-length": "1025"], + status: .ok + ) XCTAssertEqual(response, 0) } @@ -40,7 +53,11 @@ final class HTTPClientResponseTests: XCTestCase { } func testResponseInvalidInteger() { - let response = HTTPClientResponse.expectedContentLength(requestMethod: .GET, headers: ["content-length": "none"], status: .ok) + let response = HTTPClientResponse.expectedContentLength( + requestMethod: .GET, + headers: ["content-length": "none"], + status: .ok + ) XCTAssertEqual(response, nil) } } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index e8d6976c5..ad9fcfe98 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Atomics import Foundation import Logging @@ -28,6 +27,9 @@ import NIOSSL import NIOTLS import NIOTransportServices import XCTest + +@testable import AsyncHTTPClient + #if canImport(xlocale) import xlocale #elseif canImport(locale_h) @@ -52,7 +54,8 @@ func isTestingNIOTS() -> Bool { func getDefaultEventLoopGroup(numberOfThreads: Int) -> EventLoopGroup { #if canImport(Network) if #available(OSX 10.14, iOS 12.0, tvOS 12.0, watchOS 6.0, *), - isTestingNIOTS() { + isTestingNIOTS() + { return NIOTSEventLoopGroup(loopCount: numberOfThreads, defaultQoS: .default) } #endif @@ -144,7 +147,7 @@ class CountingDelegate: HTTPClientResponseDelegate { } func didFinishRequest(task: HTTPClient.Task) throws -> Int { - return self.count + self.count } } @@ -219,8 +222,8 @@ enum TemporaryFileHelpers { } else { return "/tmp" } - #endif // os - #endif // targetEnvironment + #endif // os + #endif // targetEnvironment } private static func openTemporaryFile() -> (CInt, String) { @@ -240,8 +243,10 @@ enum TemporaryFileHelpers { /// /// If the temporary directory is too long to store a UNIX domain socket path, it will `chdir` into the temporary /// directory and return a short-enough path. The iOS simulator is known to have too long paths. - internal static func withTemporaryUnixDomainSocketPathName(directory: String = temporaryDirectory, - _ body: (String) throws -> T) throws -> T { + internal static func withTemporaryUnixDomainSocketPathName( + directory: String = temporaryDirectory, + _ body: (String) throws -> T + ) throws -> T { // this is racy but we're trying to create the shortest possible path so we can't add a directory... let (fd, path) = self.openTemporaryFile() close(fd) @@ -256,10 +261,14 @@ enum TemporaryFileHelpers { shortEnoughPath = path restoreSavedCWD = false } catch SocketAddressError.unixDomainSocketPathTooLong { - FileManager.default.changeCurrentDirectoryPath(URL(fileURLWithPath: path).deletingLastPathComponent().absoluteString) + FileManager.default.changeCurrentDirectoryPath( + URL(fileURLWithPath: path).deletingLastPathComponent().absoluteString + ) shortEnoughPath = URL(fileURLWithPath: path).lastPathComponent restoreSavedCWD = true - print("WARNING: Path '\(path)' could not be used as UNIX domain socket path, using chdir & '\(shortEnoughPath)'") + print( + "WARNING: Path '\(path)' could not be used as UNIX domain socket path, using chdir & '\(shortEnoughPath)'" + ) } defer { if FileManager.default.fileExists(atPath: path) { @@ -307,11 +316,11 @@ enum TemporaryFileHelpers { } internal static func fileSize(path: String) throws -> Int? { - return try FileManager.default.attributesOfItem(atPath: path)[.size] as? Int + try FileManager.default.attributesOfItem(atPath: path)[.size] as? Int } internal static func fileExists(path: String) -> Bool { - return FileManager.default.fileExists(atPath: path) + FileManager.default.fileExists(atPath: path) } } @@ -324,9 +333,11 @@ enum TestTLS { ) } -internal final class HTTPBin where +internal final class HTTPBin +where RequestHandler.InboundIn == HTTPServerRequestPart, - RequestHandler.OutboundOut == HTTPServerResponsePart { + RequestHandler.OutboundOut == HTTPServerResponsePart +{ enum BindTarget { case unixDomainSocket(String) case localhostIPv4RandomPort @@ -393,19 +404,19 @@ internal final class HTTPBin where private let activeConnCounterHandler: ConnectionsCountHandler var activeConnections: Int { - return self.activeConnCounterHandler.currentlyActiveConnections + self.activeConnCounterHandler.currentlyActiveConnections } var createdConnections: Int { - return self.activeConnCounterHandler.createdConnections + self.activeConnCounterHandler.createdConnections } var port: Int { - return Int(self.serverChannel.localAddress!.port!) + Int(self.serverChannel.localAddress!.port!) } var socketAddress: SocketAddress { - return self.serverChannel.localAddress! + self.serverChannel.localAddress! } var baseURL: String { @@ -464,7 +475,10 @@ internal final class HTTPBin where self.serverChannel = try! ServerBootstrap(group: self.group) .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEADDR), value: 1) - .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), value: reusePort ? 1 : 0) + .serverChannelOption( + ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEPORT), + value: reusePort ? 1 : 0 + ) .serverChannelInitializer { channel in channel.pipeline.addHandler(self.activeConnCounterHandler) }.childChannelInitializer { channel in @@ -673,7 +687,11 @@ final class HTTPProxySimulator: ChannelInboundHandler, RemovableChannelHandler { init(promise: EventLoopPromise, expectedAuthorization: String?) { self.promise = promise self.expectedAuthorization = expectedAuthorization - self.head = HTTPResponseHead(version: .init(major: 1, minor: 1), status: .ok, headers: .init([("Content-Length", "0")])) + self.head = HTTPResponseHead( + version: .init(major: 1, minor: 1), + status: .ok, + headers: .init([("Content-Length", "0")]) + ) } func channelRead(context: ChannelHandlerContext, data: NIOAny) { @@ -687,7 +705,8 @@ final class HTTPProxySimulator: ChannelInboundHandler, RemovableChannelHandler { if let expectedAuthorization = self.expectedAuthorization { guard let authorization = head.headers["proxy-authorization"].first, - expectedAuthorization == authorization else { + expectedAuthorization == authorization + else { self.head.status = .proxyAuthenticationRequired return } @@ -712,7 +731,11 @@ internal struct HTTPResponseBuilder { var head: HTTPResponseHead var body: ByteBuffer? - init(_ version: HTTPVersion = HTTPVersion(major: 1, minor: 1), status: HTTPResponseStatus, headers: HTTPHeaders = HTTPHeaders()) { + init( + _ version: HTTPVersion = HTTPVersion(major: 1, minor: 1), + status: HTTPResponseStatus, + headers: HTTPHeaders = HTTPHeaders() + ) { self.head = HTTPResponseHead(version: version, status: status, headers: headers) } @@ -764,8 +787,10 @@ internal final class HTTPBinHandler: ChannelInboundHandler { for header in head.headers { let needle = "x-send-back-header-" if header.name.lowercased().starts(with: needle) { - self.responseHeaders.add(name: String(header.name.dropFirst(needle.count)), - value: header.value) + self.responseHeaders.add( + name: String(header.name.dropFirst(needle.count)), + value: header.value + ) } } } @@ -778,7 +803,12 @@ internal final class HTTPBinHandler: ChannelInboundHandler { headers = HTTPHeaders() } - context.write(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + context.write( + wrapOutboundOut( + .head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers)) + ), + promise: nil + ) for i in 0..<10 { let msg = "id: \(i)" var buf = context.channel.allocator.buffer(capacity: msg.count) @@ -793,7 +823,12 @@ internal final class HTTPBinHandler: ChannelInboundHandler { // This tests receiving chunks very fast: please do not insert delays here! let headers = HTTPHeaders([("Transfer-Encoding", "chunked")]) - context.write(self.wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + context.write( + self.wrapOutboundOut( + .head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers)) + ), + promise: nil + ) for i in 0..<10 { let msg = "id: \(i)" var buf = context.channel.allocator.buffer(capacity: msg.count) @@ -808,7 +843,12 @@ internal final class HTTPBinHandler: ChannelInboundHandler { // This tests receiving a lot of tiny chunks: they must all be sent in a single flush or the test doesn't work. let headers = HTTPHeaders([("Transfer-Encoding", "chunked")]) - context.write(self.wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + context.write( + self.wrapOutboundOut( + .head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers)) + ), + promise: nil + ) let message = ByteBuffer(integer: UInt8(ascii: "a")) // This number (10k) is load-bearing and a bit magic: it has been experimentally verified as being sufficient to blow the stack @@ -928,9 +968,12 @@ internal final class HTTPBinHandler: ChannelInboundHandler { context.close(promise: nil) return case "/custom": - context.writeAndFlush(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok))), promise: nil) + context.writeAndFlush( + wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok))), + promise: nil + ) return - case "/events/10/1": // TODO: parse path + case "/events/10/1": // TODO: parse path self.writeEvents(context: context) return case "/events/10/content-length": @@ -954,10 +997,20 @@ internal final class HTTPBinHandler: ChannelInboundHandler { case "/content-length-without-body": var headers = self.responseHeaders headers.replaceOrAdd(name: "content-length", value: "1234") - context.writeAndFlush(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers))), promise: nil) + context.writeAndFlush( + wrapOutboundOut( + .head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .ok, headers: headers)) + ), + promise: nil + ) return default: - context.write(wrapOutboundOut(.head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .notFound))), promise: nil) + context.write( + wrapOutboundOut( + .head(HTTPResponseHead(version: HTTPVersion(major: 1, minor: 1), status: .notFound)) + ), + promise: nil + ) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) return } @@ -976,18 +1029,26 @@ internal final class HTTPBinHandler: ChannelInboundHandler { response.head.headers.add(contentsOf: self.responseHeaders) context.write(wrapOutboundOut(.head(response.head)), promise: nil) if let body = response.body { - let requestInfo = RequestInfo(data: String(buffer: body), - requestNumber: self.requestId, - connectionNumber: self.connectionID) - let responseBody = try! JSONEncoder().encodeAsByteBuffer(requestInfo, - allocator: context.channel.allocator) + let requestInfo = RequestInfo( + data: String(buffer: body), + requestNumber: self.requestId, + connectionNumber: self.connectionID + ) + let responseBody = try! JSONEncoder().encodeAsByteBuffer( + requestInfo, + allocator: context.channel.allocator + ) context.write(wrapOutboundOut(.body(.byteBuffer(responseBody))), promise: nil) } else { - let requestInfo = RequestInfo(data: "", - requestNumber: self.requestId, - connectionNumber: self.connectionID) - let responseBody = try! JSONEncoder().encodeAsByteBuffer(requestInfo, - allocator: context.channel.allocator) + let requestInfo = RequestInfo( + data: "", + requestNumber: self.requestId, + connectionNumber: self.connectionID + ) + let responseBody = try! JSONEncoder().encodeAsByteBuffer( + requestInfo, + allocator: context.channel.allocator + ) context.write(wrapOutboundOut(.body(.byteBuffer(responseBody))), promise: nil) } context.eventLoop.scheduleTask(in: self.delay) { @@ -1000,8 +1061,9 @@ internal final class HTTPBinHandler: ChannelInboundHandler { self.isServingRequest = false switch result { case .success: - if self.responseHeaders[canonicalForm: "X-Close-Connection"].contains("true") || - self.shouldClose { + if self.responseHeaders[canonicalForm: "X-Close-Connection"].contains("true") + || self.shouldClose + { context.close(promise: nil) } case .failure(let error): @@ -1170,7 +1232,7 @@ struct CollectEverythingLogHandler: LogHandler { var allEntries: [Entry] { get { - return self.lock.withLock { self.logs } + self.lock.withLock { self.logs } } set { self.lock.withLock { self.logs = newValue } @@ -1179,9 +1241,13 @@ struct CollectEverythingLogHandler: LogHandler { func append(level: Logger.Level, message: Logger.Message, metadata: Logger.Metadata?) { self.lock.withLock { - self.logs.append(Entry(level: level, - message: message.description, - metadata: metadata?.mapValues { $0.description } ?? [:])) + self.logs.append( + Entry( + level: level, + message: message.description, + metadata: metadata?.mapValues { $0.description } ?? [:] + ) + ) } } } @@ -1190,16 +1256,20 @@ struct CollectEverythingLogHandler: LogHandler { self.logStore = logStore } - func log(level: Logger.Level, - message: Logger.Message, - metadata: Logger.Metadata?, - file: String, function: String, line: UInt) { + func log( + level: Logger.Level, + message: Logger.Message, + metadata: Logger.Metadata?, + file: String, + function: String, + line: UInt + ) { self.logStore.append(level: level, message: message, metadata: self.metadata.merging(metadata ?? [:]) { $1 }) } subscript(metadataKey key: String) -> Logger.Metadata.Value? { get { - return self.metadata[key] + self.metadata[key] } set { self.metadata[key] = newValue @@ -1355,7 +1425,10 @@ class HTTPEchoHandler: ChannelInboundHandler { let request = self.unwrapInboundIn(data) switch request { case .head(let requestHead): - context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: requestHead.headers))), promise: nil) + context.writeAndFlush( + self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: requestHead.headers))), + promise: nil + ) case .body(let bytes): context.writeAndFlush(self.wrapOutboundOut(.body(.byteBuffer(bytes))), promise: nil) case .end: @@ -1374,7 +1447,10 @@ final class HTTPEchoHeaders: ChannelInboundHandler { let request = self.unwrapInboundIn(data) switch request { case .head(let requestHead): - context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: requestHead.headers))), promise: nil) + context.writeAndFlush( + self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok, headers: requestHead.headers))), + promise: nil + ) case .body: break case .end: @@ -1410,7 +1486,10 @@ final class HTTP200DelayedHandler: ChannelInboundHandler { self.pendingBodyParts = pendingBodyParts - 1 } else { self.pendingBodyParts = nil - context.writeAndFlush(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok))), promise: nil) + context.writeAndFlush( + self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok))), + promise: nil + ) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) } } @@ -1421,51 +1500,51 @@ final class HTTP200DelayedHandler: ChannelInboundHandler { } private let cert = """ ------BEGIN CERTIFICATE----- -MIICmDCCAYACCQCPC8JDqMh1zzANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJ1 -czAgFw0xODEwMzExNTU1MjJaGA8yMTE4MTAwNzE1NTUyMlowDTELMAkGA1UEBhMC -dXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDiC+TGmbSP/nWWN1tj -yNfnWCU5ATjtIOfdtP6ycx8JSeqkvyNXG21kNUn14jTTU8BglGL2hfVpCbMisUdb -d3LpP8unSsvlOWwORFOViSy4YljSNM/FNoMtavuITA/sEELYgjWkz2o/uHPZHud9 -+JQwGJgqIlMa3mr2IaaUZlWN3D1u88bzJYhpt3YyxRy9+OEoOKy36KdWwhKzV3S8 -kXb0Y1GbAo68jJ9RfzeLy290mIs9qG2y1CNXWO6sxf6B//LaalizZiCfzYAVKcNR -9oNYsEJc5KB/+DsAGTzR7mL+oiU4h/vwVb2GTDat5C+PFGi6j1ujxYTRPO538ljg -dslnAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFYhA7sw8odOsRO8/DUklBOjPnmn -a078oSumgPXXw6AgcoAJv/Qthjo6CCEtrjYfcA9jaBw9/Tii7mDmqDRS5c9ZPL8+ -NEPdHjFCFBOEvlL6uHOgw0Z9Wz+5yCXnJ8oNUEgc3H2NbbzJF6sMBXSPtFS2NOK8 -OsAI9OodMrDd6+lwljrmFoCCkJHDEfE637IcsbgFKkzhO/oNCRK6OrudG4teDahz -Au4LoEYwT730QKC/VQxxEVZobjn9/sTrq9CZlbPYHxX4fz6e00sX7H9i49vk9zQ5 -5qCm9ljhrQPSa42Q62PPE2BEEGSP2KBm0J+H3vlvCD6+SNc/nMZjrRmgjrI= ------END CERTIFICATE----- -""" + -----BEGIN CERTIFICATE----- + MIICmDCCAYACCQCPC8JDqMh1zzANBgkqhkiG9w0BAQsFADANMQswCQYDVQQGEwJ1 + czAgFw0xODEwMzExNTU1MjJaGA8yMTE4MTAwNzE1NTUyMlowDTELMAkGA1UEBhMC + dXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDiC+TGmbSP/nWWN1tj + yNfnWCU5ATjtIOfdtP6ycx8JSeqkvyNXG21kNUn14jTTU8BglGL2hfVpCbMisUdb + d3LpP8unSsvlOWwORFOViSy4YljSNM/FNoMtavuITA/sEELYgjWkz2o/uHPZHud9 + +JQwGJgqIlMa3mr2IaaUZlWN3D1u88bzJYhpt3YyxRy9+OEoOKy36KdWwhKzV3S8 + kXb0Y1GbAo68jJ9RfzeLy290mIs9qG2y1CNXWO6sxf6B//LaalizZiCfzYAVKcNR + 9oNYsEJc5KB/+DsAGTzR7mL+oiU4h/vwVb2GTDat5C+PFGi6j1ujxYTRPO538ljg + dslnAgMBAAEwDQYJKoZIhvcNAQELBQADggEBAFYhA7sw8odOsRO8/DUklBOjPnmn + a078oSumgPXXw6AgcoAJv/Qthjo6CCEtrjYfcA9jaBw9/Tii7mDmqDRS5c9ZPL8+ + NEPdHjFCFBOEvlL6uHOgw0Z9Wz+5yCXnJ8oNUEgc3H2NbbzJF6sMBXSPtFS2NOK8 + OsAI9OodMrDd6+lwljrmFoCCkJHDEfE637IcsbgFKkzhO/oNCRK6OrudG4teDahz + Au4LoEYwT730QKC/VQxxEVZobjn9/sTrq9CZlbPYHxX4fz6e00sX7H9i49vk9zQ5 + 5qCm9ljhrQPSa42Q62PPE2BEEGSP2KBm0J+H3vlvCD6+SNc/nMZjrRmgjrI= + -----END CERTIFICATE----- + """ private let key = """ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDiC+TGmbSP/nWW -N1tjyNfnWCU5ATjtIOfdtP6ycx8JSeqkvyNXG21kNUn14jTTU8BglGL2hfVpCbMi -sUdbd3LpP8unSsvlOWwORFOViSy4YljSNM/FNoMtavuITA/sEELYgjWkz2o/uHPZ -Hud9+JQwGJgqIlMa3mr2IaaUZlWN3D1u88bzJYhpt3YyxRy9+OEoOKy36KdWwhKz -V3S8kXb0Y1GbAo68jJ9RfzeLy290mIs9qG2y1CNXWO6sxf6B//LaalizZiCfzYAV -KcNR9oNYsEJc5KB/+DsAGTzR7mL+oiU4h/vwVb2GTDat5C+PFGi6j1ujxYTRPO53 -8ljgdslnAgMBAAECggEBANZNWFNAnYJ2R5xmVuo/GxFk68Ujd4i4TZpPYbhkk+QG -g8I0w5htlEQQkVHfZx2CpTvq8feuAH/YhlA5qeD5WaPwq26q5qsmyV6tQGDgb9lO -w85l6ySZDbwdVOJe2il/MSB6MclSKvTGNm59chJnfHYsmvY3HHq4qsc2F+tRKYMW -pY75LgEbaTUV69J3cbC1wAeVjv0q/krND+YkhYpTxNZhbazK/FHOCvY+zFu9fg0L -zpwbn5fb6wIvqG7tXp7koa3QMn64AXmO/fb5mBd8G2vBGYnxwb7Egwdg/3Dw+BXu -ynQLP7ixWsE2KNfR9Ce1i3YvEo6QDTv2340I3dntxkECgYEA9vdaL4PGyvEbpim4 -kqz1vuug8Iq0nTVDo6jmgH1o+XdcIbW3imXtgi5zUJpj4oDD7/4aufiJZjG64i/v -phe11xeUvh5QNNOzeMymVDoJut97F97KKKTv7bG8Rpon/WzH2I0SoAkECCwmdWAJ -H3nvOCnXEkpbCqmIUvHVURPRDn8CgYEA6lCk3EzFQlbXs3Sj5op61R3Mscx7/35A -eGv5axzbENHt1so+s3Zvyyi1bo4VBcwnKVCvQjmTuLiqrc9VfX8XdbiTUNnEr2u3 -992Ja6DEJTZ9gy5WiviwYnwU2HpjwOVNBb17T0NLoRHkDZ6iXj7NZgwizOki5p3j -/hS0pObSIRkCgYEAiEdOGNIarHoHy9VR6H5QzR2xHYssx2NRA8p8B4MsnhxjVqaz -tUcxnJiNQXkwjRiJBrGthdnD2ASxH4dcMsb6rMpyZcbMc5ouewZS8j9khx4zCqUB -4RPC4eMmBb+jOZEBZlnSYUUYWHokbrij0B61BsTvzUQCoQuUElEoaSkKP3kCgYEA -mwdqXHvK076jjo9w1drvtEu4IDc8H2oH++TsrEr2QiWzaDZ9z71f8BnqGNCW5jQS -AQrqOjXgIArGmqMgXB0Xh4LsrUS4Fpx9ptiD0JsYy8pGtuGUzvQFt9OC80ve7kSI -dnDMwj+zLUmqCrzXjuWcfpUu/UaPGeiDbZuDfcteYhkCgYBLyL5JY7Qd4gVQIhFX -7Sv3sNJN3KZCQHEzut7IwojaxgpuxiFvgsoXXuYolVCQp32oWbYcE2Yke+hOKsTE -sCMAWZiSGN2Nrfea730IYAXkUm8bpEd3VxDXEEv13nxVeQof+JGMdlkldFGaBRDU -oYQsPj00S3/GA9WDapwe81Wl2A== ------END PRIVATE KEY----- -""" + -----BEGIN PRIVATE KEY----- + MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQDiC+TGmbSP/nWW + N1tjyNfnWCU5ATjtIOfdtP6ycx8JSeqkvyNXG21kNUn14jTTU8BglGL2hfVpCbMi + sUdbd3LpP8unSsvlOWwORFOViSy4YljSNM/FNoMtavuITA/sEELYgjWkz2o/uHPZ + Hud9+JQwGJgqIlMa3mr2IaaUZlWN3D1u88bzJYhpt3YyxRy9+OEoOKy36KdWwhKz + V3S8kXb0Y1GbAo68jJ9RfzeLy290mIs9qG2y1CNXWO6sxf6B//LaalizZiCfzYAV + KcNR9oNYsEJc5KB/+DsAGTzR7mL+oiU4h/vwVb2GTDat5C+PFGi6j1ujxYTRPO53 + 8ljgdslnAgMBAAECggEBANZNWFNAnYJ2R5xmVuo/GxFk68Ujd4i4TZpPYbhkk+QG + g8I0w5htlEQQkVHfZx2CpTvq8feuAH/YhlA5qeD5WaPwq26q5qsmyV6tQGDgb9lO + w85l6ySZDbwdVOJe2il/MSB6MclSKvTGNm59chJnfHYsmvY3HHq4qsc2F+tRKYMW + pY75LgEbaTUV69J3cbC1wAeVjv0q/krND+YkhYpTxNZhbazK/FHOCvY+zFu9fg0L + zpwbn5fb6wIvqG7tXp7koa3QMn64AXmO/fb5mBd8G2vBGYnxwb7Egwdg/3Dw+BXu + ynQLP7ixWsE2KNfR9Ce1i3YvEo6QDTv2340I3dntxkECgYEA9vdaL4PGyvEbpim4 + kqz1vuug8Iq0nTVDo6jmgH1o+XdcIbW3imXtgi5zUJpj4oDD7/4aufiJZjG64i/v + phe11xeUvh5QNNOzeMymVDoJut97F97KKKTv7bG8Rpon/WzH2I0SoAkECCwmdWAJ + H3nvOCnXEkpbCqmIUvHVURPRDn8CgYEA6lCk3EzFQlbXs3Sj5op61R3Mscx7/35A + eGv5axzbENHt1so+s3Zvyyi1bo4VBcwnKVCvQjmTuLiqrc9VfX8XdbiTUNnEr2u3 + 992Ja6DEJTZ9gy5WiviwYnwU2HpjwOVNBb17T0NLoRHkDZ6iXj7NZgwizOki5p3j + /hS0pObSIRkCgYEAiEdOGNIarHoHy9VR6H5QzR2xHYssx2NRA8p8B4MsnhxjVqaz + tUcxnJiNQXkwjRiJBrGthdnD2ASxH4dcMsb6rMpyZcbMc5ouewZS8j9khx4zCqUB + 4RPC4eMmBb+jOZEBZlnSYUUYWHokbrij0B61BsTvzUQCoQuUElEoaSkKP3kCgYEA + mwdqXHvK076jjo9w1drvtEu4IDc8H2oH++TsrEr2QiWzaDZ9z71f8BnqGNCW5jQS + AQrqOjXgIArGmqMgXB0Xh4LsrUS4Fpx9ptiD0JsYy8pGtuGUzvQFt9OC80ve7kSI + dnDMwj+zLUmqCrzXjuWcfpUu/UaPGeiDbZuDfcteYhkCgYBLyL5JY7Qd4gVQIhFX + 7Sv3sNJN3KZCQHEzut7IwojaxgpuxiFvgsoXXuYolVCQp32oWbYcE2Yke+hOKsTE + sCMAWZiSGN2Nrfea730IYAXkUm8bpEd3VxDXEEv13nxVeQof+JGMdlkldFGaBRDU + oYQsPj00S3/GA9WDapwe81Wl2A== + -----END PRIVATE KEY----- + """ diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index cf3578ed1..8f76b693b 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -12,11 +12,8 @@ // //===----------------------------------------------------------------------===// -/* NOT @testable */ import AsyncHTTPClient // Tests that need @testable go into HTTPClientInternalTests.swift +import AsyncHTTPClient // NOT @testable - tests that need @testable go into HTTPClientInternalTests.swift import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -30,6 +27,10 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testRequestURI() throws { let request1 = try Request(url: "https://someserver.com:8888/some/path?foo=bar") @@ -122,7 +123,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(url.scheme, "http+unix") XCTAssertEqual(url.host, "/tmp/file with spacesใจๆผขๅญ—") XCTAssertEqual(url.path, "/file/path") - XCTAssertEqual(url.absoluteString, "http+unix://%2Ftmp%2Ffile%20with%20spaces%E3%81%A8%E6%BC%A2%E5%AD%97/file/path") + XCTAssertEqual( + url.absoluteString, + "http+unix://%2Ftmp%2Ffile%20with%20spaces%E3%81%A8%E6%BC%A2%E5%AD%97/file/path" + ) } let url5 = URL(httpsURLWithSocketPath: "/tmp/file") @@ -158,7 +162,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(url.scheme, "https+unix") XCTAssertEqual(url.host, "/tmp/file with spacesใจๆผขๅญ—") XCTAssertEqual(url.path, "/file/path") - XCTAssertEqual(url.absoluteString, "https+unix://%2Ftmp%2Ffile%20with%20spaces%E3%81%A8%E6%BC%A2%E5%AD%97/file/path") + XCTAssertEqual( + url.absoluteString, + "https+unix://%2Ftmp%2Ffile%20with%20spaces%E3%81%A8%E6%BC%A2%E5%AD%97/file/path" + ) } } @@ -171,55 +178,116 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testConvenienceExecuteMethods() throws { - XCTAssertEqual(["GET"[...]], - try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["POST"[...]], - try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["PATCH"[...]], - try self.defaultClient.patch(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["PUT"[...]], - try self.defaultClient.put(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["DELETE"[...]], - try self.defaultClient.delete(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["GET"[...]], - try self.defaultClient.execute(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["CHECKOUT"[...]], - try self.defaultClient.execute(.CHECKOUT, url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) + XCTAssertEqual( + ["GET"[...]], + try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["POST"[...]], + try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["PATCH"[...]], + try self.defaultClient.patch(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["PUT"[...]], + try self.defaultClient.put(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["DELETE"[...]], + try self.defaultClient.delete(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["GET"[...]], + try self.defaultClient.execute(url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["CHECKOUT"[...]], + try self.defaultClient.execute(.CHECKOUT, url: self.defaultHTTPBinURLPrefix + "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) } func testConvenienceExecuteMethodsOverSocket() throws { - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - defer { - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) - } + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + defer { + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(["GET"[...]], - try self.defaultClient.execute(socketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["GET"[...]], - try self.defaultClient.execute(.GET, socketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["POST"[...]], - try self.defaultClient.execute(.POST, socketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - }) + XCTAssertEqual( + ["GET"[...]], + try self.defaultClient.execute(socketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["GET"[...]], + try self.defaultClient.execute(.GET, socketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["POST"[...]], + try self.defaultClient.execute(.POST, socketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + } + ) } func testConvenienceExecuteMethodsOverSecureSocket() throws { - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localSocketPathHTTPBin = HTTPBin(.http1_1(ssl: true, compress: false), bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) - } + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localSocketPathHTTPBin = HTTPBin( + .http1_1(ssl: true, compress: false), + bindTarget: .unixDomainSocket(path) + ) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(["GET"[...]], - try localClient.execute(secureSocketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["GET"[...]], - try localClient.execute(.GET, secureSocketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - XCTAssertEqual(["POST"[...]], - try localClient.execute(.POST, secureSocketPath: path, urlPath: "echo-method").wait().headers[canonicalForm: "X-Method-Used"]) - }) + XCTAssertEqual( + ["GET"[...]], + try localClient.execute(secureSocketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["GET"[...]], + try localClient.execute(.GET, secureSocketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + XCTAssertEqual( + ["POST"[...]], + try localClient.execute(.POST, secureSocketPath: path, urlPath: "echo-method").wait().headers[ + canonicalForm: "X-Method-Used" + ] + ) + } + ) } func testGet() throws { @@ -235,7 +303,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testPost() throws { - let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .string("1234")).wait() + let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .string("1234")) + .wait() let bytes = response.body.flatMap { $0.getData(at: 0, length: $0.readableBytes) } let data = try JSONDecoder().decode(RequestInfo.self, from: bytes!) @@ -247,7 +316,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let bodyData = Array("hello, world!").lazy.map { $0.uppercased().first!.asciiValue! } let erasedData = AnyRandomAccessCollection(bodyData) - let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .bytes(erasedData)).wait() + let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .bytes(erasedData)) + .wait() let bytes = response.body.flatMap { $0.getData(at: 0, length: $0.readableBytes) } let data = try JSONDecoder().decode(RequestInfo.self, from: bytes!) @@ -258,7 +328,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testPostWithFoundationDataBody() throws { let bodyData = Data("hello, world!".utf8) - let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .data(bodyData)).wait() + let response = try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + "post", body: .data(bodyData)) + .wait() let bytes = response.body.flatMap { $0.getData(at: 0, length: $0.readableBytes) } let data = try JSONDecoder().decode(RequestInfo.self, from: bytes!) @@ -268,8 +339,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testGetHttps() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -281,8 +354,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testGetHttpsWithIP() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -300,8 +375,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try group.syncShutdownGracefully()) } let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(group), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(group), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -314,8 +391,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testGetHttpsWithIPv6() throws { try XCTSkipUnless(canBindIPv6Loopback, "Requires IPv6") let localHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .localhostIPv6RandomPort) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -334,8 +413,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try group.syncShutdownGracefully()) } let localHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .localhostIPv6RandomPort) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(group), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(group), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -347,14 +428,20 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testPostHttps() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) } - let request = try Request(url: "https://localhost:\(localHTTPBin.port)/post", method: .POST, body: .string("1234")) + let request = try Request( + url: "https://localhost:\(localHTTPBin.port)/post", + method: .POST, + body: .string("1234") + ) let response = try localClient.execute(request: request).wait() let bytes = response.body.flatMap { $0.getData(at: 0, length: $0.readableBytes) } @@ -366,8 +453,13 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testHttpRedirect() throws { let httpsBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none, redirectConfiguration: .follow(max: 10, allowCycles: true))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration( + certificateVerification: .none, + redirectConfiguration: .follow(max: 10, allowCycles: true) + ) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -377,102 +469,189 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var response = try localClient.get(url: self.defaultHTTPBinURLPrefix + "redirect/302").wait() XCTAssertEqual(response.status, .ok) - response = try localClient.get(url: self.defaultHTTPBinURLPrefix + "redirect/https?port=\(httpsBin.port)").wait() + response = try localClient.get(url: self.defaultHTTPBinURLPrefix + "redirect/https?port=\(httpsBin.port)") + .wait() XCTAssertEqual(response.status, .ok) - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { httpSocketPath in - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { httpsSocketPath in - let socketHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(httpSocketPath)) - let socketHTTPSBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(httpsSocketPath)) - defer { - XCTAssertNoThrow(try socketHTTPBin.shutdown()) - XCTAssertNoThrow(try socketHTTPSBin.shutdown()) - } - - // From HTTP or HTTPS to HTTP+UNIX should fail to redirect - var targetURL = "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - var request = try Request(url: self.defaultHTTPBinURLPrefix + "redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - var response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .found) - XCTAssertEqual(response.headers.first(name: "Location"), targetURL) - - request = try Request(url: "https://localhost:\(httpsBin.port)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .found) - XCTAssertEqual(response.headers.first(name: "Location"), targetURL) - - // From HTTP or HTTPS to HTTPS+UNIX should also fail to redirect - targetURL = "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - request = try Request(url: self.defaultHTTPBinURLPrefix + "redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .found) - XCTAssertEqual(response.headers.first(name: "Location"), targetURL) - - request = try Request(url: "https://localhost:\(httpsBin.port)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .found) - XCTAssertEqual(response.headers.first(name: "Location"), targetURL) - - // ... while HTTP+UNIX to HTTP, HTTPS, or HTTP(S)+UNIX should succeed - targetURL = self.defaultHTTPBinURLPrefix + "ok" - request = try Request(url: "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "https://localhost:\(httpsBin.port)/ok" - request = try Request(url: "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - request = try Request(url: "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - request = try Request(url: "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - // ... and HTTPS+UNIX to HTTP, HTTPS, or HTTP(S)+UNIX should succeed - targetURL = self.defaultHTTPBinURLPrefix + "ok" - request = try Request(url: "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "https://localhost:\(httpsBin.port)/ok" - request = try Request(url: "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - request = try Request(url: "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) - - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - - targetURL = "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" - request = try Request(url: "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", method: .GET, headers: ["X-Target-Redirect-URL": targetURL], body: nil) + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { httpSocketPath in + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { httpsSocketPath in + let socketHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(httpSocketPath)) + let socketHTTPSBin = HTTPBin( + .http1_1(ssl: true), + bindTarget: .unixDomainSocket(httpsSocketPath) + ) + defer { + XCTAssertNoThrow(try socketHTTPBin.shutdown()) + XCTAssertNoThrow(try socketHTTPSBin.shutdown()) + } - response = try localClient.execute(request: request).wait() - XCTAssertEqual(response.status, .ok) - }) - }) + // From HTTP or HTTPS to HTTP+UNIX should fail to redirect + var targetURL = + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + var request = try Request( + url: self.defaultHTTPBinURLPrefix + "redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + var response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .found) + XCTAssertEqual(response.headers.first(name: "Location"), targetURL) + + request = try Request( + url: "https://localhost:\(httpsBin.port)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .found) + XCTAssertEqual(response.headers.first(name: "Location"), targetURL) + + // From HTTP or HTTPS to HTTPS+UNIX should also fail to redirect + targetURL = + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + request = try Request( + url: self.defaultHTTPBinURLPrefix + "redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .found) + XCTAssertEqual(response.headers.first(name: "Location"), targetURL) + + request = try Request( + url: "https://localhost:\(httpsBin.port)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .found) + XCTAssertEqual(response.headers.first(name: "Location"), targetURL) + + // ... while HTTP+UNIX to HTTP, HTTPS, or HTTP(S)+UNIX should succeed + targetURL = self.defaultHTTPBinURLPrefix + "ok" + request = try Request( + url: + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = "https://localhost:\(httpsBin.port)/ok" + request = try Request( + url: + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + request = try Request( + url: + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + request = try Request( + url: + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + // ... and HTTPS+UNIX to HTTP, HTTPS, or HTTP(S)+UNIX should succeed + targetURL = self.defaultHTTPBinURLPrefix + "ok" + request = try Request( + url: + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = "https://localhost:\(httpsBin.port)/ok" + request = try Request( + url: + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = + "http+unix://\(httpSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + request = try Request( + url: + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + + targetURL = + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/ok" + request = try Request( + url: + "https+unix://\(httpsSocketPath.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)!)/redirect/target", + method: .GET, + headers: ["X-Target-Redirect-URL": targetURL], + body: nil + ) + + response = try localClient.execute(request: request).wait() + XCTAssertEqual(response.status, .ok) + } + ) + } + ) } func testHttpHostRedirect() { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none, redirectConfiguration: .follow(max: 10, allowCycles: true))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration( + certificateVerification: .none, + redirectConfiguration: .follow(max: 10, allowCycles: true) + ) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -500,8 +679,14 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testLeadingSlashRelativeURL() throws { - let noLeadingSlashURL = URL(string: "percent%2Fencoded/hello", relativeTo: URL(string: self.defaultHTTPBinURLPrefix)!)! - let withLeadingSlashURL = URL(string: "/percent%2Fencoded/hello", relativeTo: URL(string: self.defaultHTTPBinURLPrefix)!)! + let noLeadingSlashURL = URL( + string: "percent%2Fencoded/hello", + relativeTo: URL(string: self.defaultHTTPBinURLPrefix)! + )! + let withLeadingSlashURL = URL( + string: "/percent%2Fencoded/hello", + relativeTo: URL(string: self.defaultHTTPBinURLPrefix)! + )! let noLeadingSlashURLRequest = try HTTPClient.Request(url: noLeadingSlashURL, method: .GET) let withLeadingSlashURLRequest = try HTTPClient.Request(url: withLeadingSlashURL, method: .GET) @@ -518,7 +703,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var headers = HTTPHeaders() headers.add(name: "Content-Length", value: "12") - let request = try Request(url: self.defaultHTTPBinURLPrefix + "post", method: .POST, headers: headers, body: .byteBuffer(body)) + let request = try Request( + url: self.defaultHTTPBinURLPrefix + "post", + method: .POST, + headers: headers, + body: .byteBuffer(body) + ) let response = try self.defaultClient.execute(request: request).wait() // if the library adds another content length header we'll get a bad request error. XCTAssertEqual(.ok, response.status) @@ -563,9 +753,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let progress = try TemporaryFileHelpers.withTemporaryFilePath { path -> FileDownloadDelegate.Progress in - let delegate = try FileDownloadDelegate(path: path, reportHead: { - XCTAssertEqual($0.status, .notFound) - }) + let delegate = try FileDownloadDelegate( + path: path, + reportHead: { + XCTAssertEqual($0.status, .notFound) + } + ) let progress = try self.defaultClient.execute( request: request, @@ -587,12 +780,16 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { struct CustomError: Equatable, Error {} try TemporaryFileHelpers.withTemporaryFilePath { path in - let delegate = try FileDownloadDelegate(path: path, reportHead: { task, head in - XCTAssertEqual(head.status, .ok) - task.fail(reason: CustomError()) - }, reportProgress: { _, _ in - XCTFail("should never be called") - }) + let delegate = try FileDownloadDelegate( + path: path, + reportHead: { task, head in + XCTAssertEqual(head.status, .ok) + task.fail(reason: CustomError()) + }, + reportProgress: { _, _ in + XCTFail("should never be called") + } + ) XCTAssertThrowsError( try self.defaultClient.execute( request: request, @@ -614,8 +811,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testReadTimeout() { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(timeout: HTTPClient.Configuration.Timeout(read: .milliseconds(150)))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(timeout: HTTPClient.Configuration.Timeout(read: .milliseconds(150))) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -627,8 +826,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testWriteTimeout() throws { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(timeout: HTTPClient.Configuration.Timeout(write: .nanoseconds(10)))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(timeout: HTTPClient.Configuration.Timeout(write: .nanoseconds(10))) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -636,19 +837,21 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // Create a request that writes a chunk, then waits longer than the configured write timeout, // and then writes again. This should trigger a write timeout error. - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "post", - method: .POST, - headers: ["transfer-encoding": "chunked"], - body: .stream { streamWriter in - _ = streamWriter.write(.byteBuffer(.init())) - - let promise = self.clientGroup.next().makePromise(of: Void.self) - self.clientGroup.next().scheduleTask(in: .milliseconds(3)) { - streamWriter.write(.byteBuffer(.init())).cascade(to: promise) - } + let request = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "post", + method: .POST, + headers: ["transfer-encoding": "chunked"], + body: .stream { streamWriter in + _ = streamWriter.write(.byteBuffer(.init())) + + let promise = self.clientGroup.next().makePromise(of: Void.self) + self.clientGroup.next().scheduleTask(in: .milliseconds(3)) { + streamWriter.write(.byteBuffer(.init())).cascade(to: promise) + } - return promise.futureResult - }) + return promise.futureResult + } + ) XCTAssertThrowsError(try localClient.execute(request: request).wait()) { XCTAssertEqual($0 as? HTTPClientError, .writeTimeout) @@ -684,8 +887,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let url = "http://localhost:\(port)/get" #endif - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150)))) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(timeout: .init(connect: .milliseconds(100), read: .milliseconds(150))) + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) @@ -697,7 +902,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testDeadline() { - XCTAssertThrowsError(try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "wait", deadline: .now() + .milliseconds(150)).wait()) { + XCTAssertThrowsError( + try self.defaultClient.get( + url: self.defaultHTTPBinURLPrefix + "wait", + deadline: .now() + .milliseconds(150) + ).wait() + ) { XCTAssertEqual($0 as? HTTPClientError, .deadlineExceeded) } } @@ -783,7 +993,13 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let localHTTPBin = HTTPBin(proxy: .simulate(authorization: "Basic YWxhZGRpbjpvcGVuc2VzYW1l")) let localClient = HTTPClient( eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(proxy: .server(host: "localhost", port: localHTTPBin.port, authorization: .basic(username: "aladdin", password: "opensesame"))) + configuration: .init( + proxy: .server( + host: "localhost", + port: localHTTPBin.port, + authorization: .basic(username: "aladdin", password: "opensesame") + ) + ) ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -837,8 +1053,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testEventLoopArgument() throws { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(redirectConfiguration: .follow(max: 10, allowCycles: true))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(redirectConfiguration: .follow(max: 10, allowCycles: true)) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -859,26 +1077,33 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func didFinishRequest(task: HTTPClient.Task) throws -> Bool { - return self.result + self.result } } let eventLoop = self.clientGroup.next() let delegate = EventLoopValidatingDelegate(eventLoop: eventLoop) var request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get") - var response = try localClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: eventLoop)).wait() + var response = try localClient.execute( + request: request, + delegate: delegate, + eventLoop: .delegate(on: eventLoop) + ).wait() XCTAssertEqual(true, response) // redirect request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "redirect/302") - response = try localClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: eventLoop)).wait() + response = try localClient.execute(request: request, delegate: delegate, eventLoop: .delegate(on: eventLoop)) + .wait() XCTAssertEqual(true, response) } func testDecompression() throws { let localHTTPBin = HTTPBin(.http1_1(compress: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(decompression: .enabled(limit: .none))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(decompression: .enabled(limit: .none)) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -887,7 +1112,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var body = "" for _ in 1...1000 { - body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." + body += + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." } for algorithm in [nil, "gzip", "deflate"] { @@ -929,7 +1155,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var body = "" for _ in 1...1000 { - body += "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." + body += + "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua." } for algorithm: String? in [nil] { @@ -957,7 +1184,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testDecompressionLimit() throws { let localHTTPBin = HTTPBin(.http1_1(compress: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(decompression: .enabled(limit: .ratio(1)))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(decompression: .enabled(limit: .ratio(1))) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) @@ -975,30 +1205,47 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testLoopDetectionRedirectLimit() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none, redirectConfiguration: .follow(max: 5, allowCycles: false))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration( + certificateVerification: .none, + redirectConfiguration: .follow(max: 5, allowCycles: false) + ) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) } - XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(localHTTPBin.port)/redirect/infinite1").wait(), "Should fail with redirect limit") { error in + XCTAssertThrowsError( + try localClient.get(url: "https://localhost:\(localHTTPBin.port)/redirect/infinite1").wait(), + "Should fail with redirect limit" + ) { error in XCTAssertEqual(error as? HTTPClientError, HTTPClientError.redirectCycleDetected) } } func testCountRedirectLimit() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none, redirectConfiguration: .follow(max: 10, allowCycles: true))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration( + certificateVerification: .none, + redirectConfiguration: .follow(max: 10, allowCycles: true) + ) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) } - XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(localHTTPBin.port)/redirect/infinite1").timeout(after: .seconds(10)).wait()) { error in + XCTAssertThrowsError( + try localClient.get(url: "https://localhost:\(localHTTPBin.port)/redirect/infinite1").timeout( + after: .seconds(10) + ).wait() + ) { error in XCTAssertEqual(error as? HTTPClientError, HTTPClientError.redirectLimitReached) } } @@ -1016,13 +1263,15 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { defer { XCTAssertNoThrow(try localClient.syncShutdown()) } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "https://localhost:\(localHTTPBin.port)/redirect/target", - method: .GET, - headers: [ - "X-Target-Redirect-URL": "/redirect/target", - ] - )) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "https://localhost:\(localHTTPBin.port)/redirect/target", + method: .GET, + headers: [ + "X-Target-Redirect-URL": "/redirect/target" + ] + ) + ) guard let request = maybeRequest else { return } XCTAssertThrowsError( @@ -1042,8 +1291,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func channelRead(context: ChannelHandlerContext, data: NIOAny) { if case .end = self.unwrapInboundIn(data) { - let responseHead = HTTPServerResponsePart.head(.init(version: .init(major: 1, minor: 1), - status: .ok)) + let responseHead = HTTPServerResponsePart.head( + .init( + version: .init(major: 1, minor: 1), + status: .ok + ) + ) context.write(self.wrapOutboundOut(responseHead), promise: nil) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) } @@ -1056,18 +1309,22 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } var server: Channel? - XCTAssertNoThrow(server = try ServerBootstrap(group: group) - .serverChannelOption(ChannelOptions.socket(.init(SOL_SOCKET), .init(SO_REUSEADDR)), value: 1) - .serverChannelOption(ChannelOptions.backlog, value: .init(numberOfParallelWorkers)) - .childChannelInitializer { channel in - channel.pipeline.configureHTTPServerPipeline(withPipeliningAssistance: false, - withServerUpgrade: nil, - withErrorHandling: false).flatMap { - channel.pipeline.addHandler(HTTPServer()) + XCTAssertNoThrow( + server = try ServerBootstrap(group: group) + .serverChannelOption(ChannelOptions.socket(.init(SOL_SOCKET), .init(SO_REUSEADDR)), value: 1) + .serverChannelOption(ChannelOptions.backlog, value: .init(numberOfParallelWorkers)) + .childChannelInitializer { channel in + channel.pipeline.configureHTTPServerPipeline( + withPipeliningAssistance: false, + withServerUpgrade: nil, + withErrorHandling: false + ).flatMap { + channel.pipeline.addHandler(HTTPServer()) + } } - } - .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) - .wait()) + .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) + .wait() + ) defer { XCTAssertNoThrow(try server?.close().wait()) } @@ -1100,18 +1357,28 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } let result = self.defaultClient.get(url: "http://localhost:\(web.serverPort)/foo") - XCTAssertNoThrow(try web.receiveHeadAndVerify { received in - let expected = HTTPRequestHead( - version: .http1_1, - method: .GET, - uri: "/foo", - headers: ["Host": "localhost:\(web.serverPort)"] - ) - XCTAssertEqual(expected, received) - }) + XCTAssertNoThrow( + try web.receiveHeadAndVerify { received in + let expected = HTTPRequestHead( + version: .http1_1, + method: .GET, + uri: "/foo", + headers: ["Host": "localhost:\(web.serverPort)"] + ) + XCTAssertEqual(expected, received) + } + ) XCTAssertNoThrow(try web.receiveEnd()) - XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), - status: .internalServerError)))) + XCTAssertNoThrow( + try web.writeOutbound( + .head( + .init( + version: .init(major: 1, minor: 1), + status: .internalServerError + ) + ) + ) + ) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) var response: HTTPClient.Response? @@ -1126,18 +1393,31 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try web.stop()) } let result = self.defaultClient.get(url: "http://localhost:\(web.serverPort)/foo") - XCTAssertNoThrow(try web.receiveHeadAndVerify { - XCTAssertEqual($0, HTTPRequestHead( - version: .init(major: 1, minor: 1), - method: .GET, - uri: "/foo", - headers: HTTPHeaders([("Host", "localhost:\(web.serverPort)")]) - )) - }) + XCTAssertNoThrow( + try web.receiveHeadAndVerify { + XCTAssertEqual( + $0, + HTTPRequestHead( + version: .init(major: 1, minor: 1), + method: .GET, + uri: "/foo", + headers: HTTPHeaders([("Host", "localhost:\(web.serverPort)")]) + ) + ) + } + ) XCTAssertNoThrow(try web.receiveEnd()) - XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 0), - status: .internalServerError)))) + XCTAssertNoThrow( + try web.writeOutbound( + .head( + .init( + version: .init(major: 1, minor: 0), + status: .internalServerError + ) + ) + ) + ) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) var response: HTTPClient.Response? @@ -1150,15 +1430,17 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let web = NIOHTTP1TestServer(group: self.serverGroup) let result = self.defaultClient.get(url: "http://localhost:\(web.serverPort)/foo") - XCTAssertNoThrow(try web.receiveHeadAndVerify { received in - let expected = HTTPRequestHead( - version: .http1_1, - method: .GET, - uri: "/foo", - headers: ["Host": "localhost:\(web.serverPort)"] - ) - XCTAssertEqual(expected, received) - }) + XCTAssertNoThrow( + try web.receiveHeadAndVerify { received in + let expected = HTTPRequestHead( + version: .http1_1, + method: .GET, + uri: "/foo", + headers: ["Host": "localhost:\(web.serverPort)"] + ) + XCTAssertEqual(expected, received) + } + ) XCTAssertNoThrow(try web.receiveEnd()) XCTAssertNoThrow(try web.stop()) @@ -1176,19 +1458,29 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { for _ in 0..<10 { let result = self.defaultClient.get(url: "http://localhost:\(web.serverPort)/foo") - XCTAssertNoThrow(try web.receiveHeadAndVerify { received in - let expected = HTTPRequestHead( - version: .http1_1, - method: .GET, - uri: "/foo", - headers: ["Host": "localhost:\(web.serverPort)"] - ) - XCTAssertEqual(expected, received) - }) + XCTAssertNoThrow( + try web.receiveHeadAndVerify { received in + let expected = HTTPRequestHead( + version: .http1_1, + method: .GET, + uri: "/foo", + headers: ["Host": "localhost:\(web.serverPort)"] + ) + XCTAssertEqual(expected, received) + } + ) XCTAssertNoThrow(try web.receiveEnd()) - XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 0), - status: .ok, - headers: HTTPHeaders([("connection", "close")]))))) + XCTAssertNoThrow( + try web.writeOutbound( + .head( + .init( + version: .init(major: 1, minor: 0), + status: .ok, + headers: HTTPHeaders([("connection", "close")]) + ) + ) + ) + ) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) var response: HTTPClient.Response? @@ -1207,20 +1499,34 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { for i in 0..<10 { let result = self.defaultClient.get(url: "http://localhost:\(web.serverPort)/foo") - XCTAssertNoThrow(try web.receiveHeadAndVerify { received in - let expected = HTTPRequestHead( - version: .http1_1, - method: .GET, - uri: "/foo", - headers: ["Host": "localhost:\(web.serverPort)"] - ) - XCTAssertEqual(expected, received) - }) + XCTAssertNoThrow( + try web.receiveHeadAndVerify { received in + let expected = HTTPRequestHead( + version: .http1_1, + method: .GET, + uri: "/foo", + headers: ["Host": "localhost:\(web.serverPort)"] + ) + XCTAssertEqual(expected, received) + } + ) XCTAssertNoThrow(try web.receiveEnd()) - XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 0), - status: .ok, - headers: HTTPHeaders([("connection", - i % 2 == 0 ? "close" : "keep-alive")]))))) + XCTAssertNoThrow( + try web.writeOutbound( + .head( + .init( + version: .init(major: 1, minor: 0), + status: .ok, + headers: HTTPHeaders([ + ( + "connection", + i % 2 == 0 ? "close" : "keep-alive" + ) + ]) + ) + ) + ) + ) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) var response: HTTPClient.Response? @@ -1231,8 +1537,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testStressGetHttpsSSLError() throws { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration().enableFastFailureModeForTesting()) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration().enableFastFailureModeForTesting() + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -1242,7 +1550,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { localClient.execute(request: request, delegate: TestHTTPDelegate()) } - let results = try EventLoopFuture.whenAllComplete(tasks.map { $0.futureResult }, on: localClient.eventLoopGroup.next()).wait() + let results = try EventLoopFuture.whenAllComplete( + tasks.map { $0.futureResult }, + on: localClient.eventLoopGroup.next() + ).wait() for result in results { switch result { @@ -1259,9 +1570,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // We're speaking TLS to a plain text server. This will cause the handshake to fail but given // that the bytes "HTTP/1.1" aren't the start of a valid TLS packet, we can also get // errSSLPeerProtocolVersion because the first bytes contain the version. - XCTAssert(clientError.status == errSSLHandshakeFail || - clientError.status == errSSLPeerProtocolVersion, - "unexpected NWTLSError with status \(clientError.status)") + XCTAssert( + clientError.status == errSSLHandshakeFail || clientError.status == errSSLPeerProtocolVersion, + "unexpected NWTLSError with status \(clientError.status)" + ) #endif } else { guard let clientError = error as? NIOSSLError, case NIOSSLError.handshakeFailed = clientError else { @@ -1306,7 +1618,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") #else guard let sslError = error as? NIOSSLError, - case .handshakeFailed(.sslError) = sslError else { + case .handshakeFailed(.sslError) = sslError + else { XCTFail("unexpected error \(error)") return } @@ -1339,7 +1652,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: config) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } - XCTAssertThrowsError(try localClient.get(url: "https://localhost:\(port)", deadline: .now() + .seconds(2)).wait()) { error in + XCTAssertThrowsError( + try localClient.get(url: "https://localhost:\(port)", deadline: .now() + .seconds(2)).wait() + ) { error in #if canImport(Network) guard let nwTLSError = error as? HTTPClient.NWTLSError else { XCTFail("could not cast \(error) of type \(type(of: error)) to \(HTTPClient.NWTLSError.self)") @@ -1348,7 +1663,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(nwTLSError.status, errSSLBadCert, "unexpected tls error: \(nwTLSError)") #else guard let sslError = error as? NIOSSLError, - case .handshakeFailed(.sslError) = sslError else { + case .handshakeFailed(.sslError) = sslError + else { XCTFail("unexpected error \(error)") return } @@ -1379,13 +1695,17 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let requestCount = 200 var futureResults = [EventLoopFuture]() for _ in 1...requestCount { - let req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", - method: .GET, - headers: ["X-internal-delay": "5", "Connection": "close"]) + let req = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + method: .GET, + headers: ["X-internal-delay": "5", "Connection": "close"] + ) futureResults.append(self.defaultClient.execute(request: req)) } - XCTAssertNoThrow(try EventLoopFuture.andAllComplete(futureResults, on: eventLoop) - .timeout(after: .seconds(10)).wait()) + XCTAssertNoThrow( + try EventLoopFuture.andAllComplete(futureResults, on: eventLoop) + .timeout(after: .seconds(10)).wait() + ) } func testManyConcurrentRequestsWork() { @@ -1402,8 +1722,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let q = DispatchQueue(label: "worker \(w)") q.async(group: allDone) { func go() { - allWorkersReady.signal() // tell the driver we're ready - allWorkersGo.wait() // wait for the driver to let us go + allWorkersReady.signal() // tell the driver we're ready + allWorkersGo.wait() // wait for the driver to let us go for _ in 0..]() for i in 1...100 { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .GET, headers: ["X-internal-delay": "10"]) + let request = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + method: .GET, + headers: ["X-internal-delay": "10"] + ) let preference: HTTPClient.EventLoopPreference if i <= 50 { preference = .delegateAndChannel(on: first) @@ -1640,15 +1986,18 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let seenError = DispatchGroup() seenError.enter() var maybeSecondRequest: EventLoopFuture? - XCTAssertNoThrow(maybeSecondRequest = try el.submit { - let neverSucceedingRequest = localClient.get(url: url) - let secondRequest = neverSucceedingRequest.flatMapError { error in - XCTAssertEqual(.cancelled, error as? HTTPClientError) - seenError.leave() - return localClient.get(url: url) // <== this is the main part, during the error callout, we call back in - } - return secondRequest - }.wait()) + XCTAssertNoThrow( + maybeSecondRequest = try el.submit { + let neverSucceedingRequest = localClient.get(url: url) + let secondRequest = neverSucceedingRequest.flatMapError { error in + XCTAssertEqual(.cancelled, error as? HTTPClientError) + seenError.leave() + // v this is the main part, during the error callout, we call back in + return localClient.get(url: url) + } + return secondRequest + }.wait() + ) guard let secondRequest = maybeSecondRequest else { XCTFail("couldn't get request future") @@ -1674,13 +2023,15 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try localClient.syncShutdown()) } - XCTAssertEqual(.ok, - try el.flatSubmit { () -> EventLoopFuture in - localClient.get(url: url).flatMap { firstResponse in - XCTAssertEqual(.ok, firstResponse.status) - return localClient.get(url: url) // <== interesting bit here - } - }.wait().status) + XCTAssertEqual( + .ok, + try el.flatSubmit { () -> EventLoopFuture in + localClient.get(url: url).flatMap { firstResponse in + XCTAssertEqual(.ok, firstResponse.status) + return localClient.get(url: url) // <== interesting bit here + } + }.wait().status + ) } func testMakeSecondRequestWhilstFirstIsOngoing() { @@ -1697,11 +2048,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let url = "http://127.0.0.1:\(web.serverPort)" let firstRequest = client.get(url: url) - XCTAssertNoThrow(XCTAssertNotNil(try web.readInbound())) // first request: .head + XCTAssertNoThrow(XCTAssertNotNil(try web.readInbound())) // first request: .head // Now, the first request is ongoing but not complete, let's start a second one let secondRequest = client.get(url: url) - XCTAssertEqual(.end(nil), try web.readInbound()) // first request: .end + XCTAssertEqual(.end(nil), try web.readInbound()) // first request: .end XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), status: .ok)))) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) @@ -1709,8 +2060,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(.ok, try firstRequest.wait().status) // Okay, first request done successfully, let's do the second one too. - XCTAssertNoThrow(XCTAssertNotNil(try web.readInbound())) // first request: .head - XCTAssertEqual(.end(nil), try web.readInbound()) // first request: .end + XCTAssertNoThrow(XCTAssertNotNil(try web.readInbound())) // first request: .head + XCTAssertEqual(.end(nil), try web.readInbound()) // first request: .end XCTAssertNoThrow(try web.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), status: .created)))) XCTAssertNoThrow(try web.writeOutbound(.end(nil))) @@ -1721,15 +2072,19 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // This tests just connecting to a URL where the whole URL is the UNIX domain socket path like // unix:///this/is/my/socket.sock // We don't really have a path component, so we'll have to use "/" - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - defer { - XCTAssertNoThrow(try localHTTPBin.shutdown()) + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + defer { + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + let target = "unix://\(path)" + XCTAssertEqual( + ["Yes"[...]], + try self.defaultClient.get(url: target).wait().headers[canonicalForm: "X-Is-This-Slash"] + ) } - let target = "unix://\(path)" - XCTAssertEqual(["Yes"[...]], - try self.defaultClient.get(url: target).wait().headers[canonicalForm: "X-Is-This-Slash"]) - }) + ) } func testUDSSocketAndPath() { @@ -1737,56 +2092,73 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // // 1. a "base path" which is the path to the UNIX domain socket // 2. an actual path which is the normal path in a regular URL like https://example.com/this/is/the/path - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - defer { - XCTAssertNoThrow(try localHTTPBin.shutdown()) - } - guard let target = URL(string: "/echo-uri", relativeTo: URL(string: "unix://\(path)")), - let request = try? Request(url: target) else { - XCTFail("couldn't build URL for request") - return + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + defer { + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + guard let target = URL(string: "/echo-uri", relativeTo: URL(string: "unix://\(path)")), + let request = try? Request(url: target) + else { + XCTFail("couldn't build URL for request") + return + } + XCTAssertEqual( + ["/echo-uri"[...]], + try self.defaultClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"] + ) } - XCTAssertEqual(["/echo-uri"[...]], - try self.defaultClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"]) - }) + ) } func testHTTPPlusUNIX() { // Here, we're testing a URL where the UNIX domain socket is encoded as the host name - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - defer { - XCTAssertNoThrow(try localHTTPBin.shutdown()) - } - guard let target = URL(httpURLWithSocketPath: path, uri: "/echo-uri"), - let request = try? Request(url: target) else { - XCTFail("couldn't build URL for request") - return + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + defer { + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + guard let target = URL(httpURLWithSocketPath: path, uri: "/echo-uri"), + let request = try? Request(url: target) + else { + XCTFail("couldn't build URL for request") + return + } + XCTAssertEqual( + ["/echo-uri"[...]], + try self.defaultClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"] + ) } - XCTAssertEqual(["/echo-uri"[...]], - try self.defaultClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"]) - }) + ) } func testHTTPSPlusUNIX() { // Here, we're testing a URL where the UNIX domain socket is encoded as the host name - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let localHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localHTTPBin.shutdown()) - } - guard let target = URL(httpsURLWithSocketPath: path, uri: "/echo-uri"), - let request = try? Request(url: target) else { - XCTFail("couldn't build URL for request") - return + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let localHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localHTTPBin.shutdown()) + } + guard let target = URL(httpsURLWithSocketPath: path, uri: "/echo-uri"), + let request = try? Request(url: target) + else { + XCTFail("couldn't build URL for request") + return + } + XCTAssertEqual( + ["/echo-uri"[...]], + try localClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"] + ) } - XCTAssertEqual(["/echo-uri"[...]], - try localClient.execute(request: request).wait().headers[canonicalForm: "X-Calling-URI"]) - }) + ) } func testUseExistingConnectionOnDifferentEL() throws { @@ -1800,13 +2172,20 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let eventLoops = (1...threadCount).map { _ in elg.next() } let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get") - let closingRequest = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", headers: ["Connection": "close"]) + let closingRequest = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + headers: ["Connection": "close"] + ) for (index, el) in eventLoops.enumerated() { if index.isMultiple(of: 2) { - XCTAssertNoThrow(try localClient.execute(request: request, eventLoop: .delegateAndChannel(on: el)).wait()) + XCTAssertNoThrow( + try localClient.execute(request: request, eventLoop: .delegateAndChannel(on: el)).wait() + ) } else { - XCTAssertNoThrow(try localClient.execute(request: request, eventLoop: .delegateAndChannel(on: el)).wait()) + XCTAssertNoThrow( + try localClient.execute(request: request, eventLoop: .delegateAndChannel(on: el)).wait() + ) XCTAssertNoThrow(try localClient.execute(request: closingRequest, eventLoop: .indifferent).wait()) } } @@ -1839,8 +2218,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let last = self.requestNumber.loadThenWrappingIncrement(ordering: .relaxed) switch last { case 0, 2: - context.write(self.wrapOutboundOut(.head(.init(version: .init(major: 1, minor: 1), status: .ok))), - promise: nil) + context.write( + self.wrapOutboundOut(.head(.init(version: .init(major: 1, minor: 1), status: .ok))), + promise: nil + ) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) case 1: context.close(promise: nil) @@ -1853,20 +2234,24 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let requestNumber = ManagedAtomic(0) let connectionNumber = ManagedAtomic(0) - let sharedStateServerHandler = ServerThatAcceptsThenRejects(requestNumber: requestNumber, - connectionNumber: connectionNumber) + let sharedStateServerHandler = ServerThatAcceptsThenRejects( + requestNumber: requestNumber, + connectionNumber: connectionNumber + ) var maybeServer: Channel? - XCTAssertNoThrow(maybeServer = try ServerBootstrap(group: self.serverGroup) - .serverChannelOption(ChannelOptions.socket(.init(SOL_SOCKET), .init(SO_REUSEADDR)), value: 1) - .childChannelInitializer { channel in - channel.pipeline.configureHTTPServerPipeline().flatMap { - // We're deliberately adding a handler which is shared between multiple channels. This is normally - // very verboten but this handler is specially crafted to tolerate this. - channel.pipeline.addHandler(sharedStateServerHandler) + XCTAssertNoThrow( + maybeServer = try ServerBootstrap(group: self.serverGroup) + .serverChannelOption(ChannelOptions.socket(.init(SOL_SOCKET), .init(SO_REUSEADDR)), value: 1) + .childChannelInitializer { channel in + channel.pipeline.configureHTTPServerPipeline().flatMap { + // We're deliberately adding a handler which is shared between multiple channels. This is normally + // very verboten but this handler is specially crafted to tolerate this. + channel.pipeline.addHandler(sharedStateServerHandler) + } } - } - .bind(host: "127.0.0.1", port: 0) - .wait()) + .bind(host: "127.0.0.1", port: 0) + .wait() + ) guard let server = maybeServer else { XCTFail("couldn't create server") return @@ -1902,8 +2287,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { maximumAllowedIdleTimeInConnectionPool: .milliseconds(100) ) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: configuration) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: configuration + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } @@ -1926,7 +2313,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testAvoidLeakingTLSHandshakeCompletionPromise() { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), configuration: .init(timeout: .init(connect: .milliseconds(100)))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(timeout: .init(connect: .milliseconds(100))) + ) let localHTTPBin = HTTPBin() let port = localHTTPBin.port XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -1973,9 +2363,13 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testValidationErrorsAreSurfaced() throws { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .TRACE, body: .stream { _ in - self.defaultClient.eventLoopGroup.next().makeSucceededFuture(()) - }) + let request = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + method: .TRACE, + body: .stream { _ in + self.defaultClient.eventLoopGroup.next().makeSucceededFuture(()) + } + ) let runningRequest = self.defaultClient.execute(request: request) XCTAssertThrowsError(try runningRequest.wait()) { error in XCTAssertEqual(HTTPClientError.traceRequestWithBody, error as? HTTPClientError) @@ -1993,9 +2387,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { private var bodyPartsSeenSoFar = 0 private var atEnd = false - init(headPromise: EventLoopPromise, - bodyPromises: [EventLoopPromise], - endPromise: EventLoopPromise) { + init( + headPromise: EventLoopPromise, + bodyPromises: [EventLoopPromise], + endPromise: EventLoopPromise + ) { self.headPromise = headPromise self.bodyPromises = bodyPromises self.endPromise = endPromise @@ -2011,8 +2407,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { self.bodyPartsSeenSoFar += 1 self.bodyPromises.dropFirst(myNumber).first?.succeed(bytes) ?? XCTFail("ouch, too many chunks") case .end: - context.write(self.wrapOutboundOut(.head(.init(version: .init(major: 1, minor: 1), status: .ok))), - promise: nil) + context.write( + self.wrapOutboundOut(.head(.init(version: .init(major: 1, minor: 1), status: .ok))), + promise: nil + ) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: self.endPromise) self.atEnd = true } @@ -2025,8 +2423,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { struct NotFulfilledError: Error {} self.headPromise.fail(NotFulfilledError()) - self.bodyPromises.forEach { - $0.fail(NotFulfilledError()) + for promise in self.bodyPromises { + promise.fail(NotFulfilledError()) } self.endPromise.fail(NotFulfilledError()) } @@ -2047,12 +2445,16 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let streamWriterPromise = group.next().makePromise(of: HTTPClient.Body.StreamWriter.self) func makeServer() -> Channel? { - return try? ServerBootstrap(group: group) + try? ServerBootstrap(group: group) .childChannelInitializer { channel in channel.pipeline.configureHTTPServerPipeline().flatMap { - channel.pipeline.addHandler(HTTPServer(headPromise: headPromise, - bodyPromises: bodyPromises, - endPromise: endPromise)) + channel.pipeline.addHandler( + HTTPServer( + headPromise: headPromise, + bodyPromises: bodyPromises, + endPromise: endPromise + ) + ) } } .serverChannelOption(ChannelOptions.socket(.init(SOL_SOCKET), .init(SO_REUSEADDR)), value: 1) @@ -2065,13 +2467,15 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { return nil } - return try? HTTPClient.Request(url: "http://\(localAddress.ipAddress!):\(localAddress.port!)", - method: .POST, - headers: ["transfer-encoding": "chunked"], - body: .stream { streamWriter in - streamWriterPromise.succeed(streamWriter) - return sentOffAllBodyPartsPromise.futureResult - }) + return try? HTTPClient.Request( + url: "http://\(localAddress.ipAddress!):\(localAddress.port!)", + method: .POST, + headers: ["transfer-encoding": "chunked"], + body: .stream { streamWriter in + streamWriterPromise.succeed(streamWriter) + return sentOffAllBodyPartsPromise.futureResult + } + ) } guard let server = makeServer(), let request = makeRequest(server: server) else { @@ -2103,35 +2507,45 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testUploadStreamingCallinToleratedFromOtsideEL() throws { - let request = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .POST, body: .stream(contentLength: 4) { writer in - let promise = self.defaultClient.eventLoopGroup.next().makePromise(of: Void.self) - // We have to toleare callins from any thread - DispatchQueue(label: "upload-streaming").async { - writer.write(.byteBuffer(ByteBuffer(string: "1234"))).whenComplete { _ in - promise.succeed(()) + let request = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + method: .POST, + body: .stream(contentLength: 4) { writer in + let promise = self.defaultClient.eventLoopGroup.next().makePromise(of: Void.self) + // We have to toleare callins from any thread + DispatchQueue(label: "upload-streaming").async { + writer.write(.byteBuffer(ByteBuffer(string: "1234"))).whenComplete { _ in + promise.succeed(()) + } } + return promise.futureResult } - return promise.futureResult - }) + ) XCTAssertNoThrow(try self.defaultClient.execute(request: request).wait()) } func testWeHandleUsSendingACloseHeaderCorrectly() { - guard let req1 = try? Request(url: self.defaultHTTPBinURLPrefix + "stats", - method: .GET, - headers: ["connection": "close"]), + guard + let req1 = try? Request( + url: self.defaultHTTPBinURLPrefix + "stats", + method: .GET, + headers: ["connection": "close"] + ), let statsBytes1 = try? self.defaultClient.execute(request: req1).wait().body, - let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) else { + let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) + else { XCTFail("request 1 didn't work") return } guard let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) else { + let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) + else { XCTFail("request 2 didn't work") return } guard let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) else { + let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) + else { XCTFail("request 3 didn't work") return } @@ -2147,21 +2561,27 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testWeHandleUsReceivingACloseHeaderCorrectly() { - guard let req1 = try? Request(url: self.defaultHTTPBinURLPrefix + "stats", - method: .GET, - headers: ["X-Send-Back-Header-Connection": "close"]), + guard + let req1 = try? Request( + url: self.defaultHTTPBinURLPrefix + "stats", + method: .GET, + headers: ["X-Send-Back-Header-Connection": "close"] + ), let statsBytes1 = try? self.defaultClient.execute(request: req1).wait().body, - let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) else { + let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) + else { XCTFail("request 1 didn't work") return } guard let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) else { + let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) + else { XCTFail("request 2 didn't work") return } guard let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) else { + let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) + else { XCTFail("request 3 didn't work") return } @@ -2178,22 +2598,32 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testWeHandleUsSendingACloseHeaderAmongstOtherConnectionHeadersCorrectly() { for closeHeader in [("connection", "close"), ("CoNneCTION", "ClOSe")] { - guard let req1 = try? Request(url: self.defaultHTTPBinURLPrefix + "stats", - method: .GET, - headers: ["X-Send-Back-Header-\(closeHeader.0)": - "foo,\(closeHeader.1),bar"]), + guard + let req1 = try? Request( + url: self.defaultHTTPBinURLPrefix + "stats", + method: .GET, + headers: [ + "X-Send-Back-Header-\(closeHeader.0)": + "foo,\(closeHeader.1),bar" + ] + ), let statsBytes1 = try? self.defaultClient.execute(request: req1).wait().body, - let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) else { + let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) + else { XCTFail("request 1 didn't work") return } - guard let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) else { + guard + let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, + let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) + else { XCTFail("request 2 didn't work") return } - guard let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) else { + guard + let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, + let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) + else { XCTFail("request 3 didn't work") return } @@ -2210,22 +2640,32 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testWeHandleUsReceivingACloseHeaderAmongstOtherConnectionHeadersCorrectly() { for closeHeader in [("connection", "close"), ("CoNneCTION", "ClOSe")] { - guard let req1 = try? Request(url: self.defaultHTTPBinURLPrefix + "stats", - method: .GET, - headers: ["X-Send-Back-Header-\(closeHeader.0)": - "foo,\(closeHeader.1),bar"]), + guard + let req1 = try? Request( + url: self.defaultHTTPBinURLPrefix + "stats", + method: .GET, + headers: [ + "X-Send-Back-Header-\(closeHeader.0)": + "foo,\(closeHeader.1),bar" + ] + ), let statsBytes1 = try? self.defaultClient.execute(request: req1).wait().body, - let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) else { + let stats1 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes1) + else { XCTFail("request 1 didn't work") return } - guard let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) else { + guard + let statsBytes2 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, + let stats2 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes2) + else { XCTFail("request 2 didn't work") return } - guard let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, - let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) else { + guard + let statsBytes3 = try? self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + "stats").wait().body, + let stats3 = try? JSONDecoder().decode(RequestInfo.self, from: statsBytes3) + else { XCTFail("request 3 didn't work") return } @@ -2243,28 +2683,35 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testLoggingCorrectlyAttachesRequestInformationEvenAfterDuringRedirect() { let logStore = CollectEverythingLogHandler.LogStore() - var logger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: logStore) - }) + var logger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: logStore) + } + ) logger.logLevel = .trace logger[metadataKey: "custom-request-id"] = "abcd" var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "http://localhost:\(self.defaultHTTPBin.port)/redirect/target", - method: .GET, - headers: [ - "X-Target-Redirect-URL": "/get", - ] - )) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "http://localhost:\(self.defaultHTTPBin.port)/redirect/target", + method: .GET, + headers: [ + "X-Target-Redirect-URL": "/get" + ] + ) + ) guard let request = maybeRequest else { return } - XCTAssertNoThrow(try self.defaultClient.execute( - request: request, - eventLoop: .indifferent, - deadline: nil, - logger: logger - ).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request, + eventLoop: .indifferent, + deadline: nil, + logger: logger + ).wait() + ) let logs = logStore.allEntries XCTAssertTrue(logs.allSatisfy { $0.metadata["custom-request-id"] == "abcd" }) @@ -2283,51 +2730,70 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertGreaterThan(secondRequestLogs.count, 0) XCTAssertTrue(secondRequestLogs.allSatisfy { $0.metadata["ahc-request-id"] == lastRequestID }) - logs.forEach { print($0) } + for log in logs { print(log) } } func testLoggingCorrectlyAttachesRequestInformation() { let logStore = CollectEverythingLogHandler.LogStore() - var loggerYolo001 = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: logStore) - }) + var loggerYolo001 = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: logStore) + } + ) loggerYolo001.logLevel = .trace loggerYolo001[metadataKey: "yolo-request-id"] = "yolo-001" - var loggerACME002 = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: logStore) - }) + var loggerACME002 = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: logStore) + } + ) loggerACME002.logLevel = .trace loggerACME002[metadataKey: "acme-request-id"] = "acme-002" guard let request1 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get"), - let request2 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "stats"), - let request3 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "ok") else { + let request2 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "stats"), + let request3 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "ok") + else { XCTFail("bad stuff, can't even make request structures") return } // === Request 1 (Yolo001) - XCTAssertNoThrow(try self.defaultClient.execute(request: request1, - eventLoop: .indifferent, - deadline: nil, - logger: loggerYolo001).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request1, + eventLoop: .indifferent, + deadline: nil, + logger: loggerYolo001 + ).wait() + ) let logsAfterReq1 = logStore.allEntries logStore.allEntries = [] // === Request 2 (Yolo001) - XCTAssertNoThrow(try self.defaultClient.execute(request: request2, - eventLoop: .indifferent, - deadline: nil, - logger: loggerYolo001).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request2, + eventLoop: .indifferent, + deadline: nil, + logger: loggerYolo001 + ).wait() + ) let logsAfterReq2 = logStore.allEntries logStore.allEntries = [] // === Request 3 (ACME002) - XCTAssertNoThrow(try self.defaultClient.execute(request: request3, - eventLoop: .indifferent, - deadline: nil, - logger: loggerACME002).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request3, + eventLoop: .indifferent, + deadline: nil, + logger: loggerACME002 + ).wait() + ) let logsAfterReq3 = logStore.allEntries logStore.allEntries = [] @@ -2336,176 +2802,238 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertGreaterThan(logsAfterReq2.count, 0) XCTAssertGreaterThan(logsAfterReq3.count, 0) - XCTAssert(logsAfterReq1.allSatisfy { entry in - if let httpRequestMetadata = entry.metadata["ahc-request-id"], - let yoloRequestID = entry.metadata["yolo-request-id"] { - XCTAssertNil(entry.metadata["acme-request-id"]) - XCTAssertEqual("yolo-001", yoloRequestID) - XCTAssertNotNil(Int(httpRequestMetadata)) - return true - } else { - XCTFail("log message doesn't contain the right IDs: \(entry)") - return false - } - }) - XCTAssert(logsAfterReq1.contains { entry in - // Since a new connection must be created first we expect that the request is queued - // and log message describing this is emitted. - entry.message == "Request was queued (waiting for a connection to become available)" - && entry.level == .debug - }) - XCTAssert(logsAfterReq1.contains { entry in - // After the new connection was created we expect a log message that describes that the - // request was scheduled on a connection. The connection id must be set from here on. - entry.message == "Request was scheduled on connection" - && entry.level == .debug - && entry.metadata["ahc-connection-id"] != nil - }) - - XCTAssert(logsAfterReq2.allSatisfy { entry in - if let httpRequestMetadata = entry.metadata["ahc-request-id"], - let yoloRequestID = entry.metadata["yolo-request-id"] { - XCTAssertNil(entry.metadata["acme-request-id"]) - XCTAssertEqual("yolo-001", yoloRequestID) - XCTAssertNotNil(Int(httpRequestMetadata)) - return true - } else { - XCTFail("log message doesn't contain the right IDs: \(entry)") - return false - } - }) - XCTAssertFalse(logsAfterReq2.contains { entry in - entry.message == "Request was queued (waiting for a connection to become available)" - }) - XCTAssert(logsAfterReq2.contains { entry in - entry.message == "Request was scheduled on connection" - && entry.level == .debug - && entry.metadata["ahc-connection-id"] != nil - }) - - XCTAssert(logsAfterReq3.allSatisfy { entry in - if let httpRequestMetadata = entry.metadata["ahc-request-id"], - let acmeRequestID = entry.metadata["acme-request-id"] { - XCTAssertNil(entry.metadata["yolo-request-id"]) - XCTAssertEqual("acme-002", acmeRequestID) - XCTAssertNotNil(Int(httpRequestMetadata)) - return true - } else { - XCTFail("log message doesn't contain the right IDs: \(entry)") - return false + XCTAssert( + logsAfterReq1.allSatisfy { entry in + if let httpRequestMetadata = entry.metadata["ahc-request-id"], + let yoloRequestID = entry.metadata["yolo-request-id"] + { + XCTAssertNil(entry.metadata["acme-request-id"]) + XCTAssertEqual("yolo-001", yoloRequestID) + XCTAssertNotNil(Int(httpRequestMetadata)) + return true + } else { + XCTFail("log message doesn't contain the right IDs: \(entry)") + return false + } } - }) - XCTAssertFalse(logsAfterReq3.contains { entry in - entry.message == "Request was queued (waiting for a connection to become available)" - }) - XCTAssert(logsAfterReq3.contains { entry in - entry.message == "Request was scheduled on connection" - && entry.level == .debug - && entry.metadata["ahc-connection-id"] != nil - }) + ) + XCTAssert( + logsAfterReq1.contains { entry in + // Since a new connection must be created first we expect that the request is queued + // and log message describing this is emitted. + entry.message == "Request was queued (waiting for a connection to become available)" + && entry.level == .debug + } + ) + XCTAssert( + logsAfterReq1.contains { entry in + // After the new connection was created we expect a log message that describes that the + // request was scheduled on a connection. The connection id must be set from here on. + entry.message == "Request was scheduled on connection" + && entry.level == .debug + && entry.metadata["ahc-connection-id"] != nil + } + ) + + XCTAssert( + logsAfterReq2.allSatisfy { entry in + if let httpRequestMetadata = entry.metadata["ahc-request-id"], + let yoloRequestID = entry.metadata["yolo-request-id"] + { + XCTAssertNil(entry.metadata["acme-request-id"]) + XCTAssertEqual("yolo-001", yoloRequestID) + XCTAssertNotNil(Int(httpRequestMetadata)) + return true + } else { + XCTFail("log message doesn't contain the right IDs: \(entry)") + return false + } + } + ) + XCTAssertFalse( + logsAfterReq2.contains { entry in + entry.message == "Request was queued (waiting for a connection to become available)" + } + ) + XCTAssert( + logsAfterReq2.contains { entry in + entry.message == "Request was scheduled on connection" + && entry.level == .debug + && entry.metadata["ahc-connection-id"] != nil + } + ) + + XCTAssert( + logsAfterReq3.allSatisfy { entry in + if let httpRequestMetadata = entry.metadata["ahc-request-id"], + let acmeRequestID = entry.metadata["acme-request-id"] + { + XCTAssertNil(entry.metadata["yolo-request-id"]) + XCTAssertEqual("acme-002", acmeRequestID) + XCTAssertNotNil(Int(httpRequestMetadata)) + return true + } else { + XCTFail("log message doesn't contain the right IDs: \(entry)") + return false + } + } + ) + XCTAssertFalse( + logsAfterReq3.contains { entry in + entry.message == "Request was queued (waiting for a connection to become available)" + } + ) + XCTAssert( + logsAfterReq3.contains { entry in + entry.message == "Request was scheduled on connection" + && entry.level == .debug + && entry.metadata["ahc-connection-id"] != nil + } + ) } func testNothingIsLoggedAtInfoOrHigher() { let logStore = CollectEverythingLogHandler.LogStore() - var logger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: logStore) - }) + var logger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: logStore) + } + ) logger.logLevel = .info guard let request1 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get"), - let request2 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "stats") else { + let request2 = try? HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "stats") + else { XCTFail("bad stuff, can't even make request structures") return } // === Request 1 - XCTAssertNoThrow(try self.defaultClient.execute(request: request1, - eventLoop: .indifferent, - deadline: nil, - logger: logger).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request1, + eventLoop: .indifferent, + deadline: nil, + logger: logger + ).wait() + ) XCTAssertEqual(0, logStore.allEntries.count) // === Request 2 - XCTAssertNoThrow(try self.defaultClient.execute(request: request2, - eventLoop: .indifferent, - deadline: nil, - logger: logger).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + request: request2, + eventLoop: .indifferent, + deadline: nil, + logger: logger + ).wait() + ) XCTAssertEqual(0, logStore.allEntries.count) // === Synthesized Request - XCTAssertNoThrow(try self.defaultClient.execute(.GET, - url: self.defaultHTTPBinURLPrefix + "get", - body: nil, - deadline: nil, - logger: logger).wait()) + XCTAssertNoThrow( + try self.defaultClient.execute( + .GET, + url: self.defaultHTTPBinURLPrefix + "get", + body: nil, + deadline: nil, + logger: logger + ).wait() + ) XCTAssertEqual(0, logStore.allEntries.count) XCTAssertEqual(0, self.backgroundLogStore.allEntries.filter { $0.level >= .info }.count) // === Synthesized Socket Path Request - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let backgroundLogStore = CollectEverythingLogHandler.LogStore() - var backgroundLogger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: backgroundLogStore) - }) - backgroundLogger.logLevel = .trace - - let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - backgroundActivityLogger: backgroundLogger) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) - } + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let backgroundLogStore = CollectEverythingLogHandler.LogStore() + var backgroundLogger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: backgroundLogStore) + } + ) + backgroundLogger.logLevel = .trace - XCTAssertNoThrow(try localClient.execute(.GET, - socketPath: path, - urlPath: "get", - body: nil, - deadline: nil, - logger: logger).wait()) - XCTAssertEqual(0, logStore.allEntries.count) + let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + backgroundActivityLogger: backgroundLogger + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .info }.count) - }) + XCTAssertNoThrow( + try localClient.execute( + .GET, + socketPath: path, + urlPath: "get", + body: nil, + deadline: nil, + logger: logger + ).wait() + ) + XCTAssertEqual(0, logStore.allEntries.count) - // === Synthesized Secure Socket Path Request - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let backgroundLogStore = CollectEverythingLogHandler.LogStore() - var backgroundLogger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: backgroundLogStore) - }) - backgroundLogger.logLevel = .trace - - let localSocketPathHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none), - backgroundActivityLogger: backgroundLogger) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .info }.count) } + ) + + // === Synthesized Secure Socket Path Request + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let backgroundLogStore = CollectEverythingLogHandler.LogStore() + var backgroundLogger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: backgroundLogStore) + } + ) + backgroundLogger.logLevel = .trace - XCTAssertNoThrow(try localClient.execute(.GET, - secureSocketPath: path, - urlPath: "get", - body: nil, - deadline: nil, - logger: logger).wait()) - XCTAssertEqual(0, logStore.allEntries.count) + let localSocketPathHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none), + backgroundActivityLogger: backgroundLogger + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .info }.count) - }) + XCTAssertNoThrow( + try localClient.execute( + .GET, + secureSocketPath: path, + urlPath: "get", + body: nil, + deadline: nil, + logger: logger + ).wait() + ) + XCTAssertEqual(0, logStore.allEntries.count) + + XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .info }.count) + } + ) } func testAllMethodsLog() { func checkExpectationsWithLogger(type: String, _ body: (Logger, String) throws -> T) throws -> T { let logStore = CollectEverythingLogHandler.LogStore() - var logger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: logStore) - }) + var logger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: logStore) + } + ) logger.logLevel = .trace logger[metadataKey: "req"] = "yo-\(type)" @@ -2513,86 +3041,125 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let result = try body(logger, url) XCTAssertGreaterThan(logStore.allEntries.count, 0) - logStore.allEntries.forEach { entry in + for entry in logStore.allEntries { XCTAssertEqual("yo-\(type)", entry.metadata["req"] ?? "n/a") XCTAssertNotNil(Int(entry.metadata["ahc-request-id"] ?? "n/a")) } return result } - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "GET") { logger, url in - try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "GET") { logger, url in + try self.defaultClient.get(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() + }.status + ) - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "PUT") { logger, url in - try self.defaultClient.put(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "PUT") { logger, url in + try self.defaultClient.put(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() + }.status + ) - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "POST") { logger, url in - try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "POST") { logger, url in + try self.defaultClient.post(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() + }.status + ) - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "DELETE") { logger, url in - try self.defaultClient.delete(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "DELETE") { logger, url in + try self.defaultClient.delete(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() + }.status + ) - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "PATCH") { logger, url in - try self.defaultClient.patch(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "PATCH") { logger, url in + try self.defaultClient.patch(url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() + }.status + ) - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "CHECKOUT") { logger, url in - try self.defaultClient.execute(.CHECKOUT, url: self.defaultHTTPBinURLPrefix + url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "CHECKOUT") { logger, url in + try self.defaultClient.execute(.CHECKOUT, url: self.defaultHTTPBinURLPrefix + url, logger: logger) + .wait() + }.status + ) // No background activity expected here. XCTAssertEqual(0, self.backgroundLogStore.allEntries.filter { $0.level >= .debug }.count) - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let backgroundLogStore = CollectEverythingLogHandler.LogStore() - var backgroundLogger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: backgroundLogStore) - }) - backgroundLogger.logLevel = .trace + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let backgroundLogStore = CollectEverythingLogHandler.LogStore() + var backgroundLogger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: backgroundLogStore) + } + ) + backgroundLogger.logLevel = .trace - let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - backgroundActivityLogger: backgroundLogger) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) - } + let localSocketPathHTTPBin = HTTPBin(bindTarget: .unixDomainSocket(path)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + backgroundActivityLogger: backgroundLogger + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "GET") { logger, url in - try localClient.execute(socketPath: path, urlPath: url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "GET") { logger, url in + try localClient.execute(socketPath: path, urlPath: url, logger: logger).wait() + }.status + ) - // No background activity expected here. - XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .debug }.count) - }) + // No background activity expected here. + XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .debug }.count) + } + ) - XCTAssertNoThrow(try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in - let backgroundLogStore = CollectEverythingLogHandler.LogStore() - var backgroundLogger = Logger(label: "\(#function)", factory: { _ in - CollectEverythingLogHandler(logStore: backgroundLogStore) - }) - backgroundLogger.logLevel = .trace + XCTAssertNoThrow( + try TemporaryFileHelpers.withTemporaryUnixDomainSocketPathName { path in + let backgroundLogStore = CollectEverythingLogHandler.LogStore() + var backgroundLogger = Logger( + label: "\(#function)", + factory: { _ in + CollectEverythingLogHandler(logStore: backgroundLogStore) + } + ) + backgroundLogger.logLevel = .trace - let localSocketPathHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none), - backgroundActivityLogger: backgroundLogger) - defer { - XCTAssertNoThrow(try localClient.syncShutdown()) - XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) - } + let localSocketPathHTTPBin = HTTPBin(.http1_1(ssl: true), bindTarget: .unixDomainSocket(path)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none), + backgroundActivityLogger: backgroundLogger + ) + defer { + XCTAssertNoThrow(try localClient.syncShutdown()) + XCTAssertNoThrow(try localSocketPathHTTPBin.shutdown()) + } - XCTAssertEqual(.notFound, try checkExpectationsWithLogger(type: "GET") { logger, url in - try localClient.execute(secureSocketPath: path, urlPath: url, logger: logger).wait() - }.status) + XCTAssertEqual( + .notFound, + try checkExpectationsWithLogger(type: "GET") { logger, url in + try localClient.execute(secureSocketPath: path, urlPath: url, logger: logger).wait() + }.status + ) - // No background activity expected here. - XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .debug }.count) - }) + // No background activity expected here. + XCTAssertEqual(0, backgroundLogStore.allEntries.filter { $0.level >= .debug }.count) + } + ) } func testClosingIdleConnectionsInPoolLogsInTheBackground() { @@ -2601,16 +3168,19 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertNoThrow(try self.defaultClient.syncShutdown()) XCTAssertGreaterThanOrEqual(self.backgroundLogStore.allEntries.count, 0) - XCTAssert(self.backgroundLogStore.allEntries.contains { entry in - entry.message == "Shutting down connection pool" - }) - XCTAssert(self.backgroundLogStore.allEntries.allSatisfy { entry in - entry.metadata["ahc-request-id"] == nil && - entry.metadata["ahc-request"] == nil && - entry.metadata["ahc-pool-key"] != nil - }) + XCTAssert( + self.backgroundLogStore.allEntries.contains { entry in + entry.message == "Shutting down connection pool" + } + ) + XCTAssert( + self.backgroundLogStore.allEntries.allSatisfy { entry in + entry.metadata["ahc-request-id"] == nil && entry.metadata["ahc-request"] == nil + && entry.metadata["ahc-pool-key"] != nil + } + ) - self.defaultClient = nil // so it doesn't get shut down again. + self.defaultClient = nil // so it doesn't get shut down again. } func testUploadStreamingNoLength() throws { @@ -2635,8 +3205,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTFail("Unexpected part") } - XCTAssertNoThrow(try server.readInbound()) // .body - XCTAssertNoThrow(try server.readInbound()) // .end + XCTAssertNoThrow(try server.readInbound()) // .body + XCTAssertNoThrow(try server.readInbound()) // .end XCTAssertNoThrow(try server.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), status: .ok)))) XCTAssertNoThrow(try server.writeOutbound(.end(nil))) @@ -2654,8 +3224,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } } - let httpClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(timeout: .init(connect: .milliseconds(10)))) + let httpClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(timeout: .init(connect: .milliseconds(10))) + ) defer { XCTAssertNoThrow(try httpClient.syncShutdown()) @@ -2681,11 +3253,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func didReceiveHead(task: HTTPClient.Task, _: HTTPResponseHead) -> EventLoopFuture { - return self.eventLoop.makeSucceededFuture(()) + self.eventLoop.makeSucceededFuture(()) } func didReceiveBodyPart(task: HTTPClient.Task, _: ByteBuffer) -> EventLoopFuture { - return self.eventLoop.makeSucceededFuture(()) + self.eventLoop.makeSucceededFuture(()) } func didFinishRequest(task: HTTPClient.Task) throws {} @@ -2708,8 +3280,8 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let request = try HTTPClient.Request(url: "http://localhost:\(httpServer.serverPort)/") let future = httpClient.execute(request: request, delegate: delegate) - XCTAssertNoThrow(try httpServer.readInbound()) // .head - XCTAssertNoThrow(try httpServer.readInbound()) // .end + XCTAssertNoThrow(try httpServer.readInbound()) // .head + XCTAssertNoThrow(try httpServer.readInbound()) // .end XCTAssertNoThrow(try httpServer.writeOutbound(.head(.init(version: .init(major: 1, minor: 1), status: .ok)))) XCTAssertNoThrow(try httpServer.writeOutbound(.body(.byteBuffer(ByteBuffer(string: "1234"))))) @@ -2721,15 +3293,20 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testContentLengthTooLongFails() throws { let url = self.defaultHTTPBinURLPrefix + "post" XCTAssertThrowsError( - try self.defaultClient.execute(request: - Request(url: url, + try self.defaultClient.execute( + request: + Request( + url: url, body: .stream(contentLength: 10) { streamWriter in let promise = self.defaultClient.eventLoopGroup.next().makePromise(of: Void.self) DispatchQueue(label: "content-length-test").async { streamWriter.write(.byteBuffer(ByteBuffer(string: "1"))).cascade(to: promise) } return promise.futureResult - })).wait()) { error in + } + ) + ).wait() + ) { error in XCTAssertEqual(error as! HTTPClientError, HTTPClientError.bodyLengthMismatch) } // Quickly try another request and check that it works. @@ -2751,11 +3328,16 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let url = self.defaultHTTPBinURLPrefix + "post" let tooLong = "XBAD BAD BAD NOT HTTP/1.1\r\n\r\n" XCTAssertThrowsError( - try self.defaultClient.execute(request: - Request(url: url, + try self.defaultClient.execute( + request: + Request( + url: url, body: .stream(contentLength: 1) { streamWriter in streamWriter.write(.byteBuffer(ByteBuffer(string: tooLong))) - })).wait()) { error in + } + ) + ).wait() + ) { error in XCTAssertEqual(error as! HTTPClientError, HTTPClientError.bodyLengthMismatch) } // Quickly try another request and check that it works. If we by accident wrote some extra bytes into the @@ -2829,7 +3411,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // We specify a deadline of 2 ms co that request will be timed out before all chunks are writtent, // we need to verify that second error on write after timeout does not lead to double-release. - XCTAssertThrowsError(try self.defaultClient.execute(request: request, deadline: .now() + .milliseconds(2)).wait()) + XCTAssertThrowsError( + try self.defaultClient.execute(request: request, deadline: .now() + .milliseconds(2)).wait() + ) } func testSSLHandshakeErrorPropagation() throws { @@ -3041,10 +3625,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var request = try Request(url: httpBin.baseURL) request.body = .byteBuffer(body) - XCTAssertThrowsError(try self.defaultClient.execute( - request: request, - delegate: ResponseAccumulator(request: request, maxBodySize: 10) - ).wait()) { error in + XCTAssertThrowsError( + try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait() + ) { error in XCTAssertTrue(error is ResponseAccumulator.ResponseTooBigError, "unexpected error \(error)") } } @@ -3091,10 +3677,12 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { request.body = .stream { writer in writer.write(.byteBuffer(body)) } - XCTAssertThrowsError(try self.defaultClient.execute( - request: request, - delegate: ResponseAccumulator(request: request, maxBodySize: 10) - ).wait()) { error in + XCTAssertThrowsError( + try self.defaultClient.execute( + request: request, + delegate: ResponseAccumulator(request: request, maxBodySize: 10) + ).wait() + ) { error in XCTAssertTrue(error is ResponseAccumulator.ResponseTooBigError, "unexpected error \(error)") } } @@ -3120,7 +3708,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // In this test, we test that a request can continue to stream its body after the response head and end // was received where the end is a 200. func testBiDirectionalStreamingEarly200() { - let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) } + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in + HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) + } defer { XCTAssertNoThrow(try httpBin.shutdown()) } let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) @@ -3174,7 +3764,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // This test is identical to the one above, except that we send another request immediately after. This is a regression // test for https://github.com/swift-server/async-http-client/issues/595. func testBiDirectionalStreamingEarly200DoesntPreventUsFromSendingMoreRequests() { - let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) } + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in + HTTP200DelayedHandler(bodyPartsBeforeResponse: 1) + } defer { XCTAssertNoThrow(try httpBin.shutdown()) } let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 2) @@ -3232,7 +3824,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { defer { XCTAssertNoThrow(try eventLoopGroup.syncShutdownGracefully()) } let onClosePromise = eventLoopGroup.next().makePromise(of: Void.self) - let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in ExpectClosureServerHandler(onClosePromise: onClosePromise) } + let httpBin = HTTPBin(.http1_1(ssl: false, compress: false)) { _ in + ExpectClosureServerHandler(onClosePromise: onClosePromise) + } defer { XCTAssertNoThrow(try httpBin.shutdown()) } let writeEL = eventLoopGroup.next() @@ -3290,8 +3884,10 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { tlsConfig.maximumTLSVersion = .tlsv12 tlsConfig.certificateVerification = .none let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(tlsConfiguration: tlsConfig)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(tlsConfiguration: tlsConfig) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -3366,12 +3962,16 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testRequestSpecificTLS() throws { - let configuration = HTTPClient.Configuration(tlsConfiguration: nil, - timeout: .init(), - decompression: .disabled) + let configuration = HTTPClient.Configuration( + tlsConfiguration: nil, + timeout: .init(), + decompression: .disabled + ) let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: configuration) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: configuration + ) let decoder = JSONDecoder() defer { @@ -3382,7 +3982,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // First two requests use identical TLS configurations. var tlsConfig = TLSConfiguration.makeClientConfiguration() tlsConfig.certificateVerification = .none - let firstRequest = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, tlsConfiguration: tlsConfig) + let firstRequest = try HTTPClient.Request( + url: "https://localhost:\(localHTTPBin.port)/get", + method: .GET, + tlsConfiguration: tlsConfig + ) let firstResponse = try localClient.execute(request: firstRequest).wait() guard let firstBody = firstResponse.body else { XCTFail("No request body found") @@ -3390,7 +3994,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } let firstConnectionNumber = try decoder.decode(RequestInfo.self, from: firstBody).connectionNumber - let secondRequest = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, tlsConfiguration: tlsConfig) + let secondRequest = try HTTPClient.Request( + url: "https://localhost:\(localHTTPBin.port)/get", + method: .GET, + tlsConfiguration: tlsConfig + ) let secondResponse = try localClient.execute(request: secondRequest).wait() guard let secondBody = secondResponse.body else { XCTFail("No request body found") @@ -3402,7 +4010,11 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { var tlsConfig2 = TLSConfiguration.makeClientConfiguration() tlsConfig2.certificateVerification = .none tlsConfig2.maximumTLSVersion = .tlsv1 - let thirdRequest = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, tlsConfiguration: tlsConfig2) + let thirdRequest = try HTTPClient.Request( + url: "https://localhost:\(localHTTPBin.port)/get", + method: .GET, + tlsConfiguration: tlsConfig2 + ) let thirdResponse = try localClient.execute(request: thirdRequest).wait() guard let thirdBody = thirdResponse.body else { XCTFail("No request body found") @@ -3413,8 +4025,16 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { XCTAssertEqual(firstResponse.status, .ok) XCTAssertEqual(secondResponse.status, .ok) XCTAssertEqual(thirdResponse.status, .ok) - XCTAssertEqual(firstConnectionNumber, secondConnectionNumber, "Identical TLS configurations did not use the same connection") - XCTAssertNotEqual(thirdConnectionNumber, firstConnectionNumber, "Different TLS configurations did not use different connections.") + XCTAssertEqual( + firstConnectionNumber, + secondConnectionNumber, + "Identical TLS configurations did not use the same connection" + ) + XCTAssertNotEqual( + thirdConnectionNumber, + firstConnectionNumber, + "Different TLS configurations did not use different connections." + ) } func testRequestWithHeaderTransferEncodingIdentityDoesNotFail() { @@ -3439,7 +4059,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { func testMassiveDownload() { var response: HTTPClient.Response? - XCTAssertNoThrow(response = try self.defaultClient.get(url: "\(self.defaultHTTPBinURLPrefix)mega-chunked").wait()) + XCTAssertNoThrow( + response = try self.defaultClient.get(url: "\(self.defaultHTTPBinURLPrefix)mega-chunked").wait() + ) XCTAssertEqual(.ok, response?.status) XCTAssertEqual(response?.version, .http1_1) @@ -3466,11 +4088,13 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } func testMassiveHeaderHTTP2() throws { - let bin = HTTPBin(.http2(settings: [ - .init(parameter: .maxConcurrentStreams, value: 100), - .init(parameter: .maxHeaderListSize, value: 1024 * 256), - .init(parameter: .maxFrameSize, value: 1024 * 256), - ])) + let bin = HTTPBin( + .http2(settings: [ + .init(parameter: .maxConcurrentStreams, value: 100), + .init(parameter: .maxHeaderListSize, value: 1024 * 256), + .init(parameter: .maxFrameSize, value: 1024 * 256), + ]) + ) defer { XCTAssertNoThrow(try bin.shutdown()) } let client = HTTPClient( @@ -3604,7 +4228,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { } let response = try client.get(url: self.defaultHTTPBinURLPrefix + "get").wait() XCTAssertEqual(.ok, response.status) - } catch let error as IOError where error.errnoCode == EINVAL || error.errnoCode == EPROTONOSUPPORT || error.errnoCode == ENOPROTOOPT { + } catch let error as IOError + where error.errnoCode == EINVAL || error.errnoCode == EPROTONOSUPPORT || error.errnoCode == ENOPROTOOPT + { // some old Linux kernels don't support MPTCP, skip this test in this case // see https://www.mptcp.dev/implementation.html for details about each type // of error @@ -3643,18 +4269,18 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // ! is safe, assigned above request.tlsConfiguration!.certificateVerification = .none - let response1 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + let response1 = try await client.execute(request, timeout: .hours(99)) // 99h ~= infinity XCTAssertEqual(.ok, response1.status) // For the second request, we reset the TLS config request.tlsConfiguration = nil do { - let response2 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + let response2 = try await client.execute(request, timeout: .hours(99)) // 99h ~= infinity XCTFail("shouldn't succeed, self-signed cert: \(response2)") } catch { switch error as? NIOSSLError { case .some(.handshakeFailed(_)): - () // ok + () // ok default: XCTFail("unexpected error: \(error)") } @@ -3665,7 +4291,7 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { // ! is safe, assigned above request.tlsConfiguration!.certificateVerification = .none - let response3 = try await client.execute(request, timeout: /* infinity */ .hours(99)) + let response3 = try await client.execute(request, timeout: .hours(99)) // 99h ~= infinity XCTAssertEqual(.ok, response3.status) } diff --git a/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests.swift index 854d9092c..b63eb7cba 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientUncleanSSLConnectionShutdownTests.swift @@ -155,7 +155,8 @@ final class HTTPClientUncleanSSLConnectionShutdownTests: XCTestCase { ) defer { XCTAssertNoThrow(try client.syncShutdown()) } - XCTAssertThrowsError(try client.get(url: "https://localhost:\(httpBin.port)/transferencodingtruncated").wait()) { + XCTAssertThrowsError(try client.get(url: "https://localhost:\(httpBin.port)/transferencodingtruncated").wait()) + { XCTAssertEqual($0 as? HTTPParserError, .invalidEOFState) } } @@ -184,7 +185,7 @@ final class HTTPBinForSSLUncleanShutdown { let serverChannel: Channel var port: Int { - return Int(self.serverChannel.localAddress!.port!) + Int(self.serverChannel.localAddress!.port!) } init() { @@ -231,61 +232,61 @@ private final class HTTPBinForSSLUncleanShutdownHandler: ChannelInboundHandler { switch req.uri { case "/nocontentlength": response = """ - HTTP/1.1 200 OK\r\n\ - Connection: close\r\n\ - \r\n\ - foo - """ + HTTP/1.1 200 OK\r\n\ + Connection: close\r\n\ + \r\n\ + foo + """ case "/nocontent": response = """ - HTTP/1.1 204 OK\r\n\ - Connection: close\r\n\ - \r\n - """ + HTTP/1.1 204 OK\r\n\ + Connection: close\r\n\ + \r\n + """ case "/noresponse": response = nil case "/wrongcontentlength": response = """ - HTTP/1.1 200 OK\r\n\ - Connection: close\r\n\ - Content-Length: 6\r\n\ - \r\n\ - foo - """ + HTTP/1.1 200 OK\r\n\ + Connection: close\r\n\ + Content-Length: 6\r\n\ + \r\n\ + foo + """ case "/transferencoding": response = """ - HTTP/1.1 200 OK\r\n\ - Connection: close\r\n\ - Transfer-Encoding: chunked\r\n\ - \r\n\ - 3\r\n\ - foo\r\n\ - 0\r\n\ - \r\n - """ + HTTP/1.1 200 OK\r\n\ + Connection: close\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + 3\r\n\ + foo\r\n\ + 0\r\n\ + \r\n + """ case "/transferencodingtruncated": response = """ - HTTP/1.1 200 OK\r\n\ - Connection: close\r\n\ - Transfer-Encoding: chunked\r\n\ - \r\n\ - 12\r\n\ - foo - """ + HTTP/1.1 200 OK\r\n\ + Connection: close\r\n\ + Transfer-Encoding: chunked\r\n\ + \r\n\ + 12\r\n\ + foo + """ default: response = """ - HTTP/1.1 404 OK\r\n\ - Connection: close\r\n\ - Content-Length: 9\r\n\ - \r\n\ - Not Found - """ + HTTP/1.1 404 OK\r\n\ + Connection: close\r\n\ + Content-Length: 9\r\n\ + \r\n\ + Not Found + """ } if let response = response { diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift index 476584972..d9dbd4cb1 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+FactoryTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOPosix @@ -20,18 +19,22 @@ import NIOSOCKS import NIOSSL import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_FactoryTests: XCTestCase { func testConnectionCreationTimesoutIfDeadlineIsInThePast() { let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } var server: Channel? - XCTAssertNoThrow(server = try ServerBootstrap(group: group) - .childChannelInitializer { channel in - channel.pipeline.addHandler(NeverrespondServerHandler()) - } - .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) - .wait()) + XCTAssertNoThrow( + server = try ServerBootstrap(group: group) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NeverrespondServerHandler()) + } + .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) + .wait() + ) defer { XCTAssertNoThrow(try server?.close().wait()) } @@ -45,13 +48,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { sslContextCache: .init() ) - XCTAssertThrowsError(try factory.makeChannel( - requester: ExplodingRequester(), - connectionID: 1, - deadline: .now() - .seconds(1), - eventLoop: group.next(), - logger: .init(label: "test") - ).wait() + XCTAssertThrowsError( + try factory.makeChannel( + requester: ExplodingRequester(), + connectionID: 1, + deadline: .now() - .seconds(1), + eventLoop: group.next(), + logger: .init(label: "test") + ).wait() ) { XCTAssertEqual($0 as? HTTPClientError, .connectTimeout) } @@ -62,12 +66,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } var server: Channel? - XCTAssertNoThrow(server = try ServerBootstrap(group: group) - .childChannelInitializer { channel in - channel.pipeline.addHandler(NeverrespondServerHandler()) - } - .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) - .wait()) + XCTAssertNoThrow( + server = try ServerBootstrap(group: group) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NeverrespondServerHandler()) + } + .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) + .wait() + ) defer { XCTAssertNoThrow(try server?.close().wait()) } @@ -82,13 +88,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { sslContextCache: .init() ) - XCTAssertThrowsError(try factory.makeChannel( - requester: ExplodingRequester(), - connectionID: 1, - deadline: .now() + .seconds(1), - eventLoop: group.next(), - logger: .init(label: "test") - ).wait() + XCTAssertThrowsError( + try factory.makeChannel( + requester: ExplodingRequester(), + connectionID: 1, + deadline: .now() + .seconds(1), + eventLoop: group.next(), + logger: .init(label: "test") + ).wait() ) { XCTAssertEqual($0 as? HTTPClientError, .socksHandshakeTimeout) } @@ -99,12 +106,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } var server: Channel? - XCTAssertNoThrow(server = try ServerBootstrap(group: group) - .childChannelInitializer { channel in - channel.pipeline.addHandler(NeverrespondServerHandler()) - } - .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) - .wait()) + XCTAssertNoThrow( + server = try ServerBootstrap(group: group) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NeverrespondServerHandler()) + } + .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) + .wait() + ) defer { XCTAssertNoThrow(try server?.close().wait()) } @@ -119,13 +128,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { sslContextCache: .init() ) - XCTAssertThrowsError(try factory.makeChannel( - requester: ExplodingRequester(), - connectionID: 1, - deadline: .now() + .seconds(1), - eventLoop: group.next(), - logger: .init(label: "test") - ).wait() + XCTAssertThrowsError( + try factory.makeChannel( + requester: ExplodingRequester(), + connectionID: 1, + deadline: .now() + .seconds(1), + eventLoop: group.next(), + logger: .init(label: "test") + ).wait() ) { XCTAssertEqual($0 as? HTTPClientError, .httpProxyHandshakeTimeout) } @@ -136,12 +146,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { defer { XCTAssertNoThrow(try group.syncShutdownGracefully()) } var server: Channel? - XCTAssertNoThrow(server = try ServerBootstrap(group: group) - .childChannelInitializer { channel in - channel.pipeline.addHandler(NeverrespondServerHandler()) - } - .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) - .wait()) + XCTAssertNoThrow( + server = try ServerBootstrap(group: group) + .childChannelInitializer { channel in + channel.pipeline.addHandler(NeverrespondServerHandler()) + } + .bind(to: .init(ipAddress: "127.0.0.1", port: 0)) + .wait() + ) defer { XCTAssertNoThrow(try server?.close().wait()) } @@ -158,13 +170,14 @@ class HTTPConnectionPool_FactoryTests: XCTestCase { sslContextCache: .init() ) - XCTAssertThrowsError(try factory.makeChannel( - requester: ExplodingRequester(), - connectionID: 1, - deadline: .now() + .seconds(1), - eventLoop: group.next(), - logger: .init(label: "test") - ).wait() + XCTAssertThrowsError( + try factory.makeChannel( + requester: ExplodingRequester(), + connectionID: 1, + deadline: .now() + .seconds(1), + eventLoop: group.next(), + logger: .init(label: "test") + ).wait() ) { XCTAssertEqual($0 as? HTTPClientError, .tlsHandshakeTimeout) } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift index dfeaf1d9c..914990048 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1ConnectionsTest.swift @@ -12,15 +12,20 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testCreatingConnections() { let elg = EmbeddedEventLoopGroup(loops: 4) - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -52,7 +57,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testCreatingConnectionAndFailing() { let elg = EmbeddedEventLoopGroup(loops: 4) - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -103,7 +112,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el3 = elg.next() let el4 = elg.next() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) for el in [el1, el2, el3, el4] { XCTAssertEqual(connections.startingGeneralPurposeConnections, 0) @@ -130,7 +143,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el4 = elg.next() let el5 = elg.next() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) for el in [el1, el2, el3, el4] { XCTAssertEqual(connections.startingGeneralPurposeConnections, 0) @@ -157,7 +174,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el4 = elg.next() let el5 = elg.next() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) for el in [el1, el2, el3, el4] { XCTAssertEqual(connections.startingGeneralPurposeConnections, 0) @@ -181,7 +202,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el1 = elg.next() let el2 = elg.next() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) for el in [el1, el1, el1, el1, el2] { let connID = connections.createNewConnection(on: el) @@ -228,7 +253,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testCloseConnectionIfIdle() { let elg = EmbeddedEventLoopGroup(loops: 1) - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) let el1 = elg.next() @@ -248,7 +277,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testCloseConnectionIfIdleButLeasedRaceCondition() { let elg = EmbeddedEventLoopGroup(loops: 1) - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) let el1 = elg.next() @@ -267,7 +300,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testCloseConnectionIfIdleButClosedRaceCondition() { let elg = EmbeddedEventLoopGroup(loops: 1) - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) let el1 = elg.next() @@ -288,7 +325,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el3 = elg.next() let el4 = elg.next() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: .init(), maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: .init(), + maximumConnectionUses: nil + ) for el in [el1, el2, el3, el4] { let connID = connections.createNewConnection(on: el) @@ -343,7 +384,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -372,7 +417,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2WithPendingRequestsWithRequiredEventLoop() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -411,7 +460,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2WithPendingRequestsWithRequiredEventLoopSameAsStartingConnections() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -439,7 +492,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2WithPendingRequestsWithPreferredEventLoop() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -478,7 +535,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2WithAlreadyLeasedHTTP1Connection() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() let el3 = elg.next() @@ -522,7 +583,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2WithMoreStartingConnectionsThanMaximumAllowedConccurentConnections() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 2, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 2, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -557,7 +622,11 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { func testMigrationFromHTTP2StartsEnoghOverflowConnectionsForRequiredEventLoopRequests() { let elg = EmbeddedEventLoopGroup(loops: 4) let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 1, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 1, + generator: generator, + maximumConnectionUses: nil + ) let el1 = elg.next() let el2 = elg.next() @@ -599,16 +668,23 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { let el2 = elg.next() let generator = HTTPConnectionPool.Connection.ID.Generator() - var connections = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: generator, maximumConnectionUses: nil) + var connections = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: generator, + maximumConnectionUses: nil + ) let connID1 = connections.createNewConnection(on: el1) let context = connections.migrateToHTTP2() - XCTAssertEqual(context, .init( - backingOff: [], - starting: [(connID1, el1)], - close: [] - )) + XCTAssertEqual( + context, + .init( + backingOff: [], + starting: [(connID1, el1)], + close: [] + ) + ) let connID2 = generator.next() @@ -626,8 +702,7 @@ class HTTPConnectionPool_HTTP1ConnectionsTests: XCTestCase { extension HTTPConnectionPool.HTTP1Connections.HTTP1ToHTTP2MigrationContext: Equatable { public static func == (lhs: Self, rhs: Self) -> Bool { - return lhs.close == rhs.close && - lhs.starting.elementsEqual(rhs.starting, by: { $0.0 == $1.0 && $0.1 === $1.1 }) && - lhs.backingOff.elementsEqual(rhs.backingOff, by: { $0.0 == $1.0 && $0.1 === $1.1 }) + lhs.close == rhs.close && lhs.starting.elementsEqual(rhs.starting, by: { $0.0 == $1.0 && $0.1 === $1.1 }) + && lhs.backingOff.elementsEqual(rhs.backingOff, by: { $0.0 == $1.0 && $0.1 === $1.1 }) } } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift index 367fdaffb..2be6cfa26 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP1StateTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { func testCreatingAndFailingConnections() { struct SomeError: Error, Equatable {} @@ -197,9 +198,12 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { return XCTFail("Unexpected connection action: \(action.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux - let failedConnect1 = state.failedToCreateNewConnection(HTTPClientError.connectTimeout, connectionID: connectionID) + let failedConnect1 = state.failedToCreateNewConnection( + HTTPClientError.connectTimeout, + connectionID: connectionID + ) XCTAssertEqual(failedConnect1.request, .none) guard case .scheduleBackoffTimer(connectionID, let backoffTimeAmount1, _) = failedConnect1.connection else { return XCTFail("Unexpected connection action: \(failedConnect1.connection)") @@ -212,9 +216,12 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { return XCTFail("Unexpected connection action: \(backoffDoneAction.connection)") } XCTAssertGreaterThan(newConnectionID, connectionID) - XCTAssert(connectionEL === newEventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === newEventLoop) // XCTAssertIdentical not available on Linux - let failedConnect2 = state.failedToCreateNewConnection(HTTPClientError.connectTimeout, connectionID: newConnectionID) + let failedConnect2 = state.failedToCreateNewConnection( + HTTPClientError.connectTimeout, + connectionID: newConnectionID + ) XCTAssertEqual(failedConnect2.request, .none) guard case .scheduleBackoffTimer(newConnectionID, let backoffTimeAmount2, _) = failedConnect2.connection else { return XCTFail("Unexpected connection action: \(failedConnect2.connection)") @@ -227,7 +234,9 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { guard case .failRequest(let requestToFail, let requestError, cancelTimeout: false) = failRequest.request else { return XCTFail("Unexpected request action: \(action.request)") } - XCTAssert(requestToFail.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux + + // XCTAssertIdentical not available on Linux + XCTAssert(requestToFail.__testOnly_wrapped_request() === mockRequest) XCTAssertEqual(requestError as? HTTPClientError, .connectTimeout) XCTAssertEqual(failRequest.connection, .none) @@ -257,7 +266,7 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = executeAction.connection else { return XCTFail("Unexpected connection action: \(executeAction.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux // 2. cancel request @@ -269,7 +278,9 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(state.timeoutRequest(request.id), .none, "To late timeout is ignored") // 4. succeed connection attempt - let connectedAction = state.newHTTP1ConnectionCreated(.__testOnly_connection(id: connectionID, eventLoop: connectionEL)) + let connectedAction = state.newHTTP1ConnectionCreated( + .__testOnly_connection(id: connectionID, eventLoop: connectionEL) + ) XCTAssertEqual(connectedAction.request, .none, "Request must not be executed") XCTAssertEqual(connectedAction.connection, .scheduleTimeoutTimer(connectionID, on: connectionEL)) } @@ -296,15 +307,18 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = executeAction.connection else { return XCTFail("Unexpected connection action: \(executeAction.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux // 2. connection succeeds - let connection: HTTPConnectionPool.Connection = .__testOnly_connection(id: connectionID, eventLoop: connectionEL) + let connection: HTTPConnectionPool.Connection = .__testOnly_connection( + id: connectionID, + eventLoop: connectionEL + ) let connectedAction = state.newHTTP1ConnectionCreated(connection) guard case .executeRequest(request, connection, cancelTimeout: true) = connectedAction.request else { return XCTFail("Unexpected request action: \(connectedAction.request)") } - XCTAssert(request.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux + XCTAssert(request.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux XCTAssertEqual(connectedAction.connection, .none) // 3. shutdown @@ -324,7 +338,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { let finalRequest = HTTPConnectionPool.Request(finalMockRequest) let failAction = state.executeRequest(finalRequest) XCTAssertEqual(failAction.connection, .none) - XCTAssertEqual(failAction.request, .failRequest(finalRequest, HTTPClientError.alreadyShutdown, cancelTimeout: false)) + XCTAssertEqual( + failAction.request, + .failRequest(finalRequest, HTTPClientError.alreadyShutdown, cancelTimeout: false) + ) // 5. close open connection let closeAction = state.http1ConnectionClosed(connectionID) @@ -345,7 +362,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // Add eight requests to fill all connections for _ in 0..<8 { let eventLoop = elg.next() - guard let expectedConnection = connections.newestParkedConnection(for: eventLoop) ?? connections.newestParkedConnection else { + guard + let expectedConnection = connections.newestParkedConnection(for: eventLoop) + ?? connections.newestParkedConnection + else { return XCTFail("Expected to still have connections available") } @@ -354,7 +374,8 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { let action = state.executeRequest(request) XCTAssertEqual(action.connection, .cancelTimeoutTimer(expectedConnection.id)) - guard case .executeRequest(let returnedRequest, expectedConnection, cancelTimeout: false) = action.request else { + guard case .executeRequest(let returnedRequest, expectedConnection, cancelTimeout: false) = action.request + else { return XCTFail("Expected to execute a request next, but got: \(action.request)") } @@ -428,7 +449,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // 10% of the cases enforce the eventLoop let elRequired = (0..<10).randomElement().flatMap { $0 == 0 ? true : false }! - let mockRequest = MockHTTPScheduableRequest(eventLoop: reqEventLoop, requiresEventLoopForChannel: elRequired) + let mockRequest = MockHTTPScheduableRequest( + eventLoop: reqEventLoop, + requiresEventLoopForChannel: elRequired + ) let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) @@ -440,7 +464,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssert(connEventLoop === reqEventLoop) XCTAssertEqual(action.request, .scheduleRequestTimeout(for: request, on: reqEventLoop)) - let connection: HTTPConnectionPool.Connection = .__testOnly_connection(id: connectionID, eventLoop: connEventLoop) + let connection: HTTPConnectionPool.Connection = .__testOnly_connection( + id: connectionID, + eventLoop: connEventLoop + ) let createdAction = state.newHTTP1ConnectionCreated(connection) XCTAssertEqual(createdAction.request, .executeRequest(request, connection, cancelTimeout: true)) XCTAssertEqual(createdAction.connection, .none) @@ -451,7 +478,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(state.http1ConnectionClosed(connectionID), .none) case .cancelTimeoutTimer(let connectionID): - guard let expectedConnection = connections.newestParkedConnection(for: reqEventLoop) ?? connections.newestParkedConnection else { + guard + let expectedConnection = connections.newestParkedConnection(for: reqEventLoop) + ?? connections.newestParkedConnection + else { return XCTFail("Expected to have connections available") } @@ -459,7 +489,11 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssert(expectedConnection.eventLoop === reqEventLoop) } - XCTAssertEqual(connectionID, expectedConnection.id, "Request is scheduled on the connection we expected") + XCTAssertEqual( + connectionID, + expectedConnection.id, + "Request is scheduled on the connection we expected" + ) XCTAssertNoThrow(try connections.activateConnection(connectionID)) guard case .executeRequest(let request, let connection, cancelTimeout: false) = action.request else { @@ -469,8 +503,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.execute(request.__testOnly_wrapped_request(), on: connection)) XCTAssertNoThrow(try connections.finishExecution(connection.id)) - XCTAssertEqual(state.http1ConnectionReleased(connection.id), - .init(request: .none, connection: .scheduleTimeoutTimer(connection.id, on: connection.eventLoop))) + XCTAssertEqual( + state.http1ConnectionReleased(connection.id), + .init(request: .none, connection: .scheduleTimeoutTimer(connection.id, on: connection.eventLoop)) + ) XCTAssertNoThrow(try connections.parkConnection(connectionID)) default: @@ -542,7 +578,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { // Add eight requests to fill all connections for _ in 0..<8 { let eventLoop = elg.next() - guard let expectedConnection = connections.newestParkedConnection(for: eventLoop) ?? connections.newestParkedConnection else { + guard + let expectedConnection = connections.newestParkedConnection(for: eventLoop) + ?? connections.newestParkedConnection + else { return XCTFail("Expected to still have connections available") } @@ -589,12 +628,20 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { guard let newConnection = maybeNewConnection else { return XCTFail("Expected to get a new connection") } let afterRecreationAction = state.newHTTP1ConnectionCreated(newConnection) XCTAssertEqual(afterRecreationAction.connection, .none) - guard case .executeRequest(let request, newConnection, cancelTimeout: true) = afterRecreationAction.request else { + guard + case .executeRequest(let request, newConnection, cancelTimeout: true) = afterRecreationAction + .request + else { return XCTFail("Unexpected request action: \(action.request)") } XCTAssertEqual(request.id, queuedRequestsOrder.popFirst()) - XCTAssertNoThrow(try connections.execute(queuer.get(request.id, request: request.__testOnly_wrapped_request()), on: newConnection)) + XCTAssertNoThrow( + try connections.execute( + queuer.get(request.id, request: request.__testOnly_wrapped_request()), + on: newConnection + ) + ) case .none: XCTAssert(queuer.isEmpty) @@ -730,7 +777,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(executeAction.request, .scheduleRequestTimeout(for: request, on: mockRequest.eventLoop)) - let failAction = state.failedToCreateNewConnection(HTTPClientError.httpProxyHandshakeTimeout, connectionID: connectionID) + let failAction = state.failedToCreateNewConnection( + HTTPClientError.httpProxyHandshakeTimeout, + connectionID: connectionID + ) guard case .scheduleBackoffTimer(connectionID, backoff: _, on: let timerEL) = failAction.connection else { return XCTFail("Expected to create a backoff timer") } @@ -738,7 +788,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(failAction.request, .none) let timeoutAction = state.timeoutRequest(request.id) - XCTAssertEqual(timeoutAction.request, .failRequest(request, HTTPClientError.httpProxyHandshakeTimeout, cancelTimeout: false)) + XCTAssertEqual( + timeoutAction.request, + .failRequest(request, HTTPClientError.httpProxyHandshakeTimeout, cancelTimeout: false) + ) XCTAssertEqual(timeoutAction.connection, .none) } @@ -764,7 +817,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(executeAction.request, .scheduleRequestTimeout(for: request, on: mockRequest.eventLoop)) let timeoutAction = state.timeoutRequest(request.id) - XCTAssertEqual(timeoutAction.request, .failRequest(request, HTTPClientError.connectTimeout, cancelTimeout: false)) + XCTAssertEqual( + timeoutAction.request, + .failRequest(request, HTTPClientError.connectTimeout, cancelTimeout: false) + ) XCTAssertEqual(timeoutAction.connection, .none) } @@ -802,7 +858,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(executeAction2.request, .scheduleRequestTimeout(for: request2, on: connEL1)) - let failAction = state.failedToCreateNewConnection(HTTPClientError.httpProxyHandshakeTimeout, connectionID: connectionID1) + let failAction = state.failedToCreateNewConnection( + HTTPClientError.httpProxyHandshakeTimeout, + connectionID: connectionID1 + ) guard case .scheduleBackoffTimer(connectionID1, backoff: _, on: let timerEL) = failAction.connection else { return XCTFail("Expected to create a backoff timer") } @@ -816,7 +875,10 @@ class HTTPConnectionPool_HTTP1StateMachineTests: XCTestCase { XCTAssertEqual(createdAction.connection, .none) let timeoutAction = state.timeoutRequest(request2.id) - XCTAssertEqual(timeoutAction.request, .failRequest(request2, HTTPClientError.getConnectionFromPoolTimeout, cancelTimeout: false)) + XCTAssertEqual( + timeoutAction.request, + .failRequest(request2, HTTPClientError.getConnectionFromPoolTimeout, cancelTimeout: false) + ) XCTAssertEqual(timeoutAction.connection, .none) } } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest.swift index 69bf62d81..dd56a9102 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2ConnectionsTest.swift @@ -12,11 +12,12 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { func testCreatingConnections() { let elg = EmbeddedEventLoopGroup(loops: 4) @@ -32,7 +33,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { XCTAssertTrue(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests) XCTAssertTrue(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests(for: el1)) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 100) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 100) XCTAssertEqual(conn1CreatedContext.isIdle, true) XCTAssert(conn1CreatedContext.eventLoop === el1) @@ -46,7 +50,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn2ID = connections.createNewConnection(on: el2) XCTAssertTrue(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests(for: el2)) let conn2: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn2ID, eventLoop: el2) - let (conn2Index, conn2CreatedContext) = connections.newHTTP2ConnectionEstablished(conn2, maxConcurrentStreams: 100) + let (conn2Index, conn2CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn2, + maxConcurrentStreams: 100 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 100) XCTAssertTrue(conn1CreatedContext.isIdle) XCTAssert(conn2CreatedContext.eventLoop === el2) @@ -83,7 +90,9 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { XCTAssert(conn1FailContext.eventLoop === el1) XCTAssertFalse(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests) XCTAssertFalse(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests(for: el1)) - let (replaceConn1ID, replaceConn1EL) = connections.createNewConnectionByReplacingClosedConnection(at: conn1FailIndex) + let (replaceConn1ID, replaceConn1EL) = connections.createNewConnectionByReplacingClosedConnection( + at: conn1FailIndex + ) XCTAssert(replaceConn1EL === el1) XCTAssertEqual(replaceConn1ID, 1) XCTAssertTrue(connections.hasConnectionThatCanOrWillBeAbleToExecuteRequests) @@ -336,13 +345,19 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn1ID = connections.createNewConnection(on: el1) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 100) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 100) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 100) XCTAssertEqual(leasedConn1, conn1) XCTAssertEqual(leasdConnContext1.wasIdle, true) - XCTAssertNil(connections.leaseStream(onRequired: el1), "should not be able to lease stream because they are all already leased") + XCTAssertNil( + connections.leaseStream(onRequired: el1), + "should not be able to lease stream because they are all already leased" + ) let (_, releaseContext) = connections.releaseStream(conn1ID) XCTAssertFalse(releaseContext.isIdle) @@ -354,7 +369,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { XCTAssertEqual(leasedConn, conn1) XCTAssertEqual(leaseContext.wasIdle, false) - XCTAssertNil(connections.leaseStream(onRequired: el1), "should not be able to lease stream because they are all already leased") + XCTAssertNil( + connections.leaseStream(onRequired: el1), + "should not be able to lease stream because they are all already leased" + ) } func testGoAway() { @@ -364,7 +382,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn1ID = connections.createNewConnection(on: el1) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 10) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 10 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 10) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 2) @@ -386,7 +407,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { ) ) - XCTAssertNil(connections.leaseStream(onRequired: el1), "we should not be able to lease a stream because the connection is draining") + XCTAssertNil( + connections.leaseStream(onRequired: el1), + "we should not be able to lease a stream because the connection is draining" + ) // a server can potentially send more than one connection go away and we should not crash XCTAssertTrue(connections.goAwayReceived(conn1ID)?.eventLoop === el1) @@ -445,7 +469,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn1ID = connections.createNewConnection(on: el1) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 1) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 1 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 1) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 1) @@ -454,7 +481,8 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { XCTAssertNil(connections.leaseStream(onRequired: el1), "all streams are in use") - guard let (_, newSettingsContext1) = connections.newHTTP2MaxConcurrentStreamsReceived(conn1ID, newMaxStreams: 2) else { + guard let (_, newSettingsContext1) = connections.newHTTP2MaxConcurrentStreamsReceived(conn1ID, newMaxStreams: 2) + else { return XCTFail("Expected to get a new settings context") } XCTAssertEqual(newSettingsContext1.availableStreams, 1) @@ -467,7 +495,8 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { XCTAssertEqual(leasedConn2, conn1) XCTAssertEqual(leaseContext2.wasIdle, false) - guard let (_, newSettingsContext2) = connections.newHTTP2MaxConcurrentStreamsReceived(conn1ID, newMaxStreams: 1) else { + guard let (_, newSettingsContext2) = connections.newHTTP2MaxConcurrentStreamsReceived(conn1ID, newMaxStreams: 1) + else { return XCTFail("Expected to get a new settings context") } XCTAssertEqual(newSettingsContext2.availableStreams, 0) @@ -500,7 +529,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn1ID = connections.createNewConnection(on: el1) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 1) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 1 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 1) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 1) @@ -535,7 +567,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { let conn1ID = connections.createNewConnection(on: el1) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 1) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 1 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 1) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 1) XCTAssertEqual(leasedConn1, conn1) @@ -556,9 +591,11 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { starting: [(conn1ID, el1)], backingOff: [(conn2ID, el2)] ) - XCTAssertTrue(connections.createConnectionsAfterMigrationIfNeeded( - requiredEventLoopsOfPendingRequests: [el1, el2] - ).isEmpty) + XCTAssertTrue( + connections.createConnectionsAfterMigrationIfNeeded( + requiredEventLoopsOfPendingRequests: [el1, el2] + ).isEmpty + ) XCTAssertEqual( connections.stats, @@ -574,7 +611,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { ) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 100) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 100) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 2) @@ -615,7 +655,10 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { ) let conn1: HTTPConnectionPool.Connection = .__testOnly_connection(id: conn1ID, eventLoop: el1) - let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished(conn1, maxConcurrentStreams: 100) + let (conn1Index, conn1CreatedContext) = connections.newHTTP2ConnectionEstablished( + conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(conn1CreatedContext.availableStreams, 100) let (leasedConn1, leasdConnContext1) = connections.leaseStreams(at: conn1Index, count: 2) @@ -714,9 +757,12 @@ class HTTPConnectionPool_HTTP2ConnectionsTests: XCTestCase { backingOff: [(conn3ID, el3)] ) - XCTAssertTrue(connections.createConnectionsAfterMigrationIfNeeded( - requiredEventLoopsOfPendingRequests: [el1, el2, el3] - ).isEmpty, "we still have an active connection for el1 and should not create a new one") + XCTAssertTrue( + connections.createConnectionsAfterMigrationIfNeeded( + requiredEventLoopsOfPendingRequests: [el1, el2, el3] + ).isEmpty, + "we still have an active connection for el1 and should not create a new one" + ) guard let (leasedConn, _) = connections.leaseStream(onRequired: el1) else { return XCTFail("could not lease stream on el1") diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift index 046040266..e64fd5e71 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+HTTP2StateMachineTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + private typealias Action = HTTPConnectionPool.StateMachine.Action private typealias ConnectionAction = HTTPConnectionPool.StateMachine.ConnectionAction private typealias RequestAction = HTTPConnectionPool.StateMachine.RequestAction @@ -127,14 +128,17 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { /// shutdown should only close one connection let shutdownAction = state.shutdown() XCTAssertEqual(shutdownAction.request, .none) - XCTAssertEqual(shutdownAction.connection, .cleanupConnections( - .init( - close: [conn], - cancel: [], - connectBackoff: [] - ), - isShutdown: .yes(unclean: false) - )) + XCTAssertEqual( + shutdownAction.connection, + .cleanupConnections( + .init( + close: [conn], + cancel: [], + connectBackoff: [] + ), + isShutdown: .yes(unclean: false) + ) + ) } func testConnectionFailureBackoff() { @@ -158,9 +162,12 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { return XCTFail("Unexpected connection action: \(action.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux - let failedConnect1 = state.failedToCreateNewConnection(HTTPClientError.connectTimeout, connectionID: connectionID) + let failedConnect1 = state.failedToCreateNewConnection( + HTTPClientError.connectTimeout, + connectionID: connectionID + ) XCTAssertEqual(failedConnect1.request, .none) guard case .scheduleBackoffTimer(connectionID, let backoffTimeAmount1, _) = failedConnect1.connection else { return XCTFail("Unexpected connection action: \(failedConnect1.connection)") @@ -173,9 +180,12 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { return XCTFail("Unexpected connection action: \(backoffDoneAction.connection)") } XCTAssertGreaterThan(newConnectionID, connectionID) - XCTAssert(connectionEL === newEventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === newEventLoop) // XCTAssertIdentical not available on Linux - let failedConnect2 = state.failedToCreateNewConnection(HTTPClientError.connectTimeout, connectionID: newConnectionID) + let failedConnect2 = state.failedToCreateNewConnection( + HTTPClientError.connectTimeout, + connectionID: newConnectionID + ) XCTAssertEqual(failedConnect2.request, .none) guard case .scheduleBackoffTimer(newConnectionID, let backoffTimeAmount2, _) = failedConnect2.connection else { return XCTFail("Unexpected connection action: \(failedConnect2.connection)") @@ -188,7 +198,8 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .failRequest(let requestToFail, let requestError, cancelTimeout: false) = failRequest.request else { return XCTFail("Unexpected request action: \(action.request)") } - XCTAssert(requestToFail.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux + // XCTAssertIdentical not available on Linux + XCTAssert(requestToFail.__testOnly_wrapped_request() === mockRequest) XCTAssertEqual(requestError as? HTTPClientError, .connectTimeout) XCTAssertEqual(failRequest.connection, .none) @@ -218,7 +229,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { return XCTFail("Unexpected connection action: \(action.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux // 2. initialise shutdown let shutdownAction = state.shutdown() @@ -257,11 +268,12 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = action.connection else { return XCTFail("Unexpected connection action: \(action.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux let failedConnectAction = state.failedToCreateNewConnection(SomeError(), connectionID: connectionID) XCTAssertEqual(failedConnectAction.connection, .none) - guard case .failRequestsAndCancelTimeouts(let requestsToFail, let requestError) = failedConnectAction.request else { + guard case .failRequestsAndCancelTimeouts(let requestsToFail, let requestError) = failedConnectAction.request + else { return XCTFail("Unexpected request action: \(action.request)") } XCTAssertEqualTypeAndValue(requestError, SomeError()) @@ -289,7 +301,7 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = executeAction.connection else { return XCTFail("Unexpected connection action: \(executeAction.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux // 2. cancel request let cancelAction = state.cancelRequest(request.id) @@ -329,15 +341,18 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .createConnection(let connectionID, on: let connectionEL) = executeAction.connection else { return XCTFail("Unexpected connection action: \(executeAction.connection)") } - XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux + XCTAssert(connectionEL === mockRequest.eventLoop) // XCTAssertIdentical not available on Linux // 2. connection succeeds - let connection: HTTPConnectionPool.Connection = .__testOnly_connection(id: connectionID, eventLoop: connectionEL) + let connection: HTTPConnectionPool.Connection = .__testOnly_connection( + id: connectionID, + eventLoop: connectionEL + ) let connectedAction = state.newHTTP2ConnectionEstablished(connection, maxConcurrentStreams: 100) guard case .executeRequestsAndCancelTimeouts([request], connection) = connectedAction.request else { return XCTFail("Unexpected request action: \(connectedAction.request)") } - XCTAssert(request.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux + XCTAssert(request.__testOnly_wrapped_request() === mockRequest) // XCTAssertIdentical not available on Linux XCTAssertEqual(connectedAction.connection, .none) // 3. shutdown @@ -357,7 +372,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let finalRequest = HTTPConnectionPool.Request(finalMockRequest) let failAction = state.executeRequest(finalRequest) XCTAssertEqual(failAction.connection, .none) - XCTAssertEqual(failAction.request, .failRequest(finalRequest, HTTPClientError.alreadyShutdown, cancelTimeout: false)) + XCTAssertEqual( + failAction.request, + .failRequest(finalRequest, HTTPClientError.alreadyShutdown, cancelTimeout: false) + ) // 5. close open connection let closeAction = state.http2ConnectionClosed(connectionID) @@ -416,7 +434,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { newHTTP2Connection: conn2, maxConcurrentStreams: 100 ) - XCTAssertEqual(http2ConnectAction.connection, .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil)) + XCTAssertEqual( + http2ConnectAction.connection, + .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil) + ) guard case .executeRequestsAndCancelTimeouts([request2], conn2) = http2ConnectAction.request else { return XCTFail("Unexpected request action \(http2ConnectAction.request)") } @@ -428,11 +449,17 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { let shutdownAction = http2State.shutdown() XCTAssertEqual(shutdownAction.request, .none) - XCTAssertEqual(shutdownAction.connection, .cleanupConnections(.init( - close: [conn2], - cancel: [], - connectBackoff: [] - ), isShutdown: .no)) + XCTAssertEqual( + shutdownAction.connection, + .cleanupConnections( + .init( + close: [conn2], + cancel: [], + connectBackoff: [] + ), + isShutdown: .no + ) + ) let releaseAction = http2State.http1ConnectionReleased(conn1ID) XCTAssertEqual(releaseAction.request, .none) @@ -445,7 +472,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -455,14 +486,22 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) - let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) + let connectAction = state.migrateFromHTTP1( + http1Connections: http1Conns, + requests: .init(), + newHTTP2Connection: conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(connectAction.request, .none) - XCTAssertEqual(connectAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) // execute request on idle connection let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) @@ -495,7 +534,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -505,13 +548,21 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) - let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) + let connectAction = state.migrateFromHTTP1( + http1Connections: http1Conns, + requests: .init(), + newHTTP2Connection: conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(connectAction.request, .none) - XCTAssertEqual(connectAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) // let the connection timeout let timeoutAction = state.connectionIdleTimeout(conn1ID) @@ -528,7 +579,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -537,13 +592,21 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { maximumConnectionUses: nil ) let conn1 = HTTPConnectionPool.Connection.__testOnly_connection(id: conn1ID, eventLoop: el1) - let connectAction = state.migrateFromHTTP1(http1Connections: http1Conns, requests: .init(), newHTTP2Connection: conn1, maxConcurrentStreams: 100) + let connectAction = state.migrateFromHTTP1( + http1Connections: http1Conns, + requests: .init(), + newHTTP2Connection: conn1, + maxConcurrentStreams: 100 + ) XCTAssertEqual(connectAction.request, .none) - XCTAssertEqual(connectAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) // create new http2 connection let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) @@ -568,7 +631,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -586,11 +653,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { maxConcurrentStreams: 100 ) XCTAssertEqual(connectAction.request, .none) - XCTAssertEqual(connectAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) let goAwayAction = state.http2ConnectionGoAwayReceived(conn1ID) XCTAssertEqual(goAwayAction.request, .none) @@ -603,7 +673,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -620,11 +694,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { maxConcurrentStreams: 100 ) XCTAssertEqual(connectAction.request, .none) - XCTAssertEqual(connectAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) // execute request on idle connection let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) @@ -649,7 +726,11 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // establish one idle http2 connection let idGenerator = HTTPConnectionPool.Connection.ID.Generator() - var http1Conns = HTTPConnectionPool.HTTP1Connections(maximumConcurrentConnections: 8, generator: idGenerator, maximumConnectionUses: nil) + var http1Conns = HTTPConnectionPool.HTTP1Connections( + maximumConcurrentConnections: 8, + generator: idGenerator, + maximumConnectionUses: nil + ) let conn1ID = http1Conns.createNewConnection(on: el1) var state = HTTPConnectionPool.HTTP2StateMachine( idGenerator: idGenerator, @@ -666,11 +747,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { maxConcurrentStreams: 1 ) XCTAssertEqual(connectAction1.request, .none) - XCTAssertEqual(connectAction1.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: (conn1ID, el1) - )) + XCTAssertEqual( + connectAction1.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: (conn1ID, el1) + ) + ) // execute request let mockRequest1 = MockHTTPScheduableRequest(eventLoop: el1) @@ -770,11 +854,14 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.execute(request.__testOnly_wrapped_request(), on: conn1)) } - XCTAssertEqual(migrationAction.connection, .migration( - createConnections: [], - closeConnections: [], - scheduleTimeout: nil - )) + XCTAssertEqual( + migrationAction.connection, + .migration( + createConnections: [], + closeConnections: [], + scheduleTimeout: nil + ) + ) /// remaining connections should be closed immediately without executing any request for connID in connectionIDs.dropFirst() { @@ -933,7 +1020,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .executeRequestsAndCancelTimeouts(let requests, let conn) = migrationAction.request else { return XCTFail("unexpected request action \(migrationAction.request)") } - XCTAssertEqual(migrationAction.connection, .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil)) + XCTAssertEqual( + migrationAction.connection, + .migration(createConnections: [], closeConnections: [], scheduleTimeout: nil) + ) XCTAssertEqual(conn, http2Conn) XCTAssertEqual(requests.count, 10) @@ -1030,14 +1120,20 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } // a request with new required event loop should create a new connection - let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) + let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest( + eventLoop: el2, + requiresEventLoopForChannel: true + ) let requestWithRequiredEventLoop = HTTPConnectionPool.Request(mockRequestWithRequiredEventLoop) let action2 = state.executeRequest(requestWithRequiredEventLoop) guard case .createConnection(let http1ConnId, let http1EventLoop) = action2.connection else { return XCTFail("Unexpected connection action \(action2.connection)") } XCTAssertTrue(http1EventLoop === el2) - XCTAssertEqual(action2.request, .scheduleRequestTimeout(for: requestWithRequiredEventLoop, on: mockRequestWithRequiredEventLoop.eventLoop)) + XCTAssertEqual( + action2.request, + .scheduleRequestTimeout(for: requestWithRequiredEventLoop, on: mockRequestWithRequiredEventLoop.eventLoop) + ) XCTAssertNoThrow(try connections.createConnection(http1ConnId, on: el2)) XCTAssertNoThrow(try queuer.queue(mockRequestWithRequiredEventLoop, id: requestWithRequiredEventLoop.id)) @@ -1048,7 +1144,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { guard case .executeRequest(let request2, http1Conn, cancelTimeout: true) = migrationAction2.request else { return XCTFail("unexpected request action \(migrationAction2.request)") } - guard case .migration(let createConnections, closeConnections: [], scheduleTimeout: nil) = migrationAction2.connection else { + guard + case .migration(let createConnections, closeConnections: [], scheduleTimeout: nil) = migrationAction2 + .connection + else { return XCTFail("unexpected connection action \(migrationAction2.connection)") } XCTAssertEqual(createConnections.map { $0.1.id }, [el2.id]) @@ -1102,14 +1201,20 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { } // a request with new required event loop should create a new connection - let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest(eventLoop: el2, requiresEventLoopForChannel: true) + let mockRequestWithRequiredEventLoop = MockHTTPScheduableRequest( + eventLoop: el2, + requiresEventLoopForChannel: true + ) let requestWithRequiredEventLoop = HTTPConnectionPool.Request(mockRequestWithRequiredEventLoop) let action2 = state.executeRequest(requestWithRequiredEventLoop) guard case .createConnection(let http1ConnId, let http1EventLoop) = action2.connection else { return XCTFail("Unexpected connection action \(action2.connection)") } XCTAssertTrue(http1EventLoop === el2) - XCTAssertEqual(action2.request, .scheduleRequestTimeout(for: requestWithRequiredEventLoop, on: mockRequestWithRequiredEventLoop.eventLoop)) + XCTAssertEqual( + action2.request, + .scheduleRequestTimeout(for: requestWithRequiredEventLoop, on: mockRequestWithRequiredEventLoop.eventLoop) + ) XCTAssertNoThrow(try connections.createConnection(http1ConnId, on: el2)) XCTAssertNoThrow(try queuer.queue(mockRequestWithRequiredEventLoop, id: requestWithRequiredEventLoop.id)) @@ -1131,7 +1236,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.succeedConnectionCreationHTTP1(http1ConnId)) let migrationAction2 = state.newHTTP1ConnectionCreated(http1Conn) XCTAssertEqual(migrationAction2.request, .none) - XCTAssertEqual(migrationAction2.connection, .migration(createConnections: [], closeConnections: [http1Conn], scheduleTimeout: nil)) + XCTAssertEqual( + migrationAction2.connection, + .migration(createConnections: [], closeConnections: [http1Conn], scheduleTimeout: nil) + ) // in http/1 state, we should close idle http2 connections XCTAssertNoThrow(try connections.finishExecution(http2Conn.id)) @@ -1234,10 +1342,16 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { case 0: XCTAssertEqual(executeAction.connection, .cancelTimeoutTimer(generalPurposeConnection.id)) XCTAssertNoThrow(try connections.activateConnection(generalPurposeConnection.id)) - XCTAssertEqual(executeAction.request, .executeRequest(request, generalPurposeConnection, cancelTimeout: false)) + XCTAssertEqual( + executeAction.request, + .executeRequest(request, generalPurposeConnection, cancelTimeout: false) + ) XCTAssertNoThrow(try connections.execute(mockRequest, on: generalPurposeConnection)) case 1..<100: - XCTAssertEqual(executeAction.request, .executeRequest(request, generalPurposeConnection, cancelTimeout: false)) + XCTAssertEqual( + executeAction.request, + .executeRequest(request, generalPurposeConnection, cancelTimeout: false) + ) XCTAssertEqual(executeAction.connection, .none) XCTAssertNoThrow(try connections.execute(mockRequest, on: generalPurposeConnection)) case 100..<1000: @@ -1255,7 +1369,8 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.finishExecution(generalPurposeConnection.id)) let finishAction = state.http2ConnectionStreamClosed(generalPurposeConnection.id) XCTAssertEqual(finishAction.connection, .none) - guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request else { + guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request + else { return XCTFail("Unexpected request action: \(finishAction.request)") } guard requests.count == 1, let request = requests.first else { @@ -1270,11 +1385,23 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // Next the server allows for more concurrent streams let newMaxStreams = 200 - XCTAssertNoThrow(try connections.newHTTP2ConnectionSettingsReceived(generalPurposeConnection.id, maxConcurrentStreams: newMaxStreams)) - let newMaxStreamsAction = state.newHTTP2MaxConcurrentStreamsReceived(generalPurposeConnection.id, newMaxStreams: newMaxStreams) + XCTAssertNoThrow( + try connections.newHTTP2ConnectionSettingsReceived( + generalPurposeConnection.id, + maxConcurrentStreams: newMaxStreams + ) + ) + let newMaxStreamsAction = state.newHTTP2MaxConcurrentStreamsReceived( + generalPurposeConnection.id, + newMaxStreams: newMaxStreams + ) XCTAssertEqual(newMaxStreamsAction.connection, .none) - guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = newMaxStreamsAction.request else { - return XCTFail("Unexpected request action after new max concurrent stream setting: \(newMaxStreamsAction.request)") + guard + case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = newMaxStreamsAction.request + else { + return XCTFail( + "Unexpected request action after new max concurrent stream setting: \(newMaxStreamsAction.request)" + ) } XCTAssertEqual(requests.count, 100, "Expected to execute 100 more requests") for request in requests { @@ -1291,7 +1418,8 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.finishExecution(generalPurposeConnection.id)) let finishAction = state.http2ConnectionStreamClosed(generalPurposeConnection.id) XCTAssertEqual(finishAction.connection, .none) - guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request else { + guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request + else { return XCTFail("Unexpected request action: \(finishAction.request)") } guard requests.count == 1, let request = requests.first else { @@ -1304,8 +1432,16 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { // Next the server allows for fewer concurrent streams let fewerMaxStreams = 50 - XCTAssertNoThrow(try connections.newHTTP2ConnectionSettingsReceived(generalPurposeConnection.id, maxConcurrentStreams: fewerMaxStreams)) - let fewerMaxStreamsAction = state.newHTTP2MaxConcurrentStreamsReceived(generalPurposeConnection.id, newMaxStreams: fewerMaxStreams) + XCTAssertNoThrow( + try connections.newHTTP2ConnectionSettingsReceived( + generalPurposeConnection.id, + maxConcurrentStreams: fewerMaxStreams + ) + ) + let fewerMaxStreamsAction = state.newHTTP2MaxConcurrentStreamsReceived( + generalPurposeConnection.id, + newMaxStreams: fewerMaxStreams + ) XCTAssertEqual(fewerMaxStreamsAction.connection, .none) XCTAssertEqual(fewerMaxStreamsAction.request, .none) @@ -1323,7 +1459,8 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { XCTAssertNoThrow(try connections.finishExecution(generalPurposeConnection.id)) let finishAction = state.http2ConnectionStreamClosed(generalPurposeConnection.id) XCTAssertEqual(finishAction.connection, .none) - guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request else { + guard case .executeRequestsAndCancelTimeouts(let requests, generalPurposeConnection) = finishAction.request + else { return XCTFail("Unexpected request action: \(finishAction.request)") } guard requests.count == 1, let request = requests.first else { @@ -1343,7 +1480,10 @@ class HTTPConnectionPool_HTTP2StateMachineTests: XCTestCase { switch remaining { case 1: timeoutTimerScheduled = true - XCTAssertEqual(finishAction.connection, .scheduleTimeoutTimer(generalPurposeConnection.id, on: generalPurposeConnection.eventLoop)) + XCTAssertEqual( + finishAction.connection, + .scheduleTimeoutTimer(generalPurposeConnection.id, on: generalPurposeConnection.eventLoop) + ) XCTAssertNoThrow(try connections.parkConnection(generalPurposeConnection.id)) case 2...50: XCTAssertEqual(finishAction.connection, .none) @@ -1388,13 +1528,17 @@ func XCTAssertEqualTypeAndValue( file: StaticString = #filePath, line: UInt = #line ) { - XCTAssertNoThrow(try { - let lhs = try lhs() - let rhs = try rhs() - guard let lhsAsRhs = lhs as? Right else { - XCTFail("could not cast \(lhs) of type \(type(of: lhs)) to \(type(of: rhs))", file: file, line: line) - return - } - XCTAssertEqual(lhsAsRhs, rhs, file: file, line: line) - }(), file: file, line: line) + XCTAssertNoThrow( + try { + let lhs = try lhs() + let rhs = try rhs() + guard let lhsAsRhs = lhs as? Right else { + XCTFail("could not cast \(lhs) of type \(type(of: lhs)) to \(type(of: rhs))", file: file, line: line) + return + } + XCTAssertEqual(lhsAsRhs, rhs, file: file, line: line) + }(), + file: file, + line: line + ) } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift index d84e7f442..ef59c9463 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_ManagerTests: XCTestCase { func testManagerHappyPath() { let eventLoopGroup = MultiThreadedEventLoopGroup(numberOfThreads: 4) @@ -49,15 +50,17 @@ class HTTPConnectionPool_ManagerTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -105,15 +108,17 @@ class HTTPConnectionPool_ManagerTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests.swift index f8d6044cd..d792895d3 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+RequestQueueTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOEmbedded @@ -20,6 +19,8 @@ import NIOHTTP1 import NIOSSL import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPool_RequestQueueTests: XCTestCase { func testCountAndIsEmptyWorks() { var queue = HTTPConnectionPool.RequestQueue() diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift index 53bba940c..bd9752d5d 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+StateTestUtils.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Atomics import Dispatch import NIOConcurrencyHelpers import NIOCore import NIOEmbedded +@testable import AsyncHTTPClient + /// An `EventLoopGroup` of `EmbeddedEventLoop`s. final class EmbeddedEventLoopGroup: EventLoopGroup { private let loops: [EmbeddedEventLoop] @@ -34,7 +35,7 @@ final class EmbeddedEventLoopGroup: EventLoopGroup { } internal func makeIterator() -> EventLoopIterator { - return EventLoopIterator(self.loops) + EventLoopIterator(self.loops) } internal func shutdownGracefully(queue: DispatchQueue, _ callback: @escaping (Error?) -> Void) { @@ -56,7 +57,7 @@ final class EmbeddedEventLoopGroup: EventLoopGroup { extension HTTPConnectionPool.Request: Equatable { public static func == (lhs: Self, rhs: Self) -> Bool { - return lhs.id == rhs.id + lhs.id == rhs.id } } @@ -78,15 +79,24 @@ extension HTTPConnectionPool.StateMachine.ConnectionAction: Equatable { switch (lhs, rhs) { case (.createConnection(let lhsConnID, on: let lhsEL), .createConnection(let rhsConnID, on: let rhsEL)): return lhsConnID == rhsConnID && lhsEL === rhsEL - case (.scheduleBackoffTimer(let lhsConnID, let lhsBackoff, on: let lhsEL), .scheduleBackoffTimer(let rhsConnID, let rhsBackoff, on: let rhsEL)): + case ( + .scheduleBackoffTimer(let lhsConnID, let lhsBackoff, on: let lhsEL), + .scheduleBackoffTimer(let rhsConnID, let rhsBackoff, on: let rhsEL) + ): return lhsConnID == rhsConnID && lhsBackoff == rhsBackoff && lhsEL === rhsEL case (.scheduleTimeoutTimer(let lhsConnID, on: let lhsEL), .scheduleTimeoutTimer(let rhsConnID, on: let rhsEL)): return lhsConnID == rhsConnID && lhsEL === rhsEL case (.cancelTimeoutTimer(let lhsConnID), .cancelTimeoutTimer(let rhsConnID)): return lhsConnID == rhsConnID - case (.closeConnection(let lhsConn, isShutdown: let lhsShut), .closeConnection(let rhsConn, isShutdown: let rhsShut)): + case ( + .closeConnection(let lhsConn, isShutdown: let lhsShut), + .closeConnection(let rhsConn, isShutdown: let rhsShut) + ): return lhsConn == rhsConn && lhsShut == rhsShut - case (.cleanupConnections(let lhsContext, isShutdown: let lhsShut), .cleanupConnections(let rhsContext, isShutdown: let rhsShut)): + case ( + .cleanupConnections(let lhsContext, isShutdown: let lhsShut), + .cleanupConnections(let rhsContext, isShutdown: let rhsShut) + ): return lhsContext == rhsContext && lhsShut == rhsShut case ( .migration( @@ -100,12 +110,13 @@ extension HTTPConnectionPool.StateMachine.ConnectionAction: Equatable { let rhsScheduleTimeout ) ): - return lhsCreateConnections.elementsEqual(rhsCreateConnections, by: { - $0.0 == $1.0 && $0.1 === $1.1 - }) && - lhsCloseConnections == rhsCloseConnections && - lhsScheduleTimeout?.0 == rhsScheduleTimeout?.0 && - lhsScheduleTimeout?.1 === rhsScheduleTimeout?.1 + return lhsCreateConnections.elementsEqual( + rhsCreateConnections, + by: { + $0.0 == $1.0 && $0.1 === $1.1 + } + ) && lhsCloseConnections == rhsCloseConnections && lhsScheduleTimeout?.0 == rhsScheduleTimeout?.0 + && lhsScheduleTimeout?.1 === rhsScheduleTimeout?.1 case (.none, .none): return true default: @@ -117,15 +128,27 @@ extension HTTPConnectionPool.StateMachine.ConnectionAction: Equatable { extension HTTPConnectionPool.StateMachine.RequestAction: Equatable { public static func == (lhs: Self, rhs: Self) -> Bool { switch (lhs, rhs) { - case (.executeRequest(let lhsReq, let lhsConn, let lhsReqID), .executeRequest(let rhsReq, let rhsConn, let rhsReqID)): + case ( + .executeRequest(let lhsReq, let lhsConn, let lhsReqID), + .executeRequest(let rhsReq, let rhsConn, let rhsReqID) + ): return lhsReq == rhsReq && lhsConn == rhsConn && lhsReqID == rhsReqID - case (.executeRequestsAndCancelTimeouts(let lhsReqs, let lhsConn), .executeRequestsAndCancelTimeouts(let rhsReqs, let rhsConn)): + case ( + .executeRequestsAndCancelTimeouts(let lhsReqs, let lhsConn), + .executeRequestsAndCancelTimeouts(let rhsReqs, let rhsConn) + ): return lhsReqs.elementsEqual(rhsReqs, by: { $0 == $1 }) && lhsConn == rhsConn - case (.failRequest(let lhsReq, _, cancelTimeout: let lhsReqID), .failRequest(let rhsReq, _, cancelTimeout: let rhsReqID)): + case ( + .failRequest(let lhsReq, _, cancelTimeout: let lhsReqID), + .failRequest(let rhsReq, _, cancelTimeout: let rhsReqID) + ): return lhsReq == rhsReq && lhsReqID == rhsReqID case (.failRequestsAndCancelTimeouts(let lhsReqs, _), .failRequestsAndCancelTimeouts(let rhsReqs, _)): return lhsReqs.elementsEqual(rhsReqs, by: { $0 == $1 }) - case (.scheduleRequestTimeout(for: let lhsReq, on: let lhsEL), .scheduleRequestTimeout(for: let rhsReq, on: let rhsEL)): + case ( + .scheduleRequestTimeout(for: let lhsReq, on: let lhsEL), + .scheduleRequestTimeout(for: let rhsReq, on: let rhsEL) + ): return lhsReq == rhsReq && lhsEL === rhsEL case (.none, .none): return true @@ -146,7 +169,10 @@ extension HTTPConnectionPool.HTTP2StateMachine.EstablishedConnectionAction: Equa switch (lhs, rhs) { case (.scheduleTimeoutTimer(let lhsConnID, on: let lhsEL), .scheduleTimeoutTimer(let rhsConnID, on: let rhsEL)): return lhsConnID == rhsConnID && lhsEL === rhsEL - case (.closeConnection(let lhsConn, isShutdown: let lhsShut), .closeConnection(let rhsConn, isShutdown: let rhsShut)): + case ( + .closeConnection(let lhsConn, isShutdown: let lhsShut), + .closeConnection(let rhsConn, isShutdown: let rhsShut) + ): return lhsConn == rhsConn && lhsShut == rhsShut case (.none, .none): return true diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift index a75cfb63c..a40703456 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPoolTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + class HTTPConnectionPoolTests: XCTestCase { func testOnlyOneConnectionIsUsedForSubSequentRequests() { let httpBin = HTTPBin() @@ -53,15 +54,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -111,15 +114,19 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .init(.testOnly_exact(channelOn: eventLoopGroup.next(), delegateOn: eventLoopGroup.next())), - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .init( + .testOnly_exact(channelOn: eventLoopGroup.next(), delegateOn: eventLoopGroup.next()) + ), + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -170,15 +177,19 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .init(.testOnly_exact(channelOn: eventLoopGroup.next(), delegateOn: eventLoopGroup.next())), - task: .init(eventLoop: eventLoop, logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .init( + .testOnly_exact(channelOn: eventLoopGroup.next(), delegateOn: eventLoopGroup.next()) + ), + task: .init(eventLoop: eventLoop, logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -225,15 +236,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .distantFuture, - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .distantFuture, + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -279,15 +292,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -327,15 +342,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -383,15 +400,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -429,15 +448,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequest: HTTPClient.Request? var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/wait")) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: .init(label: "test")), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } @@ -489,15 +510,17 @@ class HTTPConnectionPoolTests: XCTestCase { var maybeRequestBag: RequestBag? XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: url)) - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: XCTUnwrap(maybeRequest), - eventLoopPreference: .indifferent, - task: .init(eventLoop: eventLoopGroup.next(), logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(5), - requestOptions: .forTests(), - delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: XCTUnwrap(maybeRequest), + eventLoopPreference: .indifferent, + task: .init(eventLoop: eventLoopGroup.next(), logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(5), + requestOptions: .forTests(), + delegate: ResponseAccumulator(request: XCTUnwrap(maybeRequest)) + ) + ) guard let requestBag = maybeRequestBag else { return XCTFail("Expected to get a request") } pool.executeRequest(requestBag) @@ -521,7 +544,10 @@ class HTTPConnectionPoolTests: XCTestCase { var backoff = HTTPConnectionPool.calculateBackoff(failedAttempt: 1) // The value should be 100msยฑ3ms - XCTAssertLessThanOrEqual((backoff - .milliseconds(100)).nanoseconds.magnitude, TimeAmount.milliseconds(3).nanoseconds.magnitude) + XCTAssertLessThanOrEqual( + (backoff - .milliseconds(100)).nanoseconds.magnitude, + TimeAmount.milliseconds(3).nanoseconds.magnitude + ) // Should always increase // We stop when we get within the jitter of 60s, which is 1.8s @@ -537,7 +563,8 @@ class HTTPConnectionPoolTests: XCTestCase { // Ok, now we should be able to do a hundred increments, and always hit 60s, plus or minus 1.8s of jitter. for offset in 0..<100 { XCTAssertLessThanOrEqual( - (HTTPConnectionPool.calculateBackoff(failedAttempt: attempt + offset) - .seconds(60)).nanoseconds.magnitude, + (HTTPConnectionPool.calculateBackoff(failedAttempt: attempt + offset) - .seconds(60)).nanoseconds + .magnitude, TimeAmount.milliseconds(1800).nanoseconds.magnitude ) } diff --git a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift index 92bf42b1d..8fe879745 100644 --- a/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPRequestStateMachineTests.swift @@ -12,22 +12,29 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOHTTP1 import NIOSSL import XCTest +@testable import AsyncHTTPClient + class HTTPRequestStateMachineTests: XCTestCase { func testSimpleGETRequest() { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init([responseBody]))) @@ -36,10 +43,21 @@ class HTTPRequestStateMachineTests: XCTestCase { func testPOSTRequestWithWriterBackpressure() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "4")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "4")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [1])) let part2 = IOData.byteBuffer(ByteBuffer(bytes: [2])) @@ -62,7 +80,10 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.requestStreamFinished(promise: nil), .sendRequestEnd(nil)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init([responseBody]))) @@ -71,14 +92,25 @@ class HTTPRequestStateMachineTests: XCTestCase { func testPOSTContentLengthIsTooLong() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "4")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "4")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) let part1 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) - state.requestStreamPartReceived(part1, promise: nil).assertFailRequest(HTTPClientError.bodyLengthMismatch, .close(nil)) + state.requestStreamPartReceived(part1, promise: nil).assertFailRequest( + HTTPClientError.bodyLengthMismatch, + .close(nil) + ) // if another error happens the new one is ignored XCTAssertEqual(state.errorHappened(HTTPClientError.remoteConnectionClosed), .wait) @@ -86,9 +118,17 @@ class HTTPRequestStateMachineTests: XCTestCase { func testPOSTContentLengthIsTooShort() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "8")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "8")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(8)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) @@ -97,28 +137,51 @@ class HTTPRequestStateMachineTests: XCTestCase { func testRequestBodyStreamIsCancelledIfServerRespondsWith301() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false) + ) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .movedPermanently) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: true)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: true) + ) XCTAssertEqual(state.writabilityChanged(writable: false), .wait) XCTAssertEqual(state.writabilityChanged(writable: true), .wait) - XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") + XCTAssertEqual( + state.requestStreamPartReceived(part, promise: nil), + .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init())) - XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") - - XCTAssertEqual(state.requestStreamFinished(promise: nil), .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") + XCTAssertEqual( + state.requestStreamPartReceived(part, promise: nil), + .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) + + XCTAssertEqual( + state.requestStreamFinished(promise: nil), + .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) } func testStreamPartReceived_whenCancelled() { @@ -126,47 +189,84 @@ class HTTPRequestStateMachineTests: XCTestCase { let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestCancelled(), .failRequest(HTTPClientError.cancelled, .none)) - XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.cancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") + XCTAssertEqual( + state.requestStreamPartReceived(part, promise: nil), + .failSendBodyPart(HTTPClientError.cancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) } func testRequestBodyStreamIsCancelledIfServerRespondsWith301WhileWriteBackpressure() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) - XCTAssertEqual(state.headSent(), .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: true, startIdleTimer: false) + ) let part = IOData.byteBuffer(ByteBuffer(bytes: [0, 1, 2, 3])) XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .sendBodyPart(part, nil)) XCTAssertEqual(state.writabilityChanged(writable: false), .pauseRequestBodyStream) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .movedPermanently) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.writabilityChanged(writable: true), .wait) - XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") + XCTAssertEqual( + state.requestStreamPartReceived(part, promise: nil), + .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.close, .init())) - XCTAssertEqual(state.requestStreamPartReceived(part, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") - - XCTAssertEqual(state.requestStreamFinished(promise: nil), .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), - "Expected to drop all stream data after having received a response head, with status >= 300") + XCTAssertEqual( + state.requestStreamPartReceived(part, promise: nil), + .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) + + XCTAssertEqual( + state.requestStreamFinished(promise: nil), + .failSendStreamFinished(HTTPClientError.requestStreamCancelled, nil), + "Expected to drop all stream data after having received a response head, with status >= 300" + ) } func testRequestBodyStreamIsContinuedIfServerRespondsWith200() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .forwardResponseBodyParts(.init())) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) @@ -175,20 +275,34 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .sendBodyPart(part2, nil)) XCTAssertEqual(state.requestStreamFinished(promise: nil), .succeedRequest(.sendRequestEnd(nil), .init())) - XCTAssertEqual(state.requestStreamPartReceived(part2, promise: nil), .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil)) + XCTAssertEqual( + state.requestStreamPartReceived(part2, promise: nil), + .failSendBodyPart(HTTPClientError.requestStreamCancelled, nil) + ) } func testRequestBodyStreamIsContinuedIfServerSendHeadWithStatus200() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) @@ -201,15 +315,26 @@ class HTTPRequestStateMachineTests: XCTestCase { func testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerRespondedWith200() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .forwardResponseBodyParts(.init())) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) @@ -220,15 +345,26 @@ class HTTPRequestStateMachineTests: XCTestCase { func testRequestIsFailedIfRequestBodySizeIsWrongEvenAfterServerSendHeadWithStatus200() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/", headers: HTTPHeaders([("content-length", "12")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .POST, + uri: "/", + headers: HTTPHeaders([("content-length", "12")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(12)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part0 = IOData.byteBuffer(ByteBuffer(bytes: 0...3)) XCTAssertEqual(state.requestStreamPartReceived(part0, promise: nil), .sendBodyPart(part0, nil)) // response is coming before having send all data let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part1 = IOData.byteBuffer(ByteBuffer(bytes: 4...7)) XCTAssertEqual(state.requestStreamPartReceived(part1, promise: nil), .sendBodyPart(part1, nil)) @@ -245,7 +381,10 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.writabilityChanged(writable: true), .sendRequestHead(requestHead, sendEnd: true)) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init([responseBody]))) @@ -264,10 +403,20 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) - - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) + + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part0 = ByteBuffer(bytes: 0...3) let part1 = ByteBuffer(bytes: 4...7) let part2 = ByteBuffer(bytes: 8...11) @@ -291,10 +440,20 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) - - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) + + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part0 = ByteBuffer(bytes: 0...3) let part1 = ByteBuffer(bytes: 4...7) let part2 = ByteBuffer(bytes: 8...11) @@ -318,10 +477,20 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) - - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) + + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part0 = ByteBuffer(bytes: 0...3) let part1 = ByteBuffer(bytes: 4...7) let part2 = ByteBuffer(bytes: 8...11) @@ -336,7 +505,11 @@ class HTTPRequestStateMachineTests: XCTestCase { XCTAssertEqual(state.read(), .read) XCTAssertEqual(state.channelRead(.body(part2)), .wait) XCTAssertEqual(state.read(), .read, "Calling `read` while we wait for a channelReadComplete doesn't crash") - XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait, "Calling `demandMoreResponseBodyParts` while we wait for a channelReadComplete doesn't crash") + XCTAssertEqual( + state.demandMoreResponseBodyParts(), + .wait, + "Calling `demandMoreResponseBodyParts` while we wait for a channelReadComplete doesn't crash" + ) XCTAssertEqual(state.channelReadComplete(), .forwardResponseBodyParts(.init([part2]))) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.read(), .read) @@ -365,11 +538,17 @@ class HTTPRequestStateMachineTests: XCTestCase { // --- sending request let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) // --- receiving response let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "4"]) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let responseBody = ByteBuffer(bytes: [1, 2, 3, 4]) XCTAssertEqual(state.channelRead(.body(responseBody)), .wait) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init([responseBody]))) @@ -380,27 +559,51 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) } func testRemoteSuddenlyClosesTheConnection() { var state = HTTPRequestStateMachine(isChannelWritable: true) - let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/", headers: .init([("content-length", "4")])) + let requestHead = HTTPRequestHead( + version: .http1_1, + method: .GET, + uri: "/", + headers: .init([("content-length", "4")]) + ) let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(4)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) state.requestCancelled().assertFailRequest(HTTPClientError.cancelled, .close(nil)) - XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(.init(bytes: 1...3)), promise: nil), .failSendBodyPart(HTTPClientError.cancelled, nil)) + XCTAssertEqual( + state.requestStreamPartReceived(.byteBuffer(.init(bytes: 1...3)), promise: nil), + .failSendBodyPart(HTTPClientError.cancelled, nil) + ) } func testReadTimeoutLeadsToFailureWithEverythingAfterBeingIgnored() { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) - - let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: HTTPHeaders([("content-length", "12")])) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) + + let responseHead = HTTPResponseHead( + version: .http1_1, + status: .ok, + headers: HTTPHeaders([("content-length", "12")]) + ) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) let part0 = ByteBuffer(bytes: 0...3) XCTAssertEqual(state.channelRead(.body(part0)), .wait) state.idleReadTimeoutTriggered().assertFailRequest(HTTPClientError.readTimeout, .close(nil)) @@ -414,13 +617,19 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let continueHead = HTTPResponseHead(version: .http1_1, status: .continue) XCTAssertEqual(state.channelRead(.head(continueHead)), .wait) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init())) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -430,10 +639,16 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init())) XCTAssertEqual(state.idleReadTimeoutTriggered(), .wait, "A read timeout that fires to late must be ignored") } @@ -442,10 +657,16 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.channelRead(.end(nil)), .succeedRequest(.none, .init())) XCTAssertEqual(state.requestCancelled(), .wait, "A cancellation that happens to late is ignored") } @@ -454,9 +675,15 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) - - state.errorHappened(HTTPParserError.invalidChunkSize).assertFailRequest(HTTPParserError.invalidChunkSize, .close(nil)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) + + state.errorHappened(HTTPParserError.invalidChunkSize).assertFailRequest( + HTTPParserError.invalidChunkSize, + .close(nil) + ) XCTAssertEqual(state.requestCancelled(), .wait, "A cancellation that happens to late is ignored") } @@ -464,10 +691,16 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_0, status: .internalServerError) - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -480,11 +713,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_0, status: .internalServerError) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -498,13 +737,22 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .POST, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .stream) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: false)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: false) + ) let part1: ByteBuffer = .init(string: "foo") - XCTAssertEqual(state.requestStreamPartReceived(.byteBuffer(part1), promise: nil), .sendBodyPart(.byteBuffer(part1), nil)) + XCTAssertEqual( + state.requestStreamPartReceived(.byteBuffer(part1), promise: nil), + .sendBodyPart(.byteBuffer(part1), nil) + ) let responseHead = HTTPResponseHead(version: .http1_0, status: .ok) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -518,11 +766,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelRead(.body(body)), .wait) state.errorHappened(NIOSSLError.uncleanShutdown).assertFailRequest(NIOSSLError.uncleanShutdown, .close(nil)) @@ -534,7 +788,10 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) XCTAssertEqual(state.errorHappened(NIOSSLError.uncleanShutdown), .wait) state.channelInactive().assertFailRequest(HTTPClientError.remoteConnectionClosed, .none) @@ -545,7 +802,10 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) state.errorHappened(ArbitraryError()).assertFailRequest(ArbitraryError(), .close(nil)) XCTAssertEqual(state.channelInactive(), .wait) @@ -555,17 +815,26 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["content-length": "30"]) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.read(), .read) XCTAssertEqual(state.channelRead(.body(body)), .wait) XCTAssertEqual(state.channelReadComplete(), .forwardResponseBodyParts([body])) XCTAssertEqual(state.errorHappened(NIOSSLError.uncleanShutdown), .wait) - state.errorHappened(HTTPParserError.invalidEOFState).assertFailRequest(HTTPParserError.invalidEOFState, .close(nil)) + state.errorHappened(HTTPParserError.invalidEOFState).assertFailRequest( + HTTPParserError.invalidEOFState, + .close(nil) + ) XCTAssertEqual(state.channelInactive(), .wait) } @@ -573,11 +842,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -594,11 +869,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -615,11 +896,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -635,11 +922,17 @@ class HTTPRequestStateMachineTests: XCTestCase { var state = HTTPRequestStateMachine(isChannelWritable: true) let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) - XCTAssertEqual(state.startRequest(head: requestHead, metadata: metadata), .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.startRequest(head: requestHead, metadata: metadata), + .sendRequestHead(requestHead, sendEnd: true) + ) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["Content-Length": "50"]) let body = ByteBuffer(string: "foo bar") - XCTAssertEqual(state.channelRead(.head(responseHead)), .forwardResponseHead(responseHead, pauseRequestBodyStream: false)) + XCTAssertEqual( + state.channelRead(.head(responseHead)), + .forwardResponseHead(responseHead, pauseRequestBodyStream: false) + ) XCTAssertEqual(state.demandMoreResponseBodyParts(), .wait) XCTAssertEqual(state.channelReadComplete(), .wait) XCTAssertEqual(state.read(), .read) @@ -688,13 +981,19 @@ extension HTTPRequestStateMachine.Action: Equatable { case (.resumeRequestBodyStream, .resumeRequestBodyStream): return true - case (.forwardResponseHead(let lhsHead, let lhsPauseRequestBodyStream), .forwardResponseHead(let rhsHead, let rhsPauseRequestBodyStream)): + case ( + .forwardResponseHead(let lhsHead, let lhsPauseRequestBodyStream), + .forwardResponseHead(let rhsHead, let rhsPauseRequestBodyStream) + ): return lhsHead == rhsHead && lhsPauseRequestBodyStream == rhsPauseRequestBodyStream case (.forwardResponseBodyParts(let lhsData), .forwardResponseBodyParts(let rhsData)): return lhsData == rhsData - case (.succeedRequest(let lhsFinalAction, let lhsFinalBuffer), .succeedRequest(let rhsFinalAction, let rhsFinalBuffer)): + case ( + .succeedRequest(let lhsFinalAction, let lhsFinalBuffer), + .succeedRequest(let rhsFinalAction, let rhsFinalBuffer) + ): return lhsFinalAction == rhsFinalAction && lhsFinalBuffer == rhsFinalBuffer case (.failRequest(_, let lhsFinalAction), .failRequest(_, let rhsFinalAction)): @@ -706,10 +1005,16 @@ extension HTTPRequestStateMachine.Action: Equatable { case (.wait, .wait): return true - case (.failSendBodyPart(let lhsError as HTTPClientError, let lhsPromise), .failSendBodyPart(let rhsError as HTTPClientError, let rhsPromise)): + case ( + .failSendBodyPart(let lhsError as HTTPClientError, let lhsPromise), + .failSendBodyPart(let rhsError as HTTPClientError, let rhsPromise) + ): return lhsError == rhsError && lhsPromise?.futureResult == rhsPromise?.futureResult - case (.failSendStreamFinished(let lhsError as HTTPClientError, let lhsPromise), .failSendStreamFinished(let rhsError as HTTPClientError, let rhsPromise)): + case ( + .failSendStreamFinished(let lhsError as HTTPClientError, let lhsPromise), + .failSendStreamFinished(let rhsError as HTTPClientError, let rhsPromise) + ): return lhsError == rhsError && lhsPromise?.futureResult == rhsPromise?.futureResult default: @@ -719,7 +1024,10 @@ extension HTTPRequestStateMachine.Action: Equatable { } extension HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction: Equatable { - public static func == (lhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, rhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction) -> Bool { + public static func == ( + lhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction, + rhs: HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction + ) -> Bool { switch (lhs, rhs) { case (.close, close): return true @@ -737,7 +1045,10 @@ extension HTTPRequestStateMachine.Action.FinalSuccessfulRequestAction: Equatable } extension HTTPRequestStateMachine.Action.FinalFailedRequestAction: Equatable { - public static func == (lhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction, rhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction) -> Bool { + public static func == ( + lhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction, + rhs: HTTPRequestStateMachine.Action.FinalFailedRequestAction + ) -> Bool { switch (lhs, rhs) { case (.close(let lhsPromise), close(let rhsPromise)): return lhsPromise?.futureResult == rhsPromise?.futureResult @@ -759,7 +1070,11 @@ extension HTTPRequestStateMachine.Action { line: UInt = #line ) where Error: Swift.Error & Equatable { guard case .failRequest(let actualError, let actualFinalStreamAction) = self else { - return XCTFail("expected .failRequest(\(expectedError), \(expectedFinalStreamAction)) but got \(self)", file: file, line: line) + return XCTFail( + "expected .failRequest(\(expectedError), \(expectedFinalStreamAction)) but got \(self)", + file: file, + line: line + ) } if let actualError = actualError as? Error { XCTAssertEqual(actualError, expectedError, file: file, line: line) diff --git a/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift b/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift index e7cfed4d0..e9a0d46dc 100644 --- a/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift +++ b/Tests/AsyncHTTPClientTests/IdleTimeoutNoReuseTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,6 +26,10 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class TestIdleTimeoutNoReuse: XCTestCaseHTTPClientTestsBaseClass { func testIdleTimeoutNoReuse() throws { var req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", method: .GET) diff --git a/Tests/AsyncHTTPClientTests/LRUCacheTests.swift b/Tests/AsyncHTTPClientTests/LRUCacheTests.swift index 6392bcebe..6173c34eb 100644 --- a/Tests/AsyncHTTPClientTests/LRUCacheTests.swift +++ b/Tests/AsyncHTTPClientTests/LRUCacheTests.swift @@ -12,9 +12,10 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import XCTest +@testable import AsyncHTTPClient + class LRUCacheTests: XCTestCase { func testBasicsWork() { var cache = LRUCache(capacity: 1) diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift index 7b4eb19d9..e49c67f19 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockConnectionPool.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOHTTP1 import NIOSSL +@testable import AsyncHTTPClient + /// A mock connection pool (not creating any actual connections) that is used to validate /// connection actions returned by the `HTTPConnectionPool.StateMachine`. struct MockConnectionPool { @@ -554,7 +555,9 @@ extension MockConnectionPool { let request = HTTPConnectionPool.Request(mockRequest) let action = state.executeRequest(request) - guard case .scheduleRequestTimeout(request, on: let waitEL) = action.request, mockRequest.eventLoop === waitEL else { + guard case .scheduleRequestTimeout(request, on: let waitEL) = action.request, + mockRequest.eventLoop === waitEL + else { throw SetupError.expectedRequestToBeAddedToQueue } @@ -621,7 +624,9 @@ extension MockConnectionPool { let request = HTTPConnectionPool.Request(mockRequest) let executeAction = state.executeRequest(request) - guard case .scheduleRequestTimeout(request, on: let waitEL) = executeAction.request, mockRequest.eventLoop === waitEL else { + guard case .scheduleRequestTimeout(request, on: let waitEL) = executeAction.request, + mockRequest.eventLoop === waitEL + else { throw SetupError.expectedRequestToBeAddedToQueue } @@ -634,7 +639,10 @@ extension MockConnectionPool { // 2. the connection becomes available - let newConnection = try connections.succeedConnectionCreationHTTP2(connectionID, maxConcurrentStreams: maxConcurrentStreams) + let newConnection = try connections.succeedConnectionCreationHTTP2( + connectionID, + maxConcurrentStreams: maxConcurrentStreams + ) let action = state.newHTTP2ConnectionCreated(newConnection, maxConcurrentStreams: maxConcurrentStreams) guard case .executeRequestsAndCancelTimeouts([request], newConnection) = action.request else { @@ -674,10 +682,12 @@ final class MockHTTPScheduableRequest: HTTPSchedulableRequest { let preferredEventLoop: EventLoop let requiredEventLoop: EventLoop? - init(eventLoop: EventLoop, - logger: Logger = Logger(label: "mock"), - connectionTimeout: TimeAmount = .seconds(60), - requiresEventLoopForChannel: Bool = false) { + init( + eventLoop: EventLoop, + logger: Logger = Logger(label: "mock"), + connectionTimeout: TimeAmount = .seconds(60), + requiresEventLoopForChannel: Bool = false + ) { self.logger = logger self.connectionDeadline = .now() + connectionTimeout @@ -692,7 +702,7 @@ final class MockHTTPScheduableRequest: HTTPSchedulableRequest { } var eventLoop: EventLoop { - return self.preferredEventLoop + self.preferredEventLoop } // MARK: HTTPSchedulableRequest diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockHTTPExecutableRequest.swift b/Tests/AsyncHTTPClientTests/Mocks/MockHTTPExecutableRequest.swift index aa0dc45eb..021c69731 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockHTTPExecutableRequest.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockHTTPExecutableRequest.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + final class MockHTTPExecutableRequest: HTTPExecutableRequest { enum Event { /// ``Event`` without associated values diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift b/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift index b37ce8fa3..f85c75ce5 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockRequestExecutor.swift @@ -12,10 +12,11 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOConcurrencyHelpers import NIOCore +@testable import AsyncHTTPClient + // This is a MockRequestExecutor, that is synchronized on its EventLoop. final class MockRequestExecutor { enum Errors: Error { @@ -47,7 +48,7 @@ final class MockRequestExecutor { } var requestBodyPartsCount: Int { - return self.blockingQueue.count + self.blockingQueue.count } let eventLoop: EventLoop @@ -82,7 +83,8 @@ final class MockRequestExecutor { request.requestHeadSent() } - func receiveRequestBody(deadline: NIODeadline = .now() + .seconds(5), _ verify: (ByteBuffer) throws -> Void) throws { + func receiveRequestBody(deadline: NIODeadline = .now() + .seconds(5), _ verify: (ByteBuffer) throws -> Void) throws + { enum ReceiveAction { case value(RequestParts) case future(EventLoopFuture) @@ -155,10 +157,11 @@ final class MockRequestExecutor { func receiveResponseDemand(deadline: NIODeadline = .now() + .seconds(5)) throws { let secondsUntilDeath = deadline - NIODeadline.now() - guard self.responseBodyDemandLock.lock( - whenValue: true, - timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000) - ) + guard + self.responseBodyDemandLock.lock( + whenValue: true, + timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000) + ) else { throw TimeoutError() } @@ -168,10 +171,11 @@ final class MockRequestExecutor { func receiveCancellation(deadline: NIODeadline = .now() + .seconds(5)) throws { let secondsUntilDeath = deadline - NIODeadline.now() - guard self.cancellationLock.lock( - whenValue: true, - timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000) - ) + guard + self.cancellationLock.lock( + whenValue: true, + timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000) + ) else { throw TimeoutError() } @@ -265,8 +269,12 @@ extension MockRequestExecutor { internal func popFirst(deadline: NIODeadline) throws -> Element { let secondsUntilDeath = deadline - NIODeadline.now() - guard self.condition.lock(whenValue: true, - timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000)) else { + guard + self.condition.lock( + whenValue: true, + timeoutSeconds: .init(secondsUntilDeath.nanoseconds / 1_000_000_000) + ) + else { throw TimeoutError() } let first = self.buffer.removeFirst() diff --git a/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift b/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift index 520b51875..44e820444 100644 --- a/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift +++ b/Tests/AsyncHTTPClientTests/Mocks/MockRequestQueuer.swift @@ -12,11 +12,12 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOCore import NIOHTTP1 +@testable import AsyncHTTPClient + /// A mock request queue (not creating any timers) that is used to validate /// request actions returned by the `HTTPConnectionPool.StateMachine`. struct MockRequestQueuer { diff --git a/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift index ff9e7f45d..033214ffe 100644 --- a/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/NWWaitingHandlerTests.swift @@ -47,9 +47,14 @@ class NWWaitingHandlerTests: XCTestCase { let waitingEventHandler = NWWaitingHandler(requester: requester, connectionID: connectionID) let embedded = EmbeddedChannel(handlers: [waitingEventHandler]) - embedded.pipeline.fireUserInboundEventTriggered(NIOTSNetworkEvents.WaitingForConnectivity(transientError: .dns(1))) + embedded.pipeline.fireUserInboundEventTriggered( + NIOTSNetworkEvents.WaitingForConnectivity(transientError: .dns(1)) + ) - XCTAssertTrue(requester.waitingForConnectivityCalled, "Expected the handler to invoke .waitingForConnectivity on the requester") + XCTAssertTrue( + requester.waitingForConnectivityCalled, + "Expected the handler to invoke .waitingForConnectivity on the requester" + ) XCTAssertEqual(requester.connectionID, connectionID, "Expected the handler to pass connectionID to requester") XCTAssertEqual(requester.transientError, NWError.dns(1)) } @@ -60,7 +65,10 @@ class NWWaitingHandlerTests: XCTestCase { let embedded = EmbeddedChannel(handlers: [waitingEventHandler]) embedded.pipeline.fireUserInboundEventTriggered(NIOTSNetworkEvents.BetterPathAvailable()) - XCTAssertFalse(requester.waitingForConnectivityCalled, "Should not call .waitingForConnectivity on unrelated events") + XCTAssertFalse( + requester.waitingForConnectivityCalled, + "Should not call .waitingForConnectivity on unrelated events" + ) } func testWaitingHandlerPassesTheEventDownTheContext() { diff --git a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift index 756facb3f..026a45d4c 100644 --- a/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift +++ b/Tests/AsyncHTTPClientTests/NoBytesSentOverBodyLimitTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,6 +26,10 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class NoBytesSentOverBodyLimitTests: XCTestCaseHTTPClientTestsBaseClass { func testNoBytesSentOverBodyLimit() throws { let server = NIOHTTP1TestServer(group: self.serverGroup) diff --git a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift index fd8e45273..35a09c421 100644 --- a/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift +++ b/Tests/AsyncHTTPClientTests/RacePoolIdleConnectionsAndGetTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,10 +26,16 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class RacePoolIdleConnectionsAndGetTests: XCTestCaseHTTPClientTestsBaseClass { func testRacePoolIdleConnectionsAndGet() { - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: .init(connectionPool: .init(idleTimeout: .milliseconds(10)))) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: .init(connectionPool: .init(idleTimeout: .milliseconds(10))) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) } diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index a9b9bd0dd..365c1063c 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOConcurrencyHelpers import NIOCore @@ -21,6 +20,8 @@ import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + final class RequestBagTests: XCTestCase { func testWriteBackpressureWorks() { let embeddedEventLoop = EmbeddedEventLoop() @@ -39,7 +40,8 @@ final class RequestBagTests: XCTestCase { let expectedWrites = bytesToSent / 100 + ((bytesToSent % 100 > 0) ? 1 : 0) let writeDonePromise = embeddedEventLoop.makePromise(of: Void.self) - let requestBody: HTTPClient.Body = .stream(contentLength: Int64(bytesToSent)) { writer -> EventLoopFuture in + let requestBody: HTTPClient.Body = .stream(contentLength: Int64(bytesToSent)) { + writer -> EventLoopFuture in @Sendable func write(donePromise: EventLoopPromise) { let futureWrite: EventLoopFuture? = testState.withLockedValue { state in XCTAssertTrue(state.streamIsAllowedToWrite) @@ -67,20 +69,24 @@ final class RequestBagTests: XCTestCase { } var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "https://swift.org", method: .POST, body: requestBody)) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request(url: "https://swift.org", method: .POST, body: requestBody) + ) guard let request = maybeRequest else { return XCTFail("Expected to have a request") } let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.task.eventLoop === embeddedEventLoop) @@ -101,9 +107,11 @@ final class RequestBagTests: XCTestCase { // after starting the body stream we should have received two writes var receivedBytes = 0 for i in 0..? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.task.eventLoop === embeddedEventLoop) @@ -220,15 +236,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.eventLoop === embeddedEventLoop) @@ -253,15 +271,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.eventLoop === embeddedEventLoop) @@ -292,15 +312,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.eventLoop === embeddedEventLoop) @@ -333,15 +355,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } XCTAssert(bag.eventLoop === embeddedEventLoop) @@ -374,15 +398,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let queuer = MockTaskQueuer() @@ -408,15 +434,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) @@ -436,23 +464,27 @@ final class RequestBagTests: XCTestCase { let logger = Logger(label: "test") var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "https://swift.org", - body: .bytes([1, 2, 3, 4, 5]) - )) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "https://swift.org", + body: .bytes([1, 2, 3, 4, 5]) + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to have a request") } let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) @@ -476,11 +508,13 @@ final class RequestBagTests: XCTestCase { var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "https://swift.org", - method: .POST, - body: .byteBuffer(.init(bytes: [1])) - )) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "https://swift.org", + method: .POST, + body: .byteBuffer(.init(bytes: [1])) + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to have a request") } struct MyError: Error, Equatable {} @@ -506,15 +540,17 @@ final class RequestBagTests: XCTestCase { } let delegate = Delegate(didFinishPromise: embeddedEventLoop.makePromise()) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) @@ -545,40 +581,44 @@ final class RequestBagTests: XCTestCase { let writeSecondPartPromise = embeddedEventLoop.makePromise(of: Void.self) let firstWriteSuccess: NIOLockedValueBox = .init(false) - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request( - url: "https://swift.org", - method: .POST, - headers: ["content-length": "12"], - body: .stream(contentLength: 12) { writer -> EventLoopFuture in - return writer.write(.byteBuffer(.init(bytes: 0...3))).flatMap { _ in - firstWriteSuccess.withLockedValue { $0 = true } - - return writeSecondPartPromise.futureResult - }.flatMap { - return writer.write(.byteBuffer(.init(bytes: 4...7))) - }.always { result in - XCTAssertTrue(firstWriteSuccess.withLockedValue { $0 }) - - guard case .failure(let error) = result else { - return XCTFail("Expected the second write to fail") + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request( + url: "https://swift.org", + method: .POST, + headers: ["content-length": "12"], + body: .stream(contentLength: 12) { writer -> EventLoopFuture in + writer.write(.byteBuffer(.init(bytes: 0...3))).flatMap { _ in + firstWriteSuccess.withLockedValue { $0 = true } + + return writeSecondPartPromise.futureResult + }.flatMap { + writer.write(.byteBuffer(.init(bytes: 4...7))) + }.always { result in + XCTAssertTrue(firstWriteSuccess.withLockedValue { $0 }) + + guard case .failure(let error) = result else { + return XCTFail("Expected the second write to fail") + } + XCTAssertEqual(error as? HTTPClientError, .requestStreamCancelled) } - XCTAssertEqual(error as? HTTPClientError, .requestStreamCancelled) } - } - )) + ) + ) guard let request = maybeRequest else { return XCTFail("Expected to have a request") } let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) @@ -614,15 +654,17 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: nil, - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) @@ -670,36 +712,47 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? var redirectTriggered = false - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: .init( + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( request: request, - redirectState: RedirectState( - .follow(max: 5, allowCycles: false), - initialURL: request.url.absoluteString - )!, - execute: { request, _ in - XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") - XCTAssertFalse(redirectTriggered) - - let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) - task.promise.fail(HTTPClientError.cancelled) - redirectTriggered = true - return task - } - ), - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task( + eventLoop: embeddedEventLoop, + logger: logger + ) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) executor.runRequest(bag) XCTAssertFalse(executor.signalledDemandForResponseBody) - bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"])) + bag.receiveResponseHead( + .init( + version: .http1_1, + status: .permanentRedirect, + headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"] + ) + ) XCTAssertNil(delegate.backpressurePromise) XCTAssertTrue(executor.signalledDemandForResponseBody) executor.resetResponseStreamDemandSignal() @@ -745,36 +798,47 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? var redirectTriggered = false - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: .init( + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( request: request, - redirectState: RedirectState( - .follow(max: 5, allowCycles: false), - initialURL: request.url.absoluteString - )!, - execute: { request, _ in - XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") - XCTAssertFalse(redirectTriggered) - - let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) - task.promise.fail(HTTPClientError.cancelled) - redirectTriggered = true - return task - } - ), - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task( + eventLoop: embeddedEventLoop, + logger: logger + ) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) executor.runRequest(bag) XCTAssertFalse(executor.signalledDemandForResponseBody) - bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(4 * 1024)", "location": "https://swift.org/sswg"])) + bag.receiveResponseHead( + .init( + version: .http1_1, + status: .permanentRedirect, + headers: ["content-length": "\(4 * 1024)", "location": "https://swift.org/sswg"] + ) + ) XCTAssertNil(delegate.backpressurePromise) XCTAssertFalse(executor.signalledDemandForResponseBody) XCTAssertTrue(executor.isCancelled) @@ -794,36 +858,47 @@ final class RequestBagTests: XCTestCase { let delegate = UploadCountingDelegate(eventLoop: embeddedEventLoop) var maybeRequestBag: RequestBag? var redirectTriggered = false - XCTAssertNoThrow(maybeRequestBag = try RequestBag( - request: request, - eventLoopPreference: .delegate(on: embeddedEventLoop), - task: .init(eventLoop: embeddedEventLoop, logger: logger), - redirectHandler: .init( + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( request: request, - redirectState: RedirectState( - .follow(max: 5, allowCycles: false), - initialURL: request.url.absoluteString - )!, - execute: { request, _ in - XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") - XCTAssertFalse(redirectTriggered) - - let task = HTTPClient.Task(eventLoop: embeddedEventLoop, logger: logger) - task.promise.fail(HTTPClientError.cancelled) - redirectTriggered = true - return task - } - ), - connectionDeadline: .now() + .seconds(30), - requestOptions: .forTests(), - delegate: delegate - )) + eventLoopPreference: .delegate(on: embeddedEventLoop), + task: .init(eventLoop: embeddedEventLoop, logger: logger), + redirectHandler: .init( + request: request, + redirectState: RedirectState( + .follow(max: 5, allowCycles: false), + initialURL: request.url.absoluteString + )!, + execute: { request, _ in + XCTAssertEqual(request.url.absoluteString, "https://swift.org/sswg") + XCTAssertFalse(redirectTriggered) + + let task = HTTPClient.Task( + eventLoop: embeddedEventLoop, + logger: logger + ) + task.promise.fail(HTTPClientError.cancelled) + redirectTriggered = true + return task + } + ), + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests(), + delegate: delegate + ) + ) guard let bag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag.") } let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) executor.runRequest(bag) XCTAssertFalse(executor.signalledDemandForResponseBody) - bag.receiveResponseHead(.init(version: .http1_1, status: .permanentRedirect, headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"])) + bag.receiveResponseHead( + .init( + version: .http1_1, + status: .permanentRedirect, + headers: ["content-length": "\(3 * 1024)", "location": "https://swift.org/sswg"] + ) + ) XCTAssertNil(delegate.backpressurePromise) XCTAssertTrue(executor.signalledDemandForResponseBody) executor.resetResponseStreamDemandSignal() @@ -867,7 +942,9 @@ final class RequestBagTests: XCTestCase { do { var maybeRequest: HTTPClient.Request? - XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/", method: .POST)) + XCTAssertNoThrow( + maybeRequest = try HTTPClient.Request(url: "http://localhost:\(httpBin.port)/", method: .POST) + ) guard var request = maybeRequest else { return XCTFail("Expected to have a request here") } let writerPromise = group.any().makePromise(of: HTTPClient.Body.StreamWriter.self) diff --git a/Tests/AsyncHTTPClientTests/RequestValidationTests.swift b/Tests/AsyncHTTPClientTests/RequestValidationTests.swift index c50d3afd1..ea5a6bd66 100644 --- a/Tests/AsyncHTTPClientTests/RequestValidationTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestValidationTests.swift @@ -12,11 +12,12 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + class RequestValidationTests: XCTestCase { func testContentLengthHeaderIsRemovedFromGETIfNoBody() { var headers = HTTPHeaders([("Content-Length", "0")]) @@ -29,13 +30,17 @@ class RequestValidationTests: XCTestCase { func testContentLengthHeaderIsAddedToPOSTAndPUTWithNoBody() { var putHeaders = HTTPHeaders() var putMetadata: RequestFramingMetadata? - XCTAssertNoThrow(putMetadata = try putHeaders.validateAndSetTransportFraming(method: .PUT, bodyLength: .known(0))) + XCTAssertNoThrow( + putMetadata = try putHeaders.validateAndSetTransportFraming(method: .PUT, bodyLength: .known(0)) + ) XCTAssertEqual(putHeaders.first(name: "Content-Length"), "0") XCTAssertEqual(putMetadata?.body, .fixedSize(0)) var postHeaders = HTTPHeaders() var postMetadata: RequestFramingMetadata? - XCTAssertNoThrow(postMetadata = try postHeaders.validateAndSetTransportFraming(method: .POST, bodyLength: .known(0))) + XCTAssertNoThrow( + postMetadata = try postHeaders.validateAndSetTransportFraming(method: .POST, bodyLength: .known(0)) + ) XCTAssertEqual(postHeaders.first(name: "Content-Length"), "0") XCTAssertEqual(postMetadata?.body, .fixedSize(0)) } @@ -90,7 +95,7 @@ class RequestValidationTests: XCTestCase { func testMetadataDetectConnectionClose() { var headers = HTTPHeaders([ - ("Connection", "close"), + ("Connection", "close") ]) var metadata: RequestFramingMetadata? XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: .GET, bodyLength: .known(0))) @@ -114,7 +119,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT, .TRACE] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertTrue(headers["content-length"].isEmpty) XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -123,7 +130,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertEqual(headers["content-length"].first, "0") XCTAssertFalse(headers["transfer-encoding"].contains("chunked")) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -139,7 +148,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1)) + ) XCTAssertEqual(headers["content-length"].first, "1") XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(1)) @@ -149,7 +160,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .unknown)) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .unknown) + ) XCTAssertTrue(headers["content-length"].isEmpty) XCTAssertTrue(headers["transfer-encoding"].contains("chunked")) XCTAssertEqual(metadata?.body, .stream) @@ -159,7 +172,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1)) + ) XCTAssertEqual(headers["content-length"].first, "1") XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(1)) @@ -169,7 +184,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init() var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .unknown)) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .unknown) + ) XCTAssertTrue(headers["content-length"].isEmpty) XCTAssertTrue(headers["transfer-encoding"].contains("chunked")) XCTAssertEqual(metadata?.body, .stream) @@ -184,7 +201,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT, .TRACE] { var headers: HTTPHeaders = .init([("Content-Length", "1")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertTrue(headers["content-length"].isEmpty) XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -193,7 +212,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init([("Content-Length", "1")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertEqual(headers["content-length"].first, "0") XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -208,7 +229,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT] { var headers: HTTPHeaders = .init([("Content-Length", "1")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1)) + ) XCTAssertEqual(headers["content-length"].first, "1") XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(1)) @@ -217,7 +240,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init([("Content-Length", "1")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(1)) + ) XCTAssertEqual(headers["content-length"].first, "1") XCTAssertTrue(headers["transfer-encoding"].isEmpty) XCTAssertEqual(metadata?.body, .fixedSize(1)) @@ -232,7 +257,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.GET, .HEAD, .DELETE, .CONNECT, .TRACE] { var headers: HTTPHeaders = .init([("Transfer-Encoding", "chunked")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertTrue(headers["content-length"].isEmpty) XCTAssertFalse(headers["transfer-encoding"].contains("chunked")) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -241,7 +268,9 @@ class RequestValidationTests: XCTestCase { for method: HTTPMethod in [.POST, .PUT] { var headers: HTTPHeaders = .init([("Transfer-Encoding", "chunked")]) var metadata: RequestFramingMetadata? - XCTAssertNoThrow(metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0))) + XCTAssertNoThrow( + metadata = try headers.validateAndSetTransportFraming(method: method, bodyLength: .known(0)) + ) XCTAssertEqual(headers["content-length"].first, "0") XCTAssertFalse(headers["transfer-encoding"].contains("chunked")) XCTAssertEqual(metadata?.body, .fixedSize(0)) @@ -337,21 +366,27 @@ class RequestValidationTests: XCTestCase { func testTransferEncodingsAreOverwrittenIfBodyLengthIsFixed() { var headers: HTTPHeaders = [ - "Transfer-Encoding": "gzip, chunked", + "Transfer-Encoding": "gzip, chunked" ] XCTAssertNoThrow(try headers.validateAndSetTransportFraming(method: .POST, bodyLength: .known(1))) - XCTAssertEqual(headers, [ - "Content-Length": "1", - ]) + XCTAssertEqual( + headers, + [ + "Content-Length": "1" + ] + ) } func testTransferEncodingsAreOverwrittenIfBodyLengthIsDynamic() { var headers: HTTPHeaders = [ - "Transfer-Encoding": "gzip, chunked", + "Transfer-Encoding": "gzip, chunked" ] XCTAssertNoThrow(try headers.validateAndSetTransportFraming(method: .POST, bodyLength: .unknown)) - XCTAssertEqual(headers, [ - "Transfer-Encoding": "chunked", - ]) + XCTAssertEqual( + headers, + [ + "Transfer-Encoding": "chunked" + ] + ) } } diff --git a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift index 0af5c7243..5fd1d6720 100644 --- a/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift +++ b/Tests/AsyncHTTPClientTests/ResponseDelayGetTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,15 +26,21 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class ResponseDelayGetTests: XCTestCaseHTTPClientTestsBaseClass { func testResponseDelayGet() throws { - let req = try HTTPClient.Request(url: self.defaultHTTPBinURLPrefix + "get", - method: .GET, - headers: ["X-internal-delay": "2000"], - body: nil) + let req = try HTTPClient.Request( + url: self.defaultHTTPBinURLPrefix + "get", + method: .GET, + headers: ["X-internal-delay": "2000"], + body: nil + ) let start = NIODeadline.now() let response = try self.defaultClient.execute(request: req).wait() - XCTAssertGreaterThanOrEqual(.now() - start, .milliseconds(1_900 /* 1.9 seconds */ )) + XCTAssertGreaterThanOrEqual(.now() - start, .milliseconds(1_900)) XCTAssertEqual(response.status, .ok) } } diff --git a/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests.swift b/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests.swift index 066a631a5..1170aa444 100644 --- a/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/SOCKSEventsHandlerTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOSOCKS import XCTest +@testable import AsyncHTTPClient + class SOCKSEventsHandlerTests: XCTestCase { func testHandlerHappyPath() { let socksEventsHandler = SOCKSEventsHandler(deadline: .now() + .seconds(10)) diff --git a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift index d888769b4..6dda7d928 100644 --- a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift @@ -40,7 +40,13 @@ class MockSOCKSServer { self.channel.localAddress!.port! } - init(expectedURL: String, expectedResponse: String, misbehave: Bool = false, file: String = #filePath, line: UInt = #line) throws { + init( + expectedURL: String, + expectedResponse: String, + misbehave: Bool = false, + file: String = #filePath, + line: UInt = #line + ) throws { let elg = MultiThreadedEventLoopGroup(numberOfThreads: 1) let bootstrap: ServerBootstrap if misbehave { @@ -57,7 +63,12 @@ class MockSOCKSServer { return channel.pipeline.addHandlers([ handshakeHandler, SOCKSTestHandler(handshakeHandler: handshakeHandler), - TestHTTPServer(expectedURL: expectedURL, expectedResponse: expectedResponse, file: file, line: line), + TestHTTPServer( + expectedURL: expectedURL, + expectedResponse: expectedResponse, + file: file, + line: line + ), ]) } } @@ -86,17 +97,28 @@ class SOCKSTestHandler: ChannelInboundHandler, RemovableChannelHandler { let message = self.unwrapInboundIn(data) switch message { case .greeting: - context.writeAndFlush(.init( - ServerMessage.selectedAuthenticationMethod(.init(method: .noneRequired))), promise: nil) + context.writeAndFlush( + .init( + ServerMessage.selectedAuthenticationMethod(.init(method: .noneRequired)) + ), + promise: nil + ) case .authenticationData: context.fireErrorCaught(MockSOCKSError(description: "Received authentication data but didn't receive any.")) case .request(let request): - context.writeAndFlush(.init( - ServerMessage.response(.init(reply: .succeeded, boundAddress: request.addressType))), promise: nil) - context.channel.pipeline.addHandlers([ - ByteToMessageHandler(HTTPRequestDecoder()), - HTTPResponseEncoder(), - ], position: .after(self)).whenSuccess { + context.writeAndFlush( + .init( + ServerMessage.response(.init(reply: .succeeded, boundAddress: request.addressType)) + ), + promise: nil + ) + context.channel.pipeline.addHandlers( + [ + ByteToMessageHandler(HTTPRequestDecoder()), + HTTPResponseEncoder(), + ], + position: .after(self) + ).whenSuccess { context.channel.pipeline.removeHandler(self, promise: nil) context.channel.pipeline.removeHandler(self.handshakeHandler, promise: nil) } @@ -134,7 +156,12 @@ class TestHTTPServer: ChannelInboundHandler { break case .end: context.write(self.wrapOutboundOut(.head(.init(version: .http1_1, status: .ok))), promise: nil) - context.write(self.wrapOutboundOut(.body(.byteBuffer(context.channel.allocator.buffer(string: self.expectedResponse)))), promise: nil) + context.write( + self.wrapOutboundOut( + .body(.byteBuffer(context.channel.allocator.buffer(string: self.expectedResponse))) + ), + promise: nil + ) context.writeAndFlush(self.wrapOutboundOut(.end(nil)), promise: nil) } } diff --git a/Tests/AsyncHTTPClientTests/SSLContextCacheTests.swift b/Tests/AsyncHTTPClientTests/SSLContextCacheTests.swift index 438c643d7..c7588cc7d 100644 --- a/Tests/AsyncHTTPClientTests/SSLContextCacheTests.swift +++ b/Tests/AsyncHTTPClientTests/SSLContextCacheTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOPosix import NIOSSL import XCTest +@testable import AsyncHTTPClient + final class SSLContextCacheTests: XCTestCase { func testRequestingSSLContextWorks() { let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) @@ -27,9 +28,13 @@ final class SSLContextCacheTests: XCTestCase { XCTAssertNoThrow(try group.syncShutdownGracefully()) } - XCTAssertNoThrow(try cache.sslContext(tlsConfiguration: .makeClientConfiguration(), - eventLoop: eventLoop, - logger: HTTPClient.loggingDisabled).wait()) + XCTAssertNoThrow( + try cache.sslContext( + tlsConfiguration: .makeClientConfiguration(), + eventLoop: eventLoop, + logger: HTTPClient.loggingDisabled + ).wait() + ) } func testCacheWorks() { @@ -43,12 +48,20 @@ final class SSLContextCacheTests: XCTestCase { var firstContext: NIOSSLContext? var secondContext: NIOSSLContext? - XCTAssertNoThrow(firstContext = try cache.sslContext(tlsConfiguration: .makeClientConfiguration(), - eventLoop: eventLoop, - logger: HTTPClient.loggingDisabled).wait()) - XCTAssertNoThrow(secondContext = try cache.sslContext(tlsConfiguration: .makeClientConfiguration(), - eventLoop: eventLoop, - logger: HTTPClient.loggingDisabled).wait()) + XCTAssertNoThrow( + firstContext = try cache.sslContext( + tlsConfiguration: .makeClientConfiguration(), + eventLoop: eventLoop, + logger: HTTPClient.loggingDisabled + ).wait() + ) + XCTAssertNoThrow( + secondContext = try cache.sslContext( + tlsConfiguration: .makeClientConfiguration(), + eventLoop: eventLoop, + logger: HTTPClient.loggingDisabled + ).wait() + ) XCTAssertNotNil(firstContext) XCTAssertNotNil(secondContext) XCTAssert(firstContext === secondContext) @@ -65,16 +78,24 @@ final class SSLContextCacheTests: XCTestCase { var firstContext: NIOSSLContext? var secondContext: NIOSSLContext? - XCTAssertNoThrow(firstContext = try cache.sslContext(tlsConfiguration: .makeClientConfiguration(), - eventLoop: eventLoop, - logger: HTTPClient.loggingDisabled).wait()) + XCTAssertNoThrow( + firstContext = try cache.sslContext( + tlsConfiguration: .makeClientConfiguration(), + eventLoop: eventLoop, + logger: HTTPClient.loggingDisabled + ).wait() + ) // Second one has a _different_ TLSConfiguration. var testTLSConfig = TLSConfiguration.makeClientConfiguration() testTLSConfig.certificateVerification = .none - XCTAssertNoThrow(secondContext = try cache.sslContext(tlsConfiguration: testTLSConfig, - eventLoop: eventLoop, - logger: HTTPClient.loggingDisabled).wait()) + XCTAssertNoThrow( + secondContext = try cache.sslContext( + tlsConfiguration: testTLSConfig, + eventLoop: eventLoop, + logger: HTTPClient.loggingDisabled + ).wait() + ) XCTAssertNotNil(firstContext) XCTAssertNotNil(secondContext) XCTAssert(firstContext !== secondContext) diff --git a/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift b/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift index 4c5cd1816..587e6c64c 100644 --- a/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift +++ b/Tests/AsyncHTTPClientTests/StressGetHttpsTests.swift @@ -14,9 +14,6 @@ import AsyncHTTPClient import Atomics -#if canImport(Network) -import Network -#endif import Logging import NIOConcurrencyHelpers import NIOCore @@ -29,11 +26,17 @@ import NIOTestUtils import NIOTransportServices import XCTest +#if canImport(Network) +import Network +#endif + final class StressGetHttpsTests: XCTestCaseHTTPClientTestsBaseClass { func testStressGetHttps() throws { let localHTTPBin = HTTPBin(.http1_1(ssl: true)) - let localClient = HTTPClient(eventLoopGroupProvider: .shared(self.clientGroup), - configuration: HTTPClient.Configuration(certificateVerification: .none)) + let localClient = HTTPClient( + eventLoopGroupProvider: .shared(self.clientGroup), + configuration: HTTPClient.Configuration(certificateVerification: .none) + ) defer { XCTAssertNoThrow(try localClient.syncShutdown()) XCTAssertNoThrow(try localHTTPBin.shutdown()) @@ -43,7 +46,11 @@ final class StressGetHttpsTests: XCTestCaseHTTPClientTestsBaseClass { let requestCount = 200 var futureResults = [EventLoopFuture]() for _ in 1...requestCount { - let req = try HTTPClient.Request(url: "https://localhost:\(localHTTPBin.port)/get", method: .GET, headers: ["X-internal-delay": "100"]) + let req = try HTTPClient.Request( + url: "https://localhost:\(localHTTPBin.port)/get", + method: .GET, + headers: ["X-internal-delay": "100"] + ) futureResults.append(localClient.execute(request: req)) } XCTAssertNoThrow(try EventLoopFuture.andAllSucceed(futureResults, on: eventLoop).wait()) diff --git a/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests.swift b/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests.swift index c119c7e50..96cdf68f6 100644 --- a/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/TLSEventsHandlerTests.swift @@ -12,13 +12,14 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOSSL import NIOTLS import XCTest +@testable import AsyncHTTPClient + class TLSEventsHandlerTests: XCTestCase { func testHandlerHappyPath() { let tlsEventsHandler = TLSEventsHandler(deadline: nil) diff --git a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift index a8d3d5a5e..a631e9a93 100644 --- a/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/Transaction+StateMachineTests.swift @@ -12,12 +12,13 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import NIOCore import NIOEmbedded import NIOHTTP1 import XCTest +@testable import AsyncHTTPClient + struct NoOpAsyncSequenceProducerDelegate: NIOAsyncSequenceProducerDelegate { func produceMore() {} func didTerminate() {} @@ -37,7 +38,10 @@ final class Transaction_StateMachineTests: XCTestCase { state.requestWasQueued(queuer) let failAction = state.fail(HTTPClientError.cancelled) - guard case .failResponseHead(_, let error, let scheduler, let rexecutor, let bodyStreamContinuation) = failAction else { + guard + case .failResponseHead(_, let error, let scheduler, let rexecutor, let bodyStreamContinuation) = + failAction + else { return XCTFail("Unexpected fail action: \(failAction)") } XCTAssertEqual(error as? HTTPClientError, .cancelled) @@ -88,7 +92,10 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) let failAction = state.fail(MyError()) - guard case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = failAction else { + guard + case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = + failAction + else { return XCTFail("Unexpected fail action: \(failAction)") } XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) @@ -118,7 +125,10 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) let failAction = state.fail(MyError()) - guard case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = failAction else { + guard + case .failResponseHead(let continuation, let error, nil, nil, bodyStreamContinuation: nil) = + failAction + else { return XCTFail("Unexpected fail action: \(failAction)") } XCTAssertIdentical(scheduler as? MockTaskQueuer, queuer) @@ -203,7 +213,10 @@ final class Transaction_StateMachineTests: XCTestCase { XCTAssertEqual(state.willExecuteRequest(executor), .none) state.requestWasQueued(queuer) let head = HTTPResponseHead(version: .http1_1, status: .ok) - let receiveResponseHeadAction = state.receiveResponseHead(head, delegate: NoOpAsyncSequenceProducerDelegate()) + let receiveResponseHeadAction = state.receiveResponseHead( + head, + delegate: NoOpAsyncSequenceProducerDelegate() + ) guard case .succeedResponseHead(_, let continuation) = receiveResponseHeadAction else { return XCTFail("Unexpected action: \(receiveResponseHeadAction)") } @@ -258,7 +271,7 @@ extension Transaction.StateMachine.NextWriteAction: Equatable { public static func == (lhs: Self, rhs: Self) -> Bool { switch (lhs, rhs) { case (.writeAndWait(let lhsEx), .writeAndWait(let rhsEx)), - (.writeAndContinue(let lhsEx), .writeAndContinue(let rhsEx)): + (.writeAndContinue(let lhsEx), .writeAndContinue(let rhsEx)): if let lhsMock = lhsEx as? MockRequestExecutor, let rhsMock = rhsEx as? MockRequestExecutor { return lhsMock === rhsMock } diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 40f71d010..ff3a51d27 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -12,7 +12,6 @@ // //===----------------------------------------------------------------------===// -@testable import AsyncHTTPClient import Logging import NIOConcurrencyHelpers import NIOCore @@ -21,6 +20,8 @@ import NIOHTTP1 import NIOPosix import XCTest +@testable import AsyncHTTPClient + @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) typealias PreparedRequest = HTTPClientRequest.Prepared @@ -91,11 +92,18 @@ final class TransactionTests: XCTestCase { transaction.deadlineExceeded() struct Executor: HTTPRequestExecutor { - func writeRequestBodyPart(_: NIOCore.IOData, request: AsyncHTTPClient.HTTPExecutableRequest, promise: NIOCore.EventLoopPromise?) { + func writeRequestBodyPart( + _: NIOCore.IOData, + request: AsyncHTTPClient.HTTPExecutableRequest, + promise: NIOCore.EventLoopPromise? + ) { XCTFail() } - func finishRequestBodyStream(_ task: AsyncHTTPClient.HTTPExecutableRequest, promise: NIOCore.EventLoopPromise?) { + func finishRequestBodyStream( + _ task: AsyncHTTPClient.HTTPExecutableRequest, + promise: NIOCore.EventLoopPromise? + ) { XCTFail() } @@ -253,15 +261,17 @@ final class TransactionTests: XCTestCase { XCTAssertFalse(streamWriter.hasDemand, "Did not expect to have demand yet") transaction.resumeRequestBodyStream() - await streamWriter.demand() // wait's for the stream writer to signal demand + await streamWriter.demand() // wait's for the stream writer to signal demand transaction.pauseRequestBodyStream() let part = ByteBuffer(integer: i) streamWriter.write(part) - XCTAssertNoThrow(try executor.receiveRequestBody { - XCTAssertEqual($0, part) - }) + XCTAssertNoThrow( + try executor.receiveRequestBody { + XCTAssertEqual($0, part) + } + ) } transaction.resumeRequestBodyStream() @@ -306,11 +316,13 @@ final class TransactionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - )) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) + ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") } @@ -370,9 +382,11 @@ final class TransactionTests: XCTestCase { let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) executor.runRequest(transaction) executor.resumeRequestBodyStream() - XCTAssertNoThrow(try executor.receiveRequestBody { - XCTAssertEqual($0.getString(at: 0, length: $0.readableBytes), "Hello world!") - }) + XCTAssertNoThrow( + try executor.receiveRequestBody { + XCTAssertEqual($0.getString(at: 0, length: $0.readableBytes), "Hello world!") + } + ) XCTAssertNoThrow(try executor.receiveEndOfStream()) let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["foo": "bar"]) @@ -413,9 +427,11 @@ final class TransactionTests: XCTestCase { await writer.demand() writer.write(.init(string: "Hello world!")) - XCTAssertNoThrow(try executor.receiveRequestBody { - XCTAssertEqual($0.getString(at: 0, length: $0.readableBytes), "Hello world!") - }) + XCTAssertNoThrow( + try executor.receiveRequestBody { + XCTAssertEqual($0.getString(at: 0, length: $0.readableBytes), "Hello world!") + } + ) XCTAssertFalse(executor.isCancelled) struct WriteError: Error, Equatable {} @@ -502,11 +518,13 @@ final class TransactionTests: XCTestCase { let connectionCreator = TestConnectionCreator() let delegate = TestHTTP2ConnectionDelegate() var maybeHTTP2Connection: HTTP2Connection? - XCTAssertNoThrow(maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( - to: httpBin.port, - delegate: delegate, - on: eventLoop - )) + XCTAssertNoThrow( + maybeHTTP2Connection = try connectionCreator.createHTTP2Connection( + to: httpBin.port, + delegate: delegate, + on: eventLoop + ) + ) guard let http2Connection = maybeHTTP2Connection else { return XCTFail("Expected to have an HTTP2 connection here.") } @@ -638,7 +656,8 @@ extension Transaction { ) async -> (Transaction, _Concurrency.Task) { let transactionPromise = Promise() let task = Task { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in + try await withCheckedThrowingContinuation { + (continuation: CheckedContinuation) in let transaction = Transaction( request: request, requestOptions: requestOptions, diff --git a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift index e1d2e4592..6cdcf4f8a 100644 --- a/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift +++ b/Tests/AsyncHTTPClientTests/XCTest+AsyncAwait.swift @@ -11,21 +11,21 @@ // SPDX-License-Identifier: Apache-2.0 // //===----------------------------------------------------------------------===// -/* - * Copyright 2021, gRPC Authors All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ +// +// Copyright 2021, gRPC Authors All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// import XCTest @@ -53,7 +53,7 @@ extension XCTestCase { try await operation() } catch { XCTFail("Error thrown while executing \(function): \(error)", file: file, line: line) - Thread.callStackSymbols.forEach { print($0) } + for symbol in Thread.callStackSymbols { print(symbol) } } expectation.fulfill() } diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 2d1e57def..000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -ARG swift_version=5.7 -ARG ubuntu_version=jammy -ARG base_image=swift:$swift_version-$ubuntu_version -FROM $base_image -# needed to do again after FROM due to docker limitation -ARG swift_version -ARG ubuntu_version - -# set as UTF-8 -RUN apt-get update && apt-get install -y locales locales-all -ENV LC_ALL en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US.UTF-8 - -# dependencies -RUN apt-get update && apt-get install -y wget -RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools libz-dev curl jq # used by integration tests - -# ruby and jazzy for docs generation -RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev build-essential -# jazzy no longer works on xenial as ruby is too old. -RUN if [ "${ubuntu_version}" = "focal" ] ; then echo "gem: --no-document" > ~/.gemrc; fi -RUN if [ "${ubuntu_version}" = "focal" ] ; then gem install jazzy; fi - -# tools -RUN mkdir -p $HOME/.tools -RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile - -# swiftformat (until part of the toolchain) - -ARG swiftformat_version=0.48.8 -RUN git clone --branch $swiftformat_version --depth 1 https://github.com/nicklockwood/SwiftFormat $HOME/.tools/swift-format -RUN cd $HOME/.tools/swift-format && swift build -c release -RUN ln -s $HOME/.tools/swift-format/.build/release/swiftformat $HOME/.tools/swiftformat diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml deleted file mode 100644 index 8dbf21183..000000000 --- a/docker/docker-compose.2204.510.yaml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:22.04-5.10 - build: - args: - ubuntu_version: "jammy" - swift_version: "5.10" - - documentation-check: - image: async-http-client:22.04-5.10 - - test: - image: async-http-client:22.04-5.10 - environment: - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:22.04-5.10 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml deleted file mode 100644 index 89b410ae2..000000000 --- a/docker/docker-compose.2204.58.yaml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:22.04-5.8 - build: - args: - ubuntu_version: "jammy" - swift_version: "5.8" - - documentation-check: - image: async-http-client:22.04-5.8 - - test: - image: async-http-client:22.04-5.8 - environment: - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml deleted file mode 100644 index b125fff39..000000000 --- a/docker/docker-compose.2204.59.yaml +++ /dev/null @@ -1,22 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:22.04-5.9 - build: - args: - ubuntu_version: "jammy" - swift_version: "5.9" - - documentation-check: - image: async-http-client:22.04-5.9 - - test: - image: async-http-client:22.04-5.9 - environment: - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:22.04-5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml deleted file mode 100644 index 8dfa4c921..000000000 --- a/docker/docker-compose.2204.main.yaml +++ /dev/null @@ -1,21 +0,0 @@ -version: "3" - -services: - - runtime-setup: - image: async-http-client:22.04-main - build: - args: - base_image: "swiftlang/swift:nightly-main-jammy" - - documentation-check: - image: async-http-client:22.04-main - - test: - image: async-http-client:22.04-main - environment: - - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error - #- SANITIZER_ARG=--sanitize=thread - - shell: - image: async-http-client:22.04-main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml deleted file mode 100644 index 9ac4a6eea..000000000 --- a/docker/docker-compose.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# this file is not designed to be run directly -# instead, use the docker-compose.. files -# eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.1804.50.yaml run test -version: "3" - -services: - - runtime-setup: - image: async-http-client:default - build: - context: . - dockerfile: Dockerfile - - common: &common - image: async-http-client:default - depends_on: [runtime-setup] - volumes: - - ~/.ssh:/root/.ssh - - ..:/code:z - working_dir: /code - cap_drop: - - CAP_NET_RAW - - CAP_NET_BIND_SERVICE - - soundness: - <<: *common - command: /bin/bash -xcl "./scripts/soundness.sh" - - documentation-check: - <<: *common - command: /bin/bash -xcl "./scripts/check-docs.sh" - - test: - <<: *common - command: /bin/bash -xcl "swift test --parallel -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" - - # util - - shell: - <<: *common - entrypoint: /bin/bash - - docs: - <<: *common - command: /bin/bash -cl "./scripts/generate_docs.sh" diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh deleted file mode 100755 index 61a13a56f..000000000 --- a/scripts/check-docs.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) 2023 Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu - -raw_targets=$(sed -E -n -e 's/^.* - documentation_targets: \[(.*)\].*$/\1/p' .spi.yml) -targets=(${raw_targets//,/ }) - -for target in "${targets[@]}"; do - swift package plugin generate-documentation --target "$target" --warnings-as-errors --analyze --level detailed -done diff --git a/scripts/check_no_api_breakages.sh b/scripts/check_no_api_breakages.sh deleted file mode 100755 index 2d7028617..000000000 --- a/scripts/check_no_api_breakages.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) 2018-2022 Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -##===----------------------------------------------------------------------===## -## -## This source file is part of the SwiftNIO open source project -## -## Copyright (c) 2017-2018 Apple Inc. and the SwiftNIO project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftNIO project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu - -function usage() { - echo >&2 "Usage: $0 REPO-GITHUB-URL NEW-VERSION OLD-VERSIONS..." - echo >&2 - echo >&2 "This script requires a Swift 5.6+ toolchain." - echo >&2 - echo >&2 "Examples:" - echo >&2 - echo >&2 "Check between main and tag 1.9.0 of async-http-client:" - echo >&2 " $0 https://github.com/swift-server/async-http-client main 1.9.0" - echo >&2 - echo >&2 "Check between HEAD and commit 64cf63d7 using the provided toolchain:" - echo >&2 " xcrun --toolchain org.swift.5120190702a $0 ../some-local-repo HEAD 64cf63d7" -} - -if [[ $# -lt 3 ]]; then - usage - exit 1 -fi - -tmpdir=$(mktemp -d /tmp/.check-api_XXXXXX) -repo_url=$1 -new_tag=$2 -shift 2 - -repodir="$tmpdir/repo" -git clone "$repo_url" "$repodir" -git -C "$repodir" fetch -q origin '+refs/pull/*:refs/remotes/origin/pr/*' -cd "$repodir" -git checkout -q "$new_tag" - -for old_tag in "$@"; do - echo "Checking public API breakages from $old_tag to $new_tag" - - swift package diagnose-api-breaking-changes "$old_tag" -done - -echo done diff --git a/scripts/generate_docs.sh b/scripts/generate_docs.sh deleted file mode 100755 index 82da814d3..000000000 --- a/scripts/generate_docs.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -e - -my_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -root_path="$my_path/.." -version=$(git describe --abbrev=0 --tags || echo "main") -modules=(AsyncHTTPClient) - -if [[ "$(uname -s)" == "Linux" ]]; then - # build code if required - if [[ ! -d "$root_path/.build/x86_64-unknown-linux" ]]; then - swift build - fi - # setup source-kitten if required - mkdir -p "$root_path/.build/sourcekitten" - source_kitten_source_path="$root_path/.build/sourcekitten/source" - if [[ ! -d "$source_kitten_source_path" ]]; then - git clone https://github.com/jpsim/SourceKitten.git "$source_kitten_source_path" - fi - source_kitten_path="$source_kitten_source_path/.build/debug" - if [[ ! -d "$source_kitten_path" ]]; then - rm -rf "$source_kitten_source_path/.swift-version" - cd "$source_kitten_source_path" && swift build && cd "$root_path" - fi - # generate - for module in "${modules[@]}"; do - if [[ ! -f "$root_path/.build/sourcekitten/$module.json" ]]; then - "$source_kitten_path/sourcekitten" doc --spm --module-name $module > "$root_path/.build/sourcekitten/$module.json" - fi - done -fi - -[[ -d docs/$version ]] || mkdir -p docs/$version -[[ -d async-http-client.xcodeproj ]] || swift package generate-xcodeproj - -# run jazzy -if ! command -v jazzy > /dev/null; then - gem install jazzy --no-ri --no-rdoc -fi - -jazzy_dir="$root_path/.build/jazzy" -rm -rf "$jazzy_dir" -mkdir -p "$jazzy_dir" - -module_switcher="$jazzy_dir/README.md" -jazzy_args=(--clean - --author 'AsyncHTTPClient team' - --readme "$module_switcher" - --author_url https://github.com/swift-server/async-http-client - --github_url https://github.com/swift-server/async-http-client - --github-file-prefix "https://github.com/swift-server/async-http-client/tree/$version" - --theme fullwidth - --xcodebuild-arguments -scheme,async-http-client-Package) -cat > "$module_switcher" <<"EOF" -# AsyncHTTPClient Docs - -AsyncHTTPClient is a Swift HTTP Client package. - -To get started with AsyncHTTPClient, [`import AsyncHTTPClient`](../AsyncHTTPClient/index.html). The -most important type is [`HTTPClient`](https://swift-server.github.io/async-http-client/docs/current/AsyncHTTPClient/Classes/HTTPClient.html) -which you can use to emit log messages. - -EOF - -tmp=`mktemp -d` -for module in "${modules[@]}"; do - args=("${jazzy_args[@]}" --output "$jazzy_dir/docs/$version/$module" --docset-path "$jazzy_dir/docset/$version/$module" - --module "$module" --module-version $version - --root-url "https://swift-server.github.io/async-http-client/docs/$version/$module/") - if [[ -f "$root_path/.build/sourcekitten/$module.json" ]]; then - args+=(--sourcekitten-sourcefile "$root_path/.build/sourcekitten/$module.json") - fi - jazzy "${args[@]}" -done - -# push to github pages -if [[ $PUSH == true ]]; then - BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD) - GIT_AUTHOR=$(git --no-pager show -s --format='%an <%ae>' HEAD) - git fetch origin +gh-pages:gh-pages - git checkout gh-pages - rm -rf "docs/$version" - rm -rf "docs/current" - cp -r "$jazzy_dir/docs/$version" docs/ - cp -r "docs/$version" docs/current - git add --all docs - echo '' > index.html - git add index.html - touch .nojekyll - git add .nojekyll - changes=$(git diff-index --name-only HEAD) - if [[ -n "$changes" ]]; then - echo -e "changes detected\n$changes" - git commit --author="$GIT_AUTHOR" -m "publish $version docs" - git push origin gh-pages - else - echo "no changes detected" - fi - git checkout -f $BRANCH_NAME -fi diff --git a/scripts/soundness.sh b/scripts/soundness.sh deleted file mode 100755 index 216eab206..000000000 --- a/scripts/soundness.sh +++ /dev/null @@ -1,152 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) 2018-2022 Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" - -function replace_acceptable_years() { - # this needs to replace all acceptable forms with 'YEARS' - sed -e 's/20[12][0-9]-20[12][0-9]/YEARS/' -e 's/20[12][0-9]/YEARS/' -} - -printf "=> Checking for unacceptable language... " -# This greps for unacceptable terminology. The square bracket[s] are so that -# "git grep" doesn't find the lines that greps :). -unacceptable_terms=( - -e blacklis[t] - -e whitelis[t] - -e slav[e] - -e sanit[y] -) -if git grep --color=never -i "${unacceptable_terms[@]}" > /dev/null; then - printf "\033[0;31mUnacceptable language found.\033[0m\n" - git grep -i "${unacceptable_terms[@]}" - exit 1 -fi -printf "\033[0;32mokay.\033[0m\n" - -printf "=> Checking format... " -FIRST_OUT="$(git status --porcelain)" -swiftformat . > /dev/null 2>&1 -SECOND_OUT="$(git status --porcelain)" -if [[ "$FIRST_OUT" != "$SECOND_OUT" ]]; then - printf "\033[0;31mformatting issues!\033[0m\n" - git --no-pager diff - exit 1 -else - printf "\033[0;32mokay.\033[0m\n" -fi - -printf "=> Checking license headers\n" -tmp=$(mktemp /tmp/.async-http-client-soundness_XXXXXX) - -for language in swift-or-c bash dtrace; do - printf " * $language... " - declare -a matching_files - declare -a exceptions - expections=( ) - matching_files=( -name '*' ) - case "$language" in - swift-or-c) - exceptions=( -name c_nio_http_parser.c -o -name c_nio_http_parser.h -o -name cpp_magic.h -o -name Package.swift -o -name CNIOSHA1.h -o -name c_nio_sha1.c -o -name ifaddrs-android.c -o -name ifaddrs-android.h -o -name 'Package@swift*.swift' ) - matching_files=( -name '*.swift' -o -name '*.c' -o -name '*.h' ) - cat > "$tmp" <<"EOF" -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) YEARS Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// -EOF - ;; - bash) - matching_files=( -name '*.sh' ) - cat > "$tmp" <<"EOF" -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) YEARS Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## -EOF - ;; - dtrace) - matching_files=( -name '*.d' ) - cat > "$tmp" <<"EOF" -#!/usr/sbin/dtrace -q -s -/*===----------------------------------------------------------------------===* - * - * This source file is part of the AsyncHTTPClient open source project - * - * Copyright (c) YEARS Apple Inc. and the AsyncHTTPClient project authors - * Licensed under Apache License v2.0 - * - * See LICENSE.txt for license information - * See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors - * - * SPDX-License-Identifier: Apache-2.0 - * - *===----------------------------------------------------------------------===*/ -EOF - ;; - *) - echo >&2 "ERROR: unknown language '$language'" - ;; - esac - - expected_lines=$(cat "$tmp" | wc -l) - expected_sha=$(cat "$tmp" | shasum) - - ( - cd "$here/.." - find . \ - \( \! -path './.build/*' -a \ - \( "${matching_files[@]}" \) -a \ - \( \! \( "${exceptions[@]}" \) \) \) | while read line; do - if [[ "$(cat "$line" | replace_acceptable_years | head -n $expected_lines | shasum)" != "$expected_sha" ]]; then - printf "\033[0;31mmissing headers in file '$line'!\033[0m\n" - diff -u <(cat "$line" | replace_acceptable_years | head -n $expected_lines) "$tmp" - exit 1 - fi - done - printf "\033[0;32mokay.\033[0m\n" - ) -done - -rm "$tmp" - -# This checks for the umbrella NIO module. -printf "=> Checking for imports of umbrella NIO module... " -if git grep --color=never -i "^[ \t]*import \+NIO[ \t]*$" > /dev/null; then - printf "\033[0;31mUmbrella imports found.\033[0m\n" - git grep -i "^[ \t]*import \+NIO[ \t]*$" - exit 1 -fi -printf "\033[0;32mokay.\033[0m\n" From 5ee3708ca5f83e5cc87bd149c86ac8be8a609144 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 14 Nov 2024 16:19:19 +0000 Subject: [PATCH 132/146] remove contributors script (#782) remove contributors script --- scripts/generate_contributors_list.sh | 39 --------------------------- 1 file changed, 39 deletions(-) delete mode 100755 scripts/generate_contributors_list.sh diff --git a/scripts/generate_contributors_list.sh b/scripts/generate_contributors_list.sh deleted file mode 100755 index 00c162638..000000000 --- a/scripts/generate_contributors_list.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -##===----------------------------------------------------------------------===## -## -## This source file is part of the AsyncHTTPClient open source project -## -## Copyright (c) 2018-2019 Apple Inc. and the AsyncHTTPClient project authors -## Licensed under Apache License v2.0 -## -## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -## -## SPDX-License-Identifier: Apache-2.0 -## -##===----------------------------------------------------------------------===## - -set -eu -here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -contributors=$( cd "$here"/.. && git shortlog -es | cut -f2 | sed 's/^/- /' ) - -cat > "$here/../CONTRIBUTORS.txt" <<- EOF - For the purpose of tracking copyright, this is the list of individuals and - organizations who have contributed source code to the AsyncHTTPClient. - - For employees of an organization/company where the copyright of work done - by employees of that company is held by the company itself, only the company - needs to be listed here. - - ## COPYRIGHT HOLDERS - - - Apple Inc. (all contributors with '@apple.com') - - ### Contributors - - $contributors - - **Updating this list** - - Please do not edit this file manually. It is generated using \`./scripts/generate_contributors_list.sh\`. If a name is misspelled or appearing multiple times: add an entry in \`./.mailmap\` -EOF From c1c5f4b6cef92e2b2cd2b6466f17fbdebb93c299 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 15 Nov 2024 10:02:42 +0000 Subject: [PATCH 133/146] add .editorconfig file (#781) --- .editorconfig | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..08891d83f --- /dev/null +++ b/.editorconfig @@ -0,0 +1,8 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true \ No newline at end of file From bdaa3b18358b60e268afe82e8feb16418874f41b Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 15 Nov 2024 10:08:02 +0000 Subject: [PATCH 134/146] remove unused Swift 6 language mode workflow (#783) remove unused Swift 6 language mode workflow --- .github/workflows/pull_request.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 9d7185505..0392cb7c5 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -23,8 +23,3 @@ jobs: cxx-interop: name: Cxx interop uses: apple/swift-nio/.github/workflows/cxx_interop.yml@main - - swift-6-language-mode: - name: Swift 6 Language Mode - uses: apple/swift-nio/.github/workflows/swift_6_language_mode.yml@main - if: false # Disabled for now. From 2119f0d9cc1b334e25447fe43d3693c0e60e6234 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Tue, 26 Nov 2024 14:52:39 +0000 Subject: [PATCH 135/146] fix 784: dont crash on huge in-memory bodies (#785) fixes #784 `writeChunks` had 3 bugs: 1. An actually wrong `UnsafeMutableTransferBox` -> removed that type which should never be created 2. A loooong future chain (instead of one final promise) -> implemented 3. Potentially infinite recursion which lead to the crash in #784) -> fixed too --- .../AsyncAwait/Transaction+StateMachine.swift | 4 +- .../AsyncAwait/Transaction.swift | 74 +++++----- .../HTTP2/HTTP2ClientRequestHandler.swift | 3 + .../ConnectionPool/HTTPConnectionPool.swift | 5 +- .../HTTPExecutableRequest.swift | 2 +- Sources/AsyncHTTPClient/HTTPClient.swift | 11 +- Sources/AsyncHTTPClient/HTTPHandler.swift | 64 +++++++-- Sources/AsyncHTTPClient/UnsafeTransfer.swift | 29 ---- .../HTTP2ClientTests.swift | 79 +++++++++++ .../HTTPClientRequestTests.swift | 2 + .../HTTPClientTestUtils.swift | 129 +++++++++++++++++- .../RequestBagTests.swift | 9 +- 12 files changed, 322 insertions(+), 89 deletions(-) delete mode 100644 Sources/AsyncHTTPClient/UnsafeTransfer.swift diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift index 47b424f04..6cf0dbc07 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction+StateMachine.swift @@ -34,14 +34,14 @@ extension Transaction { case finished(error: Error?) } - fileprivate enum RequestStreamState { + fileprivate enum RequestStreamState: Sendable { case requestHeadSent case producing case paused(continuation: CheckedContinuation?) case finished } - fileprivate enum ResponseStreamState { + fileprivate enum ResponseStreamState: Sendable { // Waiting for response head. Valid transitions to: streamingBody. case waitingForResponseHead // streaming response body. Valid transitions to: finished. diff --git a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift index e420935f1..408ebeeb6 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/Transaction.swift @@ -19,7 +19,11 @@ import NIOHTTP1 import NIOSSL @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) -@usableFromInline final class Transaction: @unchecked Sendable { +@usableFromInline +final class Transaction: + // until NIOLockedValueBox learns `sending` because StateMachine cannot be Sendable + @unchecked Sendable +{ let logger: Logger let request: HTTPClientRequest.Prepared @@ -28,8 +32,7 @@ import NIOSSL let preferredEventLoop: EventLoop let requestOptions: RequestOptions - private let stateLock = NIOLock() - private var state: StateMachine + private let state: NIOLockedValueBox init( request: HTTPClientRequest.Prepared, @@ -44,7 +47,7 @@ import NIOSSL self.logger = logger self.connectionDeadline = connectionDeadline self.preferredEventLoop = preferredEventLoop - self.state = StateMachine(responseContinuation) + self.state = NIOLockedValueBox(StateMachine(responseContinuation)) } func cancel() { @@ -56,8 +59,8 @@ import NIOSSL private func writeOnceAndOneTimeOnly(byteBuffer: ByteBuffer) { // This method is synchronously invoked after sending the request head. For this reason we // can make a number of assumptions, how the state machine will react. - let writeAction = self.stateLock.withLock { - self.state.writeNextRequestPart() + let writeAction = self.state.withLockedValue { state in + state.writeNextRequestPart() } switch writeAction { @@ -99,30 +102,33 @@ import NIOSSL struct BreakTheWriteLoopError: Swift.Error {} + // FIXME: Refactor this to not use `self.state.unsafe`. private func writeRequestBodyPart(_ part: ByteBuffer) async throws { - self.stateLock.lock() - switch self.state.writeNextRequestPart() { + self.state.unsafe.lock() + switch self.state.unsafe.withValueAssumingLockIsAcquired({ state in state.writeNextRequestPart() }) { case .writeAndContinue(let executor): - self.stateLock.unlock() + self.state.unsafe.unlock() executor.writeRequestBodyPart(.byteBuffer(part), request: self, promise: nil) case .writeAndWait(let executor): try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - self.state.waitForRequestBodyDemand(continuation: continuation) - self.stateLock.unlock() + self.state.unsafe.withValueAssumingLockIsAcquired({ state in + state.waitForRequestBodyDemand(continuation: continuation) + }) + self.state.unsafe.unlock() executor.writeRequestBodyPart(.byteBuffer(part), request: self, promise: nil) } case .fail: - self.stateLock.unlock() + self.state.unsafe.unlock() throw BreakTheWriteLoopError() } } private func requestBodyStreamFinished() { - let finishAction = self.stateLock.withLock { - self.state.finishRequestBodyStream() + let finishAction = self.state.withLockedValue { state in + state.finishRequestBodyStream() } switch finishAction { @@ -150,8 +156,8 @@ extension Transaction: HTTPSchedulableRequest { var requiredEventLoop: EventLoop? { nil } func requestWasQueued(_ scheduler: HTTPRequestScheduler) { - self.stateLock.withLock { - self.state.requestWasQueued(scheduler) + self.state.withLockedValue { state in + state.requestWasQueued(scheduler) } } } @@ -165,8 +171,8 @@ extension Transaction: HTTPExecutableRequest { // MARK: Request func willExecuteRequest(_ executor: HTTPRequestExecutor) { - let action = self.stateLock.withLock { - self.state.willExecuteRequest(executor) + let action = self.state.withLockedValue { state in + state.willExecuteRequest(executor) } switch action { @@ -183,8 +189,8 @@ extension Transaction: HTTPExecutableRequest { func requestHeadSent() {} func resumeRequestBodyStream() { - let action = self.stateLock.withLock { - self.state.resumeRequestBodyStream() + let action = self.state.withLockedValue { state in + state.resumeRequestBodyStream() } switch action { @@ -214,16 +220,16 @@ extension Transaction: HTTPExecutableRequest { } func pauseRequestBodyStream() { - self.stateLock.withLock { - self.state.pauseRequestBodyStream() + self.state.withLockedValue { state in + state.pauseRequestBodyStream() } } // MARK: Response func receiveResponseHead(_ head: HTTPResponseHead) { - let action = self.stateLock.withLock { - self.state.receiveResponseHead(head, delegate: self) + let action = self.state.withLockedValue { state in + state.receiveResponseHead(head, delegate: self) } switch action { @@ -243,8 +249,8 @@ extension Transaction: HTTPExecutableRequest { } func receiveResponseBodyParts(_ buffer: CircularBuffer) { - let action = self.stateLock.withLock { - self.state.receiveResponseBodyParts(buffer) + let action = self.state.withLockedValue { state in + state.receiveResponseBodyParts(buffer) } switch action { case .none: @@ -260,8 +266,8 @@ extension Transaction: HTTPExecutableRequest { } func succeedRequest(_ buffer: CircularBuffer?) { - let succeedAction = self.stateLock.withLock { - self.state.succeedRequest(buffer) + let succeedAction = self.state.withLockedValue { state in + state.succeedRequest(buffer) } switch succeedAction { case .finishResponseStream(let source, let finalResponse): @@ -276,8 +282,8 @@ extension Transaction: HTTPExecutableRequest { } func fail(_ error: Error) { - let action = self.stateLock.withLock { - self.state.fail(error) + let action = self.state.withLockedValue { state in + state.fail(error) } self.performFailAction(action) } @@ -304,8 +310,8 @@ extension Transaction: HTTPExecutableRequest { } func deadlineExceeded() { - let action = self.stateLock.withLock { - self.state.deadlineExceeded() + let action = self.state.withLockedValue { state in + state.deadlineExceeded() } self.performDeadlineExceededAction(action) } @@ -329,8 +335,8 @@ extension Transaction: HTTPExecutableRequest { extension Transaction: NIOAsyncSequenceProducerDelegate { @usableFromInline func produceMore() { - let action = self.stateLock.withLock { - self.state.produceMore() + let action = self.state.withLockedValue { state in + state.produceMore() } switch action { case .none: diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 01a248d72..5e105c0d8 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -432,6 +432,9 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { } } +@available(*, unavailable) +extension HTTP2ClientRequestHandler: Sendable {} + extension HTTP2ClientRequestHandler: HTTPRequestExecutor { func writeRequestBodyPart(_ data: IOData, request: HTTPExecutableRequest, promise: EventLoopPromise?) { if self.eventLoop.inEventLoop { diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift index e7f1d8ce5..eebe4d029 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool.swift @@ -21,7 +21,10 @@ protocol HTTPConnectionPoolDelegate { func connectionPoolDidShutdown(_ pool: HTTPConnectionPool, unclean: Bool) } -final class HTTPConnectionPool { +final class HTTPConnectionPool: + // TODO: Refactor to use `NIOLockedValueBox` which will allow this to be checked + @unchecked Sendable +{ private let stateLock = NIOLock() private var _state: StateMachine /// The connection idle timeout timers. Protected by the stateLock diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift index d64ceedd6..e8c07e50f 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPExecutableRequest.swift @@ -132,7 +132,7 @@ import NIOSSL /// /// Use this handle to cancel the request, while it is waiting for a free connection, to execute the request. /// This protocol is only intended to be implemented by the `HTTPConnectionPool`. -protocol HTTPRequestScheduler { +protocol HTTPRequestScheduler: Sendable { /// Informs the task queuer that a request has been cancelled. func cancelRequest(_: HTTPSchedulableRequest) } diff --git a/Sources/AsyncHTTPClient/HTTPClient.swift b/Sources/AsyncHTTPClient/HTTPClient.swift index 130a59f99..f1655c7c5 100644 --- a/Sources/AsyncHTTPClient/HTTPClient.swift +++ b/Sources/AsyncHTTPClient/HTTPClient.swift @@ -222,21 +222,20 @@ public class HTTPClient { """ ) } - let errorStorageLock = NIOLock() - let errorStorage: UnsafeMutableTransferBox = .init(nil) + let errorStorage: NIOLockedValueBox = NIOLockedValueBox(nil) let continuation = DispatchWorkItem {} self.shutdown(requiresCleanClose: requiresCleanClose, queue: DispatchQueue(label: "async-http-client.shutdown")) { error in if let error = error { - errorStorageLock.withLock { - errorStorage.wrappedValue = error + errorStorage.withLockedValue { errorStorage in + errorStorage = error } } continuation.perform() } continuation.wait() - try errorStorageLock.withLock { - if let error = errorStorage.wrappedValue { + try errorStorage.withLockedValue { errorStorage in + if let error = errorStorage { throw error } } diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 7db1ce33c..0f061fbe6 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -47,25 +47,67 @@ extension HTTPClient { } @inlinable - func writeChunks(of bytes: Bytes, maxChunkSize: Int) -> EventLoopFuture - where Bytes.Element == UInt8 { - let iterator = UnsafeMutableTransferBox(bytes.chunks(ofCount: maxChunkSize).makeIterator()) - guard let chunk = iterator.wrappedValue.next() else { + func writeChunks( + of bytes: Bytes, + maxChunkSize: Int + ) -> EventLoopFuture where Bytes.Element == UInt8 { + // `StreamWriter` is has design issues, for example + // - https://github.com/swift-server/async-http-client/issues/194 + // - https://github.com/swift-server/async-http-client/issues/264 + // - We're not told the EventLoop the task runs on and the user is free to return whatever EL they + // want. + // One important consideration then is that we must lock around the iterator because we could be hopping + // between threads. + typealias Iterator = EnumeratedSequence>.Iterator + typealias Chunk = (offset: Int, element: ChunksOfCountCollection.Element) + + func makeIteratorAndFirstChunk( + bytes: Bytes + ) -> ( + iterator: NIOLockedValueBox, + chunk: Chunk + )? { + var iterator = bytes.chunks(ofCount: maxChunkSize).enumerated().makeIterator() + guard let chunk = iterator.next() else { + return nil + } + + return (NIOLockedValueBox(iterator), chunk) + } + + guard let (iterator, chunk) = makeIteratorAndFirstChunk(bytes: bytes) else { return self.write(IOData.byteBuffer(.init())) } @Sendable // can't use closure here as we recursively call ourselves which closures can't do - func writeNextChunk(_ chunk: Bytes.SubSequence) -> EventLoopFuture { - if let nextChunk = iterator.wrappedValue.next() { - return self.write(.byteBuffer(ByteBuffer(bytes: chunk))).flatMap { - writeNextChunk(nextChunk) - } + func writeNextChunk(_ chunk: Chunk, allDone: EventLoopPromise) { + if let nextElement = iterator.withLockedValue({ $0.next() }) { + self.write(.byteBuffer(ByteBuffer(bytes: chunk.element))).map { + let index = nextElement.offset + if (index + 1) % 4 == 0 { + // Let's not stack-overflow if the futures insta-complete which they at least in HTTP/2 + // mode. + // Also, we must frequently return to the EventLoop because we may get the pause signal + // from another thread. If we fail to do that promptly, we may balloon our body chunks + // into memory. + allDone.futureResult.eventLoop.execute { + writeNextChunk(nextElement, allDone: allDone) + } + } else { + writeNextChunk(nextElement, allDone: allDone) + } + }.cascadeFailure(to: allDone) } else { - return self.write(.byteBuffer(ByteBuffer(bytes: chunk))) + self.write(.byteBuffer(ByteBuffer(bytes: chunk.element))).cascade(to: allDone) } } - return writeNextChunk(chunk) + // HACK (again, we're not told the right EventLoop): Let's write 0 bytes to make the user tell us... + return self.write(.byteBuffer(ByteBuffer())).flatMapWithEventLoop { (_, loop) in + let allDone = loop.makePromise(of: Void.self) + writeNextChunk(chunk, allDone: allDone) + return allDone.futureResult + } } } diff --git a/Sources/AsyncHTTPClient/UnsafeTransfer.swift b/Sources/AsyncHTTPClient/UnsafeTransfer.swift deleted file mode 100644 index ea5af56da..000000000 --- a/Sources/AsyncHTTPClient/UnsafeTransfer.swift +++ /dev/null @@ -1,29 +0,0 @@ -//===----------------------------------------------------------------------===// -// -// This source file is part of the AsyncHTTPClient open source project -// -// Copyright (c) 2022 Apple Inc. and the AsyncHTTPClient project authors -// Licensed under Apache License v2.0 -// -// See LICENSE.txt for license information -// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors -// -// SPDX-License-Identifier: Apache-2.0 -// -//===----------------------------------------------------------------------===// - -/// ``UnsafeMutableTransferBox`` can be used to make non-`Sendable` values `Sendable` and mutable. -/// It can be used to capture local mutable values in a `@Sendable` closure and mutate them from within the closure. -/// As the name implies, the usage of this is unsafe because it disables the sendable checking of the compiler and does not add any synchronisation. -@usableFromInline -final class UnsafeMutableTransferBox { - @usableFromInline - var wrappedValue: Wrapped - - @inlinable - init(_ wrappedValue: Wrapped) { - self.wrappedValue = wrappedValue - } -} - -extension UnsafeMutableTransferBox: @unchecked Sendable {} diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 1d6c0c8f8..0580dccad 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -16,6 +16,7 @@ import AsyncHTTPClient // NOT @testable - tests that really need @testable go i import Logging import NIOCore import NIOHTTP1 +import NIOHTTP2 import NIOPosix import NIOSSL import XCTest @@ -463,6 +464,84 @@ class HTTP2ClientTests: XCTestCase { XCTAssertEqual(response?.version, .http2) XCTAssertEqual(response?.body?.readableBytes, 10_000) } + + func testSimplePost() { + let bin = HTTPBin(.http2(compress: false)) + defer { XCTAssertNoThrow(try bin.shutdown()) } + let client = self.makeDefaultHTTPClient() + defer { XCTAssertNoThrow(try client.syncShutdown()) } + var response: HTTPClient.Response? + XCTAssertNoThrow( + response = try client.post( + url: "https://localhost:\(bin.port)/post", + body: .byteBuffer(ByteBuffer(repeating: 0, count: 12345)) + ).wait() + ) + XCTAssertEqual(.ok, response?.status) + XCTAssertEqual(response?.version, .http2) + XCTAssertEqual( + String(buffer: ByteBuffer(repeating: 0, count: 12345)), + try response?.body.map { body in + try JSONDecoder().decode(RequestInfo.self, from: body) + }?.data + ) + } + + func testHugePost() { + // Regression test for https://github.com/swift-server/async-http-client/issues/784 + let group = MultiThreadedEventLoopGroup(numberOfThreads: 2) // This needs to be more than 1! + defer { + XCTAssertNoThrow(try group.syncShutdownGracefully()) + } + var serverH2Settings: HTTP2Settings = HTTP2Settings() + serverH2Settings.append(HTTP2Setting(parameter: .maxFrameSize, value: 16 * 1024 * 1024 - 1)) + serverH2Settings.append(HTTP2Setting(parameter: .initialWindowSize, value: Int(Int32.max))) + let bin = HTTPBin( + .http2(compress: false, settings: serverH2Settings) + ) + defer { XCTAssertNoThrow(try bin.shutdown()) } + var clientConfig = HTTPClient.Configuration() + clientConfig.tlsConfiguration = .clientDefault + clientConfig.tlsConfiguration?.certificateVerification = .none + clientConfig.httpVersion = .automatic + let client = HTTPClient( + eventLoopGroupProvider: .shared(group), + configuration: clientConfig, + backgroundActivityLogger: Logger(label: "HTTPClient", factory: StreamLogHandler.standardOutput(label:)) + ) + defer { XCTAssertNoThrow(try client.syncShutdown()) } + + let loop1 = group.next() + let loop2 = group.next() + precondition(loop1 !== loop2, "bug in test setup, need two distinct loops") + + XCTAssertNoThrow( + try client.execute( + request: .init(url: "https://localhost:\(bin.port)/get"), + eventLoop: .delegateAndChannel(on: loop1) // This will force the channel to live on `loop1`. + ).wait() + ) + var response: HTTPClient.Response? + let byteCount = 1024 * 1024 * 1024 // 1 GiB (unfortunately it has to be that big to trigger the bug) + XCTAssertNoThrow( + response = try client.execute( + request: HTTPClient.Request( + url: "https://localhost:\(bin.port)/post-respond-with-byte-count", + method: .POST, + body: .data(Data(repeating: 0, count: byteCount)) + ), + eventLoop: .delegate(on: loop2) + ).wait() + ) + XCTAssertEqual(.ok, response?.status) + XCTAssertEqual(response?.version, .http2) + XCTAssertEqual( + "\(byteCount)", + try response?.body.map { body in + try JSONDecoder().decode(RequestInfo.self, from: body) + }?.data + ) + } } private final class HeadReceivedCallback: HTTPClientResponseDelegate { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index a92d129a4..08e41d464 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -690,6 +690,7 @@ class HTTPClientRequestTests: XCTestCase { ).collect().wait() let expectedChunks = [ + ByteBuffer(), // We're currently emitting an empty chunk first. ByteBuffer(repeating: UInt8(ascii: "0"), count: bagOfBytesToByteBufferConversionChunkSize), ByteBuffer(repeating: UInt8(ascii: "1"), count: bagOfBytesToByteBufferConversionChunkSize), ByteBuffer(repeating: UInt8(ascii: "2"), count: bagOfBytesToByteBufferConversionChunkSize), @@ -706,6 +707,7 @@ class HTTPClientRequestTests: XCTestCase { ).collect().wait() let expectedChunks = [ + ByteBuffer(), // We're currently emitting an empty chunk first. ByteBuffer(repeating: 0, count: bagOfBytesToByteBufferConversionChunkSize), ByteBuffer(repeating: 1, count: bagOfBytesToByteBufferConversionChunkSize), ByteBuffer(repeating: 2, count: bagOfBytesToByteBufferConversionChunkSize), diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index ad9fcfe98..d9ca45d7b 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -453,6 +453,7 @@ where proxy: Proxy = .none, bindTarget: BindTarget = .localhostIPv4RandomPort, reusePort: Bool = false, + trafficShapingTargetBytesPerSecond: Int? = nil, handlerFactory: @escaping (Int) -> (RequestHandler) ) { self.mode = mode @@ -482,6 +483,13 @@ where .serverChannelInitializer { channel in channel.pipeline.addHandler(self.activeConnCounterHandler) }.childChannelInitializer { channel in + if let trafficShapingTargetBytesPerSecond = trafficShapingTargetBytesPerSecond { + try! channel.pipeline.syncOperations.addHandler( + BasicInboundTrafficShapingHandler( + targetBytesPerSecond: trafficShapingTargetBytesPerSecond + ) + ) + } do { let connectionID = connectionIDAtomic.loadThenWrappingIncrement(ordering: .relaxed) @@ -606,6 +614,7 @@ where let multiplexer = HTTP2StreamMultiplexer( mode: .server, channel: channel, + targetWindowSize: 16 * 1024 * 1024, // 16 MiB inboundStreamInitializer: { channel in do { let sync = channel.pipeline.syncOperations @@ -662,9 +671,16 @@ extension HTTPBin where RequestHandler == HTTPBinHandler { _ mode: Mode = .http1_1(ssl: false, compress: false), proxy: Proxy = .none, bindTarget: BindTarget = .localhostIPv4RandomPort, - reusePort: Bool = false + reusePort: Bool = false, + trafficShapingTargetBytesPerSecond: Int? = nil ) { - self.init(mode, proxy: proxy, bindTarget: bindTarget, reusePort: reusePort) { HTTPBinHandler(connectionID: $0) } + self.init( + mode, + proxy: proxy, + bindTarget: bindTarget, + reusePort: reusePort, + trafficShapingTargetBytesPerSecond: trafficShapingTargetBytesPerSecond + ) { HTTPBinHandler(connectionID: $0) } } } @@ -730,16 +746,31 @@ final class HTTPProxySimulator: ChannelInboundHandler, RemovableChannelHandler { internal struct HTTPResponseBuilder { var head: HTTPResponseHead var body: ByteBuffer? + var requestBodyByteCount: Int + let responseBodyIsRequestBodyByteCount: Bool init( _ version: HTTPVersion = HTTPVersion(major: 1, minor: 1), status: HTTPResponseStatus, - headers: HTTPHeaders = HTTPHeaders() + headers: HTTPHeaders = HTTPHeaders(), + responseBodyIsRequestBodyByteCount: Bool = false ) { self.head = HTTPResponseHead(version: version, status: status, headers: headers) + self.requestBodyByteCount = 0 + self.responseBodyIsRequestBodyByteCount = responseBodyIsRequestBodyByteCount } mutating func add(_ part: ByteBuffer) { + self.requestBodyByteCount += part.readableBytes + guard !self.responseBodyIsRequestBodyByteCount else { + if self.body == nil { + self.body = ByteBuffer() + self.body!.reserveCapacity(100) + } + self.body!.clear() + self.body!.writeString("\(self.requestBodyByteCount)") + return + } if var body = body { var part = part body.writeBuffer(&part) @@ -908,6 +939,13 @@ internal final class HTTPBinHandler: ChannelInboundHandler { } self.resps.append(HTTPResponseBuilder(status: .ok)) return + case "/post-respond-with-byte-count": + if req.method != .POST { + self.resps.append(HTTPResponseBuilder(status: .methodNotAllowed)) + return + } + self.resps.append(HTTPResponseBuilder(status: .ok, responseBodyIsRequestBodyByteCount: true)) + return case "/redirect/302": var headers = self.responseHeaders headers.add(name: "location", value: "/ok") @@ -1548,3 +1586,88 @@ private let key = """ oYQsPj00S3/GA9WDapwe81Wl2A== -----END PRIVATE KEY----- """ + +final class BasicInboundTrafficShapingHandler: ChannelDuplexHandler { + typealias OutboundIn = ByteBuffer + typealias InboundIn = ByteBuffer + typealias OutboundOut = ByteBuffer + + enum ReadState { + case flowingFreely + case pausing + case paused + + mutating func pause() { + switch self { + case .flowingFreely: + self = .pausing + case .pausing, .paused: + () // nothing to do + } + } + + mutating func unpause() -> Bool { + switch self { + case .flowingFreely: + return false // no extra `read` needed + case .pausing: + self = .flowingFreely + return false // no extra `read` needed + case .paused: + self = .flowingFreely + return true // yes, we need an extra read + } + } + + mutating func shouldRead() -> Bool { + switch self { + case .flowingFreely: + return true + case .pausing: + self = .paused + return false + case .paused: + return false + } + } + } + + private let targetBytesPerSecond: Int + private var currentSecondBytesSeen: Int = 0 + private var readState: ReadState = .flowingFreely + + init(targetBytesPerSecond: Int) { + self.targetBytesPerSecond = targetBytesPerSecond + } + + func evaluatePause(context: ChannelHandlerContext) { + if self.currentSecondBytesSeen >= self.targetBytesPerSecond { + self.readState.pause() + } else if self.currentSecondBytesSeen < self.targetBytesPerSecond { + if self.readState.unpause() { + context.read() + } + } + } + + func channelRead(context: ChannelHandlerContext, data: NIOAny) { + let loopBoundContext = NIOLoopBound(context, eventLoop: context.eventLoop) + defer { + context.fireChannelRead(data) + } + let buffer = Self.unwrapInboundIn(data) + let byteCount = buffer.readableBytes + self.currentSecondBytesSeen += byteCount + context.eventLoop.scheduleTask(in: .seconds(1)) { + self.currentSecondBytesSeen -= byteCount + self.evaluatePause(context: loopBoundContext.value) + } + self.evaluatePause(context: context) + } + + func read(context: ChannelHandlerContext) { + if self.readState.shouldRead() { + context.read() + } + } +} diff --git a/Tests/AsyncHTTPClientTests/RequestBagTests.swift b/Tests/AsyncHTTPClientTests/RequestBagTests.swift index 365c1063c..9aa595224 100644 --- a/Tests/AsyncHTTPClientTests/RequestBagTests.swift +++ b/Tests/AsyncHTTPClientTests/RequestBagTests.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Atomics import Logging import NIOConcurrencyHelpers import NIOCore @@ -1052,7 +1053,11 @@ class UploadCountingDelegate: HTTPClientResponseDelegate { } final class MockTaskQueuer: HTTPRequestScheduler { - private(set) var hitCancelCount = 0 + private let _hitCancelCount = ManagedAtomic(0) + + var hitCancelCount: Int { + self._hitCancelCount.load(ordering: .sequentiallyConsistent) + } let onCancelRequest: (@Sendable (HTTPSchedulableRequest) -> Void)? @@ -1061,7 +1066,7 @@ final class MockTaskQueuer: HTTPRequestScheduler { } func cancelRequest(_ request: HTTPSchedulableRequest) { - self.hitCancelCount += 1 + self._hitCancelCount.wrappingIncrement(ordering: .sequentiallyConsistent) self.onCancelRequest?(request) } } From f3a18d0679d521ca7af1a13ea3b5885e4391cf19 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Thu, 28 Nov 2024 12:51:09 +0100 Subject: [PATCH 136/146] Aligning semantic version label check name (#788) --- .github/workflows/pull_request_label.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request_label.yml b/.github/workflows/pull_request_label.yml index 86f199f32..8fd47c13f 100644 --- a/.github/workflows/pull_request_label.yml +++ b/.github/workflows/pull_request_label.yml @@ -6,7 +6,7 @@ on: jobs: semver-label-check: - name: Semantic Version label check + name: Semantic version label check runs-on: ubuntu-latest timeout-minutes: 1 steps: From dbd5c864ad1e9966e5b7078c42d9fd519169d0a5 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Fri, 13 Dec 2024 14:07:57 +0000 Subject: [PATCH 137/146] Enable MemberImportVisibility check on all targets (#794) Enable MemberImportVisibility check on all targets. Use a standard string header and footer to bracket the new block for ease of updating in the future with scripts. --- Package.swift | 11 +++++++++++ .../AsyncHTTPClient/Configuration+BrowserLike.swift | 3 +++ Sources/AsyncHTTPClient/ConnectionPool.swift | 2 ++ .../AsyncAwaitEndToEndTests.swift | 2 ++ Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift | 1 + Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift | 1 + .../AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift | 1 + .../HTTPClientInternalTests.swift | 1 + .../AsyncHTTPClientTests/HTTPClientRequestTests.swift | 1 + .../HTTPClientResponseTests.swift | 1 + Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 1 + Tests/AsyncHTTPClientTests/HTTPClientTests.swift | 1 + .../HTTPConnectionPool+ManagerTests.swift | 1 + Tests/AsyncHTTPClientTests/TransactionTests.swift | 1 + 14 files changed, 28 insertions(+) diff --git a/Package.swift b/Package.swift index bec6c9114..e4cccb6de 100644 --- a/Package.swift +++ b/Package.swift @@ -83,3 +83,14 @@ let package = Package( ), ] ) + +// --- STANDARD CROSS-REPO SETTINGS DO NOT EDIT --- // +for target in package.targets { + if target.type != .plugin { + var settings = target.swiftSettings ?? [] + // https://github.com/swiftlang/swift-evolution/blob/main/proposals/0444-member-import-visibility.md + settings.append(.enableUpcomingFeature("MemberImportVisibility")) + target.swiftSettings = settings + } +} +// --- END: STANDARD CROSS-REPO SETTINGS DO NOT EDIT --- // diff --git a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift index 39aefe975..5a0abdfad 100644 --- a/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift +++ b/Sources/AsyncHTTPClient/Configuration+BrowserLike.swift @@ -11,6 +11,9 @@ // SPDX-License-Identifier: Apache-2.0 // //===----------------------------------------------------------------------===// +import NIOCore +import NIOHTTPCompression +import NIOSSL // swift-format-ignore: DontRepeatTypeInStaticProperties extension HTTPClient.Configuration { diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index 776d1f6df..35f7a21c4 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -12,6 +12,8 @@ // //===----------------------------------------------------------------------===// +import CNIOLinux +import NIOCore import NIOSSL #if canImport(Darwin) diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index f58e07730..4bfa86d14 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -14,6 +14,8 @@ import Logging import NIOCore +import NIOFoundationCompat +import NIOHTTP1 import NIOPosix import NIOSSL import XCTest diff --git a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift index 0580dccad..d6bc2de14 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ClientTests.swift @@ -15,6 +15,7 @@ import AsyncHTTPClient // NOT @testable - tests that really need @testable go into HTTP2ClientInternalTests.swift import Logging import NIOCore +import NIOFoundationCompat import NIOHTTP1 import NIOHTTP2 import NIOPosix diff --git a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift index acf81beac..a50f1ab54 100644 --- a/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP2ConnectionTests.swift @@ -16,6 +16,7 @@ import Logging import NIOConcurrencyHelpers import NIOCore import NIOEmbedded +import NIOHPACK import NIOHTTP1 import NIOHTTP2 import NIOPosix diff --git a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift b/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift index 08dd58319..af32284b0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClient+SOCKSTests.swift @@ -15,6 +15,7 @@ import AsyncHTTPClient // NOT @testable - tests that need @testable go into HTTPClientInternalTests.swift import Logging import NIOCore +import NIOHTTP1 import NIOPosix import NIOSOCKS import XCTest diff --git a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift index 2c54d3289..5b70699a0 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientInternalTests.swift @@ -15,6 +15,7 @@ import NIOConcurrencyHelpers import NIOCore import NIOEmbedded +import NIOFoundationCompat import NIOHTTP1 import NIOPosix import NIOTestUtils diff --git a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift index 08e41d464..a2cc3b108 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientRequestTests.swift @@ -14,6 +14,7 @@ import Algorithms import NIOCore +import NIOHTTP1 import XCTest @testable import AsyncHTTPClient diff --git a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift index fd2b7ee4e..7dcc4efe6 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientResponseTests.swift @@ -14,6 +14,7 @@ import Logging import NIOCore +import NIOHTTP1 import XCTest @testable import AsyncHTTPClient diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index d9ca45d7b..da2046b81 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -18,6 +18,7 @@ import Logging import NIOConcurrencyHelpers import NIOCore import NIOEmbedded +import NIOFoundationCompat import NIOHPACK import NIOHTTP1 import NIOHTTP2 diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index 8f76b693b..fbd40ce3a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -20,6 +20,7 @@ import NIOCore import NIOEmbedded import NIOFoundationCompat import NIOHTTP1 +import NIOHTTP2 import NIOHTTPCompression import NIOPosix import NIOSSL diff --git a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift index ef59c9463..724c00b1f 100644 --- a/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPConnectionPool+ManagerTests.swift @@ -12,6 +12,7 @@ // //===----------------------------------------------------------------------===// +import Logging import NIOCore import NIOHTTP1 import NIOPosix diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index ff3a51d27..34349496d 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -16,6 +16,7 @@ import Logging import NIOConcurrencyHelpers import NIOCore import NIOEmbedded +import NIOFoundationCompat import NIOHTTP1 import NIOPosix import XCTest From f77cc00d69f7e8fa7787755a086a12b41c49f360 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Wed, 18 Dec 2024 12:52:43 +0100 Subject: [PATCH 138/146] Update release.yml (#795) Update the release.yml file with the latest label changes --- .github/release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/release.yml b/.github/release.yml index 13c29b0e6..e29eb8464 100644 --- a/.github/release.yml +++ b/.github/release.yml @@ -2,13 +2,13 @@ changelog: categories: - title: SemVer Major labels: - - semver/major + - โš ๏ธ semver/major - title: SemVer Minor labels: - - semver/minor + - ๐Ÿ†• semver/minor - title: SemVer Patch labels: - - semver/patch + - ๐Ÿ”จ semver/patch - title: Other Changes labels: - semver/none From 126518507b864b0687a9f97961060e755e4ee036 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Tue, 14 Jan 2025 16:34:21 +0000 Subject: [PATCH 139/146] Unbreak CI (#800) # Motivation The NIO 2.78 release introduced a bunch of new warnings. These warnings cause us a bunch of trouble, so we should fix them. # Modifications Mostly use a bunch of assumeIsolated() and syncOperations. # Result CI passes again. Note that Swift 6 has _many_ more warnings than this, but we expect more to come and we aren't using warnings-as-errors on that mode at the moment. We'll be cleaning that up soon. --- Package.swift | 2 +- .../HTTPConnectionPool+Factory.swift | 24 +++++----- .../FileDownloadDelegate.swift | 2 +- .../AsyncAwaitEndToEndTests.swift | 4 +- .../EmbeddedChannel+HTTPConvenience.swift | 4 +- .../HTTPClientTestUtils.swift | 10 ++-- .../HTTPClientTests.swift | 8 +++- .../AsyncHTTPClientTests/SOCKSTestUtils.swift | 46 +++++++++++-------- 8 files changed, 56 insertions(+), 44 deletions(-) diff --git a/Package.swift b/Package.swift index e4cccb6de..8bec2bd55 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( .library(name: "AsyncHTTPClient", targets: ["AsyncHTTPClient"]) ], dependencies: [ - .package(url: "https://github.com/apple/swift-nio.git", from: "2.71.0"), + .package(url: "https://github.com/apple/swift-nio.git", from: "2.78.0"), .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.27.1"), .package(url: "https://github.com/apple/swift-nio-http2.git", from: "1.19.0"), .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.13.0"), diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 0aad0c8dd..32af23830 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -249,15 +249,15 @@ extension HTTPConnectionPool.ConnectionFactory { // The proxyEstablishedFuture is set as soon as the HTTP1ProxyConnectHandler is in a // pipeline. It is created in HTTP1ProxyConnectHandler's handlerAdded method. - return proxyHandler.proxyEstablishedFuture!.flatMap { - channel.pipeline.removeHandler(proxyHandler).flatMap { - channel.pipeline.removeHandler(decoder).flatMap { - channel.pipeline.removeHandler(encoder) - } - } + return proxyHandler.proxyEstablishedFuture!.assumeIsolated().flatMap { + channel.pipeline.syncOperations.removeHandler(proxyHandler).assumeIsolated().flatMap { + channel.pipeline.syncOperations.removeHandler(decoder).assumeIsolated().flatMap { + channel.pipeline.syncOperations.removeHandler(encoder) + }.nonisolated() + }.nonisolated() }.flatMap { self.setupTLSInProxyConnectionIfNeeded(channel, deadline: deadline, logger: logger) - } + }.nonisolated() } } @@ -291,13 +291,13 @@ extension HTTPConnectionPool.ConnectionFactory { // The socksEstablishedFuture is set as soon as the SOCKSEventsHandler is in a // pipeline. It is created in SOCKSEventsHandler's handlerAdded method. - return socksEventHandler.socksEstablishedFuture!.flatMap { - channel.pipeline.removeHandler(socksEventHandler).flatMap { - channel.pipeline.removeHandler(socksConnectHandler) - } + return socksEventHandler.socksEstablishedFuture!.assumeIsolated().flatMap { + channel.pipeline.syncOperations.removeHandler(socksEventHandler).assumeIsolated().flatMap { + channel.pipeline.syncOperations.removeHandler(socksConnectHandler) + }.nonisolated() }.flatMap { self.setupTLSInProxyConnectionIfNeeded(channel, deadline: deadline, logger: logger) - } + }.nonisolated() } } diff --git a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift index 1f869506a..b21499843 100644 --- a/Sources/AsyncHTTPClient/FileDownloadDelegate.swift +++ b/Sources/AsyncHTTPClient/FileDownloadDelegate.swift @@ -167,7 +167,7 @@ public final class FileDownloadDelegate: HTTPClientResponseDelegate { } } else { let fileHandleFuture = io.openFile( - path: self.filePath, + _deprecatedPath: self.filePath, mode: .write, flags: .allowFileCreation(), eventLoop: task.eventLoop diff --git a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift index 4bfa86d14..c580164a0 100644 --- a/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift +++ b/Tests/AsyncHTTPClientTests/AsyncAwaitEndToEndTests.swift @@ -595,7 +595,9 @@ final class AsyncAwaitEndToEndTests: XCTestCase { defer { XCTAssertNoThrow(try serverGroup.syncShutdownGracefully()) } let server = ServerBootstrap(group: serverGroup) .childChannelInitializer { channel in - channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + channel.eventLoop.makeCompletedFuture { + try channel.pipeline.syncOperations.addHandler(NIOSSLServerHandler(context: sslContext)) + } } let serverChannel = try await server.bind(host: "localhost", port: 0).get() defer { XCTAssertNoThrow(try serverChannel.close().wait()) } diff --git a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift index 914d03612..397d143b0 100644 --- a/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift +++ b/Tests/AsyncHTTPClientTests/EmbeddedChannel+HTTPConvenience.swift @@ -87,8 +87,8 @@ extension EmbeddedChannel { let decoder = try self.pipeline.syncOperations.handler(type: ByteToMessageHandler.self) let encoder = try self.pipeline.syncOperations.handler(type: HTTPRequestEncoder.self) - let removeDecoderFuture = self.pipeline.removeHandler(decoder) - let removeEncoderFuture = self.pipeline.removeHandler(encoder) + let removeDecoderFuture = self.pipeline.syncOperations.removeHandler(decoder) + let removeEncoderFuture = self.pipeline.syncOperations.removeHandler(encoder) self.embeddedEventLoop.run() diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index da2046b81..1620d769a 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -544,12 +544,12 @@ where try sync.addHandler(requestDecoder) try sync.addHandler(proxySimulator) - promise.futureResult.flatMap { _ in - channel.pipeline.removeHandler(proxySimulator) + promise.futureResult.assumeIsolated().flatMap { _ in + channel.pipeline.syncOperations.removeHandler(proxySimulator) }.flatMap { _ in - channel.pipeline.removeHandler(responseEncoder) + channel.pipeline.syncOperations.removeHandler(responseEncoder) }.flatMap { _ in - channel.pipeline.removeHandler(requestDecoder) + channel.pipeline.syncOperations.removeHandler(requestDecoder) }.whenComplete { result in switch result { case .failure: @@ -653,8 +653,8 @@ where } } + try channel.pipeline.syncOperations.addHandler(sslHandler) try channel.pipeline.syncOperations.addHandler(alpnHandler) - try channel.pipeline.syncOperations.addHandler(sslHandler, position: .before(alpnHandler)) } func shutdown() throws { diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift index fbd40ce3a..546d1c3f4 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTests.swift @@ -1600,7 +1600,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let server = ServerBootstrap(group: serverGroup) .childChannelInitializer { channel in - channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + channel.eventLoop.makeCompletedFuture { + try channel.pipeline.syncOperations.addHandler(NIOSSLServerHandler(context: sslContext)) + } } let serverChannel = try server.bind(host: "localhost", port: 0).wait() defer { XCTAssertNoThrow(try serverChannel.close().wait()) } @@ -1642,7 +1644,9 @@ final class HTTPClientTests: XCTestCaseHTTPClientTestsBaseClass { let server = ServerBootstrap(group: serverGroup) .childChannelInitializer { channel in - channel.pipeline.addHandler(NIOSSLServerHandler(context: sslContext)) + channel.eventLoop.makeCompletedFuture { + try channel.pipeline.syncOperations.addHandler(NIOSSLServerHandler(context: sslContext)) + } } let serverChannel = try server.bind(host: "localhost", port: 0).wait() defer { XCTAssertNoThrow(try serverChannel.close().wait()) } diff --git a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift index 6dda7d928..ebff55a6d 100644 --- a/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/SOCKSTestUtils.swift @@ -59,17 +59,19 @@ class MockSOCKSServer { bootstrap = ServerBootstrap(group: elg) .serverChannelOption(ChannelOptions.socket(SocketOptionLevel(SOL_SOCKET), SO_REUSEADDR), value: 1) .childChannelInitializer { channel in - let handshakeHandler = SOCKSServerHandshakeHandler() - return channel.pipeline.addHandlers([ - handshakeHandler, - SOCKSTestHandler(handshakeHandler: handshakeHandler), - TestHTTPServer( - expectedURL: expectedURL, - expectedResponse: expectedResponse, - file: file, - line: line - ), - ]) + channel.eventLoop.makeCompletedFuture { + let handshakeHandler = SOCKSServerHandshakeHandler() + try channel.pipeline.syncOperations.addHandlers([ + handshakeHandler, + SOCKSTestHandler(handshakeHandler: handshakeHandler), + TestHTTPServer( + expectedURL: expectedURL, + expectedResponse: expectedResponse, + file: file, + line: line + ), + ]) + } } } self.channel = try bootstrap.bind(host: "localhost", port: 0).wait() @@ -112,15 +114,19 @@ class SOCKSTestHandler: ChannelInboundHandler, RemovableChannelHandler { ), promise: nil ) - context.channel.pipeline.addHandlers( - [ - ByteToMessageHandler(HTTPRequestDecoder()), - HTTPResponseEncoder(), - ], - position: .after(self) - ).whenSuccess { - context.channel.pipeline.removeHandler(self, promise: nil) - context.channel.pipeline.removeHandler(self.handshakeHandler, promise: nil) + + do { + try context.channel.pipeline.syncOperations.addHandlers( + [ + ByteToMessageHandler(HTTPRequestDecoder()), + HTTPResponseEncoder(), + ], + position: .after(self) + ) + context.channel.pipeline.syncOperations.removeHandler(self, promise: nil) + context.channel.pipeline.syncOperations.removeHandler(self.handshakeHandler, promise: nil) + } catch { + context.fireErrorCaught(error) } } } From e69318d4cb78ea2255d481dcde066674e4548139 Mon Sep 17 00:00:00 2001 From: Marc Prud'hommeaux Date: Tue, 14 Jan 2025 11:39:43 -0500 Subject: [PATCH 140/146] Android support (#799) This PR adds support for Android, mostly just by importing the Android module when needed. --- Sources/AsyncHTTPClient/ConnectionPool.swift | 4 +++- .../State Machine/HTTPConnectionPool+Backoff.swift | 2 ++ Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift | 2 ++ Sources/CAsyncHTTPClient/CAsyncHTTPClient.c | 2 +- Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift | 2 ++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool.swift b/Sources/AsyncHTTPClient/ConnectionPool.swift index 35f7a21c4..b5b058c2e 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool.swift @@ -20,7 +20,9 @@ import NIOSSL import Darwin.C #elseif canImport(Musl) import Musl -#elseif os(Linux) || os(FreeBSD) || os(Android) +#elseif canImport(Android) +import Android +#elseif os(Linux) || os(FreeBSD) import Glibc #else #error("unsupported target operating system") diff --git a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift index 86a54273d..71d8f15f1 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/State Machine/HTTPConnectionPool+Backoff.swift @@ -18,6 +18,8 @@ import NIOCore import func Darwin.pow #elseif canImport(Musl) import func Musl.pow +#elseif canImport(Android) +import func Android.pow #else import func Glibc.pow #endif diff --git a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift index 847a99af2..ea272a137 100644 --- a/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift +++ b/Sources/AsyncHTTPClient/HTTPClient+HTTPCookie.swift @@ -26,6 +26,8 @@ import locale_h import Darwin #elseif canImport(Musl) import Musl +#elseif canImport(Android) +import Android #elseif canImport(Glibc) import Glibc #endif diff --git a/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c b/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c index 5dfdc08a5..6342da89f 100644 --- a/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c +++ b/Sources/CAsyncHTTPClient/CAsyncHTTPClient.c @@ -31,7 +31,7 @@ bool swiftahc_cshims_strptime(const char * string, const char * format, struct t bool swiftahc_cshims_strptime_l(const char * string, const char * format, struct tm * result, void * locale) { // The pointer cast is fine as long we make sure it really points to a locale_t. -#ifdef __musl__ +#if defined(__musl__) || defined(__ANDROID__) const char * firstNonProcessed = strptime(string, format, result); #else const char * firstNonProcessed = strptime_l(string, format, result, (locale_t)locale); diff --git a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift index 1620d769a..4f08bc4f5 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClientTestUtils.swift @@ -39,6 +39,8 @@ import locale_h import Darwin #elseif canImport(Musl) import Musl +#elseif canImport(Android) +import Android #elseif canImport(Glibc) import Glibc #endif From f38c2fea867d230941f9b45fea6eda819d78fbb4 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Tue, 28 Jan 2025 17:50:38 +0000 Subject: [PATCH 141/146] Avoid precondition failure in write timeout (#803) ### Motivation: In some cases we can crash because of a precondition failure when the write timeout fires and we aren't in the running state. This can happen for example if the connection is closed whilst the write timer is active. ### Modifications: * Remove the precondition and instead take no action if the timeout fires outside of the running state. Instead we take a new `Action`, `.noAction` when the timer fires. * Clear write timeouts upon request completion. When a request completes we have no use for the idle write timer, we clear the read timer and we should clear the write one too. ### Result: Fewer crashes. The supplied tests fails without these changes and passes with either of them. --- .../HTTP1/HTTP1ClientChannelHandler.swift | 2 + .../HTTP1/HTTP1ConnectionStateMachine.swift | 2 +- .../HTTP2/HTTP2ClientRequestHandler.swift | 2 + .../HTTP1ClientChannelHandlerTests.swift | 55 +++++++++++++++++++ .../HTTP1ConnectionStateMachineTests.swift | 20 +++++++ 5 files changed, 80 insertions(+), 1 deletion(-) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift index 74a0c72d7..8203f07af 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ClientChannelHandler.swift @@ -314,6 +314,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { let oldRequest = self.request! self.request = nil self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) + self.runTimeoutAction(.clearIdleWriteTimeoutTimer, context: context) switch finalAction { case .close: @@ -353,6 +354,7 @@ final class HTTP1ClientChannelHandler: ChannelDuplexHandler { let oldRequest = self.request! self.request = nil self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) + self.runTimeoutAction(.clearIdleWriteTimeoutTimer, context: context) switch finalAction { case .close(let writePromise): diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift index aee0736ff..2cde1df3f 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP1/HTTP1ConnectionStateMachine.swift @@ -359,7 +359,7 @@ struct HTTP1ConnectionStateMachine { mutating func idleWriteTimeoutTriggered() -> Action { guard case .inRequest(var requestStateMachine, let close) = self.state else { - preconditionFailure("Invalid state: \(self.state)") + return .wait } return self.avoidingStateMachineCoW { state -> Action in diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift index 5e105c0d8..61350dfd7 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTP2/HTTP2ClientRequestHandler.swift @@ -240,6 +240,7 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { self.request!.fail(error) self.request = nil self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) + self.runTimeoutAction(.clearIdleWriteTimeoutTimer, context: context) // No matter the error reason, we must always make sure the h2 stream is closed. Only // once the h2 stream is closed, it is released from the h2 multiplexer. The // HTTPRequestStateMachine may signal finalAction: .none in the error case (as this is @@ -252,6 +253,7 @@ final class HTTP2ClientRequestHandler: ChannelDuplexHandler { self.request!.succeedRequest(finalParts) self.request = nil self.runTimeoutAction(.clearIdleReadTimeoutTimer, context: context) + self.runTimeoutAction(.clearIdleWriteTimeoutTimer, context: context) self.runSuccessfulFinalAction(finalAction, context: context) case .failSendBodyPart(let error, let writePromise), .failSendStreamFinished(let error, let writePromise): diff --git a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift index 53af0823d..df1a2926a 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ClientChannelHandlerTests.swift @@ -840,6 +840,61 @@ class HTTP1ClientChannelHandlerTests: XCTestCase { channel.writeAndFlush(request, promise: nil) XCTAssertEqual(request.events.map(\.kind), [.willExecuteRequest, .requestHeadSent]) } + + func testIdleWriteTimeoutOutsideOfRunningState() { + let embedded = EmbeddedChannel() + var maybeTestUtils: HTTP1TestTools? + XCTAssertNoThrow(maybeTestUtils = try embedded.setupHTTP1Connection()) + print("pipeline", embedded.pipeline) + guard let testUtils = maybeTestUtils else { return XCTFail("Expected connection setup works") } + + var maybeRequest: HTTPClient.Request? + XCTAssertNoThrow(maybeRequest = try HTTPClient.Request(url: "http://localhost/")) + guard var request = maybeRequest else { return XCTFail("Expected to be able to create a request") } + + // start a request stream we'll never write to + let streamPromise = embedded.eventLoop.makePromise(of: Void.self) + let streamCallback = { @Sendable (streamWriter: HTTPClient.Body.StreamWriter) -> EventLoopFuture in + streamPromise.futureResult + } + request.body = .init(contentLength: nil, stream: streamCallback) + + let accumulator = ResponseAccumulator(request: request) + var maybeRequestBag: RequestBag? + XCTAssertNoThrow( + maybeRequestBag = try RequestBag( + request: request, + eventLoopPreference: .delegate(on: embedded.eventLoop), + task: .init(eventLoop: embedded.eventLoop, logger: testUtils.logger), + redirectHandler: nil, + connectionDeadline: .now() + .seconds(30), + requestOptions: .forTests( + idleReadTimeout: .milliseconds(10), + idleWriteTimeout: .milliseconds(2) + ), + delegate: accumulator + ) + ) + guard let requestBag = maybeRequestBag else { return XCTFail("Expected to be able to create a request bag") } + + testUtils.connection.executeRequest(requestBag) + + XCTAssertNoThrow( + try embedded.receiveHeadAndVerify { + XCTAssertEqual($0.method, .GET) + XCTAssertEqual($0.uri, "/") + XCTAssertEqual($0.headers.first(name: "host"), "localhost") + } + ) + + // close the pipeline to simulate a server-side close + // note this happens before we write so the idle write timeout is still running + try! embedded.pipeline.close().wait() + + // advance time to trigger the idle write timeout + // and ensure that the state machine can tolerate this + embedded.embeddedEventLoop.advanceTime(by: .milliseconds(250)) + } } class TestBackpressureWriter { diff --git a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift index 18831d32f..1c6e9659f 100644 --- a/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTP1ConnectionStateMachineTests.swift @@ -101,6 +101,26 @@ class HTTP1ConnectionStateMachineTests: XCTestCase { XCTAssertEqual(state.read(), .read) } + func testWriteTimeoutAfterErrorDoesntCrash() { + var state = HTTP1ConnectionStateMachine() + XCTAssertEqual(state.channelActive(isWritable: true), .fireChannelActive) + + let requestHead = HTTPRequestHead(version: .http1_1, method: .GET, uri: "/") + let metadata = RequestFramingMetadata(connectionClose: false, body: .fixedSize(0)) + let newRequestAction = state.runNewRequest(head: requestHead, metadata: metadata) + XCTAssertEqual(newRequestAction, .sendRequestHead(requestHead, sendEnd: true)) + XCTAssertEqual( + state.headSent(), + .notifyRequestHeadSendSuccessfully(resumeRequestBodyStream: false, startIdleTimer: true) + ) + + struct MyError: Error, Equatable {} + XCTAssertEqual(state.errorHappened(MyError()), .failRequest(MyError(), .close(nil))) + + // Primarily we care that we don't crash here + XCTAssertEqual(state.idleWriteTimeoutTriggered(), .wait) + } + func testAConnectionCloseHeaderInTheRequestLeadsToConnectionCloseAfterRequest() { var state = HTTP1ConnectionStateMachine() XCTAssertEqual(state.channelActive(isWritable: true), .fireChannelActive) From 60fa3dcfc52d09ed0411e9c4bc99928bf63656bd Mon Sep 17 00:00:00 2001 From: Allan Shortlidge Date: Wed, 29 Jan 2025 10:42:58 -0800 Subject: [PATCH 142/146] Add missing import of Network module (#804) --- .../ConnectionPool/HTTPConnectionPool+Factory.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift index 32af23830..cb3ec0bf5 100644 --- a/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift +++ b/Sources/AsyncHTTPClient/ConnectionPool/HTTPConnectionPool+Factory.swift @@ -22,6 +22,7 @@ import NIOSSL import NIOTLS #if canImport(Network) +import Network import NIOTransportServices #endif From 81384de61c8ff7beb0156b387b296a35e3ea321a Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 30 Jan 2025 09:43:36 +0000 Subject: [PATCH 143/146] CI use 6.1 nightlies (#805) CI use 6.1 nightlies now that Swift development is happening in the 6.1 branch --- .github/workflows/main.yml | 2 +- .github/workflows/pull_request.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6e5453369..f63d89a3e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,5 +14,5 @@ jobs: linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" - linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_6_1_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 0392cb7c5..a4fde6207 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -17,7 +17,7 @@ jobs: linux_5_9_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_5_10_arguments_override: "-Xswiftc -warnings-as-errors --explicit-target-dependency-import-check error" linux_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" - linux_nightly_6_0_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" + linux_nightly_6_1_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" linux_nightly_main_arguments_override: "--explicit-target-dependency-import-check error -Xswiftc -require-explicit-sendable" cxx-interop: From 89dc8d0068eb4d3dea050eac6a0c913098c60d97 Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Thu, 6 Feb 2025 17:11:37 +0000 Subject: [PATCH 144/146] baby steps towards a Structured Concurrency API (#806) At the moment, `HTTPClient`'s entire API surface violates Structured Concurrency. Both the creation & shutdown of a HTTP client as well as making requests (#807) doesn't follow Structured Concurrency. Some of the problems are: 1. Upon return of methods, resources are still in active use in other threads/tasks 2. Cancellation doesn't always work This PR is baby steps towards a Structured Concurrency API, starting with start/shutdown of the HTTP client. Co-authored-by: Johannes Weiss --- .../AsyncAwait/HTTPClient+execute.swift | 12 +++ .../HTTPClient+StructuredConcurrency.swift | 72 +++++++++++++ Sources/AsyncHTTPClient/HTTPHandler.swift | 11 +- .../StructuredConcurrencyHelpers.swift | 80 ++++++++++++++ ...TTPClient+StructuredConcurrencyTests.swift | 100 ++++++++++++++++++ 5 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 Sources/AsyncHTTPClient/HTTPClient+StructuredConcurrency.swift create mode 100644 Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift create mode 100644 Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift diff --git a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift index fc1dbc209..3c3a6030c 100644 --- a/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift +++ b/Sources/AsyncHTTPClient/AsyncAwait/HTTPClient+execute.swift @@ -26,6 +26,10 @@ extension HTTPClient { /// - request: HTTP request to execute. /// - deadline: Point in time by which the request must complete. /// - logger: The logger to use for this request. + /// + /// - warning: This method may violates Structured Concurrency because it returns a `HTTPClientResponse` that needs to be + /// streamed by the user. This means the request, the connection and other resources are still alive when the request returns. + /// /// - Returns: The response to the request. Note that the `body` of the response may not yet have been fully received. public func execute( _ request: HTTPClientRequest, @@ -51,6 +55,10 @@ extension HTTPClient { /// - request: HTTP request to execute. /// - timeout: time the the request has to complete. /// - logger: The logger to use for this request. + /// + /// - warning: This method may violates Structured Concurrency because it returns a `HTTPClientResponse` that needs to be + /// streamed by the user. This means the request, the connection and other resources are still alive when the request returns. + /// /// - Returns: The response to the request. Note that the `body` of the response may not yet have been fully received. public func execute( _ request: HTTPClientRequest, @@ -67,6 +75,8 @@ extension HTTPClient { @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) extension HTTPClient { + /// - warning: This method may violates Structured Concurrency because it returns a `HTTPClientResponse` that needs to be + /// streamed by the user. This means the request, the connection and other resources are still alive when the request returns. private func executeAndFollowRedirectsIfNeeded( _ request: HTTPClientRequest, deadline: NIODeadline, @@ -116,6 +126,8 @@ extension HTTPClient { } } + /// - warning: This method may violates Structured Concurrency because it returns a `HTTPClientResponse` that needs to be + /// streamed by the user. This means the request, the connection and other resources are still alive when the request returns. private func executeCancellable( _ request: HTTPClientRequest.Prepared, deadline: NIODeadline, diff --git a/Sources/AsyncHTTPClient/HTTPClient+StructuredConcurrency.swift b/Sources/AsyncHTTPClient/HTTPClient+StructuredConcurrency.swift new file mode 100644 index 000000000..f7d471f10 --- /dev/null +++ b/Sources/AsyncHTTPClient/HTTPClient+StructuredConcurrency.swift @@ -0,0 +1,72 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2025 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Logging +import NIO + +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +extension HTTPClient { + #if compiler(>=6.0) + /// Start & automatically shut down a new ``HTTPClient``. + /// + /// This method allows to start & automatically dispose of a ``HTTPClient`` following the principle of Structured Concurrency. + /// The ``HTTPClient`` is guaranteed to be shut down upon return, whether `body` throws or not. + /// + /// This may be particularly useful if you cannot use the shared singleton (``HTTPClient/shared``). + public static func withHTTPClient( + eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger? = nil, + isolation: isolated (any Actor)? = #isolation, + _ body: (HTTPClient) async throws -> Return + ) async throws -> Return { + let logger = (backgroundActivityLogger ?? HTTPClient.loggingDisabled) + let httpClient = HTTPClient( + eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: logger + ) + return try await asyncDo { + try await body(httpClient) + } finally: { _ in + try await httpClient.shutdown() + } + } + #else + /// Start & automatically shut down a new ``HTTPClient``. + /// + /// This method allows to start & automatically dispose of a ``HTTPClient`` following the principle of Structured Concurrency. + /// The ``HTTPClient`` is guaranteed to be shut down upon return, whether `body` throws or not. + /// + /// This may be particularly useful if you cannot use the shared singleton (``HTTPClient/shared``). + public static func withHTTPClient( + eventLoopGroup: any EventLoopGroup = HTTPClient.defaultEventLoopGroup, + configuration: Configuration = Configuration(), + backgroundActivityLogger: Logger? = nil, + _ body: (HTTPClient) async throws -> Return + ) async throws -> Return { + let logger = (backgroundActivityLogger ?? HTTPClient.loggingDisabled) + let httpClient = HTTPClient( + eventLoopGroup: eventLoopGroup, + configuration: configuration, + backgroundActivityLogger: logger + ) + return try await asyncDo { + try await body(httpClient) + } finally: { _ in + try await httpClient.shutdown() + } + } + #endif +} diff --git a/Sources/AsyncHTTPClient/HTTPHandler.swift b/Sources/AsyncHTTPClient/HTTPHandler.swift index 0f061fbe6..38b930638 100644 --- a/Sources/AsyncHTTPClient/HTTPHandler.swift +++ b/Sources/AsyncHTTPClient/HTTPHandler.swift @@ -885,6 +885,8 @@ extension HTTPClient { /// Provides the result of this request. /// + /// - warning: This method may violates Structured Concurrency because doesn't respect cancellation. + /// /// - returns: The value of ``futureResult`` when it completes. /// - throws: The error value of ``futureResult`` if it errors. @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) @@ -892,12 +894,17 @@ extension HTTPClient { try await self.promise.futureResult.get() } - /// Cancels the request execution. + /// Initiate cancellation of a HTTP request. + /// + /// This method will return immeidately and doesn't wait for the cancellation to complete. public func cancel() { self.fail(reason: HTTPClientError.cancelled) } - /// Cancels the request execution with a custom `Error`. + /// Initiate cancellation of a HTTP request with an `error`. + /// + /// This method will return immeidately and doesn't wait for the cancellation to complete. + /// /// - Parameter error: the error that is used to fail the promise public func fail(reason error: Error) { let taskDelegate = self.lock.withLock { () -> HTTPClientTaskDelegate? in diff --git a/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift b/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift new file mode 100644 index 000000000..40ec01728 --- /dev/null +++ b/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift @@ -0,0 +1,80 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2025 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +#if compiler(>=6.0) +@inlinable +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal func asyncDo( + isolation: isolated (any Actor)? = #isolation, + _ body: () async throws -> sending R, + finally: sending @escaping ((any Error)?) async throws -> Void +) async throws -> sending R { + let result: R + do { + result = try await body() + } catch { + // `body` failed, we need to invoke `finally` with the `error`. + + // This _looks_ unstructured but isn't really because we unconditionally always await the return. + // We need to have an uncancelled task here to assure this is actually running in case we hit a + // cancellation error. + try await Task { + try await finally(error) + }.value + throw error + } + + // `body` succeeded, we need to invoke `finally` with `nil` (no error). + + // This _looks_ unstructured but isn't really because we unconditionally always await the return. + // We need to have an uncancelled task here to assure this is actually running in case we hit a + // cancellation error. + try await Task { + try await finally(nil) + }.value + return result +} +#else +@inlinable +@available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) +internal func asyncDo( + _ body: () async throws -> R, + finally: @escaping @Sendable ((any Error)?) async throws -> Void +) async throws -> R { + let result: R + do { + result = try await body() + } catch { + // `body` failed, we need to invoke `finally` with the `error`. + + // This _looks_ unstructured but isn't really because we unconditionally always await the return. + // We need to have an uncancelled task here to assure this is actually running in case we hit a + // cancellation error. + try await Task { + try await finally(error) + }.value + throw error + } + + // `body` succeeded, we need to invoke `finally` with `nil` (no error). + + // This _looks_ unstructured but isn't really because we unconditionally always await the return. + // We need to have an uncancelled task here to assure this is actually running in case we hit a + // cancellation error. + try await Task { + try await finally(nil) + }.value + return result +} +#endif diff --git a/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift b/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift new file mode 100644 index 000000000..162093383 --- /dev/null +++ b/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift @@ -0,0 +1,100 @@ +//===----------------------------------------------------------------------===// +// +// This source file is part of the AsyncHTTPClient open source project +// +// Copyright (c) 2025 Apple Inc. and the AsyncHTTPClient project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import AsyncHTTPClient +import NIO +import NIOFoundationCompat +import XCTest + +final class HTTPClientStructuredConcurrencyTests: XCTestCase { + func testDoNothingWorks() async throws { + let actual = try await HTTPClient.withHTTPClient { httpClient in + "OK" + } + XCTAssertEqual("OK", actual) + } + + func testShuttingDownTheClientInBodyLeadsToError() async { + do { + let actual = try await HTTPClient.withHTTPClient { httpClient in + try await httpClient.shutdown() + return "OK" + } + XCTFail("Expected error, got \(actual)") + } catch let error as HTTPClientError where error == .alreadyShutdown { + // OK + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testBasicRequest() async throws { + let httpBin = HTTPBin() + defer { XCTAssertNoThrow(try httpBin.shutdown()) } + + let actualBytes = try await HTTPClient.withHTTPClient { httpClient in + let response = try await httpClient.get(url: httpBin.baseURL).get() + XCTAssertEqual(response.status, .ok) + return response.body ?? ByteBuffer(string: "n/a") + } + let actual = try JSONDecoder().decode(RequestInfo.self, from: actualBytes) + + XCTAssertGreaterThanOrEqual(actual.requestNumber, 0) + XCTAssertGreaterThanOrEqual(actual.connectionNumber, 0) + } + + func testClientIsShutDownAfterReturn() async throws { + let leakedClient = try await HTTPClient.withHTTPClient { httpClient in + httpClient + } + do { + try await leakedClient.shutdown() + XCTFail("unexpected, shutdown should have failed") + } catch let error as HTTPClientError where error == .alreadyShutdown { + // OK + } catch { + XCTFail("unexpected error: \(error)") + } + } + + func testClientIsShutDownOnThrowAlso() async throws { + struct TestError: Error { + var httpClient: HTTPClient + } + + let leakedClient: HTTPClient + do { + try await HTTPClient.withHTTPClient { httpClient in + throw TestError(httpClient: httpClient) + } + XCTFail("unexpected, shutdown should have failed") + return + } catch let error as TestError { + // OK + leakedClient = error.httpClient + } catch { + XCTFail("unexpected error: \(error)") + return + } + + do { + try await leakedClient.shutdown() + XCTFail("unexpected, shutdown should have failed") + } catch let error as HTTPClientError where error == .alreadyShutdown { + // OK + } catch { + XCTFail("unexpected error: \(error)") + } + } +} From b645ad40822b5c59ac92b758c5c17af054b5b01f Mon Sep 17 00:00:00 2001 From: Johannes Weiss Date: Tue, 11 Feb 2025 12:25:31 +0000 Subject: [PATCH 145/146] fix 5.10 compile on Ubuntu 24.04 (Noble) for Intel (x86_64) (#810) Specifically Swift 5.10 _on Intel on Ubuntu Noble (24.04)_ has a crazy bug which leads to compilation failures in a `#if compiler(>=6.0)` block: https://github.com/swiftlang/swift/issues/79285 . This workaround fixes the compilation by _changing the whitespace_. Thanks @gwynne for finding this workaround! --------- Co-authored-by: Johannes Weiss --- .../AsyncHTTPClient/StructuredConcurrencyHelpers.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift b/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift index 40ec01728..25f1225e0 100644 --- a/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift +++ b/Sources/AsyncHTTPClient/StructuredConcurrencyHelpers.swift @@ -11,15 +11,18 @@ // SPDX-License-Identifier: Apache-2.0 // //===----------------------------------------------------------------------===// +// swift-format-ignore +// Note: Whitespace changes are used to workaround compiler bug +// https://github.com/swiftlang/swift/issues/79285 #if compiler(>=6.0) @inlinable @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) internal func asyncDo( isolation: isolated (any Actor)? = #isolation, - _ body: () async throws -> sending R, - finally: sending @escaping ((any Error)?) async throws -> Void -) async throws -> sending R { + // DO NOT FIX THE WHITESPACE IN THE NEXT LINE UNTIL 5.10 IS UNSUPPORTED + // https://github.com/swiftlang/swift/issues/79285 + _ body: () async throws -> sending R, finally: sending @escaping ((any Error)?) async throws -> Void) async throws -> sending R { let result: R do { result = try await body() From 3b4942f5b334e8ed7c277bc068d3d8ab6bc94ab5 Mon Sep 17 00:00:00 2001 From: Cory Benfield Date: Mon, 17 Feb 2025 15:43:34 +0000 Subject: [PATCH 146/146] Remove misuse of EmbeddedEventLoop (#812) Motivation EmbeddedEventLoop is not thread-safe, which means that outside of very rare use-cases it's not safe to use it in Swift Concurrency. Modifications Replace invalid uses of EmbeddedEventLoop with NIOAsyncTestingEventLoop Result Better safety --- ...TTPClient+StructuredConcurrencyTests.swift | 1 + .../TransactionTests.swift | 64 ++++++++++--------- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift b/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift index 162093383..a7cc1f454 100644 --- a/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift +++ b/Tests/AsyncHTTPClientTests/HTTPClient+StructuredConcurrencyTests.swift @@ -15,6 +15,7 @@ import AsyncHTTPClient import NIO import NIOFoundationCompat +import NIOHTTP1 import XCTest final class HTTPClientStructuredConcurrencyTests: XCTestCase { diff --git a/Tests/AsyncHTTPClientTests/TransactionTests.swift b/Tests/AsyncHTTPClientTests/TransactionTests.swift index 34349496d..8609597b3 100644 --- a/Tests/AsyncHTTPClientTests/TransactionTests.swift +++ b/Tests/AsyncHTTPClientTests/TransactionTests.swift @@ -33,8 +33,8 @@ final class TransactionTests: XCTestCase { // therefore we create it here as a workaround which works fine let scheduledRequestCanceled = self.expectation(description: "scheduled request canceled") XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .GET @@ -45,7 +45,7 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) let queuer = MockTaskQueuer { _ in @@ -72,8 +72,8 @@ final class TransactionTests: XCTestCase { func testDeadlineExceededWhileQueuedAndExecutorImmediatelyCancelsTask() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .GET @@ -84,7 +84,7 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) let queuer = MockTaskQueuer() @@ -127,8 +127,8 @@ final class TransactionTests: XCTestCase { func testResponseStreamingWorks() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .GET @@ -140,12 +140,12 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) let executor = MockRequestExecutor( pauseRequestBodyPartStreamAfterASingleWrite: true, - eventLoop: embeddedEventLoop + eventLoop: loop ) transaction.willExecuteRequest(executor) @@ -186,8 +186,8 @@ final class TransactionTests: XCTestCase { func testIgnoringResponseBodyWorks() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .GET @@ -199,7 +199,7 @@ final class TransactionTests: XCTestCase { } var tuple: (Transaction, Task)! = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) let transaction = tuple.0 @@ -208,9 +208,10 @@ final class TransactionTests: XCTestCase { let executor = MockRequestExecutor( pauseRequestBodyPartStreamAfterASingleWrite: true, - eventLoop: embeddedEventLoop + eventLoop: loop ) executor.runRequest(transaction) + await loop.run() let responseHead = HTTPResponseHead(version: .http1_1, status: .ok, headers: ["foo": "bar"]) XCTAssertFalse(executor.signalledDemandForResponseBody) @@ -234,8 +235,8 @@ final class TransactionTests: XCTestCase { func testWriteBackpressureWorks() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } let streamWriter = AsyncSequenceWriter() XCTAssertFalse(streamWriter.hasDemand, "Did not expect to have a demand at this point") @@ -251,12 +252,13 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) - let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + let executor = MockRequestExecutor(eventLoop: loop) executor.runRequest(transaction) + await loop.run() for i in 0..<100 { XCTAssertFalse(streamWriter.hasDemand, "Did not expect to have demand yet") @@ -364,8 +366,8 @@ final class TransactionTests: XCTestCase { func testSimplePostRequest() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .POST @@ -377,11 +379,12 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) - let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + let executor = MockRequestExecutor(eventLoop: loop) executor.runRequest(transaction) + await loop.run() executor.resumeRequestBodyStream() XCTAssertNoThrow( try executor.receiveRequestBody { @@ -403,8 +406,8 @@ final class TransactionTests: XCTestCase { func testPostStreamFails() { XCTAsyncTest { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } let writer = AsyncSequenceWriter() @@ -418,11 +421,12 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) - let executor = MockRequestExecutor(eventLoop: embeddedEventLoop) + let executor = MockRequestExecutor(eventLoop: loop) executor.runRequest(transaction) + await loop.run() executor.resumeRequestBodyStream() await writer.demand() @@ -447,8 +451,8 @@ final class TransactionTests: XCTestCase { func testResponseStreamFails() { XCTAsyncTest(timeout: 30) { - let embeddedEventLoop = EmbeddedEventLoop() - defer { XCTAssertNoThrow(try embeddedEventLoop.syncShutdownGracefully()) } + let loop = NIOAsyncTestingEventLoop() + defer { XCTAssertNoThrow(try loop.syncShutdownGracefully()) } var request = HTTPClientRequest(url: "https://localhost/") request.method = .GET @@ -460,12 +464,12 @@ final class TransactionTests: XCTestCase { } let (transaction, responseTask) = await Transaction.makeWithResultTask( request: preparedRequest, - preferredEventLoop: embeddedEventLoop + preferredEventLoop: loop ) let executor = MockRequestExecutor( pauseRequestBodyPartStreamAfterASingleWrite: true, - eventLoop: embeddedEventLoop + eventLoop: loop ) transaction.willExecuteRequest(executor)