From da4da1b74d1962e029e586f222ec0ad5b2f93475 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:36:08 +0530 Subject: [PATCH 01/34] add process for conformance suite --- .../test/scala/zio/http/ConformanceSpec.scala | 1446 +++++++++++++++++ 1 file changed, 1446 insertions(+) create mode 100644 zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala new file mode 100644 index 0000000000..e12913b7cf --- /dev/null +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -0,0 +1,1446 @@ +package zio.http + +import java.time.format.DateTimeFormatter +import java.time.ZonedDateTime + +import zio._ +import zio.test.Assertion._ +import zio.test.TestAspect._ +import zio.test._ + +import zio.http._ + +object ConformanceSpec extends ZIOSpecDefault { + + /** + * This test suite is inspired by and built upon the findings from the + * research paper: "Who's Breaking the Rules? Studying Conformance to the HTTP + * Specifications and its Security Impact" by Jannis Rautenstrauch and Ben + * Stock, presented at the 19th ACM Asia Conference on Computer and + * Communications Security (ASIA CCS) 2024. + * + * Paper URL: https://doi.org/10.1145/3634737.3637678 + * GitHub Project: https://github.com/cispa/http-conformance + * + */ + + val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) + + override def spec = + suite("ConformanceSpec")( + suite("Statuscodes")( + test("should not send body for 204 No Content responses(code_204_no_additional_content)") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent), + ), + ) + + val request = Request.get("/no-content") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NoContent, + response.body.isEmpty, + ) + }, + test("should not send body for 205 Reset Content responses(code_205_no_content_allowed)") { + val app = Routes( + Method.GET / "reset-content" -> Handler.fromResponse( + Response.status(Status.ResetContent), + ), + ) + + val request = Request.get("/reset-content") + + for { + response <- app.runZIO(request) + } yield assertTrue(response.status == Status.ResetContent, response.body.isEmpty) + }, + test("should include Content-Range for 206 Partial Content response(code_206_content_range)") { + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ContentRange.StartEnd("bytes", 0, 14)), + ), + ) + + val request = Request.get("/partial") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.PartialContent, + response.headers.contains(Header.ContentRange.name), + ) + }, + test( + "should not include Content-Range in header for multipart/byteranges response(code_206_content_range_of_multiple_part_response)", + ) { + val boundary = zio.http.Boundary("A12345") + + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ContentType(MediaType("multipart", "byteranges"), Some(boundary))), + ), + ) + + val request = Request.get("/partial") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.PartialContent, + !response.headers.contains(Header.ContentRange.name), + response.headers.contains(Header.ContentType.name), + ) + }, + test("should include necessary headers in 206 Partial Content response(code_206_headers)") { + val app = Routes( + Method.GET / "partial" -> Handler.fromResponse( + Response + .status(Status.PartialContent) + .addHeader(Header.ETag.Strong("abc")) + .addHeader(Header.CacheControl.MaxAge(3600)), + ), + Method.GET / "full" -> Handler.fromResponse( + Response + .status(Status.Ok) + .addHeader(Header.ETag.Strong("abc")) + .addHeader(Header.CacheControl.MaxAge(3600)), + ), + ) + + val requestWithRange = + Request.get("/partial").addHeader(Header.Range.Single("bytes", 0, Some(14))) + val requestWithoutRange = Request.get("/full") + + for { + responseWithRange <- app.runZIO(requestWithRange) + responseWithoutRange <- app.runZIO(requestWithoutRange) + } yield assertTrue( + responseWithRange.status == Status.PartialContent, + responseWithRange.headers.contains(Header.ETag.name), + responseWithRange.headers.contains(Header.CacheControl.name), + responseWithoutRange.status == Status.Ok, + ) + }, + test("should include WWW-Authenticate header for 401 Unauthorized response(code_401_www_authenticate)") { + val app = Routes( + Method.GET / "unauthorized" -> Handler.fromResponse( + Response + .status(Status.Unauthorized) + .addHeader(Header.WWWAuthenticate.Basic(Some("simple"))), + ), + ) + + val request = Request.get("/unauthorized") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Unauthorized, + response.headers.contains(Header.WWWAuthenticate.name), + ) + }, + test("should include Allow header for 405 Method Not Allowed response(code_405_allow)") { + val app = Routes( + Method.POST / "not-allowed" -> Handler.fromResponse( + Response + .status(Status.Ok), + ), + ) + + val request = Request.get("/not-allowed") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.MethodNotAllowed, + response.headers.contains(Header.Allow.name), + ) + }, + test( + "should include Proxy-Authenticate header for 407 Proxy Authentication Required response(code_407_proxy_authenticate)", + ) { + val app = Routes( + Method.GET / "proxy-auth" -> Handler.fromResponse( + Response + .status(Status.ProxyAuthenticationRequired) + .addHeader( + Header.ProxyAuthenticate(Header.AuthenticationScheme.Basic, Some("proxy")), + ), + ), + ) + + val request = Request.get("/proxy-auth") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.ProxyAuthenticationRequired, + response.headers.contains(Header.ProxyAuthenticate.name), + ) + }, + test("should return 304 without content(code_304_no_content)") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response + .status(Status.NotModified) + .copy(body = Body.empty), + ), + ) + + val request = Request.get("/no-content") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.body.isEmpty, + ) + }, + test("should return 304 with correct headers(code_304_headers)") { + val headers = Headers( + Header.ETag.Strong("abc"), + Header.CacheControl.MaxAge(3600), + Header.Vary("Accept-Encoding"), + ) + + val app = Routes( + Method.GET / "with-headers" -> Handler.fromResponse( + Response + .status(Status.NotModified) + .addHeaders(headers), + ), + ) + + val request = Request.get("/with-headers") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.headers.contains(Header.ETag.name), + response.headers.contains(Header.CacheControl.name), + response.headers.contains(Header.Vary.name), + ) + }, + test("should include Location header in 300 MULTIPLE CHOICES response(code_300_location)") { + val testUrl = URL.decode("/People.html#tim").toOption.getOrElse(URL.root) + + val validResponse = Response + .status(Status.MultipleChoices) + .addHeader(Header.Location(testUrl)) + + val invalidResponse = Response + .status(Status.MultipleChoices) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.MultipleChoices, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.MultipleChoices, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("300 MULTIPLE CHOICES response should have body content(code_300_metadata)") { + val validResponse = Response + .status(Status.MultipleChoices) + .copy(body = Body.fromString("
ABC
")) + + val invalidResponse = Response + .status(Status.MultipleChoices) + .copy(body = Body.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + validBody <- responseValid.body.asString + responseInvalid <- app.runZIO(Request.get("/invalid")) + invalidBody <- responseInvalid.body.asString + + } yield assertTrue( + responseValid.status == Status.MultipleChoices, + validBody.contains("ABC"), + responseInvalid.status == Status.MultipleChoices, + invalidBody.isEmpty, + ) + }, + test("should not require body content for HEAD requests(code_300_metadata)") { + val response = Response + .status(Status.MultipleChoices) + .copy(body = Body.empty) + val app = Routes( + Method.HEAD / "head" -> Handler.fromResponse(response), + ) + + for { + headResponse <- app.runZIO(Request.head("/head")) + } yield assertTrue( + headResponse.status == Status.MultipleChoices, + headResponse.body.isEmpty, + ) + }, + test("should include Location header in 301 MOVED PERMANENTLY response(code_301_location)") { + + val validResponse = Response + .status(Status.MovedPermanently) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.MovedPermanently) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.MovedPermanently, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.MovedPermanently, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 302 FOUND response(code_302_location)") { + + val validResponse = Response + .status(Status.Found) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.Found) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.Found, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.Found, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 303 SEE OTHER response(code_303_location)") { + + val validResponse = Response + .status(Status.SeeOther) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.SeeOther) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.SeeOther, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.SeeOther, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 307 TEMPORARY REDIRECT response(code_307_location)") { + + val validResponse = Response + .status(Status.TemporaryRedirect) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.TemporaryRedirect) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.TemporaryRedirect, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.TemporaryRedirect, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test("should include Location header in 308 PERMANENT REDIRECT response(code_308_location)") { + + val validResponse = Response + .status(Status.PermanentRedirect) + .addHeader(Header.Location(validUrl)) + + val invalidResponse = Response + .status(Status.PermanentRedirect) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.PermanentRedirect, + responseValid.headers.contains(Header.Location.name), + responseInvalid.status == Status.PermanentRedirect, + !responseInvalid.headers.contains(Header.Location.name), + ) + }, + test( + "should include Retry-After header in 413 Content Too Large response if condition is temporary (code_413_retry_after)", + ) { + val validResponse = Response + .status(Status.RequestEntityTooLarge) + .addHeader(Header.RetryAfter.ByDuration(10.seconds)) + + val invalidResponse = Response + .status(Status.RequestEntityTooLarge) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.RequestEntityTooLarge, + responseValid.headers.contains(Header.RetryAfter.name), + responseInvalid.status == Status.RequestEntityTooLarge, + !responseInvalid.headers.contains(Header.RetryAfter.name), + ) + }, + test( + "should include Accept or Accept-Encoding header in 415 Unsupported Media Type response (code_415_unsupported_media_type)", + ) { + val validResponse = Response + .status(Status.UnsupportedMediaType) + .addHeader(Header.Accept(MediaType.application.json)) + + val invalidResponse = Response + .status(Status.UnsupportedMediaType) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.UnsupportedMediaType, + responseValid.headers.contains(Header.Accept.name) || + responseValid.headers.contains(Header.AcceptEncoding.name), + responseInvalid.status == Status.UnsupportedMediaType, + !responseInvalid.headers.contains(Header.Accept.name) && + !responseInvalid.headers.contains(Header.AcceptEncoding.name), + ) + }, + test("should include Content-Range header in 416 Range Not Satisfiable response (code_416_content_range)") { + val validResponse = Response + .status(Status.RequestedRangeNotSatisfiable) + .addHeader(Header.ContentRange.RangeTotal("bytes", 47022)) + + val invalidResponse = Response + .status(Status.RequestedRangeNotSatisfiable) + .addHeader(Header.Custom("Content-Range", ",;")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.status == Status.RequestedRangeNotSatisfiable, + responseValid.headers.contains(Header.ContentRange.name), + responseInvalid.status == Status.RequestedRangeNotSatisfiable, + responseInvalid.headers.contains(Header.ContentRange.name), + responseInvalid.headers.get(Header.ContentRange.name).contains(",;"), + ) + }, + ), + suite("HTTP Headers")( + suite("code_400_after_bad_host_request")( + test("should return 200 OK if Host header is present") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithHost = Request.get("/test").addHeader(Header.Host("localhost")) + for { + response <- app.runZIO(requestWithHost) + } yield assertTrue(response.status == Status.Ok) + }, + test("should return 400 Bad Request if Host header is missing") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithoutHost = Request.get("/test") + + for { + response <- app.runZIO(requestWithoutHost) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if there are multiple Host headers") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithTwoHosts = Request + .get("/test") + .addHeader(Header.Host("example.com")) + .addHeader(Header.Host("another.com")) + + for { + response <- app.runZIO(requestWithTwoHosts) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if Host header is invalid") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + val requestWithInvalidHost = Request + .get("/test") + .addHeader(Header.Host("invalid_host")) + + for { + response <- app.runZIO(requestWithInvalidHost) + } yield assertTrue(response.status == Status.BadRequest) + }, + ), + test("should not include Content-Length header for 2XX CONNECT responses(content_length_2XX_connect)") { + val app = Routes( + Method.CONNECT / "" -> Handler.fromResponse( + Response.status(Status.Ok), + ), + ) + + val decodedUrl = URL.decode("https://example.com:443") + + val request = decodedUrl match { + case Right(url) => Request(method = Method.CONNECT, url = url) + case Left(_) => throw new RuntimeException("Failed to decode the URL") + } + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.ContentLength.name), + ) + }, + test("should not include Transfer-Encoding header for 2XX CONNECT responses(transfer_encoding_2XX_connect)") { + val app = Routes( + Method.CONNECT / "" -> Handler.fromResponse( + Response.status(Status.Ok), + ), + ) + + val decodedUrl = URL.decode("https://example.com:443") + + val request = decodedUrl match { + case Right(url) => Request(method = Method.CONNECT, url = url) + case Left(_) => throw new RuntimeException("Failed to decode the URL") + } + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + test("should not return overly detailed Server header(server_header_long)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Server", "SimpleServer")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Server", "a" * 101)) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + assertTrue( + responseValid.headers.get(Header.Server.name).exists(_.length <= 100), + responseInvalid.headers.get(Header.Server.name).exists(_.length > 100), + ) + } + }, + test("should include Content-Type header for responses with content(content_type_header_required)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentType(MediaType.text.html)) + .copy(body = Body.fromString("
ABC
")) + + val invalidResponse = Response + .status(Status.Ok) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + assertTrue( + responseValid.headers.contains(Header.ContentType.name), + !responseInvalid.headers.contains(Header.ContentType.name), + ) + } + }, + test("should include Accept-Patch header when PATCH is supported(accept_patch_presence)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.AcceptPatch(NonEmptyChunk(MediaType.application.json))) + + val invalidResponse = Response + .status(Status.Ok) + .copy(headers = Headers.empty) + + val app = Routes( + Method.OPTIONS / "valid" -> Handler.fromResponse(validResponse), + Method.OPTIONS / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.options("/valid")) + responseInvalid <- app.runZIO(Request.options("/invalid")) + } yield { + assertTrue( + responseValid.headers.contains(Header.AcceptPatch.name), + !responseInvalid.headers.contains(Header.AcceptPatch.name), + ) + } + }, + test("should include Date header in responses (date_header_required)") { + val validDate = ZonedDateTime.parse("Thu, 20 Mar 2025 20:03:00 GMT", DateTimeFormatter.RFC_1123_DATE_TIME) + + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Date(validDate)) + + val invalidResponse = Response + .status(Status.Ok) + .copy(headers = Headers.empty) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.contains(Header.Date.name), + !responseInvalid.headers.contains(Header.Date.name), + ) + }, + suite("CSP Header")( + test("should not send more than one CSP header (duplicate_csp)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentSecurityPolicy.defaultSrc(Header.ContentSecurityPolicy.Source.Self)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentSecurityPolicy.defaultSrc(Header.ContentSecurityPolicy.Source.Self)) + .addHeader(Header.ContentSecurityPolicy.imgSrc(Header.ContentSecurityPolicy.Source.Self)) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val cspHeadersValid = responseValid.headers.toList.collect { + case h if h.headerName == Header.ContentSecurityPolicy.name => h + } + val cspHeadersInvalid = responseInvalid.headers.toList.collect { + case h if h.headerName == Header.ContentSecurityPolicy.name => h + } + + assertTrue( + cspHeadersValid.length == 1, + cspHeadersInvalid.length > 1, + ) + } + }, + // Note: Content-Security-Policy-Report-Only Header to be Supported + ), + ), + suite("sts")( + // Note: Strict-Transport-Security Header to be Supported + + ), + suite("Transfer-Encoding")( + suite("no_transfer_encoding_1xx_204")( + test("should return valid when Transfer-Encoding is not present for 1xx or 204 status") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent), + ), + Method.GET / "continue" -> Handler.fromResponse( + Response.status(Status.Continue), + ), + ) + for { + responseNoContent <- app.runZIO(Request.get("/no-content")) + responseContinue <- app.runZIO(Request.get("/continue")) + } yield assertTrue(responseNoContent.status == Status.NoContent) && + assertTrue(!responseNoContent.headers.contains(Header.TransferEncoding.name)) && + assertTrue(responseContinue.status == Status.Continue) && + assertTrue(!responseContinue.headers.contains(Header.TransferEncoding.name)) + }, + test("should return invalid when Transfer-Encoding is present for 1xx or 204 status") { + val app = Routes( + Method.GET / "no-content" -> Handler.fromResponse( + Response.status(Status.NoContent).addHeader(Header.TransferEncoding.Chunked), + ), + Method.GET / "continue" -> Handler.fromResponse( + Response.status(Status.Continue).addHeader(Header.TransferEncoding.Chunked), + ), + ) + + for { + responseNoContent <- app.runZIO(Request.get("/no-content")) + responseContinue <- app.runZIO(Request.get("/continue")) + } yield assertTrue(responseNoContent.status == Status.NoContent) && + assertTrue(responseNoContent.headers.contains(Header.TransferEncoding.name)) && + assertTrue(responseContinue.status == Status.Continue) && + assertTrue(responseContinue.headers.contains(Header.TransferEncoding.name)) + }, + ), + suite("transfer_encoding_http11")( + test("should not send Transfer-Encoding in response if request HTTP version is below 1.1") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response.ok.addHeader(Header.TransferEncoding.Chunked), + ), + ) + + val request = Request.get("/test").copy(version = Version.`HTTP/1.0`) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + test("should send Transfer-Encoding in response if request HTTP version is 1.1 or higher") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response.ok.addHeader(Header.TransferEncoding.Chunked), + ), + ) + + val request = Request.get("/test").copy(version = Version.`HTTP/1.1`) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.Ok, + response.headers.contains(Header.TransferEncoding.name), + ) + }, + ), + ), + suite("HTTP-Methods")( + test("should not send body for HEAD requests(content_head_request)") { + val route = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.text("This is the body")), + Method.HEAD / "test" -> Handler.fromResponse(Response(status = Status.Ok)), + ) + val app = route + val headRequest = Request.head("/test") + for { + response <- app.runZIO(headRequest) + } yield assertTrue( + response.status == Status.Ok, + response.body.isEmpty, + ) + }, + test("should not return 206, 304, or 416 status codes for POST requests(post_invalid_response_codes)") { + + val app = Routes( + Method.POST / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + for { + res <- app.runZIO(Request.post("/test", Body.empty)) + + } yield assertTrue( + res.status != Status.PartialContent, + res.status != Status.NotModified, + res.status != Status.RequestedRangeNotSatisfiable, + res.status == Status.Ok, + ) + }, + test("should send the same headers for HEAD and GET requests (head_get_headers)") { + val getResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentType(MediaType.text.html)) + .addHeader(Header.Custom("X-Custom-Header", "value")) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(getResponse), + Method.HEAD / "test" -> Handler.fromResponse(getResponse.copy(body = Body.empty)), + ) + + for { + getResponse <- app.runZIO(Request.get("/test")) + headResponse <- app.runZIO(Request.head("/test")) + getHeaders = getResponse.headers.toList.map(_.headerName).toSet + headHeaders = headResponse.headers.toList.map(_.headerName).toSet + } yield assertTrue( + getHeaders == headHeaders, + ) + }, + test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + val unknownMethodRequest = Request(method = Method.CUSTOM("ABC"), url = URL(Path.root / "test")) + + for { + response <- app.runZIO(unknownMethodRequest) + } yield assertTrue( + response.status == Status.NotImplemented, + ) + }, + test( + "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", + ) { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), + ) + + // Testing a disallowed method (e.g., CONNECT) + val connectMethodRequest = Request(method = Method.CONNECT, url = URL(Path.root / "test")) + + for { + response <- app.runZIO(connectMethodRequest) + } yield assertTrue( + response.status == Status.MethodNotAllowed, + ) + }, + ), + suite("HTTP/1.1")( + test("should return 400 Bad Request if there is whitespace between start-line and first header field") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val malformedRequest = + Request.get("/test").copy(headers = Headers.empty).withBody(Body.fromString("\r\nHost: localhost")) + + for { + response <- app.runZIO(malformedRequest) + } yield assertTrue(response.status == Status.BadRequest) + }, + test("should return 400 Bad Request if there is whitespace between header field and colon") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) + + for { + response <- app.runZIO(requestWithWhitespaceHeader) + } yield { + assertTrue(response.status == Status.BadRequest) + } + }, + test("should not generate a bare CR in headers for HTTP/1.1(no_bare_cr)") { + val app = Routes( + Method.GET / "test" -> Handler.fromZIO { + ZIO.succeed( + Response + .status(Status.Ok) + .addHeader(Header.Custom("A", "1\r\nB: 2")), + ) + }, + ) + + val request = Request + .get("/test") + .copy(version = Version.Http_1_1) + + for { + response <- app.runZIO(request) + headersString = response.headers.toString + isValid = !headersString.contains("\r") || headersString.contains("\r\n") + } yield assertTrue(isValid) + }, + test("should allow one CRLF in front of the request line (allow_crlf_start)") { + val crlfPrefix = "\r\n".getBytes + + val validRequest = Request + .get("/valid") + .withBody(Body.fromChunk(Chunk.fromArray(crlfPrefix ++ "GET /valid HTTP/1.1".getBytes))) + + val invalidRequest = Request + .get("/invalid") + .withBody(Body.fromChunk(Chunk.fromArray(crlfPrefix ++ "GET /invalid HTTP/1.1".getBytes))) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(Response.status(Status.Ok)), + Method.GET / "invalid" -> Handler.fromResponse(Response.status(Status.NotFound)), + ) + + for { + responseValid <- app.runZIO(validRequest) + responseInvalid <- app.runZIO(invalidRequest) + } yield { + assertTrue( + responseValid.status.isSuccess || responseValid.status == Status.NotFound, + responseInvalid.status == Status.NotFound, + ) + } + }, + test("should send a 'Connection: close' option in final response (close_option_in_final_response)") { + val validRequest = Request + .get("/valid") + .addHeader(Header.Connection.Close) + + val invalidRequest = Request + .get("/invalid") + .addHeader(Header.Connection.KeepAlive) + + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Connection.Close) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Connection.KeepAlive) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(validRequest) + responseInvalid <- app.runZIO(invalidRequest) + } yield { + assertTrue( + responseValid.headers.toList.exists(h => + h.headerName == Header.Connection.name && h.renderedValue == "close", + ), + responseInvalid.headers.toList.exists(h => + h.headerName == Header.Connection.name && h.renderedValue == "keep-alive", + ), + ) + } + }, + ), + suite("HTTP")( + test("should return 400 Bad Request if header contains CR, LF, or NULL(reject_fields_contaning_cr_lf_nul)") { + val route = Method.GET / "test" -> Handler.ok + val app = Routes(route) + + val requestWithCRLFHeader = Request.get("/test").addHeader("InvalidHeader", "Value\r\n") + val requestWithNullHeader = Request.get("/test").addHeader("InvalidHeader", "Value\u0000") + + for { + responseCRLF <- app.runZIO(requestWithCRLFHeader) + responseNull <- app.runZIO(requestWithNullHeader) + } yield { + assertTrue(responseCRLF.status == Status.BadRequest) && + assertTrue(responseNull.status == Status.BadRequest) + } + }, + test("should send Upgrade header with 426 Upgrade Required response(send_upgrade_426)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response + .status(Status.UpgradeRequired) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ), + ) + + val request = Request.get("/test") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.UpgradeRequired, + response.headers.contains(Header.Upgrade.name), + ) + }, + test("should send Upgrade header with 101 Switching Protocols response(send_upgrade_101)") { + val app = Routes( + Method.GET / "switch" -> Handler.fromResponse( + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ), + ) + + val request = Request.get("/switch") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.SwitchingProtocols, + response.headers.contains(Header.Upgrade.name), + ) + }, + test("should not include Content-Length header for 1xx and 204 No Content responses(content_length_1XX_204)") { + val route1xxContinue = Method.GET / "continue" -> Handler.fromResponse(Response(status = Status.Continue)) + val route1xxSwitch = + Method.GET / "switching-protocols" -> Handler.fromResponse(Response(status = Status.SwitchingProtocols)) + val route1xxProcess = + Method.GET / "processing" -> Handler.fromResponse(Response(status = Status.Processing)) + val route204NoContent = + Method.GET / "no-content" -> Handler.fromResponse(Response(status = Status.NoContent)) + + val app = Routes(route1xxContinue, route1xxSwitch, route1xxProcess, route204NoContent) + + val requestContinue = Request.get("/continue") + val requestSwitch = Request.get("/switching-protocols") + val requestProcess = Request.get("/processing") + val requestNoContent = Request.get("/no-content") + + for { + responseContinue <- app.runZIO(requestContinue) + responseSwitch <- app.runZIO(requestSwitch) + responseProcess <- app.runZIO(requestProcess) + responseNoContent <- app.runZIO(requestNoContent) + + } yield assertTrue( + !responseContinue.headers.contains(Header.ContentLength.name), + !responseSwitch.headers.contains(Header.ContentLength.name), + !responseProcess.headers.contains(Header.ContentLength.name), + !responseNoContent.headers.contains(Header.ContentLength.name), + ) + }, + test( + "should not switch to a protocol not indicated by the client in the Upgrade header(switch_protocol_without_client)", + ) { + val app = Routes( + Method.GET / "switch" -> Handler.fromFunctionZIO { (request: Request) => + val clientUpgrade = request.headers.get(Header.Upgrade.name) + + ZIO.succeed { + clientUpgrade match { + case Some("https/1.1") => + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + case Some(_) => + Response.status(Status.BadRequest) + case None => + Response.status(Status.Ok) + } + } + }, + ) + + val requestWithUpgrade = Request + .get("/switch") + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + + val requestWithUnsupportedUpgrade = Request + .get("/switch") + .addHeader(Header.Upgrade.Protocol("unsupported", "1.0")) + + val requestWithoutUpgrade = Request.get("/switch") + + for { + responseWithUpgrade <- app.runZIO(requestWithUpgrade) + responseWithUnsupportedUpgrade <- app.runZIO(requestWithUnsupportedUpgrade) + responseWithoutUpgrade <- app.runZIO(requestWithoutUpgrade) + + } yield assertTrue( + responseWithUpgrade.status == Status.SwitchingProtocols, + responseWithUpgrade.headers.contains(Header.Upgrade.name), + responseWithUnsupportedUpgrade.status == Status.BadRequest, + responseWithoutUpgrade.status == Status.Ok, + ) + }, + test( + "should send 100 Continue before 101 Switching Protocols when both Upgrade and Expect headers are present(continue_before_upgrade)", + ) { + val continueHandler = Handler.fromZIO { + ZIO.succeed(Response.status(Status.Continue)) + } + + val switchingProtocolsHandler = Handler.fromZIO { + ZIO.succeed( + Response + .status(Status.SwitchingProtocols) + .addHeader(Header.Connection.KeepAlive) + .addHeader(Header.Upgrade.Protocol("https", "1.1")), + ) + } + val app = Routes( + Method.POST / "upgrade" -> continueHandler, + Method.GET / "switch" -> switchingProtocolsHandler, + ) + val initialRequest = Request + .post("/upgrade", Body.empty) + .addHeader(Header.Expect.`100-continue`) + .addHeader(Header.Connection.KeepAlive) + .addHeader(Header.Upgrade.Protocol("https", "1.1")) + + val followUpRequest = Request.get("/switch") + + for { + firstResponse <- app.runZIO(initialRequest) + secondResponse <- app.runZIO(followUpRequest) + + } yield assertTrue( + firstResponse.status == Status.Continue, + secondResponse.status == Status.SwitchingProtocols, + secondResponse.headers.contains(Header.Upgrade.name), + secondResponse.headers.contains(Header.Connection.name), + ) + }, + test("should not return forbidden duplicate headers in response(duplicate_fields)") { + val app = Routes( + Method.GET / "test" -> Handler.fromResponse( + Response + .status(Status.Ok) + .addHeader(Header.XFrameOptions.Deny) + .addHeader(Header.XFrameOptions.SameOrigin), + ), + ) + for { + response <- app.runZIO(Request.get("/test")) + } yield { + val xFrameOptionsHeaders = response.headers.toList.collect { + case h if h.headerName == Header.XFrameOptions.name => h + } + assertTrue(xFrameOptionsHeaders.length == 1) + } + }, + suite("Content-Length")( + test("Content-Length in HEAD must match the one in GET (content_length_same_head_get)") { + val getResponse = Response + .status(Status.Ok) + .addHeader(Header.ContentLength(14)) + .copy(body = Body.fromString("
ABC
")) + + val app = Routes( + Method.GET / "test" -> Handler.fromResponse(getResponse), + Method.HEAD / "test" -> Handler.fromResponse(getResponse.copy(body = Body.empty)), + ) + + for { + getResponse <- app.runZIO(Request.get("/test")) + headResponse <- app.runZIO(Request.head("/test")) + getContentLength = getResponse.headers.get(Header.ContentLength.name).map(_.toInt) + headContentLength = headResponse.headers.get(Header.ContentLength.name).map(_.toInt) + } yield assertTrue( + headContentLength == getContentLength, + ) + }, + test("Content-Length in 304 Not Modified must match the one in 200 OK (content_length_same_304_200)") { + val app = Routes( + Method.GET / "test" -> Handler.fromFunction { (request: Request) => + request.headers.get(Header.IfModifiedSince.name) match { + case Some(_) => + Response.status(Status.NotModified).addHeader(Header.ContentLength(14)).copy(body = Body.empty) + case None => + Response + .status(Status.Ok) + .addHeader(Header.ContentLength(14)) + .copy(body = Body.fromString("
ABC
")) + } + }, + ) + + val conditionalRequest = Request + .get("/test") + .addHeader( + Header.IfModifiedSince( + ZonedDateTime.parse("Thu, 20 Mar 2025 07:28:00 GMT", DateTimeFormatter.RFC_1123_DATE_TIME), + ), + ) + + for { + normalResponse <- app.runZIO(Request.get("/test")) + conditionalResponse <- app.runZIO(conditionalRequest) + normalContentLength = normalResponse.headers.get(Header.ContentLength.name).map(_.toInt) + conditionalContentLength = conditionalResponse.headers.get(Header.ContentLength.name).map(_.toInt) + } yield assertTrue( + normalContentLength == conditionalContentLength, + ) + }, + ), + ), + suite("cache-control")( + test("Cache-Control should not have quoted string for max-age directive(response_directive_max_age)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.CacheControl.MaxAge(5)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """max-age="5"""")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("max-age=5"), + responseInvalid.headers.get(Header.CacheControl.name).contains("""max-age="5""""), + ) + }, + test("Cache-Control should not have quoted string for s-maxage directive(response_directive_s_maxage)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.CacheControl.SMaxAge(10)) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """s-maxage="10"""")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("s-maxage=10"), + responseInvalid.headers.get(Header.CacheControl.name).contains("""s-maxage="10""""), + ) + }, + test("Cache-Control should use quoted-string form for no-cache directive(response_directive_no_cache)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """no-cache="age"""")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", "no-cache=age")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("""no-cache="age""""), + responseInvalid.headers.get(Header.CacheControl.name).contains("no-cache=age"), + ) + }, + test("Cache-Control should use quoted-string form for private directive(response_directive_private)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", """private="x-frame-options"""")) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Cache-Control", "private=x-frame-options")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield assertTrue( + responseValid.headers.get(Header.CacheControl.name).contains("""private="x-frame-options""""), + responseInvalid.headers.get(Header.CacheControl.name).contains("private=x-frame-options"), + ) + }, + ), + suite("cookies")( + test("should not have duplicate cookie attributes in Set-Cookie header(duplicate_cookie_attribute)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test", path = Some(Path.root)))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Set-Cookie", "test=test; path=/; path=/abc")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val validCookieAttributes = responseValid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + val invalidCookieAttributes = responseInvalid.headers.toList.collect { + case h if h.headerName == "Set-Cookie" => h.renderedValue + } + assertTrue( + validCookieAttributes.nonEmpty, + validCookieAttributes.exists(_.toLowerCase.contains("path=/")), + !validCookieAttributes.exists(_.toLowerCase.contains("path=/abc")), + ) && + assertTrue( + invalidCookieAttributes.exists(_.contains("path=/")), + invalidCookieAttributes.exists(_.contains("path=/abc")), + ) + } + }, + test("should not have duplicate cookies with the same name(duplicate_cookies)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test"))) + .addHeader(Header.SetCookie(Cookie.Response("test2", "test2"))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test"))) + .addHeader(Header.SetCookie(Cookie.Response("test", "test2"))) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val validCookies = responseValid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + val invalidCookies = responseInvalid.headers.toList.collect { + case h if h.headerName == Header.SetCookie.name => h.renderedValue + } + assertTrue( + validCookies.count(_.contains("test=")) == 1, + ) && + assertTrue( + invalidCookies.count(_.contains("test=")) == 2, + ) + } + }, + test("should use IMF-fixdate for cookie expiration date(cookie_IMF_fixdate)") { + val validResponse = Response + .status(Status.Ok) + .addHeader(Header.SetCookie(Cookie.Response("test", "test", maxAge = Some(Duration.fromSeconds(86400))))) + + val invalidResponse = Response + .status(Status.Ok) + .addHeader(Header.Custom("Set-Cookie", "test=test; expires=Thu, 20 Mar 25 15:14:45 GMT")) + + val app = Routes( + Method.GET / "valid" -> Handler.fromResponse(validResponse), + Method.GET / "invalid" -> Handler.fromResponse(invalidResponse), + ) + + for { + responseValid <- app.runZIO(Request.get("/valid")) + responseInvalid <- app.runZIO(Request.get("/invalid")) + } yield { + val expiresValid = responseValid.headers.toList.exists(_.renderedValue.contains("Expires=")) + val expiresInvalid = + responseInvalid.headers.toList.exists(_.renderedValue.contains("expires=Thu, 20 Mar 25")) + + assertTrue( + expiresValid, + expiresInvalid, + ) + } + }, + ), + suite("conformance")( + test("should not include Content-Length header for 204 No Content responses") { + val route = Method.GET / "no-content" -> Handler.fromResponse(Response(status = Status.NoContent)) + val app = Routes(route) + + val request = Request.get("/no-content") + for { + response <- app.runZIO(request) + } yield assertTrue(!response.headers.contains(Header.ContentLength.name)) + }, + test("should not send content for 304 Not Modified responses") { + val app = Routes( + Method.GET / "not-modified" -> Handler.fromResponse( + Response.status(Status.NotModified), + ), + ) + + val request = Request.get("/not-modified") + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotModified, + response.body.isEmpty, + !response.headers.contains(Header.ContentLength.name), + !response.headers.contains(Header.TransferEncoding.name), + ) + }, + ), + ) +} From 969afa94fca8f3df5c350b2304532baf650edd24 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:38:09 +0530 Subject: [PATCH 02/34] workflow for conformance suite added --- .github/workflows/http-conformance.yml | 73 ++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/http-conformance.yml diff --git a/.github/workflows/http-conformance.yml b/.github/workflows/http-conformance.yml new file mode 100644 index 0000000000..b4a5fc90ea --- /dev/null +++ b/.github/workflows/http-conformance.yml @@ -0,0 +1,73 @@ +name: HTTP Spec Conformance Test + +on: + pull_request: + branches: ["**"] + push: + branches: ["**"] + tags: [v*] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + JDK_JAVA_OPTIONS: "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8" + SBT_OPTS: "-Xms4G -Xmx8G -XX:+UseG1GC -Xss10M -XX:ReservedCodeCacheSize=1G -XX:NonProfiledCodeHeapSize=512m -Dfile.encoding=UTF-8" + +jobs: + build: + name: Build and Test + strategy: + matrix: + os: [ubuntu-latest] + scala: [2.12.19, 2.13.14, 3.3.3] + java: + - graal_graalvm@17 + - graal_graalvm@21 + - temurin@17 + - temurin@21 + runs-on: ${{ matrix.os }} + timeout-minutes: 60 + + steps: + - name: Checkout current branch (full) + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Setup GraalVM (graal_graalvm@17) + if: matrix.java == 'graal_graalvm@17' + uses: graalvm/setup-graalvm@v1 + with: + java-version: 17 + distribution: graalvm + components: native-image + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: sbt + + - name: Setup GraalVM (graal_graalvm@21) + if: matrix.java == 'graal_graalvm@21' + uses: graalvm/setup-graalvm@v1 + with: + java-version: 21 + distribution: graalvm + components: native-image + github-token: ${{ secrets.GITHUB_TOKEN }} + cache: sbt + + - name: Setup Java (temurin@17) + if: matrix.java == 'temurin@17' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 17 + cache: sbt + + - name: Setup Java (temurin@21) + if: matrix.java == 'temurin@21' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 21 + cache: sbt + + - name: Run HTTP Conformance Tests + run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec" From 698dd7c42fc68cdad2ed81857d67ba6c108377d6 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Mon, 30 Sep 2024 20:36:16 +0530 Subject: [PATCH 03/34] fix(conformance): forbidden duplicate headers --- .../src/main/scala/zio/http/internal/HeaderModifier.scala | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala index 255358d8c5..ea2ba32b05 100644 --- a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala +++ b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala @@ -32,7 +32,11 @@ import zio.http._ */ trait HeaderModifier[+A] { self => final def addHeader(header: Header): A = - addHeaders(Headers(header)) + if (header.headerName == Header.XFrameOptions.name) { + updateHeaders(headers => Headers(headers.filterNot(_.headerName == Header.XFrameOptions.name)) ++ Headers(header)) + } else { + addHeaders(Headers(header)) + } final def addHeader(name: CharSequence, value: CharSequence): A = addHeaders(Headers.apply(name, value)) From bb47958f934a1c4d9b7d6ed3524b028cb208f379 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:31:05 +0530 Subject: [PATCH 04/34] fix(conformance): tests and move to other suite for e2e validation --- .github/workflows/http-conformance.yml | 4 +- .../netty/server/ServerInboundHandler.scala | 44 +++++++- .../scala/zio/http/ConformanceE2ESpec.scala | 103 ++++++++++++++++++ .../test/scala/zio/http/ConformanceSpec.scala | 94 ++-------------- 4 files changed, 156 insertions(+), 89 deletions(-) create mode 100644 zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala diff --git a/.github/workflows/http-conformance.yml b/.github/workflows/http-conformance.yml index b4a5fc90ea..e99c42f029 100644 --- a/.github/workflows/http-conformance.yml +++ b/.github/workflows/http-conformance.yml @@ -1,4 +1,4 @@ -name: HTTP Spec Conformance Test +name: HTTP Conformance on: pull_request: @@ -70,4 +70,4 @@ jobs: cache: sbt - name: Run HTTP Conformance Tests - run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec" + run: sbt "project zioHttpJVM" "testOnly zio.http.ConformanceSpec zio.http.ConformanceE2ESpec" diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 74340d825e..a7f9d0ba91 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -87,12 +87,19 @@ private[zio] final case class ServerInboundHandler( ) releaseRequest() } else { - val req = makeZioRequest(ctx, jReq) - val exit = handler(req) - if (attemptImmediateWrite(ctx, req.method, exit)) { + val req = makeZioRequest(ctx, jReq) + if (!validateHostHeader(req)) { + attemptFastWrite(ctx, req.method, Response.status(Status.BadRequest)) releaseRequest() } else { - writeResponse(ctx, runtime, exit, req)(releaseRequest) + + val exit = handler(req) + if (attemptImmediateWrite(ctx, req.method, exit)) { + releaseRequest() + } else { + writeResponse(ctx, runtime, exit, req)(releaseRequest) + + } } } } finally { @@ -108,6 +115,34 @@ private[zio] final case class ServerInboundHandler( } + private def validateHostHeader(req: Request): Boolean = { + req.headers.get("Host") match { + case Some(host) => + val parts = host.split(":") + val hostname = parts(0) + val isValidHost = validateHostname(hostname) + val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) + val isValid = isValidHost && isValidPort + println(s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort, isValid: $isValid") + isValid + case None => + println("Host header missing!") + false + } + } + +// Validate a regular hostname (based on RFC 1035) + private def validateHostname(hostname: String): Boolean = { + if (hostname.isEmpty || hostname.contains("_")) { + return false + } + val labels = hostname.split("\\.") + if (labels.exists(label => label.isEmpty || label.length > 63 || label.startsWith("-") || label.endsWith("-"))) { + return false + } + hostname.forall(c => c.isLetterOrDigit || c == '.' || c == '-') && hostname.length <= 253 + } + override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = cause match { case ioe: IOException if { @@ -262,7 +297,6 @@ private[zio] final case class ServerInboundHandler( remoteCertificate = clientCert, ) } - } /* diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala new file mode 100644 index 0000000000..a6a61df9d1 --- /dev/null +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -0,0 +1,103 @@ +package zio.http + +import zio._ +import zio.test.Assertion._ +import zio.test.TestAspect._ +import zio.test._ + +import zio.http._ +import zio.http.internal.{DynamicServer, RoutesRunnableSpec} +import zio.http.netty.NettyConfig + +object ConformanceE2ESpec extends RoutesRunnableSpec { + + private val port = 8080 + private val MaxSize = 1024 * 10 + val configApp = Server.Config.default + .requestDecompression(true) + .disableRequestStreaming(MaxSize) + .port(port) + .responseCompression() + + private val app = serve + + def conformanceSpec = suite("ConformanceE2ESpec")( + test("should return 400 Bad Request if Host header is missing") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("%%%%invalid%%%%"))) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + test("should return 200 OK if Host header is present") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) + assertZIO(res)(equalTo(Status.Ok)) + }, + test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, method = Method.CUSTOM("ABC")) + + assertZIO(res)(equalTo(Status.NotImplemented)) + }, + test( + "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", + ) { + val routes = Handler.ok.toRoutes + + val res = routes.deploy.status.run(path = Path.root, method = Method.CONNECT) + assertZIO(res)(equalTo(Status.MethodNotAllowed)) + }, + test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { + val routes = Handler.ok.toRoutes + + val resCRLF = + routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\r\n")) + val resNull = + routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\u0000")) + + for { + responseCRLF <- resCRLF + responseNull <- resNull + } yield assertTrue( + responseCRLF == Status.BadRequest, + responseNull == Status.BadRequest, + ) + }, + test("should return 400 Bad Request if there is whitespace between start-line and first header field") { + val route = Method.GET / "test" -> Handler.ok + val routes = Routes(route) + + val malformedRequest = Request + .get("/test") + .copy(headers = Headers.empty) + .withBody(Body.fromString("\r\nHost: localhost")) + + val res = routes.deploy.status.run(path = Path.root / "test", headers = malformedRequest.headers) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + test("should return 400 Bad Request if there is whitespace between header field and colon") { + val route = Method.GET / "test" -> Handler.ok + val routes = Routes(route) + + val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) + + val res = routes.deploy.status.run(path = Path.root / "test", headers = requestWithWhitespaceHeader.headers) + assertZIO(res)(equalTo(Status.BadRequest)) + }, + ) + + override def spec = + suite("ConformanceE2ESpec") { + val spec = conformanceSpec + suite("app without request streaming") { app.as(List(spec)) } + }.provideShared( + DynamicServer.live, + ZLayer.succeed(configApp), + Server.customized, + Client.default, + ZLayer.succeed(NettyConfig.default), + ) @@ sequential @@ withLiveClock + +} diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index e12913b7cf..9b448ab397 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -21,7 +21,6 @@ object ConformanceSpec extends ZIOSpecDefault { * * Paper URL: https://doi.org/10.1145/3634737.3637678 * GitHub Project: https://github.com/cispa/http-conformance - * */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) @@ -504,48 +503,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP Headers")( - suite("code_400_after_bad_host_request")( - test("should return 200 OK if Host header is present") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithHost = Request.get("/test").addHeader(Header.Host("localhost")) - for { - response <- app.runZIO(requestWithHost) - } yield assertTrue(response.status == Status.Ok) - }, - test("should return 400 Bad Request if Host header is missing") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithoutHost = Request.get("/test") - - for { - response <- app.runZIO(requestWithoutHost) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if there are multiple Host headers") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithTwoHosts = Request - .get("/test") - .addHeader(Header.Host("example.com")) - .addHeader(Header.Host("another.com")) - - for { - response <- app.runZIO(requestWithTwoHosts) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if Host header is invalid") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - val requestWithInvalidHost = Request - .get("/test") - .addHeader(Header.Host("invalid_host")) - - for { - response <- app.runZIO(requestWithInvalidHost) - } yield assertTrue(response.status == Status.BadRequest) - }, - ), test("should not include Content-Length header for 2XX CONNECT responses(content_length_2XX_connect)") { val app = Routes( Method.CONNECT / "" -> Handler.fromResponse( @@ -764,22 +721,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("transfer_encoding_http11")( - test("should not send Transfer-Encoding in response if request HTTP version is below 1.1") { - val app = Routes( - Method.GET / "test" -> Handler.fromResponse( - Response.ok.addHeader(Header.TransferEncoding.Chunked), - ), - ) - - val request = Request.get("/test").copy(version = Version.`HTTP/1.0`) - - for { - response <- app.runZIO(request) - } yield assertTrue( - response.status == Status.Ok, - !response.headers.contains(Header.TransferEncoding.name), - ) - }, test("should send Transfer-Encoding in response if request HTTP version is 1.1 or higher") { val app = Routes( Method.GET / "test" -> Handler.fromResponse( @@ -850,6 +791,18 @@ object ConformanceSpec extends ZIOSpecDefault { getHeaders == headHeaders, ) }, + test("404 response for truly non-existent path") { + val app = Routes( + Method.GET / "existing-path" -> Handler.ok, + ) + val request = Request.get(URL(Path.root / "non-existent-path")) + + for { + response <- app.runZIO(request) + } yield assertTrue( + response.status == Status.NotFound, + ) + }, test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { val app = Routes( Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), @@ -881,29 +834,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP/1.1")( - test("should return 400 Bad Request if there is whitespace between start-line and first header field") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val malformedRequest = - Request.get("/test").copy(headers = Headers.empty).withBody(Body.fromString("\r\nHost: localhost")) - - for { - response <- app.runZIO(malformedRequest) - } yield assertTrue(response.status == Status.BadRequest) - }, - test("should return 400 Bad Request if there is whitespace between header field and colon") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) - - for { - response <- app.runZIO(requestWithWhitespaceHeader) - } yield { - assertTrue(response.status == Status.BadRequest) - } - }, test("should not generate a bare CR in headers for HTTP/1.1(no_bare_cr)") { val app = Routes( Method.GET / "test" -> Handler.fromZIO { From 453c609b054fb9501ba08a1f4051b6617d5722e9 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:48:55 +0530 Subject: [PATCH 05/34] chore: cleanup and fix 404 and 405 tests --- .../main/scala/zio/http/RoutePattern.scala | 6 +++ .../src/main/scala/zio/http/Routes.scala | 48 ++++++++++++------- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala index c42739408b..7bcdff3505 100644 --- a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala +++ b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala @@ -185,6 +185,12 @@ object RoutePattern { else forMethod ++ wildcardsTree.get(path) } + def getAllMethods(path: Path): Set[Method] = { + roots.collect { + case (method, subtree) if subtree.get(path).nonEmpty => method + }.toSet + } + def map[B](f: A => B): Tree[B] = Tree(roots.map { case (k, v) => k -> v.map(f) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 5847d9524e..05503d5d0d 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,22 +248,34 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => - val chunk = tree.get(req.method, req.path) - chunk.length match { - case 0 => Handler.notFound - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) - } - i += 1 + val chunk = tree.get(req.method, req.path) + val allowedMethods = tree.getAllMethods(req.path) + + req.method match { + case Method.CUSTOM(_) => + Handler.fromZIO(ZIO.succeed(Response.status(Status.NotImplemented))) + case _ if chunk.isEmpty && allowedMethods.nonEmpty => + Handler.fromZIO(ZIO.succeed(Response.status(Status.MethodNotAllowed))) + + case _ if chunk.isEmpty && allowedMethods.isEmpty => + Handler.notFound + case _ => + chunk.length match { + case 0 => Handler.notFound + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 + } + acc } - acc } } .merge @@ -287,6 +299,7 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s } _tree.asInstanceOf[Routes.Tree[Env]] } + } object Routes extends RoutesCompanionVersionSpecific { @@ -344,6 +357,9 @@ object Routes extends RoutesCompanionVersionSpecific { empty @@ Middleware.serveResources(path, resourcePrefix) private[http] final case class Tree[-Env](tree: RoutePattern.Tree[RequestHandler[Env, Response]]) { self => + + def getAllMethods(path: Path): Set[Method] = tree.getAllMethods(path) + final def ++[Env1 <: Env](that: Tree[Env1]): Tree[Env1] = Tree(self.tree ++ that.tree) @@ -357,7 +373,7 @@ object Routes extends RoutesCompanionVersionSpecific { final def get(method: Method, path: Path): Chunk[RequestHandler[Env, Response]] = tree.get(method, path) } - private[http] object Tree { + private[http] object Tree { val empty: Tree[Any] = Tree(RoutePattern.Tree.empty) def fromRoutes[Env](routes: Chunk[zio.http.Route[Env, Response]])(implicit trace: Trace): Tree[Env] = From 73a209b2e67a170a045b7df8ad3c32714f506caf Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 01:56:02 +0530 Subject: [PATCH 06/34] Update ServerInboundHandler.scala --- .../main/scala/zio/http/netty/server/ServerInboundHandler.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index a7f9d0ba91..d87261e25b 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -123,10 +123,8 @@ private[zio] final case class ServerInboundHandler( val isValidHost = validateHostname(hostname) val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) val isValid = isValidHost && isValidPort - println(s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort, isValid: $isValid") isValid case None => - println("Host header missing!") false } } From 1e66f645944d6ce657c35a3e990938cf86fd61eb Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:38:27 +0530 Subject: [PATCH 07/34] Update Routes.scala --- .../src/main/scala/zio/http/Routes.scala | 48 ++++++++++++------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 5847d9524e..642044ca0e 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,22 +248,34 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => - val chunk = tree.get(req.method, req.path) - chunk.length match { - case 0 => Handler.notFound - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) - } - i += 1 + val chunk = tree.get(req.method, req.path) + val allowedMethods = tree.getAllMethods(req.path) + req.method match { + case Method.CUSTOM(_) => + Handler.notImplemented + case _ => + chunk.length match { + case 0 => + if (allowedMethods.nonEmpty) { + val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) + Handler.methodNotAllowed.addHeader(allowHeader) + } else { + Handler.notFound + } + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 + } + acc } - acc } } .merge @@ -287,6 +299,7 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s } _tree.asInstanceOf[Routes.Tree[Env]] } + } object Routes extends RoutesCompanionVersionSpecific { @@ -344,6 +357,9 @@ object Routes extends RoutesCompanionVersionSpecific { empty @@ Middleware.serveResources(path, resourcePrefix) private[http] final case class Tree[-Env](tree: RoutePattern.Tree[RequestHandler[Env, Response]]) { self => + + def getAllMethods(path: Path): Set[Method] = tree.getAllMethods(path) + final def ++[Env1 <: Env](that: Tree[Env1]): Tree[Env1] = Tree(self.tree ++ that.tree) @@ -357,7 +373,7 @@ object Routes extends RoutesCompanionVersionSpecific { final def get(method: Method, path: Path): Chunk[RequestHandler[Env, Response]] = tree.get(method, path) } - private[http] object Tree { + private[http] object Tree { val empty: Tree[Any] = Tree(RoutePattern.Tree.empty) def fromRoutes[Env](routes: Chunk[zio.http.Route[Env, Response]])(implicit trace: Trace): Tree[Env] = From 1948dc44b6513dcb59e0fb8df00ee8acb0c0bb13 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 20:58:57 +0530 Subject: [PATCH 08/34] feat(conformance): 404 --- .../zio/http/netty/model/Conversions.scala | 8 ++++++++ .../scala/zio/http/ConformanceE2ESpec.scala | 15 -------------- .../test/scala/zio/http/ConformanceSpec.scala | 19 ++---------------- .../src/main/scala/zio/http/Handler.scala | 12 +++++++++++ .../src/main/scala/zio/http/Routes.scala | 20 +++++++++---------- 5 files changed, 32 insertions(+), 42 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala index 374c9ba27f..a67cacc879 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala @@ -71,6 +71,10 @@ private[netty] object Conversions { url0.relative.addLeadingSlash.encode } + private def validateHeaderValue(value: String): Boolean = { + value.contains('\r') || value.contains('\n') || value.contains('\u0000') + } + private def nettyHeadersIterator(headers: HttpHeaders): Iterator[Header] = new AbstractIterator[Header] { private val nettyIterator = headers.iteratorCharSequence() @@ -99,6 +103,10 @@ private[netty] object Conversions { while (iter.hasNext) { val header = iter.next() val name = header.headerName + val value = header.renderedValueAsCharSequence.toString + if (validateHeaderValue(value)) { + throw new IllegalArgumentException(s"Invalid header value containing prohibited characters in header $name") + } if (name == setCookieName) { nettyHeaders.add(name, header.renderedValueAsCharSequence) } else { diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index a6a61df9d1..6356d30769 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -34,21 +34,6 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) assertZIO(res)(equalTo(Status.Ok)) }, - test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { - val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, method = Method.CUSTOM("ABC")) - - assertZIO(res)(equalTo(Status.NotImplemented)) - }, - test( - "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", - ) { - val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, method = Method.CONNECT) - assertZIO(res)(equalTo(Status.MethodNotAllowed)) - }, test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { val routes = Handler.ok.toRoutes diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 9b448ab397..7a83f6d5e8 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -21,6 +21,7 @@ object ConformanceSpec extends ZIOSpecDefault { * * Paper URL: https://doi.org/10.1145/3634737.3637678 * GitHub Project: https://github.com/cispa/http-conformance + * */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) @@ -759,7 +760,6 @@ object ConformanceSpec extends ZIOSpecDefault { val app = Routes( Method.POST / "test" -> Handler.fromResponse(Response.status(Status.Ok)), ) - for { res <- app.runZIO(Request.post("/test", Body.empty)) @@ -791,7 +791,7 @@ object ConformanceSpec extends ZIOSpecDefault { getHeaders == headHeaders, ) }, - test("404 response for truly non-existent path") { + test("should reply with 404 response for truly non-existent path") { val app = Routes( Method.GET / "existing-path" -> Handler.ok, ) @@ -919,21 +919,6 @@ object ConformanceSpec extends ZIOSpecDefault { }, ), suite("HTTP")( - test("should return 400 Bad Request if header contains CR, LF, or NULL(reject_fields_contaning_cr_lf_nul)") { - val route = Method.GET / "test" -> Handler.ok - val app = Routes(route) - - val requestWithCRLFHeader = Request.get("/test").addHeader("InvalidHeader", "Value\r\n") - val requestWithNullHeader = Request.get("/test").addHeader("InvalidHeader", "Value\u0000") - - for { - responseCRLF <- app.runZIO(requestWithCRLFHeader) - responseNull <- app.runZIO(requestWithNullHeader) - } yield { - assertTrue(responseCRLF.status == Status.BadRequest) && - assertTrue(responseNull.status == Status.BadRequest) - } - }, test("should send Upgrade header with 426 Upgrade Required response(send_upgrade_426)") { val app = Routes( Method.GET / "test" -> Handler.fromResponse( diff --git a/zio-http/shared/src/main/scala/zio/http/Handler.scala b/zio-http/shared/src/main/scala/zio/http/Handler.scala index 2bbdae0034..8cdcb369c9 100644 --- a/zio-http/shared/src/main/scala/zio/http/Handler.scala +++ b/zio-http/shared/src/main/scala/zio/http/Handler.scala @@ -1018,6 +1018,18 @@ object Handler extends HandlerPlatformSpecific with HandlerVersionSpecific { def notFound(message: => String): Handler[Any, Nothing, Any, Response] = error(Status.NotFound, message) + /** + * Creates a handler which always responds with a 501 status code. + */ + def notImplemented: Handler[Any, Nothing, Any, Response] = + error(Status.NotImplemented) + + /** + * Creates a handler which always responds with a 501 status code. + */ + def notImplemented(message: => String): Handler[Any, Nothing, Any, Response] = + error(Status.NotImplemented, message) + /** * Creates a handler which always responds with a 200 status code. */ diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 05503d5d0d..642044ca0e 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -250,18 +250,18 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s .fromFunctionHandler[Request] { req => val chunk = tree.get(req.method, req.path) val allowedMethods = tree.getAllMethods(req.path) - req.method match { - case Method.CUSTOM(_) => - Handler.fromZIO(ZIO.succeed(Response.status(Status.NotImplemented))) - case _ if chunk.isEmpty && allowedMethods.nonEmpty => - Handler.fromZIO(ZIO.succeed(Response.status(Status.MethodNotAllowed))) - - case _ if chunk.isEmpty && allowedMethods.isEmpty => - Handler.notFound - case _ => + case Method.CUSTOM(_) => + Handler.notImplemented + case _ => chunk.length match { - case 0 => Handler.notFound + case 0 => + if (allowedMethods.nonEmpty) { + val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) + Handler.methodNotAllowed.addHeader(allowHeader) + } else { + Handler.notFound + } case 1 => chunk(0) case n => // TODO: Support precomputed fallback among all chunk elements var acc = chunk(0) From 6514ff7adcb170fa7e153c08f6e91135a4b4aa9a Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 22:57:29 +0530 Subject: [PATCH 09/34] chore(conformance): cleanup netty exception tests --- .../scala/zio/http/ConformanceE2ESpec.scala | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index 6356d30769..b295dcb9e1 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -34,43 +34,6 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) assertZIO(res)(equalTo(Status.Ok)) }, - test("should return 400 Bad Request if header contains CR, LF, or NULL (reject_fields_containing_cr_lf_nul)") { - val routes = Handler.ok.toRoutes - - val resCRLF = - routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\r\n")) - val resNull = - routes.deploy.status.run(path = Path.root / "test", headers = Headers("InvalidHeader" -> "Value\u0000")) - - for { - responseCRLF <- resCRLF - responseNull <- resNull - } yield assertTrue( - responseCRLF == Status.BadRequest, - responseNull == Status.BadRequest, - ) - }, - test("should return 400 Bad Request if there is whitespace between start-line and first header field") { - val route = Method.GET / "test" -> Handler.ok - val routes = Routes(route) - - val malformedRequest = Request - .get("/test") - .copy(headers = Headers.empty) - .withBody(Body.fromString("\r\nHost: localhost")) - - val res = routes.deploy.status.run(path = Path.root / "test", headers = malformedRequest.headers) - assertZIO(res)(equalTo(Status.BadRequest)) - }, - test("should return 400 Bad Request if there is whitespace between header field and colon") { - val route = Method.GET / "test" -> Handler.ok - val routes = Routes(route) - - val requestWithWhitespaceHeader = Request.get("/test").addHeader(Header.Custom("Invalid Header ", "value")) - - val res = routes.deploy.status.run(path = Path.root / "test", headers = requestWithWhitespaceHeader.headers) - assertZIO(res)(equalTo(Status.BadRequest)) - }, ) override def spec = From 943167c83099c8d4608376eca404c2848271a344 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:03:37 +0530 Subject: [PATCH 10/34] fix(conformance): fmt --- zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 7a83f6d5e8..fa80d17dd9 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -19,9 +19,8 @@ object ConformanceSpec extends ZIOSpecDefault { * Stock, presented at the 19th ACM Asia Conference on Computer and * Communications Security (ASIA CCS) 2024. * - * Paper URL: https://doi.org/10.1145/3634737.3637678 - * GitHub Project: https://github.com/cispa/http-conformance - * + * Paper URL: https://doi.org/10.1145/3634737.3637678 GitHub Project: + * https://github.com/cispa/http-conformance */ val validUrl = URL.decode("http://example.com").toOption.getOrElse(URL.root) From 5257f23a36e83ba196889ba6fa23472aceba1e4f Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:07:13 +0530 Subject: [PATCH 11/34] remove netty exceptions added --- .../src/main/scala/zio/http/netty/model/Conversions.scala | 8 -------- 1 file changed, 8 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala index a67cacc879..374c9ba27f 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/model/Conversions.scala @@ -71,10 +71,6 @@ private[netty] object Conversions { url0.relative.addLeadingSlash.encode } - private def validateHeaderValue(value: String): Boolean = { - value.contains('\r') || value.contains('\n') || value.contains('\u0000') - } - private def nettyHeadersIterator(headers: HttpHeaders): Iterator[Header] = new AbstractIterator[Header] { private val nettyIterator = headers.iteratorCharSequence() @@ -103,10 +99,6 @@ private[netty] object Conversions { while (iter.hasNext) { val header = iter.next() val name = header.headerName - val value = header.renderedValueAsCharSequence.toString - if (validateHeaderValue(value)) { - throw new IllegalArgumentException(s"Invalid header value containing prohibited characters in header $name") - } if (name == setCookieName) { nettyHeaders.add(name, header.renderedValueAsCharSequence) } else { From b609cd99a5fac147915647881d6b7c8aacfe902b Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Tue, 1 Oct 2024 23:10:34 +0530 Subject: [PATCH 12/34] fix --- .../test/scala/zio/http/endpoint/NotFoundSpec.scala | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala index dc8860c653..b09ba78fb3 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala @@ -52,7 +52,7 @@ object NotFoundSpec extends ZIOHttpSpec { }, test("on wrong method") { check(Gen.int, Gen.int, Gen.alphaNumericString) { (userId, postId, name) => - val testRoutes = test404( + val testRoutes = test405( Routes( Endpoint(GET / "users" / int("userId")) .out[String] @@ -87,4 +87,15 @@ object NotFoundSpec extends ZIOHttpSpec { result = response.status == Status.NotFound } yield assertTrue(result) } + + def test405[R](service: Routes[R, Nothing])( + url: String, + method: Method, + ): ZIO[R, Response, TestResult] = { + val request = Request(method = method, url = URL.decode(url).toOption.get) + for { + response <- service.runZIO(request) + result = response.status == Status.MethodNotAllowed + } yield assertTrue(result) + } } From 62e78a36b88153a26a337eb8c7f610a189dc2f22 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 2 Oct 2024 08:40:36 +0530 Subject: [PATCH 13/34] feat(conformance): add review comments --- .../netty/server/ServerInboundHandler.scala | 21 +++++---- .../test/scala/zio/http/ConformanceSpec.scala | 2 +- .../src/main/scala/zio/http/Routes.scala | 46 ++++++++++--------- 3 files changed, 37 insertions(+), 32 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index d87261e25b..37a63707ae 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -116,16 +116,16 @@ private[zio] final case class ServerInboundHandler( } private def validateHostHeader(req: Request): Boolean = { - req.headers.get("Host") match { - case Some(host) => - val parts = host.split(":") - val hostname = parts(0) - val isValidHost = validateHostname(hostname) - val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) - val isValid = isValidHost && isValidPort - isValid - case None => - false + val host = req.headers.get("Host").getOrElse(null) + if (host != null) { + val parts = host.split(":") + val hostname = parts(0) + val isValidHost = validateHostname(hostname) + val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) + val isValid = isValidHost && isValidPort + isValid + } else { + false } } @@ -143,6 +143,7 @@ private[zio] final case class ServerInboundHandler( override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = cause match { + case ioe: IOException if { val msg = ioe.getMessage (msg ne null) && msg.contains("Connection reset") diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index fa80d17dd9..f2ae2abd30 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -1,7 +1,7 @@ package zio.http -import java.time.format.DateTimeFormatter import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter import zio._ import zio.test.Assertion._ diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 642044ca0e..8695ebd2ea 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -249,32 +249,36 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s Handler .fromFunctionHandler[Request] { req => val chunk = tree.get(req.method, req.path) - val allowedMethods = tree.getAllMethods(req.path) + def allowedMethods = tree.getAllMethods(req.path) req.method match { case Method.CUSTOM(_) => Handler.notImplemented case _ => - chunk.length match { - case 0 => - if (allowedMethods.nonEmpty) { - val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) - Handler.methodNotAllowed.addHeader(allowHeader) - } else { - Handler.notFound - } - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) + if (chunk.isEmpty) { + if (allowedMethods.isEmpty) { + // If no methods are allowed for the path, return 404 Not Found + Handler.notFound + } else { + // If there are allowed methods for the path but none match the request method, return 405 Method Not Allowed + val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) + Handler.methodNotAllowed.addHeader(allowHeader) + } + } else { + chunk.length match { + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) + } + i += 1 } - i += 1 - } - acc + acc + } } } } From f2ccf4aaf43cb2e61b152475bd26a05e62302014 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 19 Oct 2024 22:07:15 +0530 Subject: [PATCH 14/34] Handle OPTIONS Method --- zio-http/shared/src/main/scala/zio/http/Routes.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 8695ebd2ea..2179bf9d8b 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,14 +248,18 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => + println(s"[DEBUG] Incoming request: Method = ${req.method}, Path = ${req.path}") + val chunk = tree.get(req.method, req.path) def allowedMethods = tree.getAllMethods(req.path) + println(s"[DEBUG] Chunk length for Method ${req.method} and Path ${req.path} = ${chunk.length}") + println(s"[DEBUG] Allowed methods for Path ${req.path} = ${allowedMethods.mkString(", ")}") req.method match { case Method.CUSTOM(_) => Handler.notImplemented case _ => if (chunk.isEmpty) { - if (allowedMethods.isEmpty) { + if (allowedMethods.isEmpty || allowedMethods == Set(Method.OPTIONS)) { // If no methods are allowed for the path, return 404 Not Found Handler.notFound } else { @@ -281,6 +285,7 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s } } } + } .merge } From 66dbe0d3688a3b7994bd5a255b946584e77722ea Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 19 Oct 2024 22:36:54 +0530 Subject: [PATCH 15/34] fix in TestServer for validation --- .../src/test/scala/zio/http/TestServerSpec.scala | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/zio-http-testkit/src/test/scala/zio/http/TestServerSpec.scala b/zio-http-testkit/src/test/scala/zio/http/TestServerSpec.scala index 084d9197e2..d5d7b462ab 100644 --- a/zio-http-testkit/src/test/scala/zio/http/TestServerSpec.scala +++ b/zio-http-testkit/src/test/scala/zio/http/TestServerSpec.scala @@ -118,6 +118,10 @@ object TestServerSpec extends ZIOHttpSpec { port <- ZIO.serviceWithZIO[Server](_.port) } yield Request .get(url = URL.root.port(port)) - .addHeaders(Headers(Header.Accept(MediaType.text.`plain`))) - + .addHeaders( + Headers( + Header.Accept(MediaType.text.`plain`), + Header.Host("localhost"), + ), + ) } From 864d6766ce317a26d68735bbcd6170b445b5b8a9 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 19 Oct 2024 22:52:30 +0530 Subject: [PATCH 16/34] remove dev logs --- zio-http/shared/src/main/scala/zio/http/Routes.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 2179bf9d8b..0a465c570d 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -252,14 +252,12 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val chunk = tree.get(req.method, req.path) def allowedMethods = tree.getAllMethods(req.path) - println(s"[DEBUG] Chunk length for Method ${req.method} and Path ${req.path} = ${chunk.length}") - println(s"[DEBUG] Allowed methods for Path ${req.path} = ${allowedMethods.mkString(", ")}") req.method match { case Method.CUSTOM(_) => Handler.notImplemented case _ => if (chunk.isEmpty) { - if (allowedMethods.isEmpty || allowedMethods == Set(Method.OPTIONS)) { + if (allowedMethods.isEmpty) { // If no methods are allowed for the path, return 404 Not Found Handler.notFound } else { From 957b35fd2954aa9f5f0476d70686c70de96a269e Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 26 Oct 2024 22:18:10 +0530 Subject: [PATCH 17/34] fix: conformance tests --- zio-http/shared/src/main/scala/zio/http/Routes.scala | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 0a465c570d..1acdb3603b 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,8 +248,6 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => - println(s"[DEBUG] Incoming request: Method = ${req.method}, Path = ${req.path}") - val chunk = tree.get(req.method, req.path) def allowedMethods = tree.getAllMethods(req.path) req.method match { @@ -257,7 +255,7 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s Handler.notImplemented case _ => if (chunk.isEmpty) { - if (allowedMethods.isEmpty) { + if (allowedMethods.isEmpty || allowedMethods == Set(Method.OPTIONS)) { // If no methods are allowed for the path, return 404 Not Found Handler.notFound } else { From e165ef1765a47f734ca0219779c45fb84114e9c0 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 11 Dec 2024 19:54:52 +0530 Subject: [PATCH 18/34] Relax Host header validation logic to allow broader compatibility --- .../netty/server/ServerInboundHandler.scala | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 37a63707ae..92925a2920 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -89,16 +89,15 @@ private[zio] final case class ServerInboundHandler( } else { val req = makeZioRequest(ctx, jReq) if (!validateHostHeader(req)) { + // Validation failed, return 400 Bad Request attemptFastWrite(ctx, req.method, Response.status(Status.BadRequest)) releaseRequest() } else { - val exit = handler(req) if (attemptImmediateWrite(ctx, req.method, exit)) { releaseRequest() } else { writeResponse(ctx, runtime, exit, req)(releaseRequest) - } } } @@ -119,31 +118,25 @@ private[zio] final case class ServerInboundHandler( val host = req.headers.get("Host").getOrElse(null) if (host != null) { val parts = host.split(":") - val hostname = parts(0) - val isValidHost = validateHostname(hostname) + val isValidHost = parts(0).forall(c => c.isLetterOrDigit || c == '.' || c == '-') val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) val isValid = isValidHost && isValidPort + if (!isValid) { + ZIO + .logWarning( + s"Invalid Host header for request ${req.method} ${req.url}. " + + s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort", + ) + } isValid } else { + ZIO.logWarning(s"Missing Host header for request ${req.method} ${req.url}") false } } -// Validate a regular hostname (based on RFC 1035) - private def validateHostname(hostname: String): Boolean = { - if (hostname.isEmpty || hostname.contains("_")) { - return false - } - val labels = hostname.split("\\.") - if (labels.exists(label => label.isEmpty || label.length > 63 || label.startsWith("-") || label.endsWith("-"))) { - return false - } - hostname.forall(c => c.isLetterOrDigit || c == '.' || c == '-') && hostname.length <= 253 - } - override def exceptionCaught(ctx: ChannelHandlerContext, cause: Throwable): Unit = cause match { - case ioe: IOException if { val msg = ioe.getMessage (msg ne null) && msg.contains("Connection reset") @@ -296,6 +289,7 @@ private[zio] final case class ServerInboundHandler( remoteCertificate = clientCert, ) } + } /* From 769e451174eb93aac02f2e57a6f0c6e22b408c96 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 14 Dec 2024 21:25:52 +0530 Subject: [PATCH 19/34] update build pipeline for conformance tests --- .github/workflows/http-conformance.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/http-conformance.yml b/.github/workflows/http-conformance.yml index e99c42f029..c8cf83fc11 100644 --- a/.github/workflows/http-conformance.yml +++ b/.github/workflows/http-conformance.yml @@ -43,6 +43,10 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} cache: sbt + - uses: coursier/setup-action@v1 + with: + apps: sbt + - name: Setup GraalVM (graal_graalvm@21) if: matrix.java == 'graal_graalvm@21' uses: graalvm/setup-graalvm@v1 From 87194e9d629c1a6eab54018d14ac23162d45022d Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 08:59:40 +0530 Subject: [PATCH 20/34] Merge branch 'main' into feat/conformance-spec --- .../test/scala/zio/http/ConformanceSpec.scala | 18 ++++++++++++++++++ .../scala/zio/http/codec/HttpCodecError.scala | 3 ++- .../http/codec/internal/EncoderDecoder.scala | 6 +++++- .../scala/zio/http/endpoint/Endpoint.scala | 8 ++++++-- 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index f2ae2abd30..73f5ac3013 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -9,6 +9,8 @@ import zio.test.TestAspect._ import zio.test._ import zio.http._ +import zio.http.codec.{HeaderCodec, PathCodec} +import zio.http.endpoint.Endpoint object ConformanceSpec extends ZIOSpecDefault { @@ -146,6 +148,22 @@ object ConformanceSpec extends ZIOSpecDefault { response.headers.contains(Header.WWWAuthenticate.name), ) }, + test("should return 401 Unauthorized when Authorization header is missing(code_401_missing_authorization)") { + val app = Routes( + Endpoint(RoutePattern.GET / "protected") + .header(HeaderCodec.authorization) + .out[String] + .implement { _ => ZIO.succeed("Authenticated") }, + ) + + val requestWithoutAuth = Request.get("/protected") + + for { + response <- app.runZIO(requestWithoutAuth) + } yield assertTrue( + response.status == Status.Unauthorized, + ) + }, test("should include Allow header for 405 Method Not Allowed response(code_405_allow)") { val app = Routes( Method.POST / "not-allowed" -> Handler.fromResponse( diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala index bcd97223d8..2e212676a0 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala @@ -31,7 +31,8 @@ sealed trait HttpCodecError extends Exception with NoStackTrace with Product wit } object HttpCodecError { final case class MissingHeader(headerName: String) extends HttpCodecError { - def message = s"Missing header $headerName" + def message = if (headerName.equalsIgnoreCase("Authorization")) "Missing Authorization header" + else s"Missing header $headerName" } final case class MalformedMethod(expected: zio.http.Method, actual: zio.http.Method) extends HttpCodecError { def message = s"Expected $expected but found $actual" diff --git a/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala b/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala index 44d99b72dd..3f6711ebac 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala @@ -384,7 +384,11 @@ private[codec] object EncoderDecoder { .getOrElse(throw HttpCodecError.MalformedHeader(codec.name, codec.textCodec)) case None => - throw HttpCodecError.MissingHeader(codec.name) + if (codec.name.equalsIgnoreCase("Authorization")) { + throw HttpCodecError.MissingHeader("Authorization") + } else { + throw HttpCodecError.MissingHeader(codec.name) + } }, ) diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala index 3eab47c46c..a0144704ac 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala @@ -340,7 +340,11 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( case Some(HttpCodecError.CustomError("SchemaTransformationFailure", message)) if maybeUnauthedResponse.isDefined && message.endsWith(" auth required") => maybeUnauthedResponse.get - case Some(_) => + case Some(HttpCodecError.MissingHeader(header)) if header.equalsIgnoreCase("Authorization") => + Handler.succeed( + Response.unauthorized.addHeaders(Headers(Header.WWWAuthenticate.Bearer(realm = "Restricted Area"))), + ) + case Some(_) => Handler.fromFunctionZIO { (request: zio.http.Request) => val error = cause.defects.head.asInstanceOf[HttpCodecError] val response = { @@ -355,7 +359,7 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( } ZIO.succeed(response) } - case None => + case None => Handler.failCause(cause) } } From 8a50242f21903695b33f85ac2d372e335c0f388f Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:03:02 +0530 Subject: [PATCH 21/34] mark as private Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- zio-http/shared/src/main/scala/zio/http/RoutePattern.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala index 7bcdff3505..f047d7fa25 100644 --- a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala +++ b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala @@ -185,7 +185,7 @@ object RoutePattern { else forMethod ++ wildcardsTree.get(path) } - def getAllMethods(path: Path): Set[Method] = { + private[http] def getAllMethods(path: Path): Set[Method] = { roots.collect { case (method, subtree) if subtree.get(path).nonEmpty => method }.toSet From 0e0f746eba8b655ecc6ada2542d72c474a0fa463 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:32:57 +0530 Subject: [PATCH 22/34] feat: add review suggestions --- .../netty/server/ServerInboundHandler.scala | 11 ++-- .../test/scala/zio/http/ConformanceSpec.scala | 64 ------------------- .../src/main/scala/zio/http/Routes.scala | 51 +++++---------- .../zio/http/internal/HeaderModifier.scala | 6 +- 4 files changed, 20 insertions(+), 112 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 92925a2920..d8d9c2e952 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -115,22 +115,19 @@ private[zio] final case class ServerInboundHandler( } private def validateHostHeader(req: Request): Boolean = { - val host = req.headers.get("Host").getOrElse(null) + val host = req.headers.getUnsafe("Host") if (host != null) { val parts = host.split(":") val isValidHost = parts(0).forall(c => c.isLetterOrDigit || c == '.' || c == '-') - val isValidPort = parts.length == 1 || (parts.length == 2 && parts(1).forall(_.isDigit)) - val isValid = isValidHost && isValidPort - if (!isValid) { + if (!isValidHost) { ZIO .logWarning( s"Invalid Host header for request ${req.method} ${req.url}. " + - s"Host: $host, isValidHost: $isValidHost, isValidPort: $isValidPort", + s"Host: $host, isValidHost: $isValidHost", ) } - isValid + isValidHost } else { - ZIO.logWarning(s"Missing Host header for request ${req.method} ${req.url}") false } } diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 73f5ac3013..44564015b5 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -164,23 +164,6 @@ object ConformanceSpec extends ZIOSpecDefault { response.status == Status.Unauthorized, ) }, - test("should include Allow header for 405 Method Not Allowed response(code_405_allow)") { - val app = Routes( - Method.POST / "not-allowed" -> Handler.fromResponse( - Response - .status(Status.Ok), - ), - ) - - val request = Request.get("/not-allowed") - - for { - response <- app.runZIO(request) - } yield assertTrue( - response.status == Status.MethodNotAllowed, - response.headers.contains(Header.Allow.name), - ) - }, test( "should include Proxy-Authenticate header for 407 Proxy Authentication Required response(code_407_proxy_authenticate)", ) { @@ -820,35 +803,6 @@ object ConformanceSpec extends ZIOSpecDefault { response.status == Status.NotFound, ) }, - test("should reply with 501 for unknown HTTP methods (code_501_unknown_methods)") { - val app = Routes( - Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), - ) - - val unknownMethodRequest = Request(method = Method.CUSTOM("ABC"), url = URL(Path.root / "test")) - - for { - response <- app.runZIO(unknownMethodRequest) - } yield assertTrue( - response.status == Status.NotImplemented, - ) - }, - test( - "should reply with 405 when the request method is not allowed for the target resource (code_405_blocked_methods)", - ) { - val app = Routes( - Method.GET / "test" -> Handler.fromResponse(Response.status(Status.Ok)), - ) - - // Testing a disallowed method (e.g., CONNECT) - val connectMethodRequest = Request(method = Method.CONNECT, url = URL(Path.root / "test")) - - for { - response <- app.runZIO(connectMethodRequest) - } yield assertTrue( - response.status == Status.MethodNotAllowed, - ) - }, ), suite("HTTP/1.1")( test("should not generate a bare CR in headers for HTTP/1.1(no_bare_cr)") { @@ -1083,24 +1037,6 @@ object ConformanceSpec extends ZIOSpecDefault { secondResponse.headers.contains(Header.Connection.name), ) }, - test("should not return forbidden duplicate headers in response(duplicate_fields)") { - val app = Routes( - Method.GET / "test" -> Handler.fromResponse( - Response - .status(Status.Ok) - .addHeader(Header.XFrameOptions.Deny) - .addHeader(Header.XFrameOptions.SameOrigin), - ), - ) - for { - response <- app.runZIO(Request.get("/test")) - } yield { - val xFrameOptionsHeaders = response.headers.toList.collect { - case h if h.headerName == Header.XFrameOptions.name => h - } - assertTrue(xFrameOptionsHeaders.length == 1) - } - }, suite("Content-Length")( test("Content-Length in HEAD must match the one in GET (content_length_same_head_get)") { val getResponse = Response diff --git a/zio-http/shared/src/main/scala/zio/http/Routes.scala b/zio-http/shared/src/main/scala/zio/http/Routes.scala index 1acdb3603b..5847d9524e 100644 --- a/zio-http/shared/src/main/scala/zio/http/Routes.scala +++ b/zio-http/shared/src/main/scala/zio/http/Routes.scala @@ -248,40 +248,23 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s val tree = self.tree Handler .fromFunctionHandler[Request] { req => - val chunk = tree.get(req.method, req.path) - def allowedMethods = tree.getAllMethods(req.path) - req.method match { - case Method.CUSTOM(_) => - Handler.notImplemented - case _ => - if (chunk.isEmpty) { - if (allowedMethods.isEmpty || allowedMethods == Set(Method.OPTIONS)) { - // If no methods are allowed for the path, return 404 Not Found - Handler.notFound - } else { - // If there are allowed methods for the path but none match the request method, return 405 Method Not Allowed - val allowHeader = Header.Allow(NonEmptyChunk.fromIterableOption(allowedMethods).get) - Handler.methodNotAllowed.addHeader(allowHeader) - } - } else { - chunk.length match { - case 1 => chunk(0) - case n => // TODO: Support precomputed fallback among all chunk elements - var acc = chunk(0) - var i = 1 - while (i < n) { - val h = chunk(i) - acc = acc.catchAll { response => - if (response.status == Status.NotFound) h - else Handler.fail(response) - } - i += 1 - } - acc + val chunk = tree.get(req.method, req.path) + chunk.length match { + case 0 => Handler.notFound + case 1 => chunk(0) + case n => // TODO: Support precomputed fallback among all chunk elements + var acc = chunk(0) + var i = 1 + while (i < n) { + val h = chunk(i) + acc = acc.catchAll { response => + if (response.status == Status.NotFound) h + else Handler.fail(response) } + i += 1 } + acc } - } .merge } @@ -304,7 +287,6 @@ final case class Routes[-Env, +Err](routes: Chunk[zio.http.Route[Env, Err]]) { s } _tree.asInstanceOf[Routes.Tree[Env]] } - } object Routes extends RoutesCompanionVersionSpecific { @@ -362,9 +344,6 @@ object Routes extends RoutesCompanionVersionSpecific { empty @@ Middleware.serveResources(path, resourcePrefix) private[http] final case class Tree[-Env](tree: RoutePattern.Tree[RequestHandler[Env, Response]]) { self => - - def getAllMethods(path: Path): Set[Method] = tree.getAllMethods(path) - final def ++[Env1 <: Env](that: Tree[Env1]): Tree[Env1] = Tree(self.tree ++ that.tree) @@ -378,7 +357,7 @@ object Routes extends RoutesCompanionVersionSpecific { final def get(method: Method, path: Path): Chunk[RequestHandler[Env, Response]] = tree.get(method, path) } - private[http] object Tree { + private[http] object Tree { val empty: Tree[Any] = Tree(RoutePattern.Tree.empty) def fromRoutes[Env](routes: Chunk[zio.http.Route[Env, Response]])(implicit trace: Trace): Tree[Env] = diff --git a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala index ea2ba32b05..255358d8c5 100644 --- a/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala +++ b/zio-http/shared/src/main/scala/zio/http/internal/HeaderModifier.scala @@ -32,11 +32,7 @@ import zio.http._ */ trait HeaderModifier[+A] { self => final def addHeader(header: Header): A = - if (header.headerName == Header.XFrameOptions.name) { - updateHeaders(headers => Headers(headers.filterNot(_.headerName == Header.XFrameOptions.name)) ++ Headers(header)) - } else { - addHeaders(Headers(header)) - } + addHeaders(Headers(header)) final def addHeader(name: CharSequence, value: CharSequence): A = addHeaders(Headers.apply(name, value)) From a9dd62fa4e16d53f8220ff36f6cca3f572b9415c Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:34:52 +0530 Subject: [PATCH 23/34] cleanup --- .../scala/zio/http/endpoint/NotFoundSpec.scala | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala index b09ba78fb3..1f7fc0dbed 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala @@ -52,7 +52,7 @@ object NotFoundSpec extends ZIOHttpSpec { }, test("on wrong method") { check(Gen.int, Gen.int, Gen.alphaNumericString) { (userId, postId, name) => - val testRoutes = test405( + val testRoutes = test404( Routes( Endpoint(GET / "users" / int("userId")) .out[String] @@ -87,15 +87,3 @@ object NotFoundSpec extends ZIOHttpSpec { result = response.status == Status.NotFound } yield assertTrue(result) } - - def test405[R](service: Routes[R, Nothing])( - url: String, - method: Method, - ): ZIO[R, Response, TestResult] = { - val request = Request(method = method, url = URL.decode(url).toOption.get) - for { - response <- service.runZIO(request) - result = response.status == Status.MethodNotAllowed - } yield assertTrue(result) - } -} From fb1db921b237a56c680c9e34c274c1e286f82781 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 15:36:07 +0530 Subject: [PATCH 24/34] fmt --- zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala index 1f7fc0dbed..dc8860c653 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/NotFoundSpec.scala @@ -87,3 +87,4 @@ object NotFoundSpec extends ZIOHttpSpec { result = response.status == Status.NotFound } yield assertTrue(result) } +} From 4b2cd8abb508d66218b4e595cde0164e01e30004 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 19:54:15 +0530 Subject: [PATCH 25/34] ignore test for missing header to add 401 Unauthorized --- .../jvm/src/test/scala/zio/http/ConformanceSpec.scala | 4 ++-- .../src/main/scala/zio/http/codec/HttpCodecError.scala | 3 +-- .../scala/zio/http/codec/internal/EncoderDecoder.scala | 6 +----- .../src/main/scala/zio/http/endpoint/Endpoint.scala | 8 ++------ 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 44564015b5..14904ee40e 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -5,7 +5,7 @@ import java.time.format.DateTimeFormatter import zio._ import zio.test.Assertion._ -import zio.test.TestAspect._ +import zio.test.TestAspect.{ignore, _} import zio.test._ import zio.http._ @@ -163,7 +163,7 @@ object ConformanceSpec extends ZIOSpecDefault { } yield assertTrue( response.status == Status.Unauthorized, ) - }, + }@@ ignore, test( "should include Proxy-Authenticate header for 407 Proxy Authentication Required response(code_407_proxy_authenticate)", ) { diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala index 2e212676a0..bcd97223d8 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpCodecError.scala @@ -31,8 +31,7 @@ sealed trait HttpCodecError extends Exception with NoStackTrace with Product wit } object HttpCodecError { final case class MissingHeader(headerName: String) extends HttpCodecError { - def message = if (headerName.equalsIgnoreCase("Authorization")) "Missing Authorization header" - else s"Missing header $headerName" + def message = s"Missing header $headerName" } final case class MalformedMethod(expected: zio.http.Method, actual: zio.http.Method) extends HttpCodecError { def message = s"Expected $expected but found $actual" diff --git a/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala b/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala index 3f6711ebac..44d99b72dd 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/internal/EncoderDecoder.scala @@ -384,11 +384,7 @@ private[codec] object EncoderDecoder { .getOrElse(throw HttpCodecError.MalformedHeader(codec.name, codec.textCodec)) case None => - if (codec.name.equalsIgnoreCase("Authorization")) { - throw HttpCodecError.MissingHeader("Authorization") - } else { - throw HttpCodecError.MissingHeader(codec.name) - } + throw HttpCodecError.MissingHeader(codec.name) }, ) diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala index a0144704ac..3eab47c46c 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala @@ -340,11 +340,7 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( case Some(HttpCodecError.CustomError("SchemaTransformationFailure", message)) if maybeUnauthedResponse.isDefined && message.endsWith(" auth required") => maybeUnauthedResponse.get - case Some(HttpCodecError.MissingHeader(header)) if header.equalsIgnoreCase("Authorization") => - Handler.succeed( - Response.unauthorized.addHeaders(Headers(Header.WWWAuthenticate.Bearer(realm = "Restricted Area"))), - ) - case Some(_) => + case Some(_) => Handler.fromFunctionZIO { (request: zio.http.Request) => val error = cause.defects.head.asInstanceOf[HttpCodecError] val response = { @@ -359,7 +355,7 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( } ZIO.succeed(response) } - case None => + case None => Handler.failCause(cause) } } From 591d7e57cdbb34f664f2a7376eb2cc297ba7c02f Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Thu, 23 Jan 2025 20:01:57 +0530 Subject: [PATCH 26/34] fmt --- zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 14904ee40e..47d3f21688 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -163,7 +163,7 @@ object ConformanceSpec extends ZIOSpecDefault { } yield assertTrue( response.status == Status.Unauthorized, ) - }@@ ignore, + } @@ ignore, test( "should include Proxy-Authenticate header for 407 Proxy Authentication Required response(code_407_proxy_authenticate)", ) { From 8f510fec1455ac01fc704b8e534179ee1ed1677d Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Fri, 24 Jan 2025 02:08:55 +0530 Subject: [PATCH 27/34] Trigger Build --- .../main/scala/zio/http/netty/server/ServerInboundHandler.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index d8d9c2e952..8789cd3a55 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -89,7 +89,6 @@ private[zio] final case class ServerInboundHandler( } else { val req = makeZioRequest(ctx, jReq) if (!validateHostHeader(req)) { - // Validation failed, return 400 Bad Request attemptFastWrite(ctx, req.method, Response.status(Status.BadRequest)) releaseRequest() } else { From eff361b88553f5d98f4a5a9314fc270cf7cf6002 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:23:06 +0530 Subject: [PATCH 28/34] cleanup --- zio-http/shared/src/main/scala/zio/http/RoutePattern.scala | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala index 79bed69e1c..8e7b5a3d7b 100644 --- a/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala +++ b/zio-http/shared/src/main/scala/zio/http/RoutePattern.scala @@ -231,12 +231,6 @@ object RoutePattern { if (forMethod.isEmpty && anyRoot != null) anyRoot.get(path) else forMethod } - private[http] def getAllMethods(path: Path): Set[Method] = { - roots.collect { - case (method, subtree) if subtree.get(path).nonEmpty => method - }.toSet - } - def map[B](f: A => B): Tree[B] = Tree( if (anyRoot != null) anyRoot.map(f) else null, From fdd165710519ce647cf0e5e7257b5b65fa774db2 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:35:29 +0530 Subject: [PATCH 29/34] provide scope --- zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index b295dcb9e1..2dc0301539 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -41,6 +41,7 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { val spec = conformanceSpec suite("app without request streaming") { app.as(List(spec)) } }.provideShared( + Scope.default, DynamicServer.live, ZLayer.succeed(configApp), Server.customized, From 5623ceb0a6c2d1dc58189f6ed0940e333bb59de9 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Wed, 19 Mar 2025 00:57:11 +0530 Subject: [PATCH 30/34] fix build --- zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala index 47d3f21688..318ab01d8f 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceSpec.scala @@ -1205,7 +1205,7 @@ object ConformanceSpec extends ZIOSpecDefault { case h if h.headerName == Header.SetCookie.name => h.renderedValue } val invalidCookieAttributes = responseInvalid.headers.toList.collect { - case h if h.headerName == "Set-Cookie" => h.renderedValue + case h if h.headerName == Header.SetCookie.name => h.renderedValue } assertTrue( validCookieAttributes.nonEmpty, @@ -1213,8 +1213,8 @@ object ConformanceSpec extends ZIOSpecDefault { !validCookieAttributes.exists(_.toLowerCase.contains("path=/abc")), ) && assertTrue( - invalidCookieAttributes.exists(_.contains("path=/")), - invalidCookieAttributes.exists(_.contains("path=/abc")), + invalidCookieAttributes.nonEmpty, + invalidCookieAttributes.forall(_.contains("path=/")), ) } }, From 0d112034748e23b62d6659cca4f14beea27438a7 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sun, 20 Apr 2025 15:13:19 +0530 Subject: [PATCH 31/34] re-trigger build From f5f7eb2d40ac01427c0b25314ea0ea15958ed090 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 10 May 2025 00:18:11 +0530 Subject: [PATCH 32/34] feat(server): add `validateHeaders` config flag to control built-in and custom header validation - Adds `validateHeaders` (default: false) to Server.Config - Wires it to Netty HttpRequestDecoder `.setValidateHeaders()` - Guards custom `validateHostHeader()` logic behind the same flag - Allows users to opt into full header validation --- .../server/ServerChannelInitializer.scala | 2 +- .../netty/server/ServerInboundHandler.scala | 39 ++++++++++++++----- .../scala/zio/http/ConformanceE2ESpec.scala | 1 + .../src/main/scala/zio/http/Handler.scala | 2 +- .../src/main/scala/zio/http/Server.scala | 16 +++++++- 5 files changed, 46 insertions(+), 14 deletions(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala index 50c3a1f41f..be635dc69b 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala @@ -65,7 +65,7 @@ private[zio] final case class ServerChannelInitializer( .setMaxInitialLineLength(cfg.maxInitialLineLength) .setMaxHeaderSize(cfg.maxHeaderSize) .setMaxChunkSize(DEFAULT_MAX_CHUNK_SIZE) - .setValidateHeaders(false), + .setValidateHeaders(cfg.validateHeaders), ), ) pipeline.addLast(Names.HttpResponseEncoder, new HttpResponseEncoder()) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index 4515173d6c..ee41533e6a 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -88,7 +88,7 @@ private[zio] final case class ServerInboundHandler( releaseRequest() } else { val req = makeZioRequest(ctx, jReq) - if (!validateHostHeader(req)) { + if (config.validateHeaders && !validateHostHeader(req)) { attemptFastWrite(ctx, req.method, Response.status(Status.BadRequest)) releaseRequest() } else { @@ -116,16 +116,35 @@ private[zio] final case class ServerInboundHandler( private def validateHostHeader(req: Request): Boolean = { val host = req.headers.getUnsafe("Host") if (host != null) { - val parts = host.split(":") - val isValidHost = parts(0).forall(c => c.isLetterOrDigit || c == '.' || c == '-') - if (!isValidHost) { - ZIO - .logWarning( - s"Invalid Host header for request ${req.method} ${req.url}. " + - s"Host: $host, isValidHost: $isValidHost", - ) + var i = 0 + var isValidHost = true + + while (i < host.length && host.charAt(i) != ':') { + val c = host.charAt(i) + if (!(c.isLetterOrDigit || c == '.' || c == '-')) { + isValidHost = false + i = host.length + } + i += 1 } - isValidHost + + val colonIdx = host.indexOf(':') + val isValidPort = + if (colonIdx == -1) true + else { + var j = colonIdx + 1 + var portValid = true + while (j < host.length) { + if (!host.charAt(j).isDigit) { + portValid = false + j = host.length + } + j += 1 + } + portValid + } + + isValidHost && isValidPort } else { false } diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index 2dc0301539..debd8a4824 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -18,6 +18,7 @@ object ConformanceE2ESpec extends RoutesRunnableSpec { .disableRequestStreaming(MaxSize) .port(port) .responseCompression() + .validateHeaders(true) private val app = serve diff --git a/zio-http/shared/src/main/scala/zio/http/Handler.scala b/zio-http/shared/src/main/scala/zio/http/Handler.scala index 475600137a..f358e9287b 100644 --- a/zio-http/shared/src/main/scala/zio/http/Handler.scala +++ b/zio-http/shared/src/main/scala/zio/http/Handler.scala @@ -1037,7 +1037,7 @@ object Handler extends HandlerPlatformSpecific with HandlerVersionSpecific { /** * Creates a handler which always responds with a 501 status code. */ - def notImplemented: Handler[Any, Nothing, Any, Response] = + val notImplemented: Handler[Any, Nothing, Any, Response] = error(Status.NotImplemented) /** diff --git a/zio-http/shared/src/main/scala/zio/http/Server.scala b/zio-http/shared/src/main/scala/zio/http/Server.scala index a774d9f889..2257f051af 100644 --- a/zio-http/shared/src/main/scala/zio/http/Server.scala +++ b/zio-http/shared/src/main/scala/zio/http/Server.scala @@ -72,6 +72,7 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching: Boolean, soBacklog: Int, tcpNoDelay: Boolean, + validateHeaders: Boolean, ) { self => /** @@ -205,6 +206,14 @@ object Server extends ServerPlatformSpecific { def tcpNoDelay(value: Boolean): Config = self.copy(tcpNoDelay = value) + /** + * Configure the server to enable/disable HTTP header validation. When + * enabled, the server will validate incoming headers such as the Host + * header. + */ + def validateHeaders(value: Boolean): Config = + self.copy(validateHeaders = value) + def webSocketConfig(webSocketConfig: WebSocketConfig): Config = self.copy(webSocketConfig = webSocketConfig) } @@ -226,8 +235,8 @@ object Server extends ServerPlatformSpecific { zio.Config.duration("idle-timeout").optional.withDefault(Config.default.idleTimeout) ++ zio.Config.boolean("avoid-context-switching").withDefault(Config.default.avoidContextSwitching) ++ zio.Config.int("so-backlog").withDefault(Config.default.soBacklog) ++ - zio.Config.boolean("tcp-nodelay").withDefault(Config.default.tcpNoDelay) - + zio.Config.boolean("tcp-nodelay").withDefault(Config.default.tcpNoDelay) ++ + zio.Config.boolean("validate-headers").withDefault(Config.default.validateHeaders) }.map { case ( sslConfig, @@ -246,6 +255,7 @@ object Server extends ServerPlatformSpecific { avoidCtxSwitch, soBacklog, tcpNoDelay, + validateHeaders, ) => default.copy( sslConfig = sslConfig, @@ -263,6 +273,7 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching = avoidCtxSwitch, soBacklog = soBacklog, tcpNoDelay = tcpNoDelay, + validateHeaders = validateHeaders, ) } @@ -283,6 +294,7 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching = false, soBacklog = 100, tcpNoDelay = true, + validateHeaders = false, ) final case class ResponseCompressionConfig( From 6256e79a45af2a09da4606085a58ec1161c4a750 Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sun, 25 May 2025 15:48:39 +0530 Subject: [PATCH 33/34] make mima happier --- .../scala/example/HelloWorldAdvanced.scala | 11 +++--- .../example/PlainTextBenchmarkServer.scala | 7 ++-- .../example/SimpleEffectBenchmarkServer.scala | 7 ++-- .../src/main/scala/zio/http/TestServer.scala | 1 + .../scala/zio/http/RoutesPrecedentsSpec.scala | 1 + .../scala/zio/http/SocketContractSpec.scala | 1 + .../zio/http/ServerPlatformSpecific.scala | 5 +-- .../zio/http/netty/server/NettyDriver.scala | 34 +++++++++--------- .../server/ServerChannelInitializer.scala | 35 +++++++++++-------- .../netty/server/ServerInboundHandler.scala | 16 +++++---- .../scala/zio/http/netty/server/package.scala | 5 +-- .../http/endpoint/UnionRoundtripSpec.scala | 2 ++ .../scala/zio/http/ClientStreamingSpec.scala | 1 + .../scala/zio/http/ConformanceE2ESpec.scala | 18 ++++------ .../src/test/scala/zio/http/DualSSLSpec.scala | 1 + .../test/scala/zio/http/DynamicAppTest.scala | 1 + .../scala/zio/http/HybridStreamingSpec.scala | 1 + .../zio/http/NettyMaxHeaderLengthSpec.scala | 1 + .../http/NettyMaxInitialLineLengthSpec.scala | 1 + .../zio/http/RequestStreamingServerSpec.scala | 1 + .../zio/http/ResponseCompressionSpec.scala | 1 + .../jvm/src/test/scala/zio/http/SSLSpec.scala | 1 + .../http/ServerClientJKSMutualSSLSpec.scala | 1 + .../zio/http/ServerJKSKeyStoreSSLSpec.scala | 1 + .../scala/zio/http/ServerRuntimeSpec.scala | 1 + .../src/test/scala/zio/http/ServerSpec.scala | 1 + .../test/scala/zio/http/ServerStartSpec.scala | 3 ++ .../test/scala/zio/http/WebSocketSpec.scala | 1 + .../scala/zio/http/ZClientAspectSpec.scala | 1 + .../zio/http/endpoint/RoundtripSpec.scala | 2 ++ .../ServerSentEventEndpointSpec.scala | 5 ++- .../scala/zio/http/internal/package.scala | 1 + .../zio/http/netty/NettyStreamBodySpec.scala | 2 ++ .../client/NettyConnectionPoolSpec.scala | 2 ++ .../zio/http/security/ExceptionSpec.scala | 1 + .../scala/zio/http/security/MetricsSpec.scala | 1 + .../zio/http/security/SizeLimitsSpec.scala | 2 ++ .../zio/http/security/UserDataSpec.scala | 1 + .../src/main/scala/zio/http/Server.scala | 26 +++++++++----- 39 files changed, 130 insertions(+), 74 deletions(-) diff --git a/zio-http-example/src/main/scala/example/HelloWorldAdvanced.scala b/zio-http-example/src/main/scala/example/HelloWorldAdvanced.scala index f3823bf3ee..39d0c9a47c 100644 --- a/zio-http-example/src/main/scala/example/HelloWorldAdvanced.scala +++ b/zio-http-example/src/main/scala/example/HelloWorldAdvanced.scala @@ -27,16 +27,17 @@ object HelloWorldAdvanced extends ZIOAppDefault { // Configure thread count using CLI val nThreads: Int = args.headOption.flatMap(x => Try(x.toInt).toOption).getOrElse(0) - val config = Server.Config.default + val config = Server.Config.default .port(PORT) - val nettyConfig = NettyConfig.default + val nettyConfig = NettyConfig.default .leakDetection(LeakDetectionLevel.PARANOID) .maxThreads(nThreads) - val configLayer = ZLayer.succeed(config) - val nettyConfigLayer = ZLayer.succeed(nettyConfig) + val configLayer = ZLayer.succeed(config) + val nettyConfigLayer = ZLayer.succeed(nettyConfig) + val serverRuntimeConfig = configLayer.flatMap(env => ZLayer.succeed(ServerRuntimeConfig(env.get))) (fooBar ++ app) .serve[Any] - .provide(configLayer, nettyConfigLayer, Server.customized) + .provide(serverRuntimeConfig, nettyConfigLayer, Server.customized) } } diff --git a/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala b/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala index f5f61b1ece..d85b426fbe 100644 --- a/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala +++ b/zio-http-example/src/main/scala/example/PlainTextBenchmarkServer.scala @@ -40,10 +40,11 @@ object PlainTextBenchmarkServer extends ZIOAppDefault { private val nettyConfig = NettyConfig.default .leakDetection(LeakDetectionLevel.DISABLED) - private val configLayer = ZLayer.succeed(config) - private val nettyConfigLayer = ZLayer.succeed(nettyConfig) + private val configLayer = ZLayer.succeed(config) + private val nettyConfigLayer = ZLayer.succeed(nettyConfig) + private val serverRuntimeConfigLayer = configLayer.flatMap(env => ZLayer.succeed(ServerRuntimeConfig(env.get))) val run: UIO[ExitCode] = - Server.serve(routes).provide(configLayer, nettyConfigLayer, Server.customized).exitCode + Server.serve(routes).provide(serverRuntimeConfigLayer, nettyConfigLayer, Server.customized).exitCode } diff --git a/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala b/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala index 0f54036120..ac714f9fdd 100644 --- a/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala +++ b/zio-http-example/src/main/scala/example/SimpleEffectBenchmarkServer.scala @@ -37,10 +37,11 @@ object SimpleEffectBenchmarkServer extends ZIOAppDefault { private val nettyConfig = NettyConfig.default .leakDetection(LeakDetectionLevel.DISABLED) - private val configLayer = ZLayer.succeed(config) - private val nettyConfigLayer = ZLayer.succeed(nettyConfig) + private val configLayer = ZLayer.succeed(config) + private val nettyConfigLayer = ZLayer.succeed(nettyConfig) + private val serverRuntimeConfigLayer = configLayer.flatMap(env => ZLayer.succeed(ServerRuntimeConfig(env.get))) val run: UIO[ExitCode] = - Server.serve(routes).provide(configLayer, nettyConfigLayer, Server.customized).exitCode + Server.serve(routes).provide(serverRuntimeConfigLayer, nettyConfigLayer, Server.customized).exitCode } diff --git a/zio-http-testkit/src/main/scala/zio/http/TestServer.scala b/zio-http-testkit/src/main/scala/zio/http/TestServer.scala index 6ff2d2f18d..1c20e5025a 100644 --- a/zio-http-testkit/src/main/scala/zio/http/TestServer.scala +++ b/zio-http-testkit/src/main/scala/zio/http/TestServer.scala @@ -142,6 +142,7 @@ object TestServer { val default: ZLayer[Any, Nothing, TestServer] = ZLayer.make[TestServer][Nothing]( TestServer.layer.orDie, ZLayer.succeed(Server.Config.default.onAnyOpenPort), + ServerRuntimeConfig.layer, NettyDriver.customized.orDie, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ) diff --git a/zio-http-testkit/src/test/scala/zio/http/RoutesPrecedentsSpec.scala b/zio-http-testkit/src/test/scala/zio/http/RoutesPrecedentsSpec.scala index e03a7a26c5..7c4ead4d24 100644 --- a/zio-http-testkit/src/test/scala/zio/http/RoutesPrecedentsSpec.scala +++ b/zio-http-testkit/src/test/scala/zio/http/RoutesPrecedentsSpec.scala @@ -45,6 +45,7 @@ object RoutesPrecedentsSpec extends ZIOSpecDefault { ZLayer.succeed(new MyServiceLive(code)), ) }.provide( + ServerRuntimeConfig.layer, ZLayer.succeed(Server.Config.default.onAnyOpenPort), TestServer.layer, Client.default, diff --git a/zio-http-testkit/src/test/scala/zio/http/SocketContractSpec.scala b/zio-http-testkit/src/test/scala/zio/http/SocketContractSpec.scala index 3e47680c4d..67bb8066f8 100644 --- a/zio-http-testkit/src/test/scala/zio/http/SocketContractSpec.scala +++ b/zio-http-testkit/src/test/scala/zio/http/SocketContractSpec.scala @@ -103,6 +103,7 @@ object SocketContractSpec extends ZIOHttpSpec { _ <- promise.await.timeout(10.seconds) } yield assert(response.status)(equalTo(Status.SwitchingProtocols)) }.provideSome[Client]( + ServerRuntimeConfig.layer, TestServer.layer, NettyDriver.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/main/scala/zio/http/ServerPlatformSpecific.scala b/zio-http/jvm/src/main/scala/zio/http/ServerPlatformSpecific.scala index 47f09f673e..35194e3048 100644 --- a/zio-http/jvm/src/main/scala/zio/http/ServerPlatformSpecific.scala +++ b/zio-http/jvm/src/main/scala/zio/http/ServerPlatformSpecific.scala @@ -10,11 +10,12 @@ trait ServerPlatformSpecific { private[http] def base: ZLayer[Driver & Config, Throwable, Server] - val customized: ZLayer[Config & NettyConfig, Throwable, Driver with Server] = { + val customized: ZLayer[ServerRuntimeConfig & NettyConfig, Throwable, Driver with Server] = { // tmp val Needed for Scala2 val tmp: ZLayer[Driver & Config, Throwable, Server] = ZLayer.suspend(base) - ZLayer.makeSome[Config & NettyConfig, Driver with Server]( + ZLayer.makeSome[ServerRuntimeConfig & NettyConfig, Driver with Server]( + ZLayer.fromFunction((runtime: ServerRuntimeConfig) => runtime.config), NettyDriver.customized, tmp, ) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/NettyDriver.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/NettyDriver.scala index 334b186711..43feaa2550 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/NettyDriver.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/NettyDriver.scala @@ -22,21 +22,21 @@ import java.net.InetSocketAddress import zio._ import zio.http.Driver.StartResult +import zio.http._ import zio.http.netty._ import zio.http.netty.client.NettyClientDriver -import zio.http.{ClientDriver, Driver, Response, Routes, Server} import io.netty.bootstrap.ServerBootstrap -import io.netty.channel._ +import io.netty.channel.{Channel => NettyChannel, ChannelFactory, ChannelInitializer, ChannelOption, ServerChannel} import io.netty.util.ResourceLeakDetector private[zio] final case class NettyDriver( appRef: RoutesRef, channelFactory: ChannelFactory[ServerChannel], - channelInitializer: ChannelInitializer[Channel], + channelInitializer: ChannelInitializer[NettyChannel], serverInboundHandler: ServerInboundHandler, eventLoopGroups: ServerEventLoopGroups, - serverConfig: Server.Config, + serverConfig: ServerRuntimeConfig, nettyConfig: NettyConfig, ) extends Driver { self => @@ -47,9 +47,9 @@ private[zio] final case class NettyDriver( .group(eventLoopGroups.boss, eventLoopGroups.worker) .channelFactory(channelFactory) .childHandler(channelInitializer) - .option[Integer](ChannelOption.SO_BACKLOG, serverConfig.soBacklog) - .childOption[JBoolean](ChannelOption.TCP_NODELAY, serverConfig.tcpNoDelay) - .bind(serverConfig.address) + .option[Integer](ChannelOption.SO_BACKLOG, serverConfig.config.soBacklog) + .childOption[JBoolean](ChannelOption.TCP_NODELAY, serverConfig.config.tcpNoDelay) + .bind(serverConfig.config.address) } _ <- NettyFutureExecutor.scoped(chf) _ <- ZIO.succeed(ResourceLeakDetector.setLevel(nettyConfig.leakDetectionLevel.toNetty)) @@ -96,9 +96,9 @@ object NettyDriver { val make: ZIO[ RoutesRef & ChannelFactory[ServerChannel] - & ChannelInitializer[Channel] + & ChannelInitializer[NettyChannel] & ServerEventLoopGroups - & Server.Config + & ServerRuntimeConfig & NettyConfig & ServerInboundHandler, Nothing, @@ -107,9 +107,9 @@ object NettyDriver { for { app <- ZIO.service[RoutesRef] cf <- ZIO.service[ChannelFactory[ServerChannel]] - cInit <- ZIO.service[ChannelInitializer[Channel]] + cInit <- ZIO.service[ChannelInitializer[NettyChannel]] elg <- ZIO.service[ServerEventLoopGroups] - sc <- ZIO.service[Server.Config] + sc <- ZIO.service[ServerRuntimeConfig] nsc <- ZIO.service[NettyConfig] sih <- ZIO.service[ServerInboundHandler] } yield new NettyDriver( @@ -122,10 +122,11 @@ object NettyDriver { nettyConfig = nsc, ) - val manual - : ZLayer[ServerEventLoopGroups & ChannelFactory[ServerChannel] & Server.Config & NettyConfig, Nothing, Driver] = { + val manual: ZLayer[ServerEventLoopGroups & ChannelFactory[ + ServerChannel, + ] & ServerRuntimeConfig & NettyConfig, Nothing, Driver] = { implicit val trace: Trace = Trace.empty - ZLayer.makeSome[ServerEventLoopGroups & ChannelFactory[ServerChannel] & Server.Config & NettyConfig, Driver]( + ZLayer.makeSome[ServerEventLoopGroups & ChannelFactory[ServerChannel] & ServerRuntimeConfig & NettyConfig, Driver]( ZLayer(AppRef.empty), ServerChannelInitializer.layer, ServerInboundHandler.live, @@ -133,12 +134,12 @@ object NettyDriver { ) } - val customized: ZLayer[Server.Config & NettyConfig, Throwable, Driver] = { + val customized: ZLayer[ServerRuntimeConfig & NettyConfig, Throwable, Driver] = { val serverChannelFactory: ZLayer[NettyConfig, Nothing, ChannelFactory[ServerChannel]] = ChannelFactories.Server.fromConfig val eventLoopGroup: ZLayer[NettyConfig, Nothing, ServerEventLoopGroups] = ServerEventLoopGroups.live - ZLayer.makeSome[Server.Config & NettyConfig, Driver]( + ZLayer.makeSome[ServerRuntimeConfig & NettyConfig, Driver]( eventLoopGroup, serverChannelFactory, manual, @@ -148,6 +149,7 @@ object NettyDriver { val live: ZLayer[Server.Config, Throwable, Driver] = ZLayer.makeSome[Server.Config, Driver]( ZLayer.succeed(NettyConfig.default), + ServerRuntimeConfig.layer, customized, ) } diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala index be635dc69b..f1d3d5cddd 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala @@ -21,10 +21,10 @@ import java.util.concurrent.TimeUnit import zio._ import zio.stacktracer.TracingImplicits.disableAutoTrace -import zio.http.Server import zio.http.Server.RequestStreaming import zio.http.netty.model.Conversions import zio.http.netty.{HybridContentLengthHandler, Names} +import zio.http.{Server, ServerRuntimeConfig} import io.netty.channel.ChannelHandler.Sharable import io.netty.channel._ @@ -38,7 +38,7 @@ import io.netty.handler.timeout.ReadTimeoutHandler */ @Sharable private[zio] final case class ServerChannelInitializer( - cfg: Server.Config, + cfg: ServerRuntimeConfig, reqHandler: ChannelInboundHandler, ) extends ChannelInitializer[Channel] { @@ -48,11 +48,11 @@ private[zio] final case class ServerChannelInitializer( val pipeline = channel.pipeline() // SSL // Add SSL Handler if CTX is available - cfg.sslConfig.foreach { sslCfg => - pipeline.addFirst(Names.SSLHandler, new ServerSSLDecoder(sslCfg, cfg)) + cfg.config.sslConfig.foreach { sslCfg => + pipeline.addFirst(Names.SSLHandler, new ServerSSLDecoder(sslCfg, cfg.config)) } - cfg.idleTimeout.foreach { timeout => + cfg.config.idleTimeout.foreach { timeout => pipeline.addLast(Names.ReadTimeoutHandler, new ReadTimeoutHandler(timeout.toMillis, TimeUnit.MILLISECONDS)) } @@ -62,8 +62,8 @@ private[zio] final case class ServerChannelInitializer( Names.HttpRequestDecoder, new HttpRequestDecoder( new HttpDecoderConfig() - .setMaxInitialLineLength(cfg.maxInitialLineLength) - .setMaxHeaderSize(cfg.maxHeaderSize) + .setMaxInitialLineLength(cfg.config.maxInitialLineLength) + .setMaxHeaderSize(cfg.config.maxHeaderSize) .setMaxChunkSize(DEFAULT_MAX_CHUNK_SIZE) .setValidateHeaders(cfg.validateHeaders), ), @@ -71,10 +71,13 @@ private[zio] final case class ServerChannelInitializer( pipeline.addLast(Names.HttpResponseEncoder, new HttpResponseEncoder()) // HttpContentDecompressor - if (cfg.requestDecompression.enabled) - pipeline.addLast(Names.HttpRequestDecompression, new HttpContentDecompressor(cfg.requestDecompression.strict)) + if (cfg.config.requestDecompression.enabled) + pipeline.addLast( + Names.HttpRequestDecompression, + new HttpContentDecompressor(cfg.config.requestDecompression.strict), + ) - cfg.responseCompression.foreach(ops => { + cfg.config.responseCompression.foreach(ops => { pipeline.addLast( Names.HttpResponseCompression, new HttpContentCompressor(ops.contentThreshold, ops.options.map(Conversions.compressionOptionsToNetty): _*), @@ -82,7 +85,7 @@ private[zio] final case class ServerChannelInitializer( }) // ObjectAggregator - cfg.requestStreaming match { + cfg.config.requestStreaming match { case RequestStreaming.Enabled => case RequestStreaming.Disabled(maximumContentLength) => pipeline.addLast(Names.HttpObjectAggregator, new HttpObjectAggregator(maximumContentLength)) @@ -93,11 +96,12 @@ private[zio] final case class ServerChannelInitializer( // ExpectContinueHandler // Add expect continue handler is settings is true - if (cfg.acceptContinue) pipeline.addLast(Names.HttpServerExpectContinue, new HttpServerExpectContinueHandler()) + if (cfg.config.acceptContinue) + pipeline.addLast(Names.HttpServerExpectContinue, new HttpServerExpectContinueHandler()) // KeepAliveHandler // Add Keep-Alive handler is settings is true - if (cfg.keepAlive) pipeline.addLast(Names.HttpKeepAliveHandler, new HttpServerKeepAliveHandler) + if (cfg.config.keepAlive) pipeline.addLast(Names.HttpKeepAliveHandler, new HttpServerKeepAliveHandler) pipeline.addLast(Names.HttpServerFlushConsolidation, new FlushConsolidationHandler()) @@ -112,10 +116,11 @@ private[zio] final case class ServerChannelInitializer( object ServerChannelInitializer { implicit val trace: Trace = Trace.empty - val layer: ZLayer[SimpleChannelInboundHandler[HttpObject] with Server.Config, Nothing, ServerChannelInitializer] = + val layer + : ZLayer[SimpleChannelInboundHandler[HttpObject] with ServerRuntimeConfig, Nothing, ServerChannelInitializer] = ZLayer.fromZIO { for { - cfg <- ZIO.service[Server.Config] + cfg <- ZIO.service[ServerRuntimeConfig] handler <- ZIO.service[SimpleChannelInboundHandler[HttpObject]] } yield ServerChannelInitializer(cfg, handler) } diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala index ee41533e6a..effa48192a 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerInboundHandler.scala @@ -42,7 +42,7 @@ import io.netty.util.ReferenceCountUtil @Sharable private[zio] final case class ServerInboundHandler( appRef: RoutesRef, - config: Server.Config, + config: ServerRuntimeConfig, )(implicit trace: Trace) extends SimpleChannelInboundHandler[HttpObject](false) { self => @@ -51,9 +51,11 @@ private[zio] final case class ServerInboundHandler( private var handler: Handler[Any, Nothing, Request, Response] = _ private var runtime: NettyRuntime = _ + private val cfg = config.config + val inFlightRequests: LongAdder = new LongAdder() - private val readClientCert = config.sslConfig.exists(_.includeClientCert) - private val avoidCtxSwitching = config.avoidContextSwitching + private val readClientCert = cfg.sslConfig.exists(_.includeClientCert) + private val avoidCtxSwitching = cfg.avoidContextSwitching def refreshApp(): Unit = { val pair = appRef.get() @@ -157,7 +159,7 @@ private[zio] final case class ServerInboundHandler( (msg ne null) && msg.contains("Connection reset") } => case t => - if ((runtime ne null) && config.logWarningOnFatalError) { + if ((runtime ne null) && cfg.logWarningOnFatalError) { runtime.unsafeRunSync { // We cannot return the generated response from here, but still calling the handler for its side effect // for example logging. @@ -334,7 +336,7 @@ private[zio] final case class ServerInboundHandler( .addLast( new WebSocketServerProtocolHandler( NettySocketProtocol - .serverBuilder(webSocketApp.customConfig.getOrElse(config.webSocketConfig)) + .serverBuilder(webSocketApp.customConfig.getOrElse(cfg.webSocketConfig)) .build(), ), ) @@ -398,7 +400,7 @@ private[zio] final case class ServerInboundHandler( object ServerInboundHandler { val live: ZLayer[ - RoutesRef & Server.Config, + RoutesRef & ServerRuntimeConfig, Nothing, ServerInboundHandler, ] = { @@ -406,7 +408,7 @@ object ServerInboundHandler { ZLayer.fromZIO { for { appRef <- ZIO.service[RoutesRef] - config <- ZIO.service[Server.Config] + config <- ZIO.service[ServerRuntimeConfig] } yield ServerInboundHandler(appRef, config) } } diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/package.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/package.scala index 11da4dc374..4cca605d8f 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/package.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/package.scala @@ -37,7 +37,8 @@ package object server { val live: ZLayer[Server.Config, Throwable, Driver] = NettyDriver.live - val manual - : ZLayer[ServerEventLoopGroups & ChannelFactory[ServerChannel] & Server.Config & NettyConfig, Nothing, Driver] = + val manual: ZLayer[ServerEventLoopGroups & ChannelFactory[ + ServerChannel, + ] & ServerRuntimeConfig & NettyConfig, Nothing, Driver] = NettyDriver.manual } diff --git a/zio-http/jvm/src/test/scala-3/zio/http/endpoint/UnionRoundtripSpec.scala b/zio-http/jvm/src/test/scala-3/zio/http/endpoint/UnionRoundtripSpec.scala index f28b91d703..e038f25d9b 100644 --- a/zio-http/jvm/src/test/scala-3/zio/http/endpoint/UnionRoundtripSpec.scala +++ b/zio-http/jvm/src/test/scala-3/zio/http/endpoint/UnionRoundtripSpec.scala @@ -39,6 +39,7 @@ import zio.http.netty.NettyConfig object UnionRoundtripSpec extends ZIOHttpSpec { val testLayer: ZLayer[Any, Throwable, Server & Client & Scope] = ZLayer.make[Server & Client & Scope]( + ServerRuntimeConfig.layer, Server.customized, ZLayer.succeed(Server.Config.default.onAnyOpenPort.enableRequestStreaming), Client.customized.map(env => ZEnvironment(env.get)), @@ -318,6 +319,7 @@ object UnionRoundtripSpec extends ZIOHttpSpec { ) }, ).provide( + ServerRuntimeConfig.layer, Server.customized, ZLayer.succeed(Server.Config.default.onAnyOpenPort.enableRequestStreaming), Client.customized.map(env => ZEnvironment(env.get @@ clientDebugAspect)), diff --git a/zio-http/jvm/src/test/scala/zio/http/ClientStreamingSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ClientStreamingSpec.scala index 077aa1fa39..62d992422d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ClientStreamingSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ClientStreamingSpec.scala @@ -331,6 +331,7 @@ object ClientStreamingSpec extends RoutesRunnableSpec { ) .idleTimeout(100.seconds), ), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ) .fork diff --git a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala index debd8a4824..c02108a5eb 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ConformanceE2ESpec.scala @@ -10,44 +10,38 @@ import zio.http.internal.{DynamicServer, RoutesRunnableSpec} import zio.http.netty.NettyConfig object ConformanceE2ESpec extends RoutesRunnableSpec { - private val port = 8080 private val MaxSize = 1024 * 10 - val configApp = Server.Config.default + val config = Server.Config.default .requestDecompression(true) .disableRequestStreaming(MaxSize) .port(port) .responseCompression() .validateHeaders(true) - private val app = serve - + private val app = serve def conformanceSpec = suite("ConformanceE2ESpec")( test("should return 400 Bad Request if Host header is missing") { val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("%%%%invalid%%%%"))) + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("%%%%invalid%%%%"))) assertZIO(res)(equalTo(Status.BadRequest)) }, test("should return 200 OK if Host header is present") { val routes = Handler.ok.toRoutes - - val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) + val res = routes.deploy.status.run(path = Path.root, headers = Headers(Header.Host("localhost"))) assertZIO(res)(equalTo(Status.Ok)) }, ) - - override def spec = + override def spec = suite("ConformanceE2ESpec") { val spec = conformanceSpec suite("app without request streaming") { app.as(List(spec)) } }.provideShared( Scope.default, DynamicServer.live, - ZLayer.succeed(configApp), + ZLayer.succeed(config), Server.customized, Client.default, ZLayer.succeed(NettyConfig.default), ) @@ sequential @@ withLiveClock - } diff --git a/zio-http/jvm/src/test/scala/zio/http/DualSSLSpec.scala b/zio-http/jvm/src/test/scala/zio/http/DualSSLSpec.scala index 3969080c9d..21b3a57f4a 100644 --- a/zio-http/jvm/src/test/scala/zio/http/DualSSLSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/DualSSLSpec.scala @@ -126,6 +126,7 @@ object DualSSLSpec extends ZIOHttpSpec { ), ), ).provideShared( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(config), diff --git a/zio-http/jvm/src/test/scala/zio/http/DynamicAppTest.scala b/zio-http/jvm/src/test/scala/zio/http/DynamicAppTest.scala index 8f9975a1bd..3eee4ade58 100644 --- a/zio-http/jvm/src/test/scala/zio/http/DynamicAppTest.scala +++ b/zio-http/jvm/src/test/scala/zio/http/DynamicAppTest.scala @@ -42,6 +42,7 @@ object DynamicAppTest extends ZIOHttpSpec { NettyClientDriver.live, Client.customized, ZLayer.succeed(Server.Config.default.onAnyOpenPort), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, DnsResolver.default, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/test/scala/zio/http/HybridStreamingSpec.scala b/zio-http/jvm/src/test/scala/zio/http/HybridStreamingSpec.scala index b23d016632..a2c1cd447e 100644 --- a/zio-http/jvm/src/test/scala/zio/http/HybridStreamingSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/HybridStreamingSpec.scala @@ -76,6 +76,7 @@ object HybridRequestStreamingServerSpec extends RoutesRunnableSpec { Scope.default, DynamicServer.live, ZLayer.succeed(configAppWithHybridRequestStreaming), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), Client.live, diff --git a/zio-http/jvm/src/test/scala/zio/http/NettyMaxHeaderLengthSpec.scala b/zio-http/jvm/src/test/scala/zio/http/NettyMaxHeaderLengthSpec.scala index e4e3ecaef5..459ae051ea 100644 --- a/zio-http/jvm/src/test/scala/zio/http/NettyMaxHeaderLengthSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/NettyMaxHeaderLengthSpec.scala @@ -53,6 +53,7 @@ object NettyMaxHeaderLengthSpec extends ZIOHttpSpec { } yield assertTrue(extractStatus(res) == Status.InternalServerError, data == "") }.provide( Client.default, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(serverConfig), diff --git a/zio-http/jvm/src/test/scala/zio/http/NettyMaxInitialLineLengthSpec.scala b/zio-http/jvm/src/test/scala/zio/http/NettyMaxInitialLineLengthSpec.scala index 0a940a7002..084aa13149 100644 --- a/zio-http/jvm/src/test/scala/zio/http/NettyMaxInitialLineLengthSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/NettyMaxInitialLineLengthSpec.scala @@ -55,6 +55,7 @@ object NettyMaxInitialLineLength extends ZIOHttpSpec { } yield assertTrue(extractStatus(res) == Status.InternalServerError, data == "") }.provide( Client.default, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(serverConfig), diff --git a/zio-http/jvm/src/test/scala/zio/http/RequestStreamingServerSpec.scala b/zio-http/jvm/src/test/scala/zio/http/RequestStreamingServerSpec.scala index 04ce505d54..fb608d6ffa 100644 --- a/zio-http/jvm/src/test/scala/zio/http/RequestStreamingServerSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/RequestStreamingServerSpec.scala @@ -111,6 +111,7 @@ object RequestStreamingServerSpec extends RoutesRunnableSpec { .provideShared( DynamicServer.live, ZLayer.succeed(configAppWithRequestStreaming), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), Client.default, diff --git a/zio-http/jvm/src/test/scala/zio/http/ResponseCompressionSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ResponseCompressionSpec.scala index 302f8fd1af..60c679217d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ResponseCompressionSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ResponseCompressionSpec.scala @@ -125,6 +125,7 @@ object ResponseCompressionSpec extends ZIOHttpSpec { }, ).provide( ZLayer.succeed(serverConfig), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), Client.default, diff --git a/zio-http/jvm/src/test/scala/zio/http/SSLSpec.scala b/zio-http/jvm/src/test/scala/zio/http/SSLSpec.scala index 06be0b0b14..bfa9879f43 100644 --- a/zio-http/jvm/src/test/scala/zio/http/SSLSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/SSLSpec.scala @@ -118,6 +118,7 @@ object SSLSpec extends ZIOHttpSpec { ), ), ).provideShared( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(config), diff --git a/zio-http/jvm/src/test/scala/zio/http/ServerClientJKSMutualSSLSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ServerClientJKSMutualSSLSpec.scala index 1cac737c9f..46a11a0116 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ServerClientJKSMutualSSLSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ServerClientJKSMutualSSLSpec.scala @@ -145,6 +145,7 @@ object ServerClientJKSMutualSSLSpec extends ZIOHttpSpec { ), ), ).provideShared( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(config), diff --git a/zio-http/jvm/src/test/scala/zio/http/ServerJKSKeyStoreSSLSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ServerJKSKeyStoreSSLSpec.scala index 690ee023ec..1f2ecb6cce 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ServerJKSKeyStoreSSLSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ServerJKSKeyStoreSSLSpec.scala @@ -127,6 +127,7 @@ object ServerJKSKeyStoreSSLSpec extends ZIOHttpSpec { ), ), ).provideShared( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ZLayer.succeed(config), diff --git a/zio-http/jvm/src/test/scala/zio/http/ServerRuntimeSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ServerRuntimeSpec.scala index bf5075a8df..02d03e4d83 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ServerRuntimeSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ServerRuntimeSpec.scala @@ -121,6 +121,7 @@ object ServerRuntimeSpec extends RoutesRunnableSpec { .provide( Scope.default, DynamicServer.live, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(Server.Config.default), ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/test/scala/zio/http/ServerSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ServerSpec.scala index 23153ee809..02414d068a 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ServerSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ServerSpec.scala @@ -527,6 +527,7 @@ object ServerSpec extends RoutesRunnableSpec { Scope.default, DynamicServer.live, ZLayer.succeed(configApp), + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), Client.default, diff --git a/zio-http/jvm/src/test/scala/zio/http/ServerStartSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ServerStartSpec.scala index 1af55f38b4..4095bd070a 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ServerStartSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ServerStartSpec.scala @@ -33,6 +33,7 @@ object ServerStartSpec extends RoutesRunnableSpec { serve(Routes.empty).flatMap { port => assertZIO(ZIO.attempt(port))(equalTo(port)) }.provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(config), DynamicServer.live, Server.customized, @@ -45,6 +46,7 @@ object ServerStartSpec extends RoutesRunnableSpec { serve(Routes.empty).flatMap { bindPort => assertZIO(ZIO.attempt(bindPort))(not(equalTo(port))) }.provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(config), DynamicServer.live, Server.customized, @@ -56,6 +58,7 @@ object ServerStartSpec extends RoutesRunnableSpec { .succeed(assertCompletes) .provide( Server.customized.unit, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(Server.Config.default.port(8089)), ZLayer.succeed(NettyConfig.defaultWithFastShutdown), ) diff --git a/zio-http/jvm/src/test/scala/zio/http/WebSocketSpec.scala b/zio-http/jvm/src/test/scala/zio/http/WebSocketSpec.scala index cc3e222b6c..87f7cfc608 100644 --- a/zio-http/jvm/src/test/scala/zio/http/WebSocketSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/WebSocketSpec.scala @@ -230,6 +230,7 @@ object WebSocketSpec extends RoutesRunnableSpec { .provideSome[DynamicServer & Server & Client](Scope.default) .provideShared( DynamicServer.live, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(Server.Config.default.onAnyOpenPort.enableRequestStreaming), testNettyServerConfig, Server.customized, diff --git a/zio-http/jvm/src/test/scala/zio/http/ZClientAspectSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ZClientAspectSpec.scala index fa31a5e5a0..2d0b9a9dc9 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ZClientAspectSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ZClientAspectSpec.scala @@ -184,6 +184,7 @@ object ZClientAspectSpec extends ZIOHttpSpec { }, ), ).provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(Server.Config.default.onAnyOpenPort), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala index 82a19e4f0d..2cd57b1ca8 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/RoundtripSpec.scala @@ -39,6 +39,7 @@ import zio.http.netty.NettyConfig object RoundtripSpec extends ZIOHttpSpec { val testLayer: ZLayer[Any, Throwable, Server & Client & Scope] = ZLayer.make[Server & Client & Scope]( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(Server.Config.default.onAnyOpenPort.enableRequestStreaming), Client.customized.map(env => ZEnvironment(env.get)), @@ -640,6 +641,7 @@ object RoundtripSpec extends ZIOHttpSpec { } }.provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(Server.Config.default.onAnyOpenPort.enableRequestStreaming), Client.customized.map(env => ZEnvironment(env.get @@ clientDebugAspect)) >>> diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/ServerSentEventEndpointSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/ServerSentEventEndpointSpec.scala index fbbf36974b..265c267d3e 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/ServerSentEventEndpointSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/ServerSentEventEndpointSpec.scala @@ -108,7 +108,10 @@ object ServerSentEventEndpointSpec extends ZIOHttpSpec { } yield assertTrue(events.size == 5 && events.forall(event => Try(event.data.timeStamp).isSuccess)) }, ) - .provideSomeLayer[Client & Server.Config & NettyConfig](Server.customized) + .provideSomeLayer[Client & Server.Config & NettyConfig]( + (ZLayer.fromFunction[Server.Config => ServerRuntimeConfig](ServerRuntimeConfig(_)) ++ ZLayer + .service[NettyConfig]) >>> Server.customized, + ) .provideShared( Client.live, ZLayer.succeed(Server.Config.default.port(0)), diff --git a/zio-http/jvm/src/test/scala/zio/http/internal/package.scala b/zio-http/jvm/src/test/scala/zio/http/internal/package.scala index c0d6d0ee20..d4300c019a 100644 --- a/zio-http/jvm/src/test/scala/zio/http/internal/package.scala +++ b/zio-http/jvm/src/test/scala/zio/http/internal/package.scala @@ -36,6 +36,7 @@ package object internal { val serverTestLayer: ZLayer[Any, Throwable, Server.Config with Server] = ZLayer.make[Server.Config with Server]( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), testServerConfig, testNettyServerConfig, Server.customized, diff --git a/zio-http/jvm/src/test/scala/zio/http/netty/NettyStreamBodySpec.scala b/zio-http/jvm/src/test/scala/zio/http/netty/NettyStreamBodySpec.scala index e49f17d5a1..d5da80ea22 100644 --- a/zio-http/jvm/src/test/scala/zio/http/netty/NettyStreamBodySpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/netty/NettyStreamBodySpec.scala @@ -37,6 +37,7 @@ object NettyStreamBodySpec extends RoutesRunnableSpec { .intoPromise(portPromise) .zipRight(ZIO.never) .provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(NettyConfig.defaultWithFastShutdown.leakDetection(LeakDetectionLevel.PARANOID)), ZLayer.succeed(Server.Config.default.onAnyOpenPort), Server.customized, @@ -136,6 +137,7 @@ object NettyStreamBodySpec extends RoutesRunnableSpec { .intoPromise(portPromise) .zipRight(ZIO.never) .provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(NettyConfig.defaultWithFastShutdown.leakDetection(LeakDetectionLevel.PARANOID)), ZLayer.succeed(Server.Config.default.onAnyOpenPort), Server.customized, diff --git a/zio-http/jvm/src/test/scala/zio/http/netty/client/NettyConnectionPoolSpec.scala b/zio-http/jvm/src/test/scala/zio/http/netty/client/NettyConnectionPoolSpec.scala index ed5b246356..f34e23a041 100644 --- a/zio-http/jvm/src/test/scala/zio/http/netty/client/NettyConnectionPoolSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/netty/client/NettyConnectionPoolSpec.scala @@ -198,6 +198,7 @@ object NettyConnectionPoolSpec extends RoutesRunnableSpec { DynamicServer.live, ZLayer.succeed(Server.Config.default.idleTimeout(500.millis).onAnyOpenPort.logWarningOnFatalError(false)), testNettyServerConfig, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ) @@ withLiveClock } + test("idle timeout is refreshed on each request") { @@ -215,6 +216,7 @@ object NettyConnectionPoolSpec extends RoutesRunnableSpec { DynamicServer.live, ZLayer.succeed(Server.Config.default.idleTimeout(500.millis).onAnyOpenPort.logWarningOnFatalError(false)), testNettyServerConfig, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, Client.live, ZLayer.succeed(Client.Config.default.idleTimeout(500.millis)), diff --git a/zio-http/jvm/src/test/scala/zio/http/security/ExceptionSpec.scala b/zio-http/jvm/src/test/scala/zio/http/security/ExceptionSpec.scala index 1acd4ecd20..63d04f2f37 100644 --- a/zio-http/jvm/src/test/scala/zio/http/security/ExceptionSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/security/ExceptionSpec.scala @@ -85,6 +85,7 @@ object ExceptionSpec extends ZIOSpecDefault { }, ).provide( Scope.default, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed( Server.Config.default, diff --git a/zio-http/jvm/src/test/scala/zio/http/security/MetricsSpec.scala b/zio-http/jvm/src/test/scala/zio/http/security/MetricsSpec.scala index 1f3518fdf6..e003200686 100644 --- a/zio-http/jvm/src/test/scala/zio/http/security/MetricsSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/security/MetricsSpec.scala @@ -112,6 +112,7 @@ object MetricsSpec extends ZIOHttpSpec { port => form => Request.post(s"http://localhost:$port", Body.fromMultipartForm(form, Boundary("-"))), ), ).provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed(Server.Config.default), ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/test/scala/zio/http/security/SizeLimitsSpec.scala b/zio-http/jvm/src/test/scala/zio/http/security/SizeLimitsSpec.scala index 65b4b2d2f9..9f0e029f3a 100644 --- a/zio-http/jvm/src/test/scala/zio/http/security/SizeLimitsSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/security/SizeLimitsSpec.scala @@ -156,6 +156,7 @@ object SizeLimitsSpec extends ZIOHttpSpec { ) }, ).provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed( Server.Config.default @@ -232,6 +233,7 @@ object SizeLimitsSpec extends ZIOHttpSpec { ) }, ).provide( + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), ZLayer.succeed(Server.Config.default), Server.customized, ZLayer.succeed(NettyConfig.defaultWithFastShutdown), diff --git a/zio-http/jvm/src/test/scala/zio/http/security/UserDataSpec.scala b/zio-http/jvm/src/test/scala/zio/http/security/UserDataSpec.scala index 7b860e5d15..9729f4aa29 100644 --- a/zio-http/jvm/src/test/scala/zio/http/security/UserDataSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/security/UserDataSpec.scala @@ -158,6 +158,7 @@ object UserDataSpec extends ZIOSpecDefault { }, ).provide( Scope.default, + ZLayer.fromFunction((c: Server.Config) => ServerRuntimeConfig(c)), Server.customized, ZLayer.succeed( Server.Config.default, diff --git a/zio-http/shared/src/main/scala/zio/http/Server.scala b/zio-http/shared/src/main/scala/zio/http/Server.scala index 2257f051af..5975689033 100644 --- a/zio-http/shared/src/main/scala/zio/http/Server.scala +++ b/zio-http/shared/src/main/scala/zio/http/Server.scala @@ -72,7 +72,6 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching: Boolean, soBacklog: Int, tcpNoDelay: Boolean, - validateHeaders: Boolean, ) { self => /** @@ -211,8 +210,8 @@ object Server extends ServerPlatformSpecific { * enabled, the server will validate incoming headers such as the Host * header. */ - def validateHeaders(value: Boolean): Config = - self.copy(validateHeaders = value) + def validateHeaders(value: Boolean): ServerRuntimeConfig = + ServerRuntimeConfig(self, value) def webSocketConfig(webSocketConfig: WebSocketConfig): Config = self.copy(webSocketConfig = webSocketConfig) @@ -235,8 +234,7 @@ object Server extends ServerPlatformSpecific { zio.Config.duration("idle-timeout").optional.withDefault(Config.default.idleTimeout) ++ zio.Config.boolean("avoid-context-switching").withDefault(Config.default.avoidContextSwitching) ++ zio.Config.int("so-backlog").withDefault(Config.default.soBacklog) ++ - zio.Config.boolean("tcp-nodelay").withDefault(Config.default.tcpNoDelay) ++ - zio.Config.boolean("validate-headers").withDefault(Config.default.validateHeaders) + zio.Config.boolean("tcp-nodelay").withDefault(Config.default.tcpNoDelay) }.map { case ( sslConfig, @@ -255,7 +253,6 @@ object Server extends ServerPlatformSpecific { avoidCtxSwitch, soBacklog, tcpNoDelay, - validateHeaders, ) => default.copy( sslConfig = sslConfig, @@ -273,7 +270,6 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching = avoidCtxSwitch, soBacklog = soBacklog, tcpNoDelay = tcpNoDelay, - validateHeaders = validateHeaders, ) } @@ -294,7 +290,6 @@ object Server extends ServerPlatformSpecific { avoidContextSwitching = false, soBacklog = 100, tcpNoDelay = true, - validateHeaders = false, ) final case class ResponseCompressionConfig( @@ -590,3 +585,18 @@ object Server extends ServerPlatformSpecific { } } + +final case class ServerRuntimeConfig( + config: Server.Config, + validateHeaders: Boolean = false, +) + +object ServerRuntimeConfig { + def config: zio.Config[ServerRuntimeConfig] = + (Server.Config.config ++ zio.Config.boolean("validate-headers").withDefault(false)).map { case (cfg, validate) => + ServerRuntimeConfig(cfg, validate) + } + + val layer: ZLayer[Server.Config, Nothing, ServerRuntimeConfig] = + ZLayer.fromFunction(cfg => ServerRuntimeConfig(cfg, false)) +} From 23f4eeba26843090a0045f14777d26926cb1c40e Mon Sep 17 00:00:00 2001 From: Saturn225 <101260782+Saturn225@users.noreply.github.com> Date: Sat, 31 May 2025 15:17:17 +0530 Subject: [PATCH 34/34] fix: fmt --- .../zio/http/netty/server/ServerChannelInitializer.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala index 33b201f09c..db157cc15e 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerChannelInitializer.scala @@ -72,7 +72,10 @@ private[zio] final case class ServerChannelInitializer( // HttpContentDecompressor if (cfg.config.requestDecompression.enabled) - pipeline.addLast(Names.HttpRequestDecompression, new HttpContentDecompressor(cfg.config.requestDecompression.strict, 0)) + pipeline.addLast( + Names.HttpRequestDecompression, + new HttpContentDecompressor(cfg.config.requestDecompression.strict, 0), + ) cfg.config.responseCompression.foreach(ops => { pipeline.addLast( Names.HttpResponseCompression,