[k2] add web-builtins (#1383)

add header_register_callback
add headers_sent
add send_http_103_early_hints as noop

Author: astrophysik

builtin-functions/kphp-light/stdlib/server-functions.txt

@@ -45,6 +45,10 @@ function setrawcookie ($name ::: string, $value ::: string, $expire_or_options :
 
 function headers_list () ::: string[];
 
+function headers_sent (?string &$filename = null, ?int &$line = null) ::: bool;
+
+function header_register_callback (callable():void $callback) ::: bool;
+
 function ip2long ($ip ::: string) ::: int | false;
 
 function ip2ulong ($ip ::: string) ::: string | false;
@@ -64,8 +68,7 @@ function memory_get_peak_usage ($real_usage ::: bool = false) ::: int;
 // ===== UNSUPPORTED =====
 
 /** @kphp-extern-func-info stub */
-function headers_sent (?string &$filename = null, ?int &$line = null) ::: bool;
-
+function send_http_103_early_hints($headers ::: string[]) ::: void;
 
 function get_engine_version() ::: string;
 
@@ -125,8 +128,6 @@ function setlocale ($category ::: int, $locale ::: string) ::: string | false;
 
 function debug_backtrace() ::: string[][];
 
-/** @kphp-extern-func-info stub generation-required */
-function send_http_103_early_hints($headers ::: string[]) ::: void;
 /** @kphp-extern-func-info stub generation-required */
 function ignore_user_abort ($enable ::: ?bool = null) ::: int;
 /** @kphp-extern-func-info stub generation-required */

builtin-functions/kphp-light/stdlib/system-functions.txt

@@ -34,9 +34,6 @@ function escapeshellcmd($cmd ::: string): string;
 /** @kphp-extern-func-info stub */
 function exec($command ::: string, &$output ::: mixed = [], int &$result_code = 0): string|false;
 
-/** @kphp-extern-func-info stub */
-function header_register_callback (callable():void $callback) ::: bool;
-
 /** @kphp-extern-func-info stub generation-required */
 function getenv(string $varname = '', bool $local_only = false): mixed;
 

runtime-light/server/http/http-server-state.h

@@ -14,6 +14,7 @@
 #include "runtime-common/core/allocator/script-allocator.h"
 #include "runtime-common/core/runtime-core.h"
 #include "runtime-common/core/std/containers.h"
+#include "runtime-light/coroutine/task.h"
 #include "runtime-light/streams/stream.h"
 
 namespace kphp::http {
@@ -31,6 +32,8 @@ enum status : uint16_t {
   BAD_REQUEST = 400,
 };
 
+enum class response_state : uint8_t { not_started, sending_headers, headers_sent, sending_body, completed };
+
 namespace headers {
 
 inline constexpr std::string_view HOST = "host";
@@ -60,6 +63,10 @@ struct HttpServerInstanceState final : private vk::not_copyable {
   uint64_t status_code{kphp::http::status::NO_STATUS};
   kphp::http::method http_method{kphp::http::method::other};
   kphp::http::connection_kind connection_kind{kphp::http::connection_kind::close};
+  kphp::http::response_state response_state{kphp::http::response_state::not_started};
+
+  // The headers_registered_callback function should only be invoked once
+  std::optional<kphp::coro::task<>> headers_registered_callback;
 
 private:
   kphp::stl::multimap<kphp::stl::string<kphp::memory::script_allocator>, kphp::stl::string<kphp::memory::script_allocator>, kphp::memory::script_allocator>

runtime-light/server/http/init-functions.cpp

@@ -195,6 +195,35 @@ std::string_view process_headers(const tl::K2InvokeHttp& invoke_http, PhpScriptB
   return content_type;
 }
 
+string get_http_response_body(HttpServerInstanceState& http_server_instance_st) noexcept {
+  string body{};
+  if (http_server_instance_st.http_method != kphp::http::method::head) {
+    auto& output_instance_st{OutputInstanceState::get()};
+    const auto system_buffer{output_instance_st.output_buffers.system_buffer()};
+    const auto user_buffers{output_instance_st.output_buffers.user_buffers()};
+
+    const auto body_size{std::ranges::fold_left(user_buffers | std::views::transform([](const auto& buffer) noexcept { return buffer.size(); }),
+                                                system_buffer.get().size(), std::plus<string::size_type>{})};
+    body.reserve_at_least(body_size);
+
+    body.append(system_buffer.get().buffer(), system_buffer.get().size());
+    const auto appender{[&body](const auto& buffer) noexcept { body.append(buffer.buffer(), buffer.size()); }};
+    std::ranges::for_each(user_buffers | std::views::filter([](const auto& buffer) noexcept { return buffer.size() > 0; }), appender);
+
+    const bool gzip_encoded{static_cast<bool>(http_server_instance_st.encoding & HttpServerInstanceState::ENCODING_GZIP)};
+    const bool deflate_encoded{static_cast<bool>(http_server_instance_st.encoding & HttpServerInstanceState::ENCODING_DEFLATE)};
+    // compress body if needed
+    if (gzip_encoded || deflate_encoded) {
+      auto encoded_body{kphp::zlib::encode({body.c_str(), static_cast<size_t>(body.size())}, kphp::zlib::DEFAULT_COMPRESSION_LEVEL,
+                                           gzip_encoded ? kphp::zlib::ENCODING_GZIP : kphp::zlib::ENCODING_DEFLATE)};
+      if (encoded_body.has_value()) [[likely]] {
+        body = std::move(*encoded_body);
+      }
+    }
+  }
+  return body;
+}
+
 } // namespace
 
 namespace kphp::http {
@@ -337,60 +366,59 @@ void init_server(kphp::component::stream request_stream) noexcept {
 kphp::coro::task<> finalize_server() noexcept {
   auto& http_server_instance_st{HttpServerInstanceState::get()};
 
-  string body{};
-  if (http_server_instance_st.http_method != method::head) {
-    auto& output_instance_st{OutputInstanceState::get()};
-    const auto system_buffer{output_instance_st.output_buffers.system_buffer()};
-    const auto user_buffers{output_instance_st.output_buffers.user_buffers()};
-
-    const auto body_size{std::ranges::fold_left(user_buffers | std::views::transform([](const auto& buffer) noexcept { return buffer.size(); }),
-                                                system_buffer.get().size(), std::plus<string::size_type>{})};
-    body.reserve_at_least(body_size);
-
-    body.append(system_buffer.get().buffer(), system_buffer.get().size());
-    const auto appender{[&body](const auto& buffer) noexcept { body.append(buffer.buffer(), buffer.size()); }};
-    std::ranges::for_each(user_buffers | std::views::filter([](const auto& buffer) noexcept { return buffer.size() > 0; }), appender);
-
+  string response_body{};
+  tl::HttpResponse http_response{};
+  switch (http_server_instance_st.response_state) {
+  case kphp::http::response_state::not_started:
+    http_server_instance_st.response_state = kphp::http::response_state::sending_headers;
+    if (http_server_instance_st.headers_registered_callback.has_value()) {
+      co_await *std::exchange(http_server_instance_st.headers_registered_callback, std::nullopt);
+    }
+    [[fallthrough]];
+  case kphp::http::response_state::sending_headers: {
     const bool gzip_encoded{static_cast<bool>(http_server_instance_st.encoding & HttpServerInstanceState::ENCODING_GZIP)};
     const bool deflate_encoded{static_cast<bool>(http_server_instance_st.encoding & HttpServerInstanceState::ENCODING_DEFLATE)};
-    // compress body if needed
     if (gzip_encoded || deflate_encoded) {
-      auto encoded_body{kphp::zlib::encode({body.c_str(), static_cast<size_t>(body.size())}, kphp::zlib::DEFAULT_COMPRESSION_LEVEL,
-                                           gzip_encoded ? kphp::zlib::ENCODING_GZIP : kphp::zlib::ENCODING_DEFLATE)};
-      if (encoded_body.has_value()) [[likely]] {
-        body = std::move(*encoded_body);
-
-        auto& static_SB{RuntimeContext::get().static_SB};
-        static_SB.clean() << headers::CONTENT_ENCODING.data() << ": " << (gzip_encoded ? ENCODING_GZIP.data() : ENCODING_DEFLATE.data());
-        kphp::http::header({static_SB.c_str(), static_SB.size()}, true, status::NO_STATUS);
-      }
+      auto& static_SB{RuntimeContext::get().static_SB};
+      static_SB.clean() << kphp::http::headers::CONTENT_ENCODING.data() << ": " << (gzip_encoded ? ENCODING_GZIP.data() : ENCODING_DEFLATE.data());
+      kphp::http::header({static_SB.c_str(), static_SB.size()}, true, kphp::http::status::NO_STATUS);
     }
+    // fill headers
+    http_response.http_response.headers.value.reserve(http_server_instance_st.headers().size());
+    std::transform(http_server_instance_st.headers().cbegin(), http_server_instance_st.headers().cend(),
+                   std::back_inserter(http_response.http_response.headers.value), [](const auto& header_entry) noexcept {
+                     const auto& [name, value]{header_entry};
+                     return tl::httpHeaderEntry{
+                         .is_sensitive = {}, .name = {.value = {name.data(), name.size()}}, .value = {.value = {value.data(), value.size()}}};
+                   });
+    http_server_instance_st.response_state = kphp::http::response_state::headers_sent;
+    [[fallthrough]];
   }
+  case kphp::http::response_state::headers_sent: {
+    response_body = get_http_response_body(http_server_instance_st);
+    const auto status_code{http_server_instance_st.status_code == status::NO_STATUS ? status::OK : http_server_instance_st.status_code};
+    http_response.http_response.version = tl::HttpVersion{.version = tl::HttpVersion::Version::V11};
+    http_response.http_response.status_code = {.value = static_cast<int32_t>(status_code)};
+    http_response.http_response.body = {reinterpret_cast<const std::byte*>(response_body.c_str()), response_body.size()};
+    http_server_instance_st.response_state = kphp::http::response_state::sending_body;
+    [[fallthrough]];
+  }
+  case kphp::http::response_state::sending_body: {
+    tl::storer tls{http_response.footprint()};
+    http_response.store(tls);
 
-  const auto status_code{http_server_instance_st.status_code == status::NO_STATUS ? status::OK : http_server_instance_st.status_code};
-
-  tl::HttpResponse http_response{.http_response = tl::httpResponse{.version = tl::HttpVersion{.version = tl::HttpVersion::Version::V11},
-                                                                   .status_code = {.value = static_cast<int32_t>(status_code)},
-                                                                   .headers = {},
-                                                                   .body = {reinterpret_cast<const std::byte*>(body.c_str()), body.size()}}};
-  // fill headers
-  http_response.http_response.headers.value.reserve(http_server_instance_st.headers().size());
-  std::transform(http_server_instance_st.headers().cbegin(), http_server_instance_st.headers().cend(),
-                 std::back_inserter(http_response.http_response.headers.value), [](const auto& header_entry) noexcept {
-                   const auto& [name, value]{header_entry};
-                   return tl::httpHeaderEntry{
-                       .is_sensitive = {}, .name = {.value = {name.data(), name.size()}}, .value = {.value = {value.data(), value.size()}}};
-                 });
-
-  tl::storer tls{http_response.footprint()};
-  http_response.store(tls);
-
-  if (!http_server_instance_st.request_stream.has_value()) [[unlikely]] {
-    kphp::log::error("can't send HTTP response since there is no available stream");
+    if (!http_server_instance_st.request_stream.has_value()) [[unlikely]] {
+      kphp::log::error("can't send HTTP response since there is no available stream");
+    }
+    const auto& request_stream{*http_server_instance_st.request_stream};
+    if (auto expected{co_await kphp::component::send_response(request_stream, tls.view())}; !expected) [[unlikely]] {
+      kphp::log::error("can't write HTTP response: stream -> {}, error code -> {}", request_stream.descriptor(), expected.error());
+    }
+    http_server_instance_st.response_state = kphp::http::response_state::completed;
+    [[fallthrough]];
   }
-  const auto& request_stream{*http_server_instance_st.request_stream};
-  if (auto expected{co_await kphp::component::send_response(request_stream, tls.view())}; !expected) [[unlikely]] {
-    kphp::log::error("can't write HTTP response: stream -> {}, error code -> {}", request_stream.descriptor(), expected.error());
+  case kphp::http::response_state::completed:
+    co_return;
   }
 }
 

runtime-light/stdlib/server/http-functions.cpp

@@ -103,6 +103,17 @@ void header(std::string_view header_view, bool replace, int64_t response_code) n
   }
 
   auto& http_server_instance_st{HttpServerInstanceState::get()};
+  switch (http_server_instance_st.response_state) {
+  case kphp::http::response_state::not_started:
+  case kphp::http::response_state::sending_headers:
+    break;
+  case kphp::http::response_state::headers_sent:
+  case kphp::http::response_state::sending_body:
+  case kphp::http::response_state::completed:
+    // don't add header since it will not be sent
+    return;
+  }
+
   // HTTP status special case
   if (http_status_header(header_view)) {
     if (const auto opt_status_code{valid_http_status_header(header_view)}; opt_status_code.has_value()) [[likely]] {

runtime-light/stdlib/server/http-functions.h

@@ -4,14 +4,15 @@
 
 #pragma once
 
+#include <concepts>
 #include <cstdint>
 #include <string_view>
 #include <utility>
 
 #include "runtime-common/core/runtime-core.h"
 #include "runtime-common/stdlib/server/url-functions.h"
+#include "runtime-light/coroutine/task.h"
 #include "runtime-light/server/http/http-server-state.h"
-#include "runtime-light/stdlib/diagnostics/logs.h"
 
 namespace kphp::http {
 
@@ -48,12 +49,45 @@ inline array<string> f$headers_list() noexcept {
 
 inline bool f$headers_sent([[maybe_unused]] Optional<std::optional<std::reference_wrapper<string>>> filename = {},
                            [[maybe_unused]] Optional<std::optional<std::reference_wrapper<string>>> line = {}) noexcept {
-  kphp::log::warning("called stub headers_sent");
-  return false;
+  const auto& http_server_instance_st{HttpServerInstanceState::get()};
+  switch (http_server_instance_st.response_state) {
+  case kphp::http::response_state::not_started:
+  case kphp::http::response_state::sending_headers:
+    return false;
+  case kphp::http::response_state::headers_sent:
+  case kphp::http::response_state::sending_body:
+  case kphp::http::response_state::completed:
+    return true;
+  }
 }
 
-template<typename F>
-bool f$header_register_callback(F&& /*unused*/) noexcept {
-  kphp::log::warning("called stub header_register_callback");
+template<std::invocable F>
+bool f$header_register_callback(F&& f) noexcept {
+  auto& http_server_instance_st{HttpServerInstanceState::get()};
+  switch (http_server_instance_st.response_state) {
+  case kphp::http::response_state::not_started:
+    break;
+  case kphp::http::response_state::sending_headers:
+  case kphp::http::response_state::headers_sent:
+  case kphp::http::response_state::sending_body:
+  case kphp::http::response_state::completed:
+    return false;
+  }
+
+  auto headers_callback_task{std::invoke(
+      [](F f) noexcept -> kphp::coro::task<> {
+        if constexpr (kphp::coro::is_async_function_v<F>) {
+          co_await std::invoke(std::move(f));
+        } else {
+          std::invoke(std::move(f));
+        }
+      },
+      std::forward<F>(f))};
+
+  http_server_instance_st.headers_registered_callback.emplace(std::move(headers_callback_task));
   return true;
 }
+
+inline void f$send_http_103_early_hints([[maybe_unused]] const array<string>& headers) noexcept {
+  // noop
+}

tests/python/tests/http_server/test_gzip_header_reset.py

@@ -8,7 +8,6 @@ def test_single_gzip_buffer(self):
         response = self.gzip_request("gzip")
         self.assertEqual(response.headers["Content-Encoding"], "gzip")
 
-    @pytest.mark.k2_skip
     def test_single_gzip_buffer_closed(self):
         response = self.gzip_request("reset")
         with self.assertRaises(KeyError):

tests/python/tests/http_server/test_header_register_callback.py

@@ -5,14 +5,14 @@
 from python.lib.testcase import WebServerAutoTestCase
 
 
-@pytest.mark.k2_skip_suite
 class TestHeaderRegisterCallback(WebServerAutoTestCase):
 
+    @pytest.mark.k2_skip
     def test_rpc_in_callback(self):
         self.web_server.http_request(uri='/test_header_register_callback?act_in_callback=rpc&port={}'.format(str(typing.cast(KphpServer, self.web_server).master_port)), timeout=5)
         self.web_server.assert_log(["test_header_register_callback/rpc_in_callback"], timeout=5)
 
     def test_exit_in_callback(self):
-        response = self.web_server.http_request(uri='/test_header_register_callback?act_in_callback=exit&port={}'.format(str(typing.cast(KphpServer, self.web_server).master_port)), timeout=5)
+        response = self.web_server.http_request(uri='/test_header_register_callback?act_in_callback=exit', timeout=5)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.content, b"test_header_register_callback/exit_in_callback")