Merge pull request #832 from Altinity/backports/24.8.14/79975

24.8.14 Backport of ClickHouse#79975 Allow to add `http_response_headers` in `http_handlers` of any kind

Author: zvonand

docs/en/interfaces/http.md

@@ -791,7 +791,42 @@ $ curl -vv -H 'XXX:xxx' 'http://localhost:8123/get_relative_path_static_handler'
 * Connection #0 to host localhost left intact
 ```
 
-## Valid JSON/XML response on exception during HTTP streaming {valid-output-on-exception-http-streaming} 
+## HTTP Response Headers {#http-response-headers}
+
+ClickHouse allows you to configure custom HTTP response headers that can be applied to any kind of handler that can be configured. These headers can be set using the `http_response_headers` setting, which accepts key-value pairs representing header names and their values. This feature is particularly useful for implementing custom security headers, CORS policies, or any other HTTP header requirements across your ClickHouse HTTP interface.
+
+For example, you can configure headers for:
+- Regular query endpoints
+- Web UI
+- Health check.
+
+It is also possible to specify `common_http_response_headers`. These will be applied to all http handlers defined in the configuration.
+
+The headers will be included in the HTTP response for every configured handler.
+
+In the example below, every server response will contain two custom headers: `X-My-Common-Header` and `X-My-Custom-Header`.
+
+```xml
+<clickhouse>
+    <http_handlers>
+        <common_http_response_headers>
+            <X-My-Common-Header>Common header</X-My-Common-Header>
+        </common_http_response_headers>
+        <rule>
+            <methods>GET</methods>
+            <url>/ping</url>
+            <handler>
+                <type>ping</type>
+                <http_response_headers>
+                    <X-My-Custom-Header>Custom indeed</X-My-Custom-Header>
+                </http_response_headers>
+            </handler>
+        </rule>
+    </http_handlers>
+</clickhouse>
+```
+
+## Valid JSON/XML response on exception during HTTP streaming {#valid-output-on-exception-http-streaming}
 
 While query execution over HTTP an exception can happen when part of the data has already been sent. Usually an exception is sent to the client in plain text
 even if some specific data format was used to output data and the output may become invalid in terms of specified data format.

src/Server/HTTPHandler.cpp

@@ -895,11 +895,14 @@ std::string PredefinedQueryHandler::getQuery(HTTPServerRequest & request, HTMLFo
 
 HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix)
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers)
 {
     auto query_param_name = config.getString(config_prefix + ".handler.query_param_name", "query");
 
     HTTPResponseHeaderSetup http_response_headers_override = parseHTTPResponseHeaders(config, config_prefix);
+    if (http_response_headers_override.has_value())
+        http_response_headers_override.value().insert(common_headers.begin(), common_headers.end());
 
     auto creator = [&server, query_param_name, http_response_headers_override]() -> std::unique_ptr<DynamicQueryHandler>
     { return std::make_unique<DynamicQueryHandler>(server, query_param_name, http_response_headers_override); };
@@ -932,7 +935,8 @@ static inline CompiledRegexPtr getCompiledRegex(const std::string & expression)
 
 HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix)
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers)
 {
     if (!config.has(config_prefix + ".handler.query"))
         throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no path '{}.handler.query' in configuration file.", config_prefix);
@@ -958,6 +962,8 @@ HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
     }
 
     HTTPResponseHeaderSetup http_response_headers_override = parseHTTPResponseHeaders(config, config_prefix);
+    if (http_response_headers_override.has_value())
+        http_response_headers_override.value().insert(common_headers.begin(), common_headers.end());
 
     std::shared_ptr<HandlingRuleHTTPHandlerFactory<PredefinedQueryHandler>> factory;
 

src/Server/HTTPHandlerFactory.cpp

@@ -13,6 +13,7 @@
 #include "InterserverIOHTTPHandler.h"
 #include "WebUIRequestHandler.h"
 
+#include <iostream>
 
 namespace DB
 {
@@ -31,27 +32,35 @@ class RedirectRequestHandler : public HTTPRequestHandler
 {
 private:
     std::string url;
+    std::unordered_map<String, String> http_response_headers_override;
 
 public:
-    explicit RedirectRequestHandler(std::string url_)
-        : url(std::move(url_))
+    explicit RedirectRequestHandler(std::string url_, std::unordered_map<String, String> http_response_headers_override_ = {})
+        : url(std::move(url_)), http_response_headers_override(http_response_headers_override_)
     {
     }
 
     void handleRequest(HTTPServerRequest &, HTTPServerResponse & response, const ProfileEvents::Event &) override
     {
+        applyHTTPResponseHeaders(response, http_response_headers_override);
         response.redirect(url);
     }
 };
 
 HTTPRequestHandlerFactoryPtr createRedirectHandlerFactory(
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix)
+    const std::string & config_prefix,
+    std::unordered_map<String, String> common_headers)
 {
     std::string url = config.getString(config_prefix + ".handler.location");
 
+    auto headers = parseHTTPResponseHeadersWithCommons(config, config_prefix, common_headers);
+
     auto factory = std::make_shared<HandlingRuleHTTPHandlerFactory<RedirectRequestHandler>>(
-        [my_url = std::move(url)]() { return std::make_unique<RedirectRequestHandler>(my_url); });
+        [my_url = std::move(url), headers_override = std::move(headers)]()
+        {
+            return std::make_unique<RedirectRequestHandler>(my_url, headers_override);
+        });
 
     factory->addFiltersFromConfig(config, config_prefix);
     return factory;
@@ -78,6 +87,33 @@ static auto createPingHandlerFactory(IServer & server)
     return std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(std::move(creator));
 }
 
+static auto createPingHandlerFactory(IServer & server, const Poco::Util::AbstractConfiguration & config, const String & config_prefix,
+                                     std::unordered_map<String, String> common_headers)
+{
+    auto creator = [&server,&config,config_prefix,common_headers]() -> std::unique_ptr<StaticRequestHandler>
+    {
+        constexpr auto ping_response_expression = "Ok.\n";
+
+        auto headers = parseHTTPResponseHeadersWithCommons(config, config_prefix, "text/html; charset=UTF-8", common_headers);
+
+        return std::make_unique<StaticRequestHandler>(
+            server, ping_response_expression, headers);
+    };
+    return std::make_shared<HandlingRuleHTTPHandlerFactory<StaticRequestHandler>>(std::move(creator));
+}
+
+template <typename UIRequestHandler>
+static auto createWebUIHandlerFactory(IServer & server, const Poco::Util::AbstractConfiguration & config, const String & config_prefix,
+                                      std::unordered_map<String, String> common_headers)
+{
+    auto creator = [&server,&config,config_prefix,common_headers]() -> std::unique_ptr<UIRequestHandler>
+    {
+        auto headers = parseHTTPResponseHeadersWithCommons(config, config_prefix, "text/html; charset=UTF-8", common_headers);
+        return std::make_unique<UIRequestHandler>(server, headers);
+    };
+    return std::make_shared<HandlingRuleHTTPHandlerFactory<UIRequestHandler>>(std::move(creator));
+}
+
 static inline auto createHandlersFactoryFromConfig(
     IServer & server,
     const Poco::Util::AbstractConfiguration & config,
@@ -90,6 +126,19 @@ static inline auto createHandlersFactoryFromConfig(
     Poco::Util::AbstractConfiguration::Keys keys;
     config.keys(prefix, keys);
 
+    std::unordered_map<String, String> common_headers_override;
+
+    if (std::find(keys.begin(), keys.end(), "common_http_response_headers") != keys.end())
+    {
+        auto common_headers_prefix = prefix + ".common_http_response_headers";
+        Poco::Util::AbstractConfiguration::Keys headers_keys;
+        config.keys(common_headers_prefix, headers_keys);
+        for (const auto & header_key : headers_keys)
+        {
+            common_headers_override[header_key] = config.getString(common_headers_prefix + "." + header_key);
+        }
+    }
+
     for (const auto & key : keys)
     {
         if (key == "defaults")
@@ -106,58 +155,75 @@ static inline auto createHandlersFactoryFromConfig(
 
             if (handler_type == "static")
             {
-                main_handler_factory->addHandler(createStaticHandlerFactory(server, config, prefix + "." + key));
+                main_handler_factory->addHandler(createStaticHandlerFactory(server, config, prefix + "." + key, common_headers_override));
             }
             else if (handler_type == "redirect")
             {
-                main_handler_factory->addHandler(createRedirectHandlerFactory(config, prefix + "." + key));
+                main_handler_factory->addHandler(createRedirectHandlerFactory(config, prefix + "." + key, common_headers_override));
             }
             else if (handler_type == "dynamic_query_handler")
             {
-                main_handler_factory->addHandler(createDynamicHandlerFactory(server, config, prefix + "." + key));
+                main_handler_factory->addHandler(createDynamicHandlerFactory(server, config, prefix + "." + key, common_headers_override));
             }
             else if (handler_type == "predefined_query_handler")
             {
-                main_handler_factory->addHandler(createPredefinedHandlerFactory(server, config, prefix + "." + key));
+                main_handler_factory->addHandler(createPredefinedHandlerFactory(server, config, prefix + "." + key, common_headers_override));
             }
             else if (handler_type == "prometheus")
             {
                 main_handler_factory->addHandler(
-                    createPrometheusHandlerFactoryForHTTPRule(server, config, prefix + "." + key, async_metrics));
+                    createPrometheusHandlerFactoryForHTTPRule(server, config, prefix + "." + key, async_metrics, common_headers_override));
             }
             else if (handler_type == "replicas_status")
             {
-                main_handler_factory->addHandler(createReplicasStatusHandlerFactory(server, config, prefix + "." + key));
+                main_handler_factory->addHandler(createReplicasStatusHandlerFactory(server, config, prefix + "." + key, common_headers_override));
             }
             else if (handler_type == "ping")
             {
-                auto handler = createPingHandlerFactory(server);
-                handler->addFiltersFromConfig(config, prefix + "." + key);
+                const String config_prefix = prefix + "." + key;
+                auto handler = createPingHandlerFactory(server, config, config_prefix, common_headers_override);
+                handler->addFiltersFromConfig(config, config_prefix);
                 main_handler_factory->addHandler(std::move(handler));
             }
             else if (handler_type == "play")
             {
-                auto handler = std::make_shared<HandlingRuleHTTPHandlerFactory<PlayWebUIRequestHandler>>(server);
+                auto handler = createWebUIHandlerFactory<PlayWebUIRequestHandler>(server, config, prefix + "." + key, common_headers_override);
                 handler->addFiltersFromConfig(config, prefix + "." + key);
                 main_handler_factory->addHandler(std::move(handler));
             }
             else if (handler_type == "dashboard")
             {
-                auto handler = std::make_shared<HandlingRuleHTTPHandlerFactory<DashboardWebUIRequestHandler>>(server);
+                auto handler = createWebUIHandlerFactory<DashboardWebUIRequestHandler>(server, config, prefix + "." + key, common_headers_override);
                 handler->addFiltersFromConfig(config, prefix + "." + key);
                 main_handler_factory->addHandler(std::move(handler));
             }
             else if (handler_type == "binary")
             {
-                auto handler = std::make_shared<HandlingRuleHTTPHandlerFactory<BinaryWebUIRequestHandler>>(server);
+                auto handler = createWebUIHandlerFactory<BinaryWebUIRequestHandler>(server, config, prefix + "." + key, common_headers_override);
+                handler->addFiltersFromConfig(config, prefix + "." + key);
+                main_handler_factory->addHandler(std::move(handler));
+            }
+            else if (handler_type == "js")
+            {
+                // NOTE: JavaScriptWebUIRequestHandler only makes sense for paths other then /js/uplot.js, /js/lz-string.js
+                // because these paths are hardcoded in dashboard.html
+                const auto & path = config.getString(prefix + "." + key + ".url", "");
+                if (path != "/js/uplot.js" && path != "/js/lz-string.js")
+                {
+                    throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
+                                    "Handler type 'js' is only supported for url '/js/'. "
+                                    "Configured path here: {}", path);
+                }
+
+                auto handler = createWebUIHandlerFactory<JavaScriptWebUIRequestHandler>(server, config, prefix + "." + key, common_headers_override);
                 handler->addFiltersFromConfig(config, prefix + "." + key);
                 main_handler_factory->addHandler(std::move(handler));
             }
             else
                 throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Unknown handler type '{}' in config here: {}.{}.handler.type",
                     handler_type, prefix, key);
         }
-        else
+        else if (key != "common_http_response_headers")
             throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, "Unknown element in config: "
                 "{}.{}, must be 'rule' or 'defaults'", prefix, key);
     }

src/Server/HTTPHandlerFactory.h

@@ -110,19 +110,23 @@ class HandlingRuleHTTPHandlerFactory : public HTTPRequestHandlerFactory
 
 HTTPRequestHandlerFactoryPtr createStaticHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix);
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers);
 
 HTTPRequestHandlerFactoryPtr createDynamicHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix);
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers);
 
 HTTPRequestHandlerFactoryPtr createPredefinedHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix);
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers);
 
 HTTPRequestHandlerFactoryPtr createReplicasStatusHandlerFactory(IServer & server,
     const Poco::Util::AbstractConfiguration & config,
-    const std::string & config_prefix);
+    const std::string & config_prefix,
+    std::unordered_map<String, String> & common_headers);
 
 /// @param server - used in handlers to check IServer::isCancelled()
 /// @param config - not the same as server.config(), since it can be newer

src/Server/HTTPResponseHeaderWriter.cpp

@@ -66,4 +66,25 @@ void applyHTTPResponseHeaders(Poco::Net::HTTPResponse & response, const std::uno
         response.set(header_name, header_value);
 }
 
+std::unordered_map<String, String> parseHTTPResponseHeadersWithCommons(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const std::string & default_content_type,
+    const std::unordered_map<String, String> & common_headers)
+{
+    auto headers = parseHTTPResponseHeaders(config, config_prefix, default_content_type);
+    headers.insert(common_headers.begin(), common_headers.end());
+    return headers;
+}
+
+std::unordered_map<String, String> parseHTTPResponseHeadersWithCommons(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const std::unordered_map<String, String> & common_headers)
+{
+    auto headers = parseHTTPResponseHeaders(config, config_prefix).value_or(std::unordered_map<String, String>{});
+    headers.insert(common_headers.begin(), common_headers.end());
+    return headers;
+}
+
 }

src/Server/HTTPResponseHeaderWriter.h

@@ -22,4 +22,16 @@ std::unordered_map<String, String> parseHTTPResponseHeaders(const std::string &
 void applyHTTPResponseHeaders(Poco::Net::HTTPResponse & response, const HTTPResponseHeaderSetup & setup);
 
 void applyHTTPResponseHeaders(Poco::Net::HTTPResponse & response, const std::unordered_map<String, String> & setup);
+
+std::unordered_map<String, String> parseHTTPResponseHeadersWithCommons(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const std::unordered_map<String, String> & common_headers);
+
+std::unordered_map<String, String> parseHTTPResponseHeadersWithCommons(
+    const Poco::Util::AbstractConfiguration & config,
+    const std::string & config_prefix,
+    const std::string & default_content_type,
+    const std::unordered_map<String, String> & common_headers);
+
 }

src/Server/PrometheusRequestHandler.cpp

@@ -3,6 +3,7 @@
 #include <Common/logger_useful.h>
 #include <Common/setThreadName.h>
 #include <IO/HTTPCommon.h>
+#include <Server/HTTPResponseHeaderWriter.h>
 #include <Server/HTTP/WriteBufferFromHTTPServerResponse.h>
 #include <Server/HTTP/sendExceptionToHTTPClient.h>
 #include <Server/IServer.h>
@@ -303,13 +304,15 @@ PrometheusRequestHandler::PrometheusRequestHandler(
     IServer & server_,
     const PrometheusRequestHandlerConfig & config_,
     const AsynchronousMetrics & async_metrics_,
-    std::shared_ptr<PrometheusMetricsWriter> metrics_writer_)
+    std::shared_ptr<PrometheusMetricsWriter> metrics_writer_,
+    std::unordered_map<String, String> response_headers_)
     : server(server_)
     , config(config_)
     , async_metrics(async_metrics_)
     , metrics_writer(metrics_writer_)
     , log(getLogger("PrometheusRequestHandler"))
 {
+    response_headers = response_headers_;
     createImpl();
 }
 
@@ -341,6 +344,7 @@ void PrometheusRequestHandler::createImpl()
 void PrometheusRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event_)
 {
     setThreadName("PrometheusHndlr");
+    applyHTTPResponseHeaders(response, response_headers);
 
     try
     {

src/Server/PrometheusRequestHandler.h

@@ -19,7 +19,8 @@ class PrometheusRequestHandler : public HTTPRequestHandler
         IServer & server_,
         const PrometheusRequestHandlerConfig & config_,
         const AsynchronousMetrics & async_metrics_,
-        std::shared_ptr<PrometheusMetricsWriter> metrics_writer_);
+        std::shared_ptr<PrometheusMetricsWriter> metrics_writer_,
+        std::unordered_map<String, String> response_headers_ = {});
     ~PrometheusRequestHandler() override;
 
     void handleRequest(HTTPServerRequest & request, HTTPServerResponse & response, const ProfileEvents::Event & write_event_) override;
@@ -59,6 +60,7 @@ class PrometheusRequestHandler : public HTTPRequestHandler
     std::unique_ptr<WriteBufferFromHTTPServerResponse> write_buffer_from_response;
     bool response_finalized = false;
     ProfileEvents::Event write_event;
+    std::unordered_map<String, String> response_headers;
 };
 
 }