Support JWT in clickhouse-client (by upstream)

zvonand · zvonand · commit ecdfbcd73683 · 2024-10-30T18:37:39.000Z
Fix fasttest

Fix fasttest

Fix

Update src/Client/ConnectionPool.h

Co-authored-by: Alexander Tokmakov &lt;tavplubix@clickhouse.com&gt;

Update src/Storages/StorageReplicatedMergeTree.cpp

Co-authored-by: Alexander Tokmakov &lt;tavplubix@clickhouse.com&gt;

Forbid creating a user with a name equal to `JWT_AUTHENTICAION_MARKER`

Cleanup

Pass `connection_host` by ref

Refactor with exceptions

Lint

Add new AuthenticationType to minimize conflicts

Fix fastest

Fix client failure

Add docs
diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
@@ -193,6 +193,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 - `--hardware-utilization` — Print hardware utilization information in progress bar.
 - `--print-profile-events` – Print `ProfileEvents` packets.
 - `--profile-events-delay-ms` – Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet).
+- `--jwt` – If specified, enables authorization via JSON Web Token. Server JWT authorization is available only in ClickHouse Cloud.
 
 Instead of `--host`, `--port`, `--user` and `--password` options, ClickHouse client also supports connection strings (see next section).
 
diff --git a/docs/ru/interfaces/cli.md b/docs/ru/interfaces/cli.md
@@ -141,6 +141,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
 -   `--secure` — если указано, будет использован безопасный канал.
 -   `--history_file` - путь к файлу с историей команд.
 -   `--param_<name>` — значение параметра для [запроса с параметрами](#cli-queries-with-parameters).
+-   `--jwt` – авторизация с использованием JSON Web Token. Доступно только в ClickHouse Cloud.
 
 Вместо параметров `--host`, `--port`, `--user` и `--password` клиент ClickHouse также поддерживает строки подключения (смотри следующий раздел).
 
diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
@@ -63,6 +63,7 @@ namespace ErrorCodes
     extern const int NETWORK_ERROR;
     extern const int AUTHENTICATION_FAILED;
     extern const int NO_ELEMENTS_IN_CONFIG;
+    extern const int USER_EXPIRED;
 }
 
 
@@ -73,6 +74,12 @@ void Client::processError(const String & query) const
         fmt::print(stderr, "Received exception from server (version {}):\n{}\n",
                 server_version,
                 getExceptionMessage(*server_exception, print_stack_trace, true));
+
+        if (server_exception->code() == ErrorCodes::USER_EXPIRED)
+        {
+            server_exception->rethrow();
+        }
+
         if (is_interactive)
         {
             fmt::print(stderr, "\n");
@@ -936,6 +943,7 @@ void Client::addOptions(OptionsDescription & options_description)
         ("ssh-key-file", po::value<std::string>(), "File containing ssh private key needed for authentication. If not set does password authentication.")
         ("ssh-key-passphrase", po::value<std::string>(), "Passphrase for imported ssh key.")
         ("quota_key", po::value<std::string>(), "A string to differentiate quotas when the user have keyed quotas configured on server")
+        ("jwt", po::value<std::string>(), "Use JWT for authentication")
 
         ("max_client_network_bandwidth", po::value<int>(), "the maximum speed of data exchange over the network for the client in bytes per second.")
         ("compression", po::value<bool>(), "enable or disable compression (enabled by default for remote communication and disabled for localhost communication).")
@@ -1093,6 +1101,12 @@ void Client::processOptions(const OptionsDescription & options_description,
         config().setBool("no-warnings", true);
     if (options.count("fake-drop"))
         fake_drop = true;
+    if (options.count("jwt"))
+    {
+        if (!options["user"].defaulted())
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "User and JWT flags can't be specified together");
+        config().setString("jwt", options["jwt"].as<std::string>());
+    }
     if (options.count("accept-invalid-certificate"))
     {
         config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");
diff --git a/src/Access/Authentication.cpp b/src/Access/Authentication.cpp
@@ -107,6 +107,9 @@ bool Authentication::areCredentialsValid(
             case AuthenticationType::HTTP:
                 throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
 
+            case AuthenticationType::JWT:
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
             case AuthenticationType::KERBEROS:
                 return external_authenticators.checkKerberosCredentials(auth_data.getKerberosRealm(), *gss_acceptor_context);
 
@@ -144,6 +147,9 @@ bool Authentication::areCredentialsValid(
             case AuthenticationType::SSL_CERTIFICATE:
                 throw Authentication::Require<BasicCredentials>("ClickHouse X.509 Authentication");
 
+            case AuthenticationType::JWT:
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
             case AuthenticationType::SSH_KEY:
                 throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
 
@@ -180,6 +186,9 @@ bool Authentication::areCredentialsValid(
             case AuthenticationType::SSH_KEY:
                 throw Authentication::Require<SshCredentials>("Ssh Keys Authentication");
 
+            case AuthenticationType::JWT:
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
             case AuthenticationType::BCRYPT_PASSWORD:
                 return checkPasswordBcrypt(basic_credentials->getPassword(), auth_data.getPasswordHashBinary());
 
@@ -209,6 +218,9 @@ bool Authentication::areCredentialsValid(
             case AuthenticationType::HTTP:
                 throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
 
+            case AuthenticationType::JWT:
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
             case AuthenticationType::KERBEROS:
                 throw Authentication::Require<GSSAcceptorContext>(auth_data.getKerberosRealm());
 
@@ -236,6 +248,9 @@ bool Authentication::areCredentialsValid(
             case AuthenticationType::HTTP:
                 throw Authentication::Require<BasicCredentials>("ClickHouse Basic Authentication");
 
+            case AuthenticationType::JWT:
+                throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+
             case AuthenticationType::KERBEROS:
                 throw Authentication::Require<GSSAcceptorContext>(auth_data.getKerberosRealm());
 
diff --git a/src/Access/AuthenticationData.cpp b/src/Access/AuthenticationData.cpp
@@ -126,6 +126,7 @@ void AuthenticationData::setPassword(const String & password_)
         case AuthenticationType::BCRYPT_PASSWORD:
         case AuthenticationType::NO_PASSWORD:
         case AuthenticationType::LDAP:
+        case AuthenticationType::JWT:
         case AuthenticationType::KERBEROS:
         case AuthenticationType::SSL_CERTIFICATE:
         case AuthenticationType::SSH_KEY:
@@ -231,6 +232,7 @@ void AuthenticationData::setPasswordHashBinary(const Digest & hash)
 
         case AuthenticationType::NO_PASSWORD:
         case AuthenticationType::LDAP:
+        case AuthenticationType::JWT:
         case AuthenticationType::KERBEROS:
         case AuthenticationType::SSL_CERTIFICATE:
         case AuthenticationType::SSH_KEY:
@@ -302,6 +304,10 @@ std::shared_ptr<ASTAuthenticationData> AuthenticationData::toAST() const
             node->children.push_back(std::make_shared<ASTLiteral>(getLDAPServerName()));
             break;
         }
+        case AuthenticationType::JWT:
+        {
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "JWT is available only in ClickHouse Cloud");
+        }
         case AuthenticationType::KERBEROS:
         {
             const auto & realm = getKerberosRealm();
diff --git a/src/Access/Common/AuthenticationType.cpp b/src/Access/Common/AuthenticationType.cpp
@@ -72,6 +72,11 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty
             static const auto info = make_info(Keyword::HTTP);
             return info;
         }
+        case AuthenticationType::JWT:
+        {
+            static const auto info = make_info(Keyword::JWT);
+            return info;
+        }
         case AuthenticationType::MAX:
             break;
     }
diff --git a/src/Access/Common/AuthenticationType.h b/src/Access/Common/AuthenticationType.h
@@ -41,6 +41,9 @@ enum class AuthenticationType
     /// Authentication through HTTP protocol
     HTTP,
 
+    /// JSON Web Token
+    JWT,
+
     MAX,
 };
 
diff --git a/src/Access/User.cpp b/src/Access/User.cpp
@@ -33,6 +33,8 @@ void User::setName(const String & name_)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);
     if (startsWith(name_, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);
+    if (name_.starts_with(EncodedUserInfo::JWT_AUTHENTICAION_MARKER))
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);
     name = name_;
 }
 
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
@@ -109,6 +109,7 @@ namespace ErrorCodes
     extern const int USER_SESSION_LIMIT_EXCEEDED;
     extern const int NOT_IMPLEMENTED;
     extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;
+    extern const int USER_EXPIRED;
 }
 
 }
@@ -2243,7 +2244,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 catch (...)
                 {
                     // Surprisingly, this is a client error. A server error would
-                    // have been reported without throwing (see onReceiveSeverException()).
+                    // have been reported without throwing (see onReceiveExceptionFromServer()).
                     client_exception = std::make_unique<Exception>(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());
                     have_error = true;
                 }
@@ -2616,6 +2617,9 @@ void ClientBase::runInteractive()
         }
         catch (const Exception & e)
         {
+            if (e.code() == ErrorCodes::USER_EXPIRED)
+                break;
+
             /// We don't need to handle the test hints in the interactive mode.
             std::cerr << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl;
             client_exception.reset(e.clone());
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
@@ -129,6 +129,7 @@ class ClientBase : public Poco::Util::Application, public IHints<2>
                                 const std::vector<Arguments> & hosts_and_ports_arguments) = 0;
     virtual void processConfig() = 0;
 
+    /// Returns true if query processing was successful.
     bool processQueryText(const String & text);
 
     virtual void readArguments(
diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp
@@ -68,6 +68,7 @@ Connection::Connection(const String & host_, UInt16 port_,
     const String & default_database_,
     const String & user_, const String & password_,
     const ssh::SSHKey & ssh_private_key_,
+    const String & jwt_,
     const String & quota_key_,
     const String & cluster_,
     const String & cluster_secret_,
@@ -78,6 +79,7 @@ Connection::Connection(const String & host_, UInt16 port_,
     , user(user_), password(password_)
     , ssh_private_key(ssh_private_key_)
     , quota_key(quota_key_)
+    , jwt(jwt_)
     , cluster(cluster_)
     , cluster_secret(cluster_secret_)
     , client_name(client_name_)
@@ -343,6 +345,11 @@ void Connection::sendHello()
         performHandshakeForSSHAuth();
     }
 #endif
+    else if (!jwt.empty())
+    {
+        writeStringBinary(EncodedUserInfo::JWT_AUTHENTICAION_MARKER, *out);
+        writeStringBinary(jwt, *out);
+    }
     else
     {
         writeStringBinary(user, *out);
@@ -1286,6 +1293,7 @@ ServerConnectionPtr Connection::createConnection(const ConnectionParameters & pa
         parameters.user,
         parameters.password,
         parameters.ssh_private_key,
+        parameters.jwt,
         parameters.quota_key,
         "", /* cluster */
         "", /* cluster_secret */
diff --git a/src/Client/Connection.h b/src/Client/Connection.h
@@ -54,6 +54,7 @@ class Connection : public IServerConnection
         const String & default_database_,
         const String & user_, const String & password_,
         const ssh::SSHKey & ssh_private_key_,
+        const String & jwt_,
         const String & quota_key_,
         const String & cluster_,
         const String & cluster_secret_,
@@ -172,6 +173,7 @@ class Connection : public IServerConnection
     String password;
     ssh::SSHKey ssh_private_key;
     String quota_key;
+    String jwt;
 
     /// For inter-server authorization
     String cluster;
diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp
@@ -53,31 +53,11 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
     /// changed the default value to "default" to fix the issue when the user in the prompt is blank
     user = config.getString("user", "default");
 
-    if (!config.has("ssh-key-file"))
+    if (config.has("jwt"))
     {
-        bool password_prompt = false;
-        if (config.getBool("ask-password", false))
-        {
-            if (config.has("password"))
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Specified both --password and --ask-password. Remove one of them");
-            password_prompt = true;
-        }
-        else
-        {
-            password = config.getString("password", "");
-            /// if the value of --password is omitted, the password will be set implicitly to "\n"
-            if (password == ASK_PASSWORD)
-                password_prompt = true;
-        }
-        if (password_prompt)
-        {
-            std::string prompt{"Password for user (" + user + "): "};
-            char buf[1000] = {};
-            if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0))
-                password = result;
-        }
+        jwt = config.getString("jwt");
     }
-    else
+    else if (config.has("ssh-key-file"))
     {
 #if USE_SSH
         std::string filename = config.getString("ssh-key-file");
@@ -103,6 +83,30 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "SSH is disabled, because ClickHouse is built without OpenSSL");
 #endif
     }
+    else
+    {
+        bool password_prompt = false;
+        if (config.getBool("ask-password", false))
+        {
+            if (config.has("password"))
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Specified both --password and --ask-password. Remove one of them");
+            password_prompt = true;
+        }
+        else
+        {
+            password = config.getString("password", "");
+            /// if the value of --password is omitted, the password will be set implicitly to "\n"
+            if (password == ASK_PASSWORD)
+                password_prompt = true;
+        }
+        if (password_prompt)
+        {
+            std::string prompt{"Password for user (" + user + "): "};
+            char buf[1000] = {};
+            if (auto * result = readpassphrase(prompt.c_str(), buf, sizeof(buf), 0))
+                password = result;
+        }
+    }
 
     quota_key = config.getString("quota_key", "");
 
@@ -140,7 +144,7 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati
 }
 
 UInt16 ConnectionParameters::getPortFromConfig(const Poco::Util::AbstractConfiguration & config,
-                                               std::string connection_host)
+                                               const std::string & connection_host)
 {
     bool is_secure = enableSecureConnection(config, connection_host);
     return config.getInt("port",
diff --git a/src/Client/ConnectionParameters.h b/src/Client/ConnectionParameters.h
@@ -21,6 +21,7 @@ struct ConnectionParameters
     std::string password;
     std::string quota_key;
     ssh::SSHKey ssh_private_key;
+    std::string jwt;
     Protocol::Secure security = Protocol::Secure::Disable;
     Protocol::Compression compression = Protocol::Compression::Enable;
     ConnectionTimeouts timeouts;
@@ -29,7 +30,7 @@ struct ConnectionParameters
     ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host);
     ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host, std::optional<UInt16> port);
 
-    static UInt16 getPortFromConfig(const Poco::Util::AbstractConfiguration & config, std::string connection_host);
+    static UInt16 getPortFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & connection_host);
 
     /// Ask to enter the user's password if password option contains this value.
     /// "\n" is used because there is hardly a chance that a user would use '\n' as password.
diff --git a/src/Client/ConnectionPool.h b/src/Client/ConnectionPool.h
@@ -123,7 +123,7 @@ class ConnectionPool : public IConnectionPool, private PoolBase<Connection>
     {
         return std::make_shared<Connection>(
             host, port,
-            default_database, user, password, ssh::SSHKey(), quota_key,
+            default_database, user, password, ssh::SSHKey(), /*jwt*/ "", quota_key,
             cluster, cluster_secret,
             client_name, compression, secure);
     }
diff --git a/src/Core/Protocol.h b/src/Core/Protocol.h
@@ -62,6 +62,9 @@ const char USER_INTERSERVER_MARKER[] = " INTERSERVER SECRET ";
 /// Marker of the SSH keys based authentication (passed in the user name)
 const char SSH_KEY_AUTHENTICAION_MARKER[] = " SSH KEY AUTHENTICATION ";
 
+/// Market for JSON Web Token authentication
+const char JWT_AUTHENTICAION_MARKER[] = " JWT AUTHENTICATION ";
+
 };
 
 namespace Protocol
diff --git a/src/Interpreters/SessionLog.cpp b/src/Interpreters/SessionLog.cpp
@@ -86,14 +86,15 @@ ColumnsDescription SessionLogElement::getColumnsDescription()
             AUTH_TYPE_NAME_AND_VALUE(AuthType::SHA256_PASSWORD),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::DOUBLE_SHA1_PASSWORD),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::LDAP),
+            AUTH_TYPE_NAME_AND_VALUE(AuthType::JWT),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::KERBEROS),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::SSH_KEY),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::SSL_CERTIFICATE),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::BCRYPT_PASSWORD),
             AUTH_TYPE_NAME_AND_VALUE(AuthType::HTTP),
         });
 #undef AUTH_TYPE_NAME_AND_VALUE
-    static_assert(static_cast<int>(AuthenticationType::MAX) == 10);
+    static_assert(static_cast<int>(AuthenticationType::MAX) == 11);
 
     auto interface_type_column = std::make_shared<DataTypeEnum8>(
         DataTypeEnum8::Values
diff --git a/src/Parsers/Access/ASTAuthenticationData.cpp b/src/Parsers/Access/ASTAuthenticationData.cpp
@@ -89,6 +89,12 @@ void ASTAuthenticationData::formatImpl(const FormatSettings & settings, FormatSt
                 password = true;
                 break;
             }
+            case AuthenticationType::JWT:
+            {
+                prefix = "CLAIMS";
+                parameter = true;
+                break;
+            }
             case AuthenticationType::LDAP:
             {
                 prefix = "SERVER";
diff --git a/src/Parsers/CommonParsers.h b/src/Parsers/CommonParsers.h
diff --git a/src/Server/grpc_protos/clickhouse_grpc.proto b/src/Server/grpc_protos/clickhouse_grpc.proto
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
diff --git a/utils/grpc-client/pb2/clickhouse_grpc_pb2.py b/utils/grpc-client/pb2/clickhouse_grpc_pb2.py

Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,11 @@ const AuthenticationTypeInfo & AuthenticationTypeInfo::get(AuthenticationType ty`
`72`	`72`	`static const auto info = make_info(Keyword::HTTP);`
`73`	`73`	`return info;`
`74`	`74`	`}`
	`75`	`+ case AuthenticationType::JWT:`
	`76`	`+ {`
	`77`	`+ static const auto info = make_info(Keyword::JWT);`
	`78`	`+ return info;`
	`79`	`+ }`
`75`	`80`	`case AuthenticationType::MAX:`
`76`	`81`	`break;`
`77`	`82`	`}`
Original file line number	Diff line number	Diff line change
`@@ -33,6 +33,8 @@ void User::setName(const String & name_)`
`33`	`33`	`throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);`
`34`	`34`	`if (startsWith(name_, EncodedUserInfo::SSH_KEY_AUTHENTICAION_MARKER))`
`35`	`35`	`throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);`
	`36`	`+ if (name_.starts_with(EncodedUserInfo::JWT_AUTHENTICAION_MARKER))`
	`37`	`+ throw Exception(ErrorCodes::BAD_ARGUMENTS, "User name '{}' is reserved", name_);`
`36`	`38`	`name = name_;`
`37`	`39`	`}`
`38`	`40`
Original file line number	Diff line number	Diff line change
`@@ -109,6 +109,7 @@ namespace ErrorCodes`
`109`	`109`	`extern const int USER_SESSION_LIMIT_EXCEEDED;`
`110`	`110`	`extern const int NOT_IMPLEMENTED;`
`111`	`111`	`extern const int CANNOT_READ_FROM_FILE_DESCRIPTOR;`
	`112`	`+ extern const int USER_EXPIRED;`
`112`	`113`	`}`
`113`	`114`
`114`	`115`	`}`
`@@ -2243,7 +2244,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)`
`2243`	`2244`	`catch (...)`
`2244`	`2245`	`{`
`2245`	`2246`	`// Surprisingly, this is a client error. A server error would`
`2246`		`- // have been reported without throwing (see onReceiveSeverException()).`
	`2247`	`+ // have been reported without throwing (see onReceiveExceptionFromServer()).`
`2247`	`2248`	`client_exception = std::make_unique<Exception>(getCurrentExceptionMessageAndPattern(print_stack_trace), getCurrentExceptionCode());`
`2248`	`2249`	`have_error = true;`
`2249`	`2250`	`}`
`@@ -2616,6 +2617,9 @@ void ClientBase::runInteractive()`
`2616`	`2617`	`}`
`2617`	`2618`	`catch (const Exception & e)`
`2618`	`2619`	`{`
	`2620`	`+ if (e.code() == ErrorCodes::USER_EXPIRED)`
	`2621`	`+ break;`
	`2622`	`+`
`2619`	`2623`	`/// We don't need to handle the test hints in the interactive mode.`
`2620`	`2624`	`std::cerr << "Exception on client:" << std::endl << getExceptionMessage(e, print_stack_trace, true) << std::endl << std::endl;`
`2621`	`2625`	`client_exception.reset(e.clone());`