cql3: implement metadata::calculate_metadata_id()

CQLv5 introduced metadata_id, which is a checksum computed from column
names and types, to track schema changes in prepared statements. This
commit introduces calculate_metadata_id to compute such id for given
metadata.

Please note that calculate_metadata_id() produces different hashes
than Cassandra's computeResultMetadataId(). We use SHA256 truncated to
128 bits instead of MD5. There are also two smaller technical
differences: calculate_metadata_id() doesn't add unneeded zeros and it
adds a length of a string when an sstring is being fed to the hasher.
The difference is intentional because MD5 has known vulnerabilities,
moreover we don't want to introduce any dependency between our
metadata_id and Cassandra's.

This change:
 - Add cql_metadata_id_type
 - Implement metadata::calculate_metadata_id()
 - Add boost tests to confirm correctness of the function

Author: andrzej-jackowski-scylladb

cql3/result_set.cc

@@ -8,7 +8,9 @@
  * SPDX-License-Identifier: (LicenseRef-ScyllaDB-Source-Available-1.0 and Apache-2.0)
  */
 
+#include <cstdint>
 #include "utils/assert.hh"
+#include "utils/hashers.hh"
 #include "cql3/result_set.hh"
 
 namespace cql3 {
@@ -71,6 +73,18 @@ lw_shared_ptr<const service::pager::paging_state> metadata::paging_state() const
     return _paging_state;
 }
 
+// Metadata_id is a checksum computed from given metadata to track schema changes in prepared statements.
+// Originally introduced in CQLv5.
+cql3::cql_metadata_id_type metadata::calculate_metadata_id() const {
+    auto h = sha256_hasher();
+    for (uint32_t i = 0; i < _column_info->_column_count; ++i) {
+        feed_hash(h, _column_info->_names[i]->name->name());
+        feed_hash(h, _column_info->_names[i]->type->name());
+    }
+    // Return first 16 bytes to have the same length as Cassandra's MD5
+    return cql_metadata_id_type(h.finalize().substr(0, 16));
+}
+
 prepared_metadata::prepared_metadata(const std::vector<lw_shared_ptr<column_specification>>& names,
                                      const std::vector<uint16_t>& partition_key_bind_indices,
                                      bool is_conditional)

cql3/result_set.hh

@@ -11,6 +11,7 @@
 #pragma once
 
 #include <vector>
+#include "cql3/statements/prepared_statement.hh"
 #include "utils/chunked_vector.hh"
 #include "enum_set.hh"
 #include "service/pager/paging_state.hh"
@@ -89,6 +90,8 @@ public:
     const std::vector<lw_shared_ptr<column_specification>>& get_names() const {
         return _column_info->_names;
     }
+
+    cql3::cql_metadata_id_type calculate_metadata_id() const;
 };
 
 ::shared_ptr<const cql3::metadata> make_empty_metadata();

cql3/statements/prepared_statement.hh

@@ -18,13 +18,21 @@
 #include <vector>
 
 #include "exceptions/exceptions.hh"
+#include "types/types.hh"
 
 namespace cql3 {
 
 class prepare_context;
 class column_specification;
 class cql_statement;
 
+struct cql_metadata_id_type {
+    explicit cql_metadata_id_type(bytes&& metadata_id) : _metadata_id(std::move(metadata_id)) {}
+    bytes _metadata_id;
+
+    bool operator==(const cql_metadata_id_type& other) const = default;
+};
+
 namespace statements {
 
 struct invalidated_prepared_usage_attempt {
@@ -61,3 +69,10 @@ public:
 }
 
 }
+
+template <> struct fmt::formatter<cql3::cql_metadata_id_type> {
+    constexpr auto parse(format_parse_context& ctx) { return ctx.begin(); }
+    auto format(const cql3::cql_metadata_id_type& m, fmt::format_context& ctx) const {
+        return fmt::format_to(ctx.out(), "{}", m._metadata_id);
+    }
+};

test/boost/schema_change_test.cc

@@ -7,9 +7,12 @@
  */
 
 
+#include "bytes.hh"
+#include <boost/intrusive/detail/hook_traits.hpp>
 #include <iostream>
 #include <fmt/ranges.h>
 #include <seastar/core/thread.hh>
+#include <utility>
 #undef SEASTAR_TESTING_MAIN
 #include <seastar/testing/test_case.hh>
 #include <seastar/util/defer.hh>
@@ -30,6 +33,7 @@
 #include "test/lib/exception_utils.hh"
 #include "test/lib/log.hh"
 #include "cdc/cdc_extension.hh"
+#include "test/lib/test_utils.hh"
 
 BOOST_AUTO_TEST_SUITE(schema_change_test)
 
@@ -1160,4 +1164,139 @@ SEASTAR_TEST_CASE(test_system_schema_version_is_stable) {
     });
 }
 
+// The purpose of this check is to make sure that we don't accidentally change the metadata_id.
+// The metadata_id should be stable to avoid ping-pong when driver connects to a mixed cluster.
+void verify_metadata_id_is_stable(cql3::cql_metadata_id_type metadata_id, sstring known_hash) {
+    BOOST_REQUIRE_EQUAL(metadata_id._metadata_id, from_hex(known_hash));
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_from_empty_metadata) {
+    auto m = cql3::metadata{std::vector<lw_shared_ptr<cql3::column_specification>>{}};
+    auto metadata_id = m.calculate_metadata_id();
+    BOOST_REQUIRE_EQUAL(metadata_id._metadata_id.size(), 16);
+    verify_metadata_id_is_stable(metadata_id, "e3b0c44298fc1c149afbf4c8996fb924");
+}
+
+cql3::cql_metadata_id_type compute_metadata_id(std::vector<std::pair<sstring, shared_ptr<const abstract_type>>> columns, sstring ks = "ks", sstring cf = "cf") {
+    std::vector<lw_shared_ptr<cql3::column_specification>> columns_specification;
+    for (const auto& column : columns) {
+        columns_specification.push_back(make_lw_shared(cql3::column_specification(ks, cf, make_shared<cql3::column_identifier>(column.first, false), column.second)));
+    }
+    return cql3::metadata{columns_specification}.calculate_metadata_id();
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_different_keyspace_and_table) {
+    const auto c = std::make_pair("id", uuid_type);
+    auto h1 = compute_metadata_id({c}, "ks1", "cf1");
+    auto h2 = compute_metadata_id({c}, "ks2", "cf2");
+
+    BOOST_REQUIRE_EQUAL(h1, h2);
+    verify_metadata_id_is_stable(h1, "d0c38eb409a57bb14497c35b80dfaaf1");
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_different_column_name) {
+    auto h1 = compute_metadata_id({{"id", uuid_type}});
+    auto h2 = compute_metadata_id({{"id2", uuid_type}});
+
+    BOOST_REQUIRE_NE(h1, h2);
+    verify_metadata_id_is_stable(h1, "d0c38eb409a57bb14497c35b80dfaaf1");
+    verify_metadata_id_is_stable(h2, "ae0bc2741d0480f0ebf4ee18a9bca7c7");
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_different_column_type) {
+    const auto column_name = "id";
+    auto h1 = compute_metadata_id({{column_name, uuid_type}});
+    auto h2 = compute_metadata_id({{column_name, int32_type}});
+
+    BOOST_REQUIRE_NE(h1, h2);
+    verify_metadata_id_is_stable(h1, "d0c38eb409a57bb14497c35b80dfaaf1");
+    verify_metadata_id_is_stable(h2, "b62d95c978e2e2498100ad8d20979868");
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_different_column_number) {
+    const auto c1 = std::make_pair("val1", int32_type);
+    const auto c2 = std::make_pair("val2", int32_type);
+    auto h1 = compute_metadata_id({c1});
+    auto h2 = compute_metadata_id({c1, c2});
+
+    BOOST_REQUIRE_NE(h1, h2);
+    verify_metadata_id_is_stable(h1, "f38171ab2b2e4d98e3f76a4640de5b32");
+    verify_metadata_id_is_stable(h2, "31c5cb5d0d41fbc426266248cc37941a");
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_different_column_order) {
+    const auto c1 = std::make_pair("val1", int32_type);
+    const auto c2 = std::make_pair("val2", int32_type);
+    auto h1 = compute_metadata_id({c1, c2});
+    auto h2 = compute_metadata_id({c2, c1});
+
+    BOOST_REQUIRE_NE(h1, h2);
+    verify_metadata_id_is_stable(h1, "31c5cb5d0d41fbc426266248cc37941a");
+    verify_metadata_id_is_stable(h2, "b52512f2b76d3e0695dcaf7b0a71efac");
+}
+
+BOOST_AUTO_TEST_CASE(metadata_id_with_udt) {
+
+    auto compute_metadata_id_for_type = [&](
+        const std::vector<bytes>& names,
+        const std::vector<data_type>& types,
+        const char* udt_name = "udt_name",
+        const bool multi_cell = true) {
+        BOOST_REQUIRE_EQUAL(names.size(), types.size());
+        return compute_metadata_id({{
+            "val1",
+            user_type_impl::get_instance("ks", udt_name, names, types, multi_cell)}}
+        );
+    };
+
+    auto h1 = compute_metadata_id_for_type({"f1"}, {int32_type});
+
+    // Different field number
+    auto h2 = compute_metadata_id_for_type({"f1", "f2"}, {int32_type, int32_type});
+    BOOST_REQUIRE_NE(h1, h2);
+
+    // Different field name
+    auto h3 = compute_metadata_id_for_type({"f2"}, {int32_type});
+    BOOST_REQUIRE_NE(h1, h3);
+
+    // Different field type
+    auto h4 = compute_metadata_id_for_type({"f1"}, {float_type});
+    BOOST_REQUIRE_NE(h1, h4);
+
+    // Different UDT name
+    auto h5 = compute_metadata_id_for_type({"f1"}, {int32_type}, "different_udt_name");
+    BOOST_REQUIRE_NE(h1, h5);
+
+    // False multi_cell mark
+    auto h6 = compute_metadata_id_for_type({"f1"}, {int32_type}, "udt_name", false);
+    BOOST_REQUIRE_NE(h1, h6);
+
+    verify_metadata_id_is_stable(h1, "9e556a9632191ac829c961c94719073a");
+    verify_metadata_id_is_stable(h2, "f0a58cd95fed3009b67ff6b4bda1fae1");
+    verify_metadata_id_is_stable(h3, "6a99234baebad33d9b9081cbdef9cd8b");
+    verify_metadata_id_is_stable(h4, "72780d64c71ec0265bb48194ec5b0f75");
+    verify_metadata_id_is_stable(h5, "767b01cdb5a61f90af9d824338de40e9");
+    verify_metadata_id_is_stable(h6, "02f16bdc4b235791a44983fe56618006");
+}
+
+cql3::cql_metadata_id_type get_metadata_id(cql_test_env& e, sstring const& table) {
+    auto msg = e.execute_cql(format("SELECT * FROM {};", table)).get();
+    auto rows = dynamic_pointer_cast<cql_transport::messages::result_message::rows>(msg);
+    return rows->rs().get_metadata().calculate_metadata_id();
+}
+
+SEASTAR_TEST_CASE(metadata_id_unchanged) {
+    return do_with_cql_env_thread([] (cql_test_env& e) {
+        cquery_nofail(e, "CREATE TABLE t(p int PRIMARY KEY, c1 int)");
+        cquery_nofail(e, "INSERT INTO  t(p, c1) VALUES (0, 0)");
+        const auto initial_metadata_id = get_metadata_id(e, "t");
+
+        cquery_nofail(e, "ALTER TABLE t ADD (c2 int)");
+        BOOST_REQUIRE_NE(initial_metadata_id, get_metadata_id(e, "t"));
+
+        cquery_nofail(e, "ALTER TABLE t DROP c2");
+        BOOST_REQUIRE_EQUAL(initial_metadata_id, get_metadata_id(e, "t"));
+    });
+}
+
 BOOST_AUTO_TEST_SUITE_END()