Merge pull request ClickHouse#79350 from sachinkumarsingh092/string-bytes-functions

feat: Add stringBytesUniq and stringBytesEntropy functions

Author: alexey-milovidov

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -2836,6 +2836,8 @@ stochasticlinearregression
 stochasticlogisticregression
 storages
 storig
+stringBytesEntropy
+stringBytesUniq
 stringCompare
 stringJaccardIndex
 stringJaccardIndexUTF

docs/en/sql-reference/functions/string-functions.md

@@ -2637,3 +2637,67 @@ Result:
 1. │ [417784657,728683856,3071092609] │
    └──────────────────────────────────┘
 ```
+
+## stringBytesUniq {#stringbytesuniq}
+
+Counts the number of distinct bytes in a string.
+
+**Syntax**
+
+```sql
+stringBytesUniq(s)
+```
+
+**Arguments**
+
+- `s` — The string to analyze. [String](../data-types/string.md).
+
+**Returned value**
+
+- The number of distinct bytes in the string. [UInt16](../data-types/int-uint.md).
+
+**Example**
+
+```sql
+SELECT stringBytesUniq('Hello');
+```
+
+Result:
+
+```result
+┌─stringBytesUniq('Hello')─┐
+│                        4 │
+└──────────────────────────┘
+```
+
+## stringBytesEntropy {#stringbytesentropy}
+
+Calculates Shannon's entropy of byte distribution in a string.
+
+**Syntax**
+
+```sql
+stringBytesEntropy(s)
+```
+
+**Arguments**
+
+- `s` — The string to analyze. [String](../data-types/string.md).
+
+**Returned value**
+
+- Shannon's entropy of byte distribution in the string. [Float64](../data-types/float.md).
+
+**Example**
+
+```sql
+SELECT stringBytesEntropy('Hello, world!');
+```
+
+Result:
+
+```result
+┌─stringBytesEntropy('Hello, world!')─┐
+│                         3.07049960  │
+└─────────────────────────────────────┘
+```

src/Functions/stringBytes.h

@@ -0,0 +1,83 @@
+#pragma once
+
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/IFunction.h>
+#include <Common/BitHelpers.h>
+
+#include <cmath>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int ILLEGAL_COLUMN;
+}
+
+template <typename Impl, typename Name>
+class FunctionStringBytes : public IFunction
+{
+public:
+    static constexpr auto name = Name::name;
+    using ResultType = typename Impl::ResultType;
+
+    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionStringBytes>(); }
+
+    String getName() const override { return name; }
+
+    size_t getNumberOfArguments() const override { return 1; }
+
+    bool useDefaultImplementationForConstants() const override { return true; }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; }
+
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
+    {
+        if (!isString(arguments[0].type))
+            throw Exception(
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "Illegal type {} of argument of function {}",
+                arguments[0].type->getName(),
+                getName());
+
+        if constexpr (std::is_same_v<ResultType, UInt16>)
+            return std::make_shared<DataTypeUInt16>();
+        else if constexpr (std::is_same_v<ResultType, Float64>)
+            return std::make_shared<DataTypeFloat64>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    {
+        const ColumnPtr column = arguments[0].column;
+        const ColumnString * col_str = checkAndGetColumn<ColumnString>(column.get());
+
+        if (!col_str)
+            throw Exception(
+                ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", arguments[0].column->getName(), getName());
+
+        auto col_res = ColumnVector<ResultType>::create();
+        auto & vec_res = col_res->getData();
+        vec_res.resize(input_rows_count);
+
+        const ColumnString::Chars & data = col_str->getChars();
+        const ColumnString::Offsets & offsets = col_str->getOffsets();
+
+        size_t prev_offset = 0;
+        for (size_t i = 0; i < input_rows_count; ++i)
+        {
+            const UInt8 * data_ptr = data.data() + prev_offset;
+            const size_t size = offsets[i] - prev_offset - 1;
+
+            vec_res[i] = Impl::process(data_ptr, size);
+            prev_offset = offsets[i];
+        }
+
+        return col_res;
+    }
+};
+
+}

src/Functions/stringBytesEntropy.cpp

@@ -0,0 +1,82 @@
+#include <Columns/ColumnConst.h>
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/IFunction.h>
+#include <Functions/stringBytes.h>
+#include <Common/BitHelpers.h>
+#include <Common/PODArray.h>
+
+#include <cmath>
+
+namespace DB
+{
+
+struct StringBytesEntropyImpl
+{
+    using ResultType = Float64;
+
+    static ResultType process(const UInt8 * data, size_t size)
+    {
+        if (size == 0)
+            return 0;
+
+        std::array<UInt32, 256> counters{};
+        const UInt8 * end = data + size;
+
+        for (; data < end; ++data)
+            counters[*data]++;
+
+        Float64 entropy = 0.0;
+
+        for (size_t byte = 0; byte < 256; ++byte)
+        {
+            UInt32 count = counters[byte];
+            if (count > 0)
+            {
+                Float64 p = static_cast<Float64>(count) / size;
+                entropy -= p * std::log2(p);
+            }
+        }
+
+        return entropy;
+    }
+};
+
+struct NameStringBytesEntropy
+{
+    static constexpr auto name = "stringBytesEntropy";
+};
+
+using FunctionStringBytesEntropy = FunctionStringBytes<StringBytesEntropyImpl, NameStringBytesEntropy>;
+
+REGISTER_FUNCTION(StringBytesEntropy)
+{
+    FunctionDocumentation::Description description = "Calculates Shannon's entropy of byte distribution in a string.";
+    FunctionDocumentation::Syntax syntax = "stringBytesEntropy(s);";
+    FunctionDocumentation::Arguments arguments = {
+        {"s", "The string to analyze. [String](../../sql-reference/data-types/string.md))"}
+    };
+    FunctionDocumentation::ReturnedValue returned_value = "The Shannon entropy of the byte distribution. [Float64](../../sql-reference/data-types/float.md).";
+    FunctionDocumentation::Examples examples = {
+        {"Example",
+         "SELECT stringBytesEntropy('Hello, world!');",
+         "3.180832987205441"}
+    };
+    FunctionDocumentation::Category category = FunctionDocumentation::Category::String;
+
+    FunctionDocumentation function_documentation = {
+        .description = description,
+        .syntax = syntax,
+        .arguments = arguments,
+        .returned_value = returned_value,
+        .examples = examples,
+        .category = category
+    };
+
+    factory.registerFunction<FunctionStringBytesEntropy>(function_documentation);
+}
+
+}

src/Functions/stringBytesUniq.cpp

@@ -0,0 +1,73 @@
+#include <Columns/ColumnConst.h>
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnVector.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/stringBytes.h>
+#include <Functions/IFunction.h>
+#include <Common/BitHelpers.h>
+#include <Common/PODArray.h>
+
+#include <bit>
+#include <cmath>
+
+namespace DB
+{
+
+struct StringBytesUniqImpl
+{
+    using ResultType = UInt16;
+
+    static ResultType process(const UInt8 * data, size_t size)
+    {
+        UInt64 mask[4] = {0};
+        const UInt8 * end = data + size;
+
+        for (; data < end; ++data)
+        {
+            UInt8 byte = *data;
+            mask[byte >> 6] |= (1ULL << (byte & 0x3F));
+        }
+
+        return std::popcount(mask[0]) + std::popcount(mask[1]) + std::popcount(mask[2]) + std::popcount(mask[3]);
+    }
+};
+
+
+struct NameStringBytesUniq
+{
+    static constexpr auto name = "stringBytesUniq";
+};
+
+
+using FunctionStringBytesUniq = FunctionStringBytes<StringBytesUniqImpl, NameStringBytesUniq>;
+
+REGISTER_FUNCTION(StringBytesUniq)
+{
+    FunctionDocumentation::Description description = "Counts the number of distinct bytes in a string.";
+    FunctionDocumentation::Syntax syntax = "stringBytesUniq(s);";
+    FunctionDocumentation::Arguments arguments = {
+        {"s", "The string to analyze. [String](../../sql-reference/data-types/string.md)"}
+    };
+    FunctionDocumentation::ReturnedValue returned_value = "The number of distinct bytes in the string. [UInt16](../../sql-reference/data-types/int-uint.md).";
+    FunctionDocumentation::Examples examples = {
+        {"Example",
+         "SELECT stringBytesUniq('Hello, world!');",
+         "10"}
+    };
+    FunctionDocumentation::Category category = FunctionDocumentation::Category::String;
+
+    FunctionDocumentation function_documentation = {
+        .description = description,
+        .syntax = syntax,
+        .arguments = arguments,
+        .returned_value = returned_value,
+        .examples = examples,
+        .category = category
+    };
+
+    factory.registerFunction<FunctionStringBytesUniq>(function_documentation);
+}
+
+}

tests/performance/string_bytes_functions.xml

@@ -0,0 +1,11 @@
+<test>
+    <create_query>DROP TABLE IF EXISTS test_string_bytes</create_query>
+    <create_query>CREATE TABLE test_string_bytes(s String) ENGINE Memory</create_query>
+
+    <fill_query>INSERT INTO test_string_bytes SELECT reinterpretAsString(arrayJoin(range(0, 256))) FROM numbers(1000000)</fill_query>
+
+    <query>SELECT sum(stringBytesUniq(s)) FROM test_string_bytes FORMAT Null</query>
+    <query>SELECT sum(stringBytesEntropy(s)) FROM test_string_bytes FORMAT Null</query>
+
+    <drop_query>DROP TABLE IF EXISTS test_string_bytes</drop_query>
+</test>