Merge remote-tracking branch 'origin/master' into split_kafka_tests

Author: ilejn

ci/jobs/scripts/check_style/aspell-ignore/en/aspell-dict.txt

@@ -1887,8 +1887,10 @@ geospatial
 getClientHTTPHeader
 getMacro
 getMaxTableNameLengthForDatabase
+getMergeTreeSetting
 getOSKernelVersion
 getServerPort
+getServerSetting
 getSetting
 getSettingOrDefault
 getSizeOfEnumType
@@ -2233,6 +2235,7 @@ mebibytes
 memtable
 memtables
 mergeTreeIndex
+mergeTreeProjection
 mergeable
 mergetree
 messageID

ci/jobs/scripts/workflow_hooks/filter_job.py

@@ -83,17 +83,18 @@ def should_skip_job(job_name):
             f"Skipped, labeled with '{Labels.CI_FUNCTIONAL_FLAKY}' - run stateless test jobs only",
         )
 
-    if Labels.CI_INTEGRATION in _info_cache.pr_labels and (
+    if Labels.CI_INTEGRATION in _info_cache.pr_labels and not (
         job_name.startswith(JobNames.INTEGRATION) or job_name in JobConfigs.builds_for_tests
     ):
         return (
             True,
             f"Skipped, labeled with '{Labels.CI_INTEGRATION}' - run integration test jobs only",
         )
 
-    if Labels.CI_FUNCTIONAL in _info_cache.pr_labels and (
+    if Labels.CI_FUNCTIONAL in _info_cache.pr_labels and not (
         job_name.startswith(JobNames.STATELESS)
-        or job_name.startswith(JobNames.STATEFUL or job_name in JobConfigs.builds_for_tests)
+        or job_name.startswith(JobNames.STATEFUL)
+        or job_name in JobConfigs.builds_for_tests
     ):
         return (
             True,

contrib/zstd

@@ -4671,3 +4671,67 @@ Result:
 │                                         206 │
 └─────────────────────────────────────────────┘
 ```
+
+## getServerSetting {#getserversetting}
+
+Returns the current value of one of the server settings
+
+**Syntax**
+
+```sql
+getServerSetting('server_setting');
+```
+
+**Parameter**
+
+- `server_setting` — The setting name. [String](../data-types/string.md).
+
+**Returned value**
+
+- The server setting's current value.
+
+**Example**
+
+```sql
+SELECT getServerSetting('allow_use_jemalloc_memory');
+```
+
+Result:
+
+```text
+┌─getServerSetting('allow_use_jemalloc_memory')─┐
+│ true                                          │
+└───────────────────────────────────────────────┘
+```
+
+## getMergeTreeSetting {#getmergetreesetting}
+
+Returns the current value of one of the merge tree settings
+
+**Syntax**
+
+```sql
+getMergeTreeSetting('merge_tree_setting');
+```
+
+**Parameter**
+
+- `merge_tree_setting` — The setting name. [String](../data-types/string.md).
+
+**Returned value**
+
+- The merge tree setting's current value.
+
+**Example**
+
+```sql
+SELECT getMergeTreeSetting('index_granularity');
+```
+
+Result:
+
+```text
+┌─getMergeTree(index_granularity')─┐
+│                     8192         │
+└──────────────────────────────────┘
+```

docs/en/sql-reference/functions/other-functions.md

@@ -137,6 +137,49 @@ As mentioned before, we could review the `system.query_log` table. On the `proje
 SELECT query, projections FROM system.query_log WHERE query_id='<query_id>'
 ```
 
+## Normal projection with `_part_offset` field {#normal-projection-with-part-offset-field}
+
+Creating a table with a normal projection that utilizes the `_part_offset` field:
+
+```sql
+CREATE TABLE events
+(
+    `event_time` DateTime,
+    `event_id` UInt64,
+    `user_id` UInt64,
+    `huge_string` String,
+    PROJECTION order_by_user_id
+    (
+        SELECT
+            _part_offset
+        ORDER BY user_id
+    )
+)
+ENGINE = MergeTree()
+ORDER BY (event_id);
+```
+
+Inserting some sample data:
+
+```sql
+INSERT INTO events SELECT * FROM generateRandom() LIMIT 100000;
+```
+
+### Using `_part_offset` as a secondary index {#normal-projection-secondary-index}
+
+The `_part_offset` field preserves its value through merges and mutations, making it valuable for secondary indexing. We can leverage this in queries:
+
+```sql
+SELECT
+    count()
+FROM events
+WHERE (_part, _part_offset) IN (
+    SELECT _part, _part_offset
+    FROM events
+    WHERE user_id = 42
+)
+```
+
 # Manipulating Projections
 
 The following operations with [projections](/engines/table-engines/mergetree-family/mergetree.md/#projections) are available:

docs/en/sql-reference/statements/alter/projection.md

@@ -0,0 +1,70 @@
+---
+description: 'Represents the contents of some projection in MergeTree tables.
+  It can be used for introspection.'
+sidebar_label: 'mergeTreeProjection'
+sidebar_position: 77
+slug: /sql-reference/table-functions/mergeTreeProjection
+title: 'mergeTreeProjection'
+---
+
+# mergeTreeProjection Table Function
+
+Represents the contents of some projection in MergeTree tables. It can be used for introspection.
+
+```sql
+mergeTreeProjection(database, table, projection)
+```
+
+**Arguments**
+
+- `database`- The database name to read projection from.
+- `table`- The table name to read projection from.
+- `projection` - The projection to read from.
+
+**Returned Value**
+
+A table object with columns provided by given projection.
+
+## Usage Example {#usage-example}
+
+```sql
+CREATE TABLE test
+(
+    `user_id` UInt64,
+    `item_id` UInt64,
+    PROJECTION order_by_item_id
+    (
+        SELECT _part_offset
+        ORDER BY item_id
+    )
+)
+ENGINE = MergeTree
+ORDER BY user_id;
+
+INSERT INTO test SELECT number, 100 - number FROM numbers(5);
+```
+
+```sql
+SELECT *, _part_offset FROM mergeTreeProjection(currentDatabase(), test, order_by_item_id);
+```
+
+```text
+   ┌─item_id─┬─_parent_part_offset─┬─_part_offset─┐
+1. │      96 │                   4 │            0 │
+2. │      97 │                   3 │            1 │
+3. │      98 │                   2 │            2 │
+4. │      99 │                   1 │            3 │
+5. │     100 │                   0 │            4 │
+   └─────────┴─────────────────────┴──────────────┘
+```
+
+```sql
+DESCRIBE mergeTreeProjection(currentDatabase(), test, order_by_item_id) SETTINGS describe_compact_output = 1;
+```
+
+```text
+   ┌─name────────────────┬─type───┐
+1. │ item_id             │ UInt64 │
+2. │ _parent_part_offset │ UInt64 │
+   └─────────────────────┴────────┘
+```

docs/en/sql-reference/table-functions/mergeTreeProjection.md

@@ -313,7 +313,9 @@ class Benchmark : public Poco::Util::Application
         std::lock_guard lock(mutex);
 
         log << "\nQueries executed: " << num;
-        if (queries.size() > 1)
+        if (max_iterations > 1)
+            log << " (" << (num * 100.0 / max_iterations) << "%)";
+        else if (queries.size() > 1)
             log << " (" << (num * 100.0 / queries.size()) << "%)";
         log << ".\n" << flush;
     }

programs/benchmark/Benchmark.cpp

@@ -314,14 +314,12 @@ namespace ServerSetting
     extern const ServerSettingsUInt64 max_prefixes_deserialization_thread_pool_size;
     extern const ServerSettingsUInt64 max_prefixes_deserialization_thread_pool_free_size;
     extern const ServerSettingsUInt64 prefixes_deserialization_thread_pool_thread_pool_queue_size;
-    extern const ServerSettingsUInt64 page_cache_block_size;
     extern const ServerSettingsUInt64 page_cache_history_window_ms;
     extern const ServerSettingsString page_cache_policy;
     extern const ServerSettingsDouble page_cache_size_ratio;
     extern const ServerSettingsUInt64 page_cache_min_size;
     extern const ServerSettingsUInt64 page_cache_max_size;
     extern const ServerSettingsDouble page_cache_free_memory_ratio;
-    extern const ServerSettingsUInt64 page_cache_lookahead_blocks;
     extern const ServerSettingsUInt64 page_cache_shards;
     extern const ServerSettingsUInt64 os_cpu_busy_time_threshold;
     extern const ServerSettingsFloat min_os_cpu_wait_time_ratio_to_drop_connection;
@@ -1170,8 +1168,6 @@ try
     if (server_settings[ServerSetting::page_cache_max_size] != 0)
     {
         global_context->setPageCache(
-            server_settings[ServerSetting::page_cache_block_size],
-            server_settings[ServerSetting::page_cache_lookahead_blocks],
             std::chrono::milliseconds(Int64(server_settings[ServerSetting::page_cache_history_window_ms])),
             server_settings[ServerSetting::page_cache_policy],
             server_settings[ServerSetting::page_cache_size_ratio],

programs/server/Server.cpp

@@ -553,7 +553,7 @@
 let params = default_params;
 
 /// Palette generation for charts
-function generatePalette(numColors) {
+function generateColor(l, c, h) {
     // oklch() does not work in firefox<=125 inside <canvas> element so we convert it back to rgb for now.
     // Based on https://github.com/color-js/color.js/blob/main/src/spaces/oklch.js
     const multiplyMatrices = (A, B) => {
@@ -598,13 +598,17 @@
         ], xyz)
     }
 
-    const oklch2rgb = lch =>  srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch))))
+    const oklch2rgb = lch => srgbLinear2rgb(xyz2rgbLinear(oklab2xyz(oklch2oklab(lch))))
+
+    let rgb = oklch2rgb([l, c, h]);
+    return `rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`;
+}
+
+function generatePalette(numColors) {
 
     palette = [];
     for (let i = 0; i < numColors; i++) {
-        //palette.push(`oklch(${theme != 'dark' ? 0.75 : 0.5}, 0.15, ${360 * i / numColors})`);
-        let rgb = oklch2rgb([theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors]);
-        palette.push(`rgb(${rgb[0] * 255}, ${rgb[1] * 255}, ${rgb[2] * 255})`);
+        palette.push(generateColor(theme != 'dark' ? 0.75 : 0.5, 0.15, 360 * i / numColors));
     }
     return palette;
 }
@@ -1146,6 +1150,15 @@
     return {reply, error};
 }
 
+function stringHash(str) {
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        hash = ((hash << 5) - hash) + str.charCodeAt(i);
+        hash = hash & hash;
+    }
+    return hash;
+}
+
 async function draw(idx, chart, url_params, query) {
     if (plots[idx]) {
         plots[idx].destroy();
@@ -1238,9 +1251,12 @@
         title_div.style.display = 'block';
     }
 
-    const [line_color, fill_color, grid_color, axes_color] = theme != 'dark'
-        ? ["#ff8888", "#ffeeee", "#eeeedd", "#2c3235"]
-        : ["#886644", "#004455", "#2c3235", "#c7d0d9"];
+    const color_rotate = Math.abs(stringHash(query));
+
+    const line_color = theme != 'dark' ? generateColor(0.75, 0.14, (21 + color_rotate) % 360) : generateColor(0.53, 0.07, (56 + color_rotate) % 360);
+    const fill_color = theme != 'dark' ? generateColor(0.96, 0.02, (21 + color_rotate) % 360) : generateColor(0.36, 0.07, (56 + color_rotate) % 360);
+    const grid_color = theme != 'dark' ? "#eeeedd" : "#2c3235";
+    const axes_color = theme != 'dark' ? "#2c3235" : "#c7d0d9";
 
     let sync = uPlot.sync("sync");
 

programs/server/dashboard.html

@@ -54,6 +54,11 @@ using QueryTreeNodeConstRawPtrWithHashSet = std::unordered_set<QueryTreeNodeCons
 template <typename Value>
 using QueryTreeNodePtrWithHashMap = std::unordered_map<QueryTreeNodePtrWithHash, Value>;
 
+class ColumnNode;
+using ColumnNodePtr = std::shared_ptr<ColumnNode>;
+using ColumnNodePtrWithHash = QueryTreeNodeWithHash<ColumnNodePtr>;
+using ColumnNodePtrWithHashSet = std::unordered_set<ColumnNodePtrWithHash>;
+
 }
 
 template <typename T, bool compare_aliases, bool compare_types>