Merge branch '25.8' into backport/25.8/86357

Author: kssenii

.gitmodules

@@ -124,7 +124,7 @@
 	url = https://github.com/anrieff/libcpuid
 [submodule "contrib/openldap"]
 	path = contrib/openldap
-	url = https://github.com/ClickHouse/openldap
+	url = https://github.com/openldap/openldap
 [submodule "contrib/AMQP-CPP"]
 	path = contrib/AMQP-CPP
 	url = https://github.com/ClickHouse/AMQP-CPP

cmake/autogenerated_versions.txt

@@ -2,11 +2,11 @@
 
 # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54502)
+SET(VERSION_REVISION 54503)
 SET(VERSION_MAJOR 25)
 SET(VERSION_MINOR 8)
-SET(VERSION_PATCH 2)
-SET(VERSION_GITHASH 4f2b50b8c9293522f2e9b593de8f709290fa8312)
-SET(VERSION_DESCRIBE v25.8.2.1-lts)
-SET(VERSION_STRING 25.8.2.1)
+SET(VERSION_PATCH 3)
+SET(VERSION_GITHASH 874146507b059896a7ca4ca257ee847fe05dbe05)
+SET(VERSION_DESCRIBE v25.8.3.1-lts)
+SET(VERSION_STRING 25.8.3.1)
 # end of autochange

contrib/openldap

@@ -969,6 +969,8 @@ To enable JSON logging support, use the following snippet:
 <logger>
     <formatting>
         <type>json</type>
+        <!-- Can be configured on a per-channel basis (log, errorlog, console, syslog), or globally for all channels (then just omit it). -->
+        <!-- <channel></channel> -->
         <names>
             <date_time>date_time</date_time>
             <thread_name>thread_name</thread_name>

docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md

@@ -0,0 +1,86 @@
+---
+description: 'System table containing information about metadata files read from Iceberg tables. Each entry
+  represents either a root metadata file, metadata extracted from an Avro file, or an entry of some Avro file.'
+keywords: ['system table', 'iceberg_metadata_log']
+slug: /operations/system-tables/iceberg_metadata_log
+title: 'system.iceberg_metadata_log'
+---
+
+import SystemTableCloud from '@site/docs/_snippets/_system_table_cloud.md';
+
+# system.iceberg_metadata_log
+
+The `system.iceberg_metadata_log` table records metadata access and parsing events for Iceberg tables read by ClickHouse. It provides detailed information about each metadata file or entry processed, which is useful for debugging, auditing, and understanding Iceberg table structure evolution.
+
+## Purpose {#purpose}
+
+This table logs every metadata file and entry read from Iceberg tables, including root metadata files, manifest lists, and manifest entries. It helps users trace how ClickHouse interprets Iceberg table metadata and diagnose issues related to schema evolution, file resolution, or query planning.
+
+:::note
+This table is primarily intended for debugging purposes.
+:::note
+
+## Columns {#columns}
+| Name           | Type      | Description                                                                                   |
+|----------------|-----------|----------------------------------------------------------------------------------------------|
+| `event_date`   | [Date](../../sql-reference/data-types/date.md)      | Date of the log entry.                                                                       |
+| `event_time`   | [DateTime](../../sql-reference/data-types/datetime.md)  | Timestamp of the event.                                                                      |
+| `query_id`     | [String](../../sql-reference/data-types/string.md)    | Query ID that triggered the metadata read.                                                   |
+| `content_type` | [Enum8](../../sql-reference/data-types/enum.md)     | Type of metadata content (see below).                                                        |
+| `table_path`   | [String](../../sql-reference/data-types/string.md)    | Path to the Iceberg table.                                                                   |
+| `file_path`    | [String](../../sql-reference/data-types/string.md)    | Path to the root metadata JSON file, Avro manifest list, or manifest file.                   |
+| `content`      | [String](../../sql-reference/data-types/string.md)    | Content in JSON format (raw metadata from .json, Avro metadata, or Avro entry).              |
+| `row_in_file`  | [Nullable](../../sql-reference/data-types/nullable.md)([UInt64](../../sql-reference/data-types/int-uint.md)) | Row number in the file, if applicable. Present for `ManifestListEntry` and `ManifestFileEntry` content types. |
+
+## `content_type` values {#content-type-values}
+
+- `None`: No content.
+- `Metadata`: Root metadata file.
+- `ManifestListMetadata`: Manifest list metadata.
+- `ManifestListEntry`: Entry in a manifest list.
+- `ManifestFileMetadata`: Manifest file metadata.
+- `ManifestFileEntry`: Entry in a manifest file.
+
+<SystemTableCloud/>
+
+## Controlling log verbosity {#controlling-log-verbosity}
+
+You can control which metadata events are logged using the [`iceberg_metadata_log_level`](../../operations/settings/settings.md#iceberg_metadata_log_level) setting.
+
+To log all metadata used in the current query:
+
+```sql
+SELECT * FROM my_iceberg_table SETTINGS iceberg_metadata_log_level = 'manifest_file_entry';
+
+SYSTEM FLUSH LOGS iceberg_metadata_log;
+
+SELECT content_type, file_path, row_in_file
+FROM system.iceberg_metadata_log
+WHERE query_id = '{previous_query_id}';
+```
+
+To log only the root metadata JSON file used in the current query:
+
+```sql
+SELECT * FROM my_iceberg_table SETTINGS iceberg_metadata_log_level = 'metadata';
+
+SYSTEM FLUSH LOGS iceberg_metadata_log;
+
+SELECT content_type, file_path, row_in_file
+FROM system.iceberg_metadata_log
+WHERE query_id = '{previous_query_id}';
+```
+
+See more information in the description of the [`iceberg_metadata_log_level`](../../operations/settings/settings.md#iceberg_metadata_log_level) setting.
+
+### Good To Know {#good-to-know}
+
+- Use `iceberg_metadata_log_level` at the query level only when you need to investigate your Iceberg table in detail. Otherwise, you may populate the log table with excessive metadata and experience performance degradation.
+- The table may contain duplicate entries, as it is intended primarily for debugging and does not guarantee uniqueness per entity.
+- If you use a `content_type` more verbose than `ManifestListMetadata`, the Iceberg metadata cache is disabled for manifest lists.
+- Similarly, if you use a `content_type` more verbose than `ManifestFileMetadata`, the Iceberg metadata cache is disabled for manifest files.
+
+## See also {#see-also}
+- [Iceberg Table Engine](../../engines/table-engines/integrations/iceberg.md)
+- [Iceberg Table Function](../../sql-reference/table-functions/iceberg.md)
+- [system.iceberg_history](./iceberg_history.md)

docs/en/operations/system-tables/iceberg_metadata_log.md

@@ -1223,7 +1223,7 @@ try
     std::vector<ProtocolServerAdapter> servers_to_start_before_tables;
 
     /// Wait for all threads to avoid possible use-after-free (for example logging objects can be already destroyed).
-    SCOPE_EXIT({
+    SCOPE_EXIT_SAFE({
         Stopwatch watch;
         LOG_INFO(log, "Waiting for background threads");
         GlobalThreadPool::instance().shutdown();
@@ -1274,7 +1274,7 @@ try
 
     /// NOTE: global context should be destroyed *before* GlobalThreadPool::shutdown()
     /// Otherwise GlobalThreadPool::shutdown() will hang, since Context holds some threads.
-    SCOPE_EXIT({
+    SCOPE_EXIT_SAFE({
         async_metrics.stop();
 
         /** Ask to cancel background jobs all table engines,

programs/server/Server.cpp

@@ -72,10 +72,12 @@
           </logger>
         </levels>
         -->
+
         <!-- Structured log formatting:
-        You can specify log format(for now, JSON only). In that case, the console log will be printed
-        in specified format like JSON.
-        For example, as below:
+
+        You can specify log format(for now, JSON only).
+        It can be done either on a per-channel (log, errorlog, console, syslog) level (set `channel`, i.e. `<channel>console</channel>') or for all channels (omit `channel`).
+        The log will be printed in specified format like JSON, example:
 
         {"date_time":"1650918987.180175","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
         {"date_time_utc":"2024-11-06T09:06:09Z","thread_name":"#1","thread_id":"254545","level":"Trace","query_id":"","logger_name":"BaseDaemon","message":"Received signal 2","source_file":"../base/daemon/BaseDaemon.cpp; virtual void SignalListener::run()","source_line":"192"}
@@ -91,8 +93,10 @@
         However, if you comment out all the tags under <names>, the program will print default values for as
         below.
         -->
-        <!-- <formatting>
+        <!--
+        <formatting>
             <type>json</type>
+            <channel></channel>
             <names>
                 <date_time>date_time</date_time>
                 <date_time_utc>date_time_utc</date_time_utc>
@@ -105,7 +109,8 @@
                 <source_file>source_file</source_file>
                 <source_line>source_line</source_line>
             </names>
-        </formatting> -->
+        </formatting>
+        -->
     </logger>
 
     <url_scheme_mappers>
@@ -1281,6 +1286,16 @@
         <flush_on_crash>false</flush_on_crash>
     </asynchronous_metric_log>
 
+    <iceberg_metadata_log>
+        <database>system</database>
+        <table>iceberg_metadata_log</table>
+        <flush_interval_milliseconds>2000</flush_interval_milliseconds>
+        <max_size_rows>1048576</max_size_rows>
+        <reserved_size_rows>8192</reserved_size_rows>
+        <buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
+        <flush_on_crash>false</flush_on_crash>
+    </iceberg_metadata_log>
+
     <!--
         OpenTelemetry log contains OpenTelemetry trace spans.
 

programs/server/config.xml

@@ -255,6 +255,25 @@ char * ColumnArray::serializeValueIntoMemory(size_t n, char * memory) const
     return memory;
 }
 
+std::optional<size_t> ColumnArray::getSerializedValueSize(size_t n) const
+{
+    const auto & offsets_data = getOffsets();
+
+    size_t pos = offsets_data[n - 1];
+    size_t end = offsets_data[n];
+
+    size_t res = sizeof(offsets_data[0]);
+    for (; pos < end; ++pos)
+    {
+        auto element_size = getData().getSerializedValueSize(pos);
+        if (!element_size)
+            return std::nullopt;
+        res += *element_size;
+    }
+
+    return res;
+}
+
 
 const char * ColumnArray::deserializeAndInsertFromArena(const char * pos)
 {

src/Columns/ColumnArray.cpp

@@ -80,6 +80,7 @@ class ColumnArray final : public COWHelper<IColumnHelper<ColumnArray>, ColumnArr
     void insertData(const char * pos, size_t length) override;
     StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
     char * serializeValueIntoMemory(size_t, char * memory) const override;
+    std::optional<size_t> getSerializedValueSize(size_t n) const override;
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;

src/Columns/ColumnArray.h

@@ -196,6 +196,7 @@ class ColumnDynamic final : public COWHelper<IColumnHelper<ColumnDynamic>, Colum
     StringRef serializeValueIntoArena(size_t n, Arena & arena, char const *& begin) const override;
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
+    std::optional<size_t> getSerializedValueSize(size_t) const override { return std::nullopt; }
 
     void updateHashWithValue(size_t n, SipHash & hash) const override;