Merge pull request #31968 from teskje/wallclock-lag-rollup-view

catalog: add wallclock lag rollup view

Author: teskje

doc/user/content/sql/system-catalog/mz_internal.md

@@ -1224,6 +1224,17 @@ operations in the system.
 | `created_at`             | [`timestamp with time zone`] | The time at which the subscription was created.                                                                            |
 | `referenced_object_ids`  | [`text list`]                | The IDs of objects referenced by the subscription. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects)             |
 
+## `mz_wallclock_global_lag`
+
+The `mz_wallclock_global_lag` view contains the most recently recorded wallclock lag
+for each table, source, index, materialized view, and sink in the system.
+
+<!-- RELATION_SPEC mz_internal.mz_wallclock_global_lag -->
+| Field         | Type         | Meaning
+| --------------| -------------| --------
+| `object_id`   | [`text`]     | The ID of the table, source, materialized view, index, or sink. Corresponds to [`mz_objects.id`](../mz_catalog/#mz_objects).
+| `lag`         | [`interval`] | The amount of time the object's write frontier lags behind wallclock time.
+
 ## `mz_wallclock_lag_history`
 
 The `mz_wallclock_lag_history` table records the historical wallclock lag,

src/catalog/src/builtin.rs

@@ -3678,6 +3678,19 @@ WHERE occurred_at + '1 day' > mz_now()",
         access: vec![PUBLIC_SELECT],
     });
 
+pub static MZ_WALLCLOCK_GLOBAL_LAG: LazyLock<BuiltinView> = LazyLock::new(|| BuiltinView {
+    name: "mz_wallclock_global_lag",
+    schema: MZ_INTERNAL_SCHEMA,
+    oid: oid::VIEW_MZ_WALLCLOCK_GLOBAL_LAG_OID,
+    column_defs: None,
+    sql: "
+SELECT DISTINCT ON (object_id) object_id, lag
+FROM mz_internal.mz_wallclock_global_lag_recent_history
+WHERE occurred_at + '5 minutes' > mz_now()
+ORDER BY object_id, occurred_at DESC",
+    access: vec![PUBLIC_SELECT],
+});
+
 pub static MZ_MATERIALIZED_VIEW_REFRESHES: LazyLock<BuiltinSource> =
     LazyLock::new(|| BuiltinSource {
         name: "mz_materialized_view_refreshes",
@@ -9686,6 +9699,7 @@ pub static BUILTINS_STATIC: LazyLock<Vec<Builtin<NameReference>>> = LazyLock::ne
         Builtin::Source(&MZ_WALLCLOCK_LAG_HISTORY),
         Builtin::View(&MZ_WALLCLOCK_GLOBAL_LAG_HISTORY),
         Builtin::View(&MZ_WALLCLOCK_GLOBAL_LAG_RECENT_HISTORY),
+        Builtin::View(&MZ_WALLCLOCK_GLOBAL_LAG),
         Builtin::Source(&MZ_MATERIALIZED_VIEW_REFRESHES),
         Builtin::Source(&MZ_COMPUTE_DEPENDENCIES),
         Builtin::Source(&MZ_COMPUTE_OPERATOR_HYDRATION_STATUSES_PER_WORKER),

src/environmentd/tests/testdata/http/ws

@@ -773,3 +773,4 @@ pub const INDEX_MZ_CLUSTER_DEPLOYMENT_LINEAGE_IND_OID: u32 = 17050;
 pub const INDEX_MZ_CLUSTER_REPLICA_FRONTIERS_IND_OID: u32 = 17051;
 pub const INDEX_MZ_COMPUTE_HYDRATION_TIMES_IND_OID: u32 = 17052;
 pub const VIEW_MZ_MAPPABLE_OBJECTS_OID: u32 = 17053;
+pub const VIEW_MZ_WALLCLOCK_GLOBAL_LAG_OID: u32 = 17054;

src/pgrepr-consts/src/oid.rs

@@ -637,6 +637,12 @@ SELECT position, name, type FROM objects WHERE schema = 'mz_internal' AND object
 4  created_at  timestamp␠with␠time␠zone
 5  referenced_object_ids  list
 
+query ITT
+SELECT position, name, type FROM objects WHERE schema = 'mz_internal' AND object = 'mz_wallclock_global_lag' ORDER BY position
+----
+1  object_id  text
+2  lag  interval
+
 query ITT
 SELECT position, name, type FROM objects WHERE schema = 'mz_internal' AND object = 'mz_wallclock_lag_history' ORDER BY position
 ----
@@ -770,6 +776,7 @@ mz_storage_shards
 mz_storage_usage_by_shard
 mz_subscriptions
 mz_type_pg_metadata
+mz_wallclock_global_lag
 mz_wallclock_global_lag_history
 mz_wallclock_global_lag_recent_history
 mz_wallclock_lag_history

test/sqllogictest/autogenerated/mz_internal.slt

@@ -721,6 +721,10 @@ mz_type_pg_metadata
 BASE TABLE
 materialize
 mz_internal
+mz_wallclock_global_lag
+VIEW
+materialize
+mz_internal
 mz_wallclock_global_lag_history
 VIEW
 materialize

test/sqllogictest/information_schema_tables.slt

@@ -37,8 +37,8 @@ mz_arrangement_heap_capacity_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangemen
 mz_arrangement_heap_size_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_heap_size_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_heap_size_raw"␠("operator_id",␠"worker_id")
 mz_arrangement_records_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_records_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_records_raw"␠("operator_id",␠"worker_id")
 mz_arrangement_sharing_raw_s2_primary_idx  CREATE␠INDEX␠"mz_arrangement_sharing_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_arrangement_sharing_raw"␠("operator_id",␠"worker_id")
-mz_cluster_deployment_lineage_ind  CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s722␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id")
-mz_cluster_replica_frontiers_ind  CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s717␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id")
+mz_cluster_deployment_lineage_ind  CREATE␠INDEX␠"mz_cluster_deployment_lineage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s723␠AS␠"mz_internal"."mz_cluster_deployment_lineage"]␠("cluster_id")
+mz_cluster_replica_frontiers_ind  CREATE␠INDEX␠"mz_cluster_replica_frontiers_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s718␠AS␠"mz_catalog"."mz_cluster_replica_frontiers"]␠("object_id")
 mz_cluster_replica_history_ind  CREATE␠INDEX␠"mz_cluster_replica_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s584␠AS␠"mz_internal"."mz_cluster_replica_history"]␠("dropped_at")
 mz_cluster_replica_metrics_history_ind  CREATE␠INDEX␠"mz_cluster_replica_metrics_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s496␠AS␠"mz_internal"."mz_cluster_replica_metrics_history"]␠("replica_id")
 mz_cluster_replica_metrics_ind  CREATE␠INDEX␠"mz_cluster_replica_metrics_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s495␠AS␠"mz_internal"."mz_cluster_replica_metrics"]␠("replica_id")
@@ -51,17 +51,17 @@ mz_clusters_ind  CREATE␠INDEX␠"mz_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON
 mz_columns_ind  CREATE␠INDEX␠"mz_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s461␠AS␠"mz_catalog"."mz_columns"]␠("name")
 mz_comments_ind  CREATE␠INDEX␠"mz_comments_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s511␠AS␠"mz_internal"."mz_comments"]␠("id")
 mz_compute_dataflow_global_ids_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_dataflow_global_ids_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_dataflow_global_ids_per_worker"␠("id",␠"worker_id")
-mz_compute_dependencies_ind  CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s705␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id")
+mz_compute_dependencies_ind  CREATE␠INDEX␠"mz_compute_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s706␠AS␠"mz_internal"."mz_compute_dependencies"]␠("dependency_id")
 mz_compute_error_counts_raw_s2_primary_idx  CREATE␠INDEX␠"mz_compute_error_counts_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_error_counts_raw"␠("export_id",␠"worker_id")
 mz_compute_exports_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_exports_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_exports_per_worker"␠("export_id",␠"worker_id")
 mz_compute_frontiers_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_frontiers_per_worker"␠("export_id",␠"worker_id")
-mz_compute_hydration_times_ind  CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s713␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id")
+mz_compute_hydration_times_ind  CREATE␠INDEX␠"mz_compute_hydration_times_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s714␠AS␠"mz_internal"."mz_compute_hydration_times"]␠("replica_id")
 mz_compute_hydration_times_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_hydration_times_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_hydration_times_per_worker"␠("export_id",␠"worker_id")
 mz_compute_import_frontiers_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_import_frontiers_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_import_frontiers_per_worker"␠("export_id",␠"import_id",␠"worker_id")
 mz_compute_lir_mapping_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_compute_lir_mapping_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_lir_mapping_per_worker"␠("global_id",␠"lir_id",␠"worker_id")
 mz_compute_operator_durations_histogram_raw_s2_primary_idx  CREATE␠INDEX␠"mz_compute_operator_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_compute_operator_durations_histogram_raw"␠("id",␠"worker_id",␠"duration_ns")
 mz_connections_ind  CREATE␠INDEX␠"mz_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s492␠AS␠"mz_catalog"."mz_connections"]␠("schema_id")
-mz_console_cluster_utilization_overview_ind  CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s709␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id")
+mz_console_cluster_utilization_overview_ind  CREATE␠INDEX␠"mz_console_cluster_utilization_overview_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s710␠AS␠"mz_internal"."mz_console_cluster_utilization_overview"]␠("cluster_id")
 mz_continual_tasks_ind  CREATE␠INDEX␠"mz_continual_tasks_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s514␠AS␠"mz_internal"."mz_continual_tasks"]␠("id")
 mz_databases_ind  CREATE␠INDEX␠"mz_databases_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s459␠AS␠"mz_catalog"."mz_databases"]␠("name")
 mz_dataflow_addresses_per_worker_s2_primary_idx  CREATE␠INDEX␠"mz_dataflow_addresses_per_worker_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_dataflow_addresses_per_worker"␠("id",␠"worker_id")
@@ -77,7 +77,7 @@ mz_message_batch_counts_received_raw_s2_primary_idx  CREATE␠INDEX␠"mz_messag
 mz_message_batch_counts_sent_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_batch_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_batch_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
 mz_message_counts_received_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_counts_received_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_received_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
 mz_message_counts_sent_raw_s2_primary_idx  CREATE␠INDEX␠"mz_message_counts_sent_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_message_counts_sent_raw"␠("channel_id",␠"from_worker_id",␠"to_worker_id")
-mz_notices_ind  CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s789␠AS␠"mz_internal"."mz_notices"]␠("id")
+mz_notices_ind  CREATE␠INDEX␠"mz_notices_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s790␠AS␠"mz_internal"."mz_notices"]␠("id")
 mz_object_dependencies_ind  CREATE␠INDEX␠"mz_object_dependencies_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s458␠AS␠"mz_internal"."mz_object_dependencies"]␠("object_id")
 mz_object_history_ind  CREATE␠INDEX␠"mz_object_history_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s522␠AS␠"mz_internal"."mz_object_history"]␠("id")
 mz_object_lifetimes_ind  CREATE␠INDEX␠"mz_object_lifetimes_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s523␠AS␠"mz_internal"."mz_object_lifetimes"]␠("id")
@@ -86,14 +86,14 @@ mz_objects_ind  CREATE␠INDEX␠"mz_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[
 mz_peek_durations_histogram_raw_s2_primary_idx  CREATE␠INDEX␠"mz_peek_durations_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_peek_durations_histogram_raw"␠("worker_id",␠"type",␠"duration_ns")
 mz_recent_activity_log_thinned_ind  CREATE␠INDEX␠"mz_recent_activity_log_thinned_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s683␠AS␠"mz_internal"."mz_recent_activity_log_thinned"]␠("sql_hash")
 mz_recent_sql_text_ind  CREATE␠INDEX␠"mz_recent_sql_text_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s679␠AS␠"mz_internal"."mz_recent_sql_text"]␠("sql_hash")
-mz_recent_storage_usage_ind  CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s781␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id")
+mz_recent_storage_usage_ind  CREATE␠INDEX␠"mz_recent_storage_usage_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s782␠AS␠"mz_catalog"."mz_recent_storage_usage"]␠("object_id")
 mz_roles_ind  CREATE␠INDEX␠"mz_roles_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s481␠AS␠"mz_catalog"."mz_roles"]␠("id")
 mz_scheduling_elapsed_raw_s2_primary_idx  CREATE␠INDEX␠"mz_scheduling_elapsed_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_elapsed_raw"␠("id",␠"worker_id")
 mz_scheduling_parks_histogram_raw_s2_primary_idx  CREATE␠INDEX␠"mz_scheduling_parks_histogram_raw_s2_primary_idx"␠IN␠CLUSTER␠[s2]␠ON␠"mz_introspection"."mz_scheduling_parks_histogram_raw"␠("worker_id",␠"slept_for_ns",␠"requested_ns")
 mz_schemas_ind  CREATE␠INDEX␠"mz_schemas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s460␠AS␠"mz_catalog"."mz_schemas"]␠("database_id")
 mz_secrets_ind  CREATE␠INDEX␠"mz_secrets_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s491␠AS␠"mz_catalog"."mz_secrets"]␠("name")
 mz_show_all_objects_ind  CREATE␠INDEX␠"mz_show_all_objects_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s568␠AS␠"mz_internal"."mz_show_all_objects"]␠("schema_id")
-mz_show_cluster_replicas_ind  CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s720␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster")
+mz_show_cluster_replicas_ind  CREATE␠INDEX␠"mz_show_cluster_replicas_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s721␠AS␠"mz_internal"."mz_show_cluster_replicas"]␠("cluster")
 mz_show_clusters_ind  CREATE␠INDEX␠"mz_show_clusters_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s570␠AS␠"mz_internal"."mz_show_clusters"]␠("name")
 mz_show_columns_ind  CREATE␠INDEX␠"mz_show_columns_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s569␠AS␠"mz_internal"."mz_show_columns"]␠("id")
 mz_show_connections_ind  CREATE␠INDEX␠"mz_show_connections_ind"␠IN␠CLUSTER␠[s2]␠ON␠[s578␠AS␠"mz_internal"."mz_show_connections"]␠("schema_id")

test/sqllogictest/mz_catalog_server_index_accounting.slt

@@ -1163,3 +1163,4 @@ SELECT oid, name FROM mz_objects WHERE id LIKE 's%' AND oid < 20000 ORDER BY oid
 17051  mz_cluster_replica_frontiers_ind
 17052  mz_compute_hydration_times_ind
 17053  mz_mappable_objects
+17054  mz_wallclock_global_lag

test/sqllogictest/oid.slt

@@ -693,6 +693,7 @@ mz_source_statuses                       ""
 mz_sql_text_redacted                     ""
 mz_aws_privatelink_connection_statuses   ""
 mz_statement_execution_history_redacted  ""
+mz_wallclock_global_lag                  ""
 mz_wallclock_global_lag_history          ""
 mz_wallclock_global_lag_recent_history   ""
 pg_class_all_databases                   ""

test/testdrive/catalog.td

@@ -96,3 +96,16 @@ snk          true  true
 src          true  true
 src_progress true  true
 tbl          true  true
+
+> SELECT o.name, l.lag >= '0s', l.lag < '10s'
+  FROM mz_internal.mz_wallclock_global_lag l
+  JOIN mz_objects o ON o.id = l.object_id
+  WHERE l.object_id LIKE 'u%'
+idx          true  true
+idx_const    true  true
+mv           true  true
+mv_const     true  true
+snk          true  true
+src          true  true
+src_progress true  true
+tbl          true  true