add current_storage_usage log streams topic (#42682)

atrakh · Convex, Inc. · commit f27bb87cb783 · 2025-11-04T21:05:07.000Z
Adds a new log stream topic called `current_storage_usage` to report snapshots of storage usage metrics at the current timestamp

GitOrigin-RevId: f15b27d1a8f8fd29ceb8bf5e025ef4380d6a67ec
diff --git a/crates/common/src/log_streaming.rs b/crates/common/src/log_streaming.rs
@@ -159,6 +159,15 @@ pub enum StructuredLogEvent {
     ScheduledJobLag {
         lag_seconds: Duration,
     },
+    /// Topic for storage usage metrics. These are periodic snapshots of current
+    /// storage state aggregated across all tables.
+    CurrentStorageUsage {
+        total_document_size_bytes: u64,
+        total_index_size_bytes: u64,
+        total_vector_storage_bytes: u64,
+        total_file_storage_bytes: u64,
+        total_backup_storage_bytes: u64,
+    },
     // User-specified topics -- not yet implemented.
     // See here for more details: https://www.notion.so/Log-Streaming-in-Convex-19a1dfadd6924c33b29b2796b0f5b2e2
     // User {
@@ -361,6 +370,21 @@ impl LogEvent {
                         "lag_seconds": lag_seconds.as_secs()
                     })
                 },
+                StructuredLogEvent::CurrentStorageUsage {
+                    total_document_size_bytes,
+                    total_index_size_bytes,
+                    total_vector_storage_bytes,
+                    total_file_storage_bytes,
+                    total_backup_storage_bytes,
+                } => serialize_map!({
+                    "_timestamp": ms,
+                    "_topic": "_current_storage_usage",
+                    "total_document_size_bytes": total_document_size_bytes,
+                    "total_index_size_bytes": total_index_size_bytes,
+                    "total_vector_storage_bytes": total_vector_storage_bytes,
+                    "total_file_storage_bytes": total_file_storage_bytes,
+                    "total_backup_storage_bytes": total_backup_storage_bytes,
+                }),
             },
             LogEventFormatVersion::V2 => match &self.event {
                 StructuredLogEvent::Verification => {
@@ -498,6 +522,23 @@ impl LogEvent {
                         "lag_seconds": lag_seconds.as_secs()
                     })
                 },
+                StructuredLogEvent::CurrentStorageUsage {
+                    total_document_size_bytes,
+                    total_index_size_bytes,
+                    total_vector_storage_bytes,
+                    total_file_storage_bytes,
+                    total_backup_storage_bytes,
+                } => {
+                    serialize_map!({
+                        "timestamp": ms,
+                        "topic": "current_storage_usage",
+                        "total_document_size_bytes": total_document_size_bytes,
+                        "total_index_size_bytes": total_index_size_bytes,
+                        "total_vector_storage_bytes": total_vector_storage_bytes,
+                        "total_file_storage_bytes": total_file_storage_bytes,
+                        "total_backup_storage_bytes": total_backup_storage_bytes,
+                    })
+                },
             },
         }
     }
@@ -838,6 +879,17 @@ mod tests {
         lag_seconds: u64,
     }
 
+    #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
+    #[allow(dead_code)]
+    struct StorageUsageEvent {
+        timestamp: u64,
+        total_document_size_bytes: u64,
+        total_index_size_bytes: u64,
+        total_vector_storage_bytes: u64,
+        total_file_storage_bytes: u64,
+        total_backup_storage_bytes: u64,
+    }
+
     // Union type for all log events, discriminated by topic field
     #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ToSchema)]
     #[serde(tag = "topic")]
@@ -855,6 +907,8 @@ mod tests {
         SchedulerStats(SchedulerStatsEvent),
         #[serde(rename = "scheduled_job_lag")]
         ScheduledJobLag(ScheduledJobLagEvent),
+        #[serde(rename = "current_storage_usage")]
+        CurrentStorageUsage(StorageUsageEvent),
     }
 
     #[test]
diff --git a/crates/database/src/table_usage.rs b/crates/database/src/table_usage.rs
@@ -9,7 +9,7 @@ use itertools::{
 use value::TableName;
 
 /// Counts the amount of storage used by documents and indexes in a table.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct TableUsage {
     /// Bytes used by documents in this table
     pub document_size: u64,
@@ -23,7 +23,7 @@ pub struct TableUsage {
 
 /// `TableUsage` for all tables in a database. `T` is the fully qualified name
 /// of a table.
-#[derive(Debug)]
+#[derive(Debug, Clone)]
 pub struct TablesUsage<T>(pub BTreeMap<T, TableUsage>);
 
 #[derive(Debug, PartialEq)]
diff --git a/crates/log_streaming/src/sinks/utils.rs b/crates/log_streaming/src/sinks/utils.rs
@@ -25,7 +25,8 @@ pub fn default_log_filter(event: &LogEvent) -> bool {
         | StructuredLogEvent::FunctionExecution { .. }
         | StructuredLogEvent::DeploymentAuditLog { .. }
         | StructuredLogEvent::SchedulerStats { .. }
-        | StructuredLogEvent::ScheduledJobLag { .. } => true,
+        | StructuredLogEvent::ScheduledJobLag { .. }
+        | StructuredLogEvent::CurrentStorageUsage { .. } => true,
         StructuredLogEvent::Exception { .. } => false,
     }
 }
diff --git a/npm-packages/@convex-dev/platform/log-stream-openapi.json b/npm-packages/@convex-dev/platform/log-stream-openapi.json
@@ -430,6 +430,27 @@
                 }
               }
             ]
+          },
+          {
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/StorageUsageEvent"
+              },
+              {
+                "type": "object",
+                "required": [
+                  "topic"
+                ],
+                "properties": {
+                  "topic": {
+                    "type": "string",
+                    "enum": [
+                      "current_storage_usage"
+                    ]
+                  }
+                }
+              }
+            ]
           }
         ]
       },
@@ -477,6 +498,49 @@
           }
         }
       },
+      "StorageUsageEvent": {
+        "type": "object",
+        "required": [
+          "timestamp",
+          "total_document_size_bytes",
+          "total_index_size_bytes",
+          "total_vector_storage_bytes",
+          "total_file_storage_bytes",
+          "total_backup_storage_bytes"
+        ],
+        "properties": {
+          "timestamp": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "total_backup_storage_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "total_document_size_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "total_file_storage_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "total_index_size_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          },
+          "total_vector_storage_bytes": {
+            "type": "integer",
+            "format": "int64",
+            "minimum": 0
+          }
+        }
+      },
       "VerificationEvent": {
         "type": "object",
         "required": [
diff --git a/npm-packages/@convex-dev/platform/src/generatedLogStreamApi.ts b/npm-packages/@convex-dev/platform/src/generatedLogStreamApi.ts
@@ -93,6 +93,9 @@ export interface components {
         }) | (components["schemas"]["ScheduledJobLagEvent"] & {
             /** @enum {string} */
             topic: "scheduled_job_lag";
+        }) | (components["schemas"]["StorageUsageEvent"] & {
+            /** @enum {string} */
+            topic: "current_storage_usage";
         });
         ScheduledJobLagEvent: {
             /** Format: int64 */
@@ -108,6 +111,20 @@ export interface components {
             /** Format: int64 */
             timestamp: number;
         };
+        StorageUsageEvent: {
+            /** Format: int64 */
+            timestamp: number;
+            /** Format: int64 */
+            total_backup_storage_bytes: number;
+            /** Format: int64 */
+            total_document_size_bytes: number;
+            /** Format: int64 */
+            total_file_storage_bytes: number;
+            /** Format: int64 */
+            total_index_size_bytes: number;
+            /** Format: int64 */
+            total_vector_storage_bytes: number;
+        };
         VerificationEvent: {
             message: string;
             /** Format: int64 */
@@ -126,6 +143,7 @@ export type FunctionExecutionEvent = components['schemas']['FunctionExecutionEve
 export type LogStreamEvent = components['schemas']['LogStreamEvent'];
 export type ScheduledJobLagEvent = components['schemas']['ScheduledJobLagEvent'];
 export type SchedulerStatsEvent = components['schemas']['SchedulerStatsEvent'];
+export type StorageUsageEvent = components['schemas']['StorageUsageEvent'];
 export type VerificationEvent = components['schemas']['VerificationEvent'];
 export type $defs = Record<string, never>;
 export type operations = Record<string, never>;
diff --git a/npm-packages/docs/docs/production/integrations/log-streams/log-streams.mdx b/npm-packages/docs/docs/production/integrations/log-streams/log-streams.mdx
@@ -88,7 +88,7 @@ type-safe pipelines ingesting log events.
 All events will have the following three fields:
 
 - `topic`: string, categorizes a log event, one of
-  `["verification", "console", "function_execution", "audit_log"]`
+  `["verification", "console", "function_execution", "audit_log", "scheduler_stats", "current_storage_usage"]`
 - `timestamp`: number, Unix epoch timestamp in milliseconds as an integer
 - `convex`: An object containing metadata related to your Convex deployment,
   including `deployment_name`, `deployment_type`, `project_name`, and
@@ -245,6 +245,46 @@ Schema:
   of the oldest overdue scheduled job, in seconds.
 - `num_running_jobs`: number, the number of scheduled jobs currently running
 
+### `current_storage_usage` events
+
+These events are periodically sent with snapshots of the current storage usage
+across your deployment. They provide aggregated totals for all storage types.
+
+These events are not currently sent for self-hosted deployments.
+
+For calculating billing costs:
+
+- Database Storage Bytes: `total_document_size_bytes + total_index_size_bytes`
+- File Storage: `total_file_storage_bytes + total_backup_storage_bytes`
+- Vector Storage: `total_vector_storage_bytes`
+
+Schema:
+
+- `topic`: `"current_storage_usage"`
+- `timestamp`: Unix epoch timestamp in milliseconds
+- `total_document_size_bytes`: number, total size in bytes of all documents
+  stored in database tables
+- `total_index_size_bytes`: number, total size in bytes of all database indexes
+- `total_vector_storage_bytes`: number, total size in bytes of vector index
+  storage
+- `total_file_storage_bytes`: number, total size in bytes of file storage
+- `total_backup_storage_bytes`: number, total size in bytes of snapshot/backup
+  storage
+
+Example event:
+
+```json
+{
+  "topic": "current_storage_usage",
+  "timestamp": 1715973841548,
+  "total_document_size_bytes": 104857600,
+  "total_index_size_bytes": 10485760,
+  "total_vector_storage_bytes": 5242880,
+  "total_file_storage_bytes": 52428800,
+  "total_backup_storage_bytes": 209715200
+}
+```
+
 ### `audit_log` events
 
 These events represent changes to your deployment, which also show up in the

Original file line number	Diff line number	Diff line change
`@@ -25,7 +25,8 @@ pub fn default_log_filter(event: &LogEvent) -> bool {`
`25`	`25`	`\| StructuredLogEvent::FunctionExecution { .. }`
`26`	`26`	`\| StructuredLogEvent::DeploymentAuditLog { .. }`
`27`	`27`	`\| StructuredLogEvent::SchedulerStats { .. }`
`28`		`- \| StructuredLogEvent::ScheduledJobLag { .. } => true,`
	`28`	`+ \| StructuredLogEvent::ScheduledJobLag { .. }`
	`29`	`+ \| StructuredLogEvent::CurrentStorageUsage { .. } => true,`
`29`	`30`	`StructuredLogEvent::Exception { .. } => false,`
`30`	`31`	`}`
`31`	`32`	`}`