diff --git a/.changeset/beige-camels-reply.md b/.changeset/beige-camels-reply.md new file mode 100644 index 000000000..baac818bc --- /dev/null +++ b/.changeset/beige-camels-reply.md @@ -0,0 +1,12 @@ +--- +'@powersync/service-module-postgres-storage': minor +'@powersync/service-module-mongodb-storage': minor +'@powersync/service-core-tests': minor +'@powersync/service-module-postgres': minor +'@powersync/service-module-mongodb': minor +'@powersync/service-core': minor +'@powersync/service-module-mysql': minor +'@powersync/service-types': minor +--- + +Refactored Metrics to use a MetricsEngine which is telemetry framework agnostic. diff --git a/modules/module-mongodb-storage/test/src/setup.ts b/modules/module-mongodb-storage/test/src/setup.ts index 43946a1aa..0cdd8be9b 100644 --- a/modules/module-mongodb-storage/test/src/setup.ts +++ b/modules/module-mongodb-storage/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-mongodb/src/module/MongoModule.ts b/modules/module-mongodb/src/module/MongoModule.ts index f6bf30a5f..f2bce5082 100644 --- a/modules/module-mongodb/src/module/MongoModule.ts +++ b/modules/module-mongodb/src/module/MongoModule.ts @@ -37,11 +37,14 @@ export class MongoModule extends replication.ReplicationModule {} + /** * Combines base config with normalized connection settings */ diff --git a/modules/module-mongodb/src/replication/ChangeStream.ts b/modules/module-mongodb/src/replication/ChangeStream.ts index 4bb5c828f..3a47ca435 100644 --- a/modules/module-mongodb/src/replication/ChangeStream.ts +++ b/modules/module-mongodb/src/replication/ChangeStream.ts @@ -8,14 +8,7 @@ import { ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; -import { - BSON_DESERIALIZE_DATA_OPTIONS, - Metrics, - SaveOperationTag, - SourceEntityDescriptor, - SourceTable, - storage -} from '@powersync/service-core'; +import { MetricsEngine, SaveOperationTag, SourceEntityDescriptor, SourceTable, storage } from '@powersync/service-core'; import { DatabaseInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules'; import { MongoLSN } from '../common/MongoLSN.js'; import { PostImagesOption } from '../types/types.js'; @@ -23,10 +16,12 @@ import { escapeRegExp } from '../utils.js'; import { MongoManager } from './MongoManager.js'; import { constructAfterRecord, createCheckpoint, getCacheIdentifier, getMongoRelation } from './MongoRelation.js'; import { CHECKPOINTS_COLLECTION } from './replication-utils.js'; +import { ReplicationMetric } from '@powersync/service-types'; export interface ChangeStreamOptions { connections: MongoManager; storage: storage.SyncRulesBucketStorage; + metrics: MetricsEngine; abort_signal: AbortSignal; } @@ -59,6 +54,7 @@ export class ChangeStream { private connections: MongoManager; private readonly client: mongo.MongoClient; private readonly defaultDb: mongo.Db; + private readonly metrics: MetricsEngine; private abort_signal: AbortSignal; @@ -66,6 +62,7 @@ export class ChangeStream { constructor(options: ChangeStreamOptions) { this.storage = options.storage; + this.metrics = options.metrics; this.group_id = options.storage.group_id; this.connections = options.connections; this.client = this.connections.client; @@ -318,7 +315,7 @@ export class ChangeStream { } at += docBatch.length; - Metrics.getInstance().rows_replicated_total.add(docBatch.length); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(docBatch.length); const duration = performance.now() - lastBatch; lastBatch = performance.now(); logger.info( @@ -446,7 +443,7 @@ export class ChangeStream { return null; } - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); if (change.operationType == 'insert') { const baseRecord = constructAfterRecord(change.fullDocument); return await batch.save({ diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts index a9b2fb6dd..c8e8fc8f0 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicationJob.ts @@ -71,6 +71,7 @@ export class ChangeStreamReplicationJob extends replication.AbstractReplicationJ const stream = new ChangeStream({ abort_signal: this.abortController.signal, storage: this.options.storage, + metrics: this.options.metrics, connections: connectionManager }); await stream.replicate(); diff --git a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts index 9f6aae168..99930e2eb 100644 --- a/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts +++ b/modules/module-mongodb/src/replication/ChangeStreamReplicator.ts @@ -20,6 +20,7 @@ export class ChangeStreamReplicator extends replication.AbstractReplicator { // Executes for every test file container.registerDefaults(); - - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-mysql/src/module/MySQLModule.ts b/modules/module-mysql/src/module/MySQLModule.ts index 32d6fbc2f..ced2464f8 100644 --- a/modules/module-mysql/src/module/MySQLModule.ts +++ b/modules/module-mysql/src/module/MySQLModule.ts @@ -25,9 +25,7 @@ export class MySQLModule extends replication.ReplicationModule { - await super.initialize(context); - } + async onInitialized(context: system.ServiceContextContainer): Promise {} protected createRouteAPIAdapter(): api.RouteAPI { return new MySQLRouteAPIAdapter(this.resolveConfig(this.decodedConfig!)); @@ -42,6 +40,7 @@ export class MySQLModule extends replication.ReplicationModule(); - constructor(protected options: BinLogStreamOptions) { + constructor(private options: BinLogStreamOptions) { this.storage = options.storage; this.connections = options.connections; this.syncRules = options.storage.getParsedSyncRules({ defaultSchema: this.defaultSchema }); @@ -74,6 +82,10 @@ export class BinLogStream { return this.connections.connectionTag; } + private get metrics() { + return this.options.metrics; + } + get connectionId() { const { connectionId } = this.connections; // Default to 1 if not set @@ -335,6 +347,8 @@ AND table_type = 'BASE TABLE';`, after: record, afterReplicaId: getUuidReplicaIdentityBson(record, table.replicaIdColumns) }); + + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); } await batch.flush(); } @@ -461,7 +475,7 @@ AND table_type = 'BASE TABLE';`, }); break; case zongji_utils.eventIsXid(evt): - Metrics.getInstance().transactions_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(1); // Need to commit with a replicated GTID with updated next position await batch.commit( new common.ReplicatedGTID({ @@ -606,7 +620,7 @@ AND table_type = 'BASE TABLE';`, ): Promise { switch (payload.type) { case storage.SaveOperationTag.INSERT: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const record = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ tag: storage.SaveOperationTag.INSERT, @@ -617,7 +631,7 @@ AND table_type = 'BASE TABLE';`, afterReplicaId: getUuidReplicaIdentityBson(record, payload.sourceTable.replicaIdColumns) }); case storage.SaveOperationTag.UPDATE: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); // "before" may be null if the replica id columns are unchanged // It's fine to treat that the same as an insert. const beforeUpdated = payload.previous_data @@ -637,7 +651,7 @@ AND table_type = 'BASE TABLE';`, }); case storage.SaveOperationTag.DELETE: - Metrics.getInstance().rows_replicated_total.add(1); + this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1); const beforeDeleted = common.toSQLiteRow(payload.data, payload.columns); return await batch.save({ diff --git a/modules/module-mysql/test/src/BinLogStream.test.ts b/modules/module-mysql/test/src/BinLogStream.test.ts index 6bde81601..a0db934cf 100644 --- a/modules/module-mysql/test/src/BinLogStream.test.ts +++ b/modules/module-mysql/test/src/BinLogStream.test.ts @@ -1,10 +1,11 @@ -import { Metrics, storage } from '@powersync/service-core'; -import { putOp, removeOp } from '@powersync/service-core-tests'; +import { storage } from '@powersync/service-core'; +import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { v4 as uuid } from 'uuid'; import { describe, expect, test } from 'vitest'; import { BinlogStreamTestContext } from './BinlogStreamUtils.js'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; +import { ReplicationMetric } from '@powersync/service-types'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -35,9 +36,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); const testId = uuid(); @@ -47,9 +47,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -68,9 +67,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -80,9 +78,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: testId, description: 'test1' })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -172,11 +169,15 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const testId = uuid(); await connectionManager.query(`INSERT INTO test_data(id, description) VALUES('${testId}','test1')`); + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + await context.replicateSnapshot(); context.startStreaming(); + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: testId, description: 'test1' })]); + expect(endRowCount - startRowCount).toEqual(1); }); test('snapshot with date values', async () => { @@ -229,9 +230,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -258,9 +258,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { timestamp: '2023-03-06T15:47:00.000Z' }) ]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(2); expect(endTxCount - startTxCount).toEqual(2); }); @@ -274,9 +273,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -284,9 +282,8 @@ function defineBinlogStreamTests(factory: storage.TestStorageFactory) { const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index fcffea264..7844c8e85 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -4,6 +4,8 @@ import { MySQLConnectionManager } from '@module/replication/MySQLConnectionManag import { logger } from '@powersync/lib-services-framework'; import { BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, InternalOpId, OplogEntry, ProtocolOpId, @@ -11,7 +13,7 @@ import { storage, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import mysqlPromise from 'mysql2/promise'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -44,7 +46,10 @@ export class BinlogStreamTestContext { constructor( public factory: BucketStorageFactory, public connectionManager: MySQLConnectionManager - ) {} + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } async dispose() { this.abortController.abort(); @@ -97,6 +102,7 @@ export class BinlogStreamTestContext { } const options: BinLogStreamOptions = { storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abortSignal: this.abortController.signal }; diff --git a/modules/module-mysql/test/src/setup.ts b/modules/module-mysql/test/src/setup.ts index 43946a1aa..8d0b885e6 100644 --- a/modules/module-mysql/test/src/setup.ts +++ b/modules/module-mysql/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-postgres-storage/test/src/setup.ts b/modules/module-postgres-storage/test/src/setup.ts index 802007e89..25b983ea4 100644 --- a/modules/module-postgres-storage/test/src/setup.ts +++ b/modules/module-postgres-storage/test/src/setup.ts @@ -1,16 +1,10 @@ import { container } from '@powersync/lib-services-framework'; -import { Metrics } from '@powersync/service-core'; import { beforeAll } from 'vitest'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - // The metrics need to be initialized before they can be used - await Metrics.initialise({ - disable_telemetry_sharing: true, - powersync_instance_id: 'test', - internal_metrics_endpoint: 'unused.for.tests.com' - }); - Metrics.getInstance().resetCounters(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-postgres/src/module/PostgresModule.ts b/modules/module-postgres/src/module/PostgresModule.ts index d9b7ff470..7dfb06107 100644 --- a/modules/module-postgres/src/module/PostgresModule.ts +++ b/modules/module-postgres/src/module/PostgresModule.ts @@ -19,6 +19,7 @@ import { WalStreamReplicator } from '../replication/WalStreamReplicator.js'; import * as types from '../types/types.js'; import { PostgresConnectionConfig } from '../types/types.js'; import { baseUri, NormalizedBasePostgresConnectionConfig } from '@powersync/lib-service-postgres'; +import { ReplicationMetric } from '@powersync/service-types'; export class PostgresModule extends replication.ReplicationModule { constructor() { @@ -29,9 +30,7 @@ export class PostgresModule extends replication.ReplicationModule { - await super.initialize(context); - + async onInitialized(context: system.ServiceContextContainer): Promise { const client_auth = context.configuration.base_config.client_auth; if (client_auth?.supabase && client_auth?.supabase_jwt_secret == null) { @@ -45,13 +44,14 @@ export class PostgresModule extends replication.ReplicationModule { while (!done) { const count = - ((await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0) - - startRowCount; + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; if (count >= stopAfter) { break; diff --git a/modules/module-postgres/test/src/setup.ts b/modules/module-postgres/test/src/setup.ts index 43946a1aa..8d0b885e6 100644 --- a/modules/module-postgres/test/src/setup.ts +++ b/modules/module-postgres/test/src/setup.ts @@ -1,13 +1,12 @@ import { container } from '@powersync/lib-services-framework'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER } from '@powersync/service-core-tests'; import { beforeAll, beforeEach } from 'vitest'; beforeAll(async () => { // Executes for every test file container.registerDefaults(); - await test_utils.initMetrics(); }); beforeEach(async () => { - await test_utils.resetMetrics(); + METRICS_HELPER.resetMetrics(); }); diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index c6b83393c..b79009c8d 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -1,5 +1,5 @@ import * as bson from 'bson'; -import { afterEach, describe, expect, test } from 'vitest'; +import { afterEach, beforeAll, describe, expect, test } from 'vitest'; import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js'; import { env } from './env.js'; import { @@ -15,8 +15,8 @@ import * as pgwire from '@powersync/service-jpgwire'; import { SqliteRow } from '@powersync/service-sync-rules'; import { PgManager } from '@module/replication/PgManager.js'; -import { storage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { createCoreReplicationMetrics, initializeCoreReplicationMetrics, storage } from '@powersync/service-core'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as mongo_storage from '@powersync/service-module-mongodb-storage'; import * as postgres_storage from '@powersync/service-module-postgres-storage'; import * as timers from 'node:timers/promises'; @@ -49,6 +49,11 @@ function defineSlowTests(factory: storage.TestStorageFactory) { let abortController: AbortController | undefined; let streamPromise: Promise | undefined; + beforeAll(async () => { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + }); + afterEach(async () => { // This cleans up, similar to WalStreamTestContext.dispose(). // These tests are a little more complex than what is supported by WalStreamTestContext. @@ -98,7 +103,8 @@ bucket_definitions: const options: WalStreamOptions = { abort_signal: abortController.signal, connections, - storage: storage + storage: storage, + metrics: METRICS_HELPER.metricsEngine }; walStream = new WalStream(options); @@ -344,13 +350,14 @@ bucket_definitions: const connections = new PgManager(TEST_CONNECTION_OPTIONS, {}); const replicationConnection = await connections.replicationConnection(); - abortController = new AbortController(); - const options: WalStreamOptions = { - abort_signal: abortController.signal, - connections, - storage: storage - }; - walStream = new WalStream(options); + abortController = new AbortController(); + const options: WalStreamOptions = { + abort_signal: abortController.signal, + connections, + storage: storage, + metrics: METRICS_HELPER.metricsEngine + }; + walStream = new WalStream(options); await storage.clear(); diff --git a/modules/module-postgres/test/src/wal_stream.test.ts b/modules/module-postgres/test/src/wal_stream.test.ts index fea606643..109460acd 100644 --- a/modules/module-postgres/test/src/wal_stream.test.ts +++ b/modules/module-postgres/test/src/wal_stream.test.ts @@ -1,12 +1,13 @@ import { MissingReplicationSlotError } from '@module/replication/WalStream.js'; -import { Metrics, storage } from '@powersync/service-core'; -import { putOp, removeOp } from '@powersync/service-core-tests'; +import { storage } from '@powersync/service-core'; +import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests'; import { pgwireRows } from '@powersync/service-jpgwire'; import * as crypto from 'crypto'; import { describe, expect, test } from 'vitest'; import { env } from './env.js'; import { INITIALIZED_MONGO_STORAGE_FACTORY, INITIALIZED_POSTGRES_STORAGE_FACTORY } from './util.js'; import { WalStreamTestContext } from './wal_stream_utils.js'; +import { ReplicationMetric } from '@powersync/service-types'; const BASIC_SYNC_RULES = ` bucket_definitions: @@ -40,9 +41,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -55,9 +55,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_data', { id: test_id, description: 'test1', num: 1152921504606846976n })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -77,9 +76,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -90,9 +88,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([putOp('test_DATA', { id: test_id, description: 'test1' })]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; expect(endRowCount - startRowCount).toEqual(1); expect(endTxCount - startTxCount).toEqual(1); }); @@ -274,9 +271,8 @@ bucket_definitions: await context.replicateSnapshot(); - const startRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const startTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const startTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; context.startStreaming(); @@ -287,9 +283,8 @@ bucket_definitions: const data = await context.getBucketData('global[]'); expect(data).toMatchObject([]); - const endRowCount = (await Metrics.getInstance().getMetricValueForTests('powersync_rows_replicated_total')) ?? 0; - const endTxCount = - (await Metrics.getInstance().getMetricValueForTests('powersync_transactions_replicated_total')) ?? 0; + const endRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; + const endTxCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.TRANSACTIONS_REPLICATED)) ?? 0; // There was a transaction, but we should not replicate any actual data expect(endRowCount - startRowCount).toEqual(0); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 25b8dd1d2..74c2663f9 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -2,12 +2,14 @@ import { PgManager } from '@module/replication/PgManager.js'; import { PUBLICATION_NAME, WalStream, WalStreamOptions } from '@module/replication/WalStream.js'; import { BucketStorageFactory, + createCoreReplicationMetrics, + initializeCoreReplicationMetrics, InternalOpId, OplogEntry, storage, SyncRulesBucketStorage } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -41,7 +43,10 @@ export class WalStreamTestContext implements AsyncDisposable { constructor( public factory: BucketStorageFactory, public connectionManager: PgManager - ) {} + ) { + createCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + initializeCoreReplicationMetrics(METRICS_HELPER.metricsEngine); + } async [Symbol.asyncDispose]() { await this.dispose(); @@ -101,6 +106,7 @@ export class WalStreamTestContext implements AsyncDisposable { } const options: WalStreamOptions = { storage: this.storage, + metrics: METRICS_HELPER.metricsEngine, connections: this.connectionManager, abort_signal: this.abortController.signal }; diff --git a/packages/service-core-tests/package.json b/packages/service-core-tests/package.json index a65258b81..0a1e92f5c 100644 --- a/packages/service-core-tests/package.json +++ b/packages/service-core-tests/package.json @@ -23,6 +23,7 @@ "vitest": "^3.0.5" }, "devDependencies": { - "typescript": "^5.7.3" + "typescript": "^5.7.3", + "@opentelemetry/sdk-metrics": "^1.30.1" } } diff --git a/packages/service-core-tests/src/test-utils/MetricsHelper.ts b/packages/service-core-tests/src/test-utils/MetricsHelper.ts new file mode 100644 index 000000000..7e1553a18 --- /dev/null +++ b/packages/service-core-tests/src/test-utils/MetricsHelper.ts @@ -0,0 +1,54 @@ +import { MetricsEngine, MetricsFactory, OpenTelemetryMetricsFactory } from '@powersync/service-core'; +import { + AggregationTemporality, + InMemoryMetricExporter, + MeterProvider, + PeriodicExportingMetricReader +} from '@opentelemetry/sdk-metrics'; + +export class MetricsHelper { + public factory: MetricsFactory; + public metricsEngine: MetricsEngine; + private meterProvider: MeterProvider; + private exporter: InMemoryMetricExporter; + private metricReader: PeriodicExportingMetricReader; + + constructor() { + this.exporter = new InMemoryMetricExporter(AggregationTemporality.CUMULATIVE); + + this.metricReader = new PeriodicExportingMetricReader({ + exporter: this.exporter, + exportIntervalMillis: 100 // Export quickly for tests + }); + this.meterProvider = new MeterProvider({ + readers: [this.metricReader] + }); + const meter = this.meterProvider.getMeter('powersync-tests'); + this.factory = new OpenTelemetryMetricsFactory(meter); + this.metricsEngine = new MetricsEngine({ + factory: this.factory, + disable_telemetry_sharing: true + }); + } + + resetMetrics() { + this.exporter.reset(); + } + + async getMetricValueForTests(name: string): Promise { + await this.metricReader.forceFlush(); + const metrics = this.exporter.getMetrics(); + // Use the latest reported ResourceMetric + const scoped = metrics[metrics.length - 1]?.scopeMetrics?.[0]?.metrics; + const metric = scoped?.find((metric) => metric.descriptor.name == name); + if (metric == null) { + throw new Error( + `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}` + ); + } + const point = metric.dataPoints[metric.dataPoints.length - 1]; + return point?.value as number; + } +} + +export const METRICS_HELPER = new MetricsHelper(); diff --git a/packages/service-core-tests/src/test-utils/metrics-utils.ts b/packages/service-core-tests/src/test-utils/metrics-utils.ts deleted file mode 100644 index 8f61d250b..000000000 --- a/packages/service-core-tests/src/test-utils/metrics-utils.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Metrics } from '@powersync/service-core'; - -export const initMetrics = async () => { - await Metrics.initialise({ - disable_telemetry_sharing: true, - powersync_instance_id: 'test', - internal_metrics_endpoint: 'unused.for.tests.com' - }); - await resetMetrics(); -}; - -export const resetMetrics = async () => { - Metrics.getInstance().resetCounters(); -}; diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index 1e4cbea0a..1b174d84c 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,4 +1,4 @@ export * from './bucket-validation.js'; export * from './general-utils.js'; -export * from './metrics-utils.js'; +export * from './MetricsHelper.js'; export * from './stream_utils.js'; diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index c30302402..8e3dec5e7 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -1,5 +1,5 @@ import { - CheckpointLine, + createCoreAPIMetrics, storage, StreamingSyncCheckpoint, StreamingSyncCheckpointDiff, @@ -13,6 +13,7 @@ import * as timers from 'timers/promises'; import { fileURLToPath } from 'url'; import { expect, test } from 'vitest'; import * as test_utils from '../test-utils/test-utils-index.js'; +import { METRICS_HELPER } from '../test-utils/test-utils-index.js'; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -37,7 +38,8 @@ export const SYNC_SNAPSHOT_PATH = path.resolve(__dirname, '../__snapshots/sync.t * ``` */ export function registerSyncTests(factory: storage.TestStorageFactory) { - const tracker = new sync.RequestTracker(); + createCoreAPIMetrics(METRICS_HELPER.metricsEngine); + const tracker = new sync.RequestTracker(METRICS_HELPER.metricsEngine); const syncContext = new sync.SyncContext({ maxBuckets: 10, maxParameterQueryResults: 10, diff --git a/packages/service-core/package.json b/packages/service-core/package.json index 6bbeb9c67..94b516ee0 100644 --- a/packages/service-core/package.json +++ b/packages/service-core/package.json @@ -17,11 +17,11 @@ }, "dependencies": { "@js-sdsl/ordered-set": "^4.4.2", - "@opentelemetry/api": "~1.8.0", - "@opentelemetry/exporter-metrics-otlp-http": "^0.51.1", - "@opentelemetry/exporter-prometheus": "^0.51.1", - "@opentelemetry/resources": "^1.24.1", - "@opentelemetry/sdk-metrics": "1.24.1", + "@opentelemetry/api": "~1.9.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.57.2", + "@opentelemetry/exporter-prometheus": "^0.57.2", + "@opentelemetry/resources": "^1.30.1", + "@opentelemetry/sdk-metrics": "1.30.1", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-jsonbig": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", diff --git a/packages/service-core/src/api/api-index.ts b/packages/service-core/src/api/api-index.ts index 0f90b1738..2074af6ae 100644 --- a/packages/service-core/src/api/api-index.ts +++ b/packages/service-core/src/api/api-index.ts @@ -1,3 +1,4 @@ export * from './diagnostics.js'; export * from './RouteAPI.js'; export * from './schema.js'; +export * from './api-metrics.js'; diff --git a/packages/service-core/src/api/api-metrics.ts b/packages/service-core/src/api/api-metrics.ts new file mode 100644 index 000000000..5f57cfb14 --- /dev/null +++ b/packages/service-core/src/api/api-metrics.ts @@ -0,0 +1,35 @@ +import { MetricsEngine } from '../metrics/MetricsEngine.js'; +import { APIMetric } from '@powersync/service-types'; + +/** + * Create and register the core API metrics. + * @param engine + */ +export function createCoreAPIMetrics(engine: MetricsEngine): void { + engine.createCounter({ + name: APIMetric.DATA_SYNCED_BYTES, + description: 'Uncompressed size of synced data', + unit: 'bytes' + }); + + engine.createCounter({ + name: APIMetric.OPERATIONS_SYNCED, + description: 'Number of operations synced' + }); + + engine.createUpDownCounter({ + name: APIMetric.CONCURRENT_CONNECTIONS, + description: 'Number of concurrent sync connections' + }); +} + +/** + * Initialise the core API metrics. This should be called after the metrics have been created. + * @param engine + */ +export function initializeCoreAPIMetrics(engine: MetricsEngine): void { + const concurrent_connections = engine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS); + + // Initialize the metric, so that it reports a value before connections have been opened. + concurrent_connections.add(0); +} diff --git a/packages/service-core/src/index.ts b/packages/service-core/src/index.ts index e0c8a2864..09e3e9d79 100644 --- a/packages/service-core/src/index.ts +++ b/packages/service-core/src/index.ts @@ -12,8 +12,8 @@ export * as entry from './entry/entry-index.js'; // Re-export framework for easy use of Container API export * as framework from '@powersync/lib-services-framework'; -export * from './metrics/Metrics.js'; -export * as metrics from './metrics/Metrics.js'; +export * from './metrics/metrics-index.js'; +export * as metrics from './metrics/metrics-index.js'; export * from './migrations/migrations-index.js'; export * as migrations from './migrations/migrations-index.js'; diff --git a/packages/service-core/src/metrics/Metrics.ts b/packages/service-core/src/metrics/Metrics.ts deleted file mode 100644 index 7a1682160..000000000 --- a/packages/service-core/src/metrics/Metrics.ts +++ /dev/null @@ -1,255 +0,0 @@ -import { Attributes, Counter, ObservableGauge, UpDownCounter, ValueType } from '@opentelemetry/api'; -import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; -import { Resource } from '@opentelemetry/resources'; -import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; -import { logger, ServiceAssertionError } from '@powersync/lib-services-framework'; -import * as storage from '../storage/storage-index.js'; -import * as util from '../util/util-index.js'; - -export interface MetricsOptions { - disable_telemetry_sharing: boolean; - powersync_instance_id: string; - internal_metrics_endpoint: string; -} - -export class Metrics { - private static instance: Metrics; - - private prometheusExporter: PrometheusExporter; - private meterProvider: MeterProvider; - - // Metrics - // 1. Data processing / month - - // 1a. Postgres -> PowerSync - // Record on replication pod - public data_replicated_bytes: Counter; - // 1b. PowerSync -> clients - // Record on API pod - public data_synced_bytes: Counter; - // Unused for pricing - // Record on replication pod - public rows_replicated_total: Counter; - // Unused for pricing - // Record on replication pod - public transactions_replicated_total: Counter; - // Unused for pricing - // Record on replication pod - public chunks_replicated_total: Counter; - - // 2. Sync operations / month - - // Record on API pod - public operations_synced_total: Counter; - - // 3. Data hosted on PowerSync sync service - - // Record on replication pod - // 3a. Replication storage -> raw data as received from Postgres. - public replication_storage_size_bytes: ObservableGauge; - // 3b. Operations storage -> transformed history, as will be synced to clients - public operation_storage_size_bytes: ObservableGauge; - // 3c. Parameter storage -> used for parameter queries - public parameter_storage_size_bytes: ObservableGauge; - - // 4. Peak concurrent connections - - // Record on API pod - public concurrent_connections: UpDownCounter; - - private constructor(meterProvider: MeterProvider, prometheusExporter: PrometheusExporter) { - this.meterProvider = meterProvider; - this.prometheusExporter = prometheusExporter; - const meter = meterProvider.getMeter('powersync'); - - this.data_replicated_bytes = meter.createCounter('powersync_data_replicated_bytes_total', { - description: 'Uncompressed size of replicated data', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.data_synced_bytes = meter.createCounter('powersync_data_synced_bytes_total', { - description: 'Uncompressed size of synced data', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.rows_replicated_total = meter.createCounter('powersync_rows_replicated_total', { - description: 'Total number of replicated rows', - valueType: ValueType.INT - }); - - this.transactions_replicated_total = meter.createCounter('powersync_transactions_replicated_total', { - description: 'Total number of replicated transactions', - valueType: ValueType.INT - }); - - this.chunks_replicated_total = meter.createCounter('powersync_chunks_replicated_total', { - description: 'Total number of replication chunks', - valueType: ValueType.INT - }); - - this.operations_synced_total = meter.createCounter('powersync_operations_synced_total', { - description: 'Number of operations synced', - valueType: ValueType.INT - }); - - this.replication_storage_size_bytes = meter.createObservableGauge('powersync_replication_storage_size_bytes', { - description: 'Size of current data stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.operation_storage_size_bytes = meter.createObservableGauge('powersync_operation_storage_size_bytes', { - description: 'Size of operations stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.parameter_storage_size_bytes = meter.createObservableGauge('powersync_parameter_storage_size_bytes', { - description: 'Size of parameter data stored in PowerSync', - unit: 'bytes', - valueType: ValueType.INT - }); - - this.concurrent_connections = meter.createUpDownCounter('powersync_concurrent_connections', { - description: 'Number of concurrent sync connections', - valueType: ValueType.INT - }); - } - - // Generally only useful for tests. Note: gauges are ignored here. - resetCounters() { - this.data_replicated_bytes.add(0); - this.data_synced_bytes.add(0); - this.rows_replicated_total.add(0); - this.transactions_replicated_total.add(0); - this.chunks_replicated_total.add(0); - this.operations_synced_total.add(0); - this.concurrent_connections.add(0); - } - - public static getInstance(): Metrics { - if (!Metrics.instance) { - throw new ServiceAssertionError('Metrics have not been initialized'); - } - - return Metrics.instance; - } - - public static async initialise(options: MetricsOptions): Promise { - if (Metrics.instance) { - return; - } - logger.info('Configuring telemetry.'); - - logger.info( - ` -Attention: -PowerSync collects completely anonymous telemetry regarding usage. -This information is used to shape our roadmap to better serve our customers. -You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL: -https://docs.powersync.com/self-hosting/telemetry -Anonymous telemetry is currently: ${options.disable_telemetry_sharing ? 'disabled' : 'enabled'} - `.trim() - ); - - const configuredExporters: MetricReader[] = []; - - const port: number = util.env.METRICS_PORT ?? 0; - const prometheusExporter = new PrometheusExporter({ port: port, preventServerStart: true }); - configuredExporters.push(prometheusExporter); - - if (!options.disable_telemetry_sharing) { - logger.info('Sharing anonymous telemetry'); - const periodicExporter = new PeriodicExportingMetricReader({ - exporter: new OTLPMetricExporter({ - url: options.internal_metrics_endpoint - }), - exportIntervalMillis: 1000 * 60 * 5 // 5 minutes - }); - - configuredExporters.push(periodicExporter); - } - - const meterProvider = new MeterProvider({ - resource: new Resource({ - ['service']: 'PowerSync', - ['instance_id']: options.powersync_instance_id - }), - readers: configuredExporters - }); - - if (port > 0) { - await prometheusExporter.startServer(); - } - - Metrics.instance = new Metrics(meterProvider, prometheusExporter); - - logger.info('Telemetry configuration complete.'); - } - - public async shutdown(): Promise { - await this.meterProvider.shutdown(); - } - - public configureApiMetrics() { - // Initialize the metric, so that it reports a value before connections - // have been opened. - this.concurrent_connections.add(0); - } - - public configureReplicationMetrics(bucketStorage: storage.BucketStorageFactory) { - // Rate limit collection of these stats, since it may be an expensive query - const MINIMUM_INTERVAL = 60_000; - - let cachedRequest: Promise | undefined = undefined; - let cacheTimestamp = 0; - - function getMetrics() { - if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) { - cachedRequest = bucketStorage.getStorageMetrics().catch((e) => { - logger.error(`Failed to get storage metrics`, e); - return null; - }); - cacheTimestamp = Date.now(); - } - return cachedRequest; - } - - this.operation_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.operations_size_bytes); - } - }); - - this.parameter_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.parameters_size_bytes); - } - }); - - this.replication_storage_size_bytes.addCallback(async (result) => { - const metrics = await getMetrics(); - if (metrics) { - result.observe(metrics.replication_size_bytes); - } - }); - } - - public async getMetricValueForTests(name: string): Promise { - const metrics = await this.prometheusExporter.collect(); - const scoped = metrics.resourceMetrics.scopeMetrics[0].metrics; - const metric = scoped.find((metric) => metric.descriptor.name == name); - if (metric == null) { - throw new Error( - `Cannot find metric ${name}. Options: ${scoped.map((metric) => metric.descriptor.name).join(',')}` - ); - } - const point = metric.dataPoints[metric.dataPoints.length - 1]; - return point?.value as number; - } -} diff --git a/packages/service-core/src/metrics/MetricsEngine.ts b/packages/service-core/src/metrics/MetricsEngine.ts new file mode 100644 index 000000000..f98cf587b --- /dev/null +++ b/packages/service-core/src/metrics/MetricsEngine.ts @@ -0,0 +1,98 @@ +import { logger, ServiceAssertionError } from '@powersync/lib-services-framework'; +import { Counter, UpDownCounter, ObservableGauge, MetricMetadata, MetricsFactory } from './metrics-interfaces.js'; + +export interface MetricsEngineOptions { + factory: MetricsFactory; + disable_telemetry_sharing: boolean; +} + +export class MetricsEngine { + private counters: Map; + private upDownCounters: Map; + private observableGauges: Map; + + constructor(private options: MetricsEngineOptions) { + this.counters = new Map(); + this.upDownCounters = new Map(); + this.observableGauges = new Map(); + } + + private get factory(): MetricsFactory { + return this.options.factory; + } + + createCounter(metadata: MetricMetadata): Counter { + if (this.counters.has(metadata.name)) { + logger.warn(`Counter with name ${metadata.name} already created and registered, skipping.`); + return this.counters.get(metadata.name)!; + } + + const counter = this.factory.createCounter(metadata); + this.counters.set(metadata.name, counter); + return counter; + } + createUpDownCounter(metadata: MetricMetadata): UpDownCounter { + if (this.upDownCounters.has(metadata.name)) { + logger.warn(`UpDownCounter with name ${metadata.name} already created and registered, skipping.`); + return this.upDownCounters.get(metadata.name)!; + } + + const upDownCounter = this.factory.createUpDownCounter(metadata); + this.upDownCounters.set(metadata.name, upDownCounter); + return upDownCounter; + } + + createObservableGauge(metadata: MetricMetadata): ObservableGauge { + if (this.observableGauges.has(metadata.name)) { + logger.warn(`ObservableGauge with name ${metadata.name} already created and registered, skipping.`); + return this.observableGauges.get(metadata.name)!; + } + + const observableGauge = this.factory.createObservableGauge(metadata); + this.observableGauges.set(metadata.name, observableGauge); + return observableGauge; + } + + getCounter(name: string): Counter { + const counter = this.counters.get(name); + if (!counter) { + throw new ServiceAssertionError(`Counter '${name}' has not been created and registered yet.`); + } + return counter; + } + + getUpDownCounter(name: string): UpDownCounter { + const upDownCounter = this.upDownCounters.get(name); + if (!upDownCounter) { + throw new ServiceAssertionError(`UpDownCounter '${name}' has not been created and registered yet.`); + } + return upDownCounter; + } + + getObservableGauge(name: string): ObservableGauge { + const observableGauge = this.observableGauges.get(name); + if (!observableGauge) { + throw new ServiceAssertionError(`ObservableGauge '${name}' has not been created and registered yet.`); + } + return observableGauge; + } + + public async start(): Promise { + logger.info( + ` +Attention: +PowerSync collects completely anonymous telemetry regarding usage. +This information is used to shape our roadmap to better serve our customers. +You can learn more, including how to opt-out if you'd not like to participate in this anonymous program, by visiting the following URL: +https://docs.powersync.com/self-hosting/lifecycle-maintenance/telemetry + `.trim() + ); + logger.info(`Anonymous telemetry is currently: ${this.options.disable_telemetry_sharing ? 'disabled' : 'enabled'}`); + + logger.info('Successfully started Metrics Engine.'); + } + + public async shutdown(): Promise { + logger.info('Successfully shut down Metrics Engine.'); + } +} diff --git a/packages/service-core/src/metrics/metrics-index.ts b/packages/service-core/src/metrics/metrics-index.ts new file mode 100644 index 000000000..6f13dd7d0 --- /dev/null +++ b/packages/service-core/src/metrics/metrics-index.ts @@ -0,0 +1,5 @@ +export * from './MetricsEngine.js'; +export * from './metrics-interfaces.js'; +export * from './register-metrics.js'; +export * from './open-telemetry/OpenTelemetryMetricsFactory.js'; +export * from './open-telemetry/util.js'; diff --git a/packages/service-core/src/metrics/metrics-interfaces.ts b/packages/service-core/src/metrics/metrics-interfaces.ts new file mode 100644 index 000000000..d28475954 --- /dev/null +++ b/packages/service-core/src/metrics/metrics-interfaces.ts @@ -0,0 +1,41 @@ +export interface Counter { + /** + * Increment the counter by the given value. Only positive numbers are valid. + * @param value + */ + add(value: number): void; +} + +export interface UpDownCounter { + /** + * Increment or decrement(if negative) the counter by the given value. + * @param value + */ + add(value: number): void; +} + +export interface ObservableGauge { + /** + * Set a value provider that provides the value for the gauge at the time of observation. + * @param valueProvider + */ + setValueProvider(valueProvider: () => Promise): void; +} + +export enum Precision { + INT = 'int', + DOUBLE = 'double' +} + +export interface MetricMetadata { + name: string; + description?: string; + unit?: string; + precision?: Precision; +} + +export interface MetricsFactory { + createCounter(metadata: MetricMetadata): Counter; + createUpDownCounter(metadata: MetricMetadata): UpDownCounter; + createObservableGauge(metadata: MetricMetadata): ObservableGauge; +} diff --git a/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts b/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts new file mode 100644 index 000000000..13013cb50 --- /dev/null +++ b/packages/service-core/src/metrics/open-telemetry/OpenTelemetryMetricsFactory.ts @@ -0,0 +1,66 @@ +import { Meter, ValueType } from '@opentelemetry/api'; +import { + Counter, + ObservableGauge, + UpDownCounter, + MetricMetadata, + MetricsFactory, + Precision +} from '../metrics-interfaces.js'; + +export class OpenTelemetryMetricsFactory implements MetricsFactory { + private meter: Meter; + + constructor(meter: Meter) { + this.meter = meter; + } + + createCounter(metadata: MetricMetadata): Counter { + return this.meter.createCounter(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + } + + createObservableGauge(metadata: MetricMetadata): ObservableGauge { + const gauge = this.meter.createObservableGauge(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + + return { + setValueProvider(valueProvider: () => Promise) { + gauge.addCallback(async (result) => { + const value = await valueProvider(); + + if (value) { + result.observe(value); + } + }); + } + }; + } + + createUpDownCounter(metadata: MetricMetadata): UpDownCounter { + return this.meter.createUpDownCounter(metadata.name, { + description: metadata.description, + unit: metadata.unit, + valueType: this.toValueType(metadata.precision) + }); + } + + private toValueType(precision?: Precision): ValueType { + if (!precision) { + return ValueType.INT; + } + + switch (precision) { + case Precision.INT: + return ValueType.INT; + case Precision.DOUBLE: + return ValueType.DOUBLE; + } + } +} diff --git a/packages/service-core/src/metrics/open-telemetry/util.ts b/packages/service-core/src/metrics/open-telemetry/util.ts new file mode 100644 index 000000000..af4af4065 --- /dev/null +++ b/packages/service-core/src/metrics/open-telemetry/util.ts @@ -0,0 +1,80 @@ +import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { ServiceContext } from '../../system/ServiceContext.js'; +import { OpenTelemetryMetricsFactory } from './OpenTelemetryMetricsFactory.js'; +import { MetricsFactory } from '../metrics-interfaces.js'; +import { logger } from '@powersync/lib-services-framework'; + +export interface RuntimeMetadata { + [key: string]: string | number | undefined; +} + +export function createOpenTelemetryMetricsFactory(context: ServiceContext): MetricsFactory { + const { configuration, lifeCycleEngine, storageEngine } = context; + const configuredExporters: MetricReader[] = []; + + if (configuration.telemetry.prometheus_port) { + const prometheusExporter = new PrometheusExporter({ + port: configuration.telemetry.prometheus_port, + preventServerStart: true + }); + configuredExporters.push(prometheusExporter); + + lifeCycleEngine.withLifecycle(prometheusExporter, { + start: async () => { + await prometheusExporter.startServer(); + logger.info(`Prometheus metric export enabled on port:${configuration.telemetry.prometheus_port}`); + } + }); + } + + if (!configuration.telemetry.disable_telemetry_sharing) { + const periodicExporter = new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: configuration.telemetry.internal_service_endpoint + }), + exportIntervalMillis: 1000 * 60 * 5 // 5 minutes + }); + + configuredExporters.push(periodicExporter); + } + + let resolvedMetadata: (metadata: RuntimeMetadata) => void; + const runtimeMetadata: Promise = new Promise((resolve) => { + resolvedMetadata = resolve; + }); + + lifeCycleEngine.withLifecycle(null, { + start: async () => { + const bucketStorage = storageEngine.activeBucketStorage; + try { + const instanceId = await bucketStorage.getPowerSyncInstanceId(); + resolvedMetadata({ ['instance_id']: instanceId }); + } catch (err) { + resolvedMetadata({ ['instance_id']: 'Unknown' }); + } + } + }); + + const meterProvider = new MeterProvider({ + resource: new Resource( + { + ['service']: 'PowerSync' + }, + runtimeMetadata + ), + readers: configuredExporters + }); + + lifeCycleEngine.withLifecycle(meterProvider, { + stop: async () => { + await meterProvider.shutdown(); + } + }); + + const meter = meterProvider.getMeter('powersync'); + + return new OpenTelemetryMetricsFactory(meter); +} diff --git a/packages/service-core/src/metrics/register-metrics.ts b/packages/service-core/src/metrics/register-metrics.ts new file mode 100644 index 000000000..5fef5a8f4 --- /dev/null +++ b/packages/service-core/src/metrics/register-metrics.ts @@ -0,0 +1,56 @@ +import { ServiceContextContainer } from '../system/ServiceContext.js'; +import { createOpenTelemetryMetricsFactory } from './open-telemetry/util.js'; +import { MetricsEngine } from './MetricsEngine.js'; +import { createCoreAPIMetrics, initializeCoreAPIMetrics } from '../api/api-metrics.js'; +import { createCoreReplicationMetrics, initializeCoreReplicationMetrics } from '../replication/replication-metrics.js'; +import { createCoreStorageMetrics, initializeCoreStorageMetrics } from '../storage/storage-metrics.js'; + +export enum MetricModes { + API = 'api', + REPLICATION = 'replication', + STORAGE = 'storage' +} + +export type MetricsRegistrationOptions = { + service_context: ServiceContextContainer; + modes: MetricModes[]; +}; + +export const registerMetrics = async (options: MetricsRegistrationOptions) => { + const { service_context, modes } = options; + + const metricsFactory = createOpenTelemetryMetricsFactory(service_context); + const metricsEngine = new MetricsEngine({ + factory: metricsFactory, + disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing + }); + service_context.register(MetricsEngine, metricsEngine); + + if (modes.includes(MetricModes.API)) { + createCoreAPIMetrics(metricsEngine); + initializeCoreAPIMetrics(metricsEngine); + } + + if (modes.includes(MetricModes.REPLICATION)) { + createCoreReplicationMetrics(metricsEngine); + initializeCoreReplicationMetrics(metricsEngine); + } + + if (modes.includes(MetricModes.STORAGE)) { + createCoreStorageMetrics(metricsEngine); + + // This requires an instantiated bucket storage, which is only created when the lifecycle starts + service_context.storageEngine.registerListener({ + storageActivated: (bucketStorage) => { + initializeCoreStorageMetrics(metricsEngine, bucketStorage); + } + }); + } + + service_context.lifeCycleEngine.withLifecycle(metricsEngine, { + start: async () => { + await metricsEngine.start(); + }, + stop: () => metricsEngine.shutdown() + }); +}; diff --git a/packages/service-core/src/replication/AbstractReplicationJob.ts b/packages/service-core/src/replication/AbstractReplicationJob.ts index 8226e5422..e38cbd3fb 100644 --- a/packages/service-core/src/replication/AbstractReplicationJob.ts +++ b/packages/service-core/src/replication/AbstractReplicationJob.ts @@ -2,10 +2,12 @@ import { container, logger } from '@powersync/lib-services-framework'; import winston from 'winston'; import * as storage from '../storage/storage-index.js'; import { ErrorRateLimiter } from './ErrorRateLimiter.js'; +import { MetricsEngine } from '../metrics/MetricsEngine.js'; export interface AbstractReplicationJobOptions { id: string; storage: storage.SyncRulesBucketStorage; + metrics: MetricsEngine; lock: storage.ReplicationLock; rateLimiter: ErrorRateLimiter; } diff --git a/packages/service-core/src/replication/AbstractReplicator.ts b/packages/service-core/src/replication/AbstractReplicator.ts index ee2822263..cb6cc870f 100644 --- a/packages/service-core/src/replication/AbstractReplicator.ts +++ b/packages/service-core/src/replication/AbstractReplicator.ts @@ -7,6 +7,7 @@ import { SyncRulesProvider } from '../util/config/sync-rules/sync-rules-provider import { AbstractReplicationJob } from './AbstractReplicationJob.js'; import { ErrorRateLimiter } from './ErrorRateLimiter.js'; import { ConnectionTestResult } from './ReplicationModule.js'; +import { MetricsEngine } from '../metrics/MetricsEngine.js'; // 5 minutes const PING_INTERVAL = 1_000_000_000n * 300n; @@ -19,6 +20,7 @@ export interface CreateJobOptions { export interface AbstractReplicatorOptions { id: string; storageEngine: StorageEngine; + metricsEngine: MetricsEngine; syncRuleProvider: SyncRulesProvider; /** * This limits the effect of retries when there is a persistent issue. @@ -33,6 +35,7 @@ export interface AbstractReplicatorOptions { */ export abstract class AbstractReplicator { protected logger: winston.Logger; + /** * Map of replication jobs by sync rule id. Usually there is only one running job, but there could be two when * transitioning to a new set of sync rules. @@ -72,6 +75,10 @@ export abstract class AbstractReplicator { this.runLoop().catch((e) => { this.logger.error('Data source fatal replication error', e); diff --git a/packages/service-core/src/replication/ReplicationModule.ts b/packages/service-core/src/replication/ReplicationModule.ts index 410c022ab..5c69074b5 100644 --- a/packages/service-core/src/replication/ReplicationModule.ts +++ b/packages/service-core/src/replication/ReplicationModule.ts @@ -64,6 +64,14 @@ export abstract class ReplicationModule */ protected abstract createReplicator(context: system.ServiceContext): AbstractReplicator; + /** + * Any additional initialization specific to the module should be added here. Will be called if necessary after the + * main initialization has been completed + * @param context + * @protected + */ + protected abstract onInitialized(context: system.ServiceContext): Promise; + public abstract testConnection(config: TConfig): Promise; /** @@ -93,6 +101,8 @@ export abstract class ReplicationModule context.replicationEngine?.register(this.createReplicator(context)); context.routerEngine?.registerAPI(this.createRouteAPIAdapter()); + + await this.onInitialized(context); } protected decodeConfig(config: TConfig): void { diff --git a/packages/service-core/src/replication/replication-index.ts b/packages/service-core/src/replication/replication-index.ts index 0b37534c9..eb1f4705b 100644 --- a/packages/service-core/src/replication/replication-index.ts +++ b/packages/service-core/src/replication/replication-index.ts @@ -3,3 +3,4 @@ export * from './AbstractReplicator.js'; export * from './ErrorRateLimiter.js'; export * from './ReplicationEngine.js'; export * from './ReplicationModule.js'; +export * from './replication-metrics.js'; diff --git a/packages/service-core/src/replication/replication-metrics.ts b/packages/service-core/src/replication/replication-metrics.ts new file mode 100644 index 000000000..d589696ec --- /dev/null +++ b/packages/service-core/src/replication/replication-metrics.ts @@ -0,0 +1,45 @@ +import { MetricsEngine } from '../metrics/metrics-index.js'; +import { ReplicationMetric } from '@powersync/service-types'; + +/** + * Create and register the core replication metrics. + * @param engine + */ +export function createCoreReplicationMetrics(engine: MetricsEngine): void { + engine.createCounter({ + name: ReplicationMetric.DATA_REPLICATED_BYTES, + description: 'Uncompressed size of replicated data', + unit: 'bytes' + }); + + engine.createCounter({ + name: ReplicationMetric.ROWS_REPLICATED, + description: 'Total number of replicated rows' + }); + + engine.createCounter({ + name: ReplicationMetric.TRANSACTIONS_REPLICATED, + description: 'Total number of replicated transactions' + }); + + engine.createCounter({ + name: ReplicationMetric.CHUNKS_REPLICATED, + description: 'Total number of replication chunks' + }); +} + +/** + * Initialise the core replication metrics. This should be called after the metrics have been created. + * @param engine + */ +export function initializeCoreReplicationMetrics(engine: MetricsEngine): void { + const data_replicated_bytes = engine.getCounter(ReplicationMetric.DATA_REPLICATED_BYTES); + const rows_replicated_total = engine.getCounter(ReplicationMetric.ROWS_REPLICATED); + const transactions_replicated_total = engine.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED); + const chunks_replicated_total = engine.getCounter(ReplicationMetric.CHUNKS_REPLICATED); + + data_replicated_bytes.add(0); + rows_replicated_total.add(0); + transactions_replicated_total.add(0); + chunks_replicated_total.add(0); +} diff --git a/packages/service-core/src/routes/endpoints/socket-route.ts b/packages/service-core/src/routes/endpoints/socket-route.ts index 52d548867..006ac4225 100644 --- a/packages/service-core/src/routes/endpoints/socket-route.ts +++ b/packages/service-core/src/routes/endpoints/socket-route.ts @@ -2,18 +2,19 @@ import { ErrorCode, errors, logger, schema } from '@powersync/lib-services-frame import { RequestParameters } from '@powersync/service-sync-rules'; import { serialize } from 'bson'; -import { Metrics } from '../../metrics/Metrics.js'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; import { SocketRouteGenerator } from '../router-socket.js'; import { SyncRoutes } from './sync-stream.js'; +import { APIMetric } from '@powersync/service-types'; + export const syncStreamReactive: SocketRouteGenerator = (router) => router.reactiveStream(SyncRoutes.STREAM, { validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }), handler: async ({ context, params, responder, observer, initialN, signal: upstreamSignal }) => { const { service_context } = context; - const { routerEngine, syncContext } = service_context; + const { routerEngine, metricsEngine, syncContext } = service_context; // Create our own controller that we can abort directly const controller = new AbortController(); @@ -69,8 +70,8 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => controller.abort(); }); - Metrics.getInstance().concurrent_connections.add(1); - const tracker = new sync.RequestTracker(); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); + const tracker = new sync.RequestTracker(metricsEngine); try { for await (const data of sync.streamResponse({ syncContext: syncContext, @@ -147,7 +148,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) => operations_synced: tracker.operationsSynced, data_synced_bytes: tracker.dataSyncedBytes }); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/routes/endpoints/sync-stream.ts b/packages/service-core/src/routes/endpoints/sync-stream.ts index eb1f4b4fa..7ff4140ed 100644 --- a/packages/service-core/src/routes/endpoints/sync-stream.ts +++ b/packages/service-core/src/routes/endpoints/sync-stream.ts @@ -5,10 +5,11 @@ import { Readable } from 'stream'; import * as sync from '../../sync/sync-index.js'; import * as util from '../../util/util-index.js'; -import { Metrics } from '../../metrics/Metrics.js'; import { authUser } from '../auth.js'; import { routeDefinition } from '../router.js'; +import { APIMetric } from '@powersync/service-types'; + export enum SyncRoutes { STREAM = '/sync/stream' } @@ -20,7 +21,7 @@ export const syncStreamed = routeDefinition({ validator: schema.createTsCodecValidator(util.StreamingSyncRequest, { allowAdditional: true }), handler: async (payload) => { const { service_context } = payload.context; - const { routerEngine, storageEngine, syncContext } = service_context; + const { routerEngine, storageEngine, metricsEngine, syncContext } = service_context; const headers = payload.request.headers; const userAgent = headers['x-user-agent'] ?? headers['user-agent']; const clientId = payload.params.client_id; @@ -49,9 +50,9 @@ export const syncStreamed = routeDefinition({ const syncRules = bucketStorage.getParsedSyncRules(routerEngine!.getAPI().getParseSyncRulesOptions()); const controller = new AbortController(); - const tracker = new sync.RequestTracker(); + const tracker = new sync.RequestTracker(metricsEngine); try { - Metrics.getInstance().concurrent_connections.add(1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(1); const stream = Readable.from( sync.transformToBytesTracked( sync.ndjson( @@ -96,7 +97,7 @@ export const syncStreamed = routeDefinition({ data: stream, afterSend: async () => { controller.abort(); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); logger.info(`Sync stream complete`, { user_id: syncParams.user_id, client_id: clientId, @@ -108,7 +109,7 @@ export const syncStreamed = routeDefinition({ }); } catch (ex) { controller.abort(); - Metrics.getInstance().concurrent_connections.add(-1); + metricsEngine.getUpDownCounter(APIMetric.CONCURRENT_CONNECTIONS).add(-1); } } }); diff --git a/packages/service-core/src/storage/storage-index.ts b/packages/service-core/src/storage/storage-index.ts index b802b0c82..9485f85d5 100644 --- a/packages/service-core/src/storage/storage-index.ts +++ b/packages/service-core/src/storage/storage-index.ts @@ -6,6 +6,7 @@ export * from './SourceEntity.js'; export * from './SourceTable.js'; export * from './StorageEngine.js'; export * from './StorageProvider.js'; +export * from './storage-metrics.js'; export * from './WriteCheckpointAPI.js'; export * from './BucketStorageFactory.js'; export * from './BucketStorageBatch.js'; diff --git a/packages/service-core/src/storage/storage-metrics.ts b/packages/service-core/src/storage/storage-metrics.ts new file mode 100644 index 000000000..5ce8f66c1 --- /dev/null +++ b/packages/service-core/src/storage/storage-metrics.ts @@ -0,0 +1,67 @@ +import { MetricsEngine } from '../metrics/MetricsEngine.js'; +import { logger } from '@powersync/lib-services-framework'; +import { BucketStorageFactory, StorageMetrics } from './BucketStorageFactory.js'; +import { StorageMetric } from '@powersync/service-types'; + +export function createCoreStorageMetrics(engine: MetricsEngine): void { + engine.createObservableGauge({ + name: StorageMetric.REPLICATION_SIZE_BYTES, + description: 'Size of current data stored in PowerSync', + unit: 'bytes' + }); + + engine.createObservableGauge({ + name: StorageMetric.OPERATION_SIZE_BYTES, + description: 'Size of operations stored in PowerSync', + unit: 'bytes' + }); + + engine.createObservableGauge({ + name: StorageMetric.PARAMETER_SIZE_BYTES, + description: 'Size of parameter data stored in PowerSync', + unit: 'bytes' + }); +} + +export function initializeCoreStorageMetrics(engine: MetricsEngine, storage: BucketStorageFactory): void { + const replication_storage_size_bytes = engine.getObservableGauge(StorageMetric.REPLICATION_SIZE_BYTES); + const operation_storage_size_bytes = engine.getObservableGauge(StorageMetric.OPERATION_SIZE_BYTES); + const parameter_storage_size_bytes = engine.getObservableGauge(StorageMetric.PARAMETER_SIZE_BYTES); + + const MINIMUM_INTERVAL = 60_000; + + let cachedRequest: Promise | undefined = undefined; + let cacheTimestamp = 0; + + const getMetrics = () => { + if (cachedRequest == null || Date.now() - cacheTimestamp > MINIMUM_INTERVAL) { + cachedRequest = storage.getStorageMetrics().catch((e) => { + logger.error(`Failed to get storage metrics`, e); + return null; + }); + cacheTimestamp = Date.now(); + } + return cachedRequest; + }; + + replication_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.replication_size_bytes; + } + }); + + operation_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.operations_size_bytes; + } + }); + + parameter_storage_size_bytes.setValueProvider(async () => { + const metrics = await getMetrics(); + if (metrics) { + return metrics.parameters_size_bytes; + } + }); +} diff --git a/packages/service-core/src/sync/RequestTracker.ts b/packages/service-core/src/sync/RequestTracker.ts index 81d717d26..d6137e640 100644 --- a/packages/service-core/src/sync/RequestTracker.ts +++ b/packages/service-core/src/sync/RequestTracker.ts @@ -1,4 +1,6 @@ -import { Metrics } from '../metrics/Metrics.js'; +import { MetricsEngine } from '../metrics/MetricsEngine.js'; + +import { APIMetric } from '@powersync/service-types'; /** * Record sync stats per request stream. @@ -7,15 +9,19 @@ export class RequestTracker { operationsSynced = 0; dataSyncedBytes = 0; + constructor(private metrics: MetricsEngine) { + this.metrics = metrics; + } + addOperationsSynced(operations: number) { this.operationsSynced += operations; - Metrics.getInstance().operations_synced_total.add(operations); + this.metrics.getCounter(APIMetric.OPERATIONS_SYNCED).add(operations); } addDataSynced(bytes: number) { this.dataSyncedBytes += bytes; - Metrics.getInstance().data_synced_bytes.add(bytes); + this.metrics.getCounter(APIMetric.DATA_SYNCED_BYTES).add(bytes); } } diff --git a/packages/service-core/src/system/ServiceContext.ts b/packages/service-core/src/system/ServiceContext.ts index 2cc82802b..40887b474 100644 --- a/packages/service-core/src/system/ServiceContext.ts +++ b/packages/service-core/src/system/ServiceContext.ts @@ -1,7 +1,7 @@ import { LifeCycledSystem, MigrationManager, ServiceIdentifier, container } from '@powersync/lib-services-framework'; import { framework } from '../index.js'; -import * as metrics from '../metrics/Metrics.js'; +import * as metrics from '../metrics/MetricsEngine.js'; import { PowerSyncMigrationManager } from '../migrations/PowerSyncMigrationManager.js'; import * as replication from '../replication/replication-index.js'; import * as routes from '../routes/routes-index.js'; @@ -12,7 +12,7 @@ import { SyncContext } from '../sync/SyncContext.js'; export interface ServiceContext { configuration: utils.ResolvedPowerSyncConfig; lifeCycleEngine: LifeCycledSystem; - metrics: metrics.Metrics | null; + metricsEngine: metrics.MetricsEngine; replicationEngine: replication.ReplicationEngine | null; routerEngine: routes.RouterEngine | null; storageEngine: storage.StorageEngine; @@ -37,6 +37,11 @@ export class ServiceContextContainer implements ServiceContext { configuration }); + this.lifeCycleEngine.withLifecycle(this.storageEngine, { + start: (storageEngine) => storageEngine.start(), + stop: (storageEngine) => storageEngine.shutDown() + }); + this.syncContext = new SyncContext({ maxDataFetchConcurrency: configuration.api_parameters.max_data_fetch_concurrency, maxBuckets: configuration.api_parameters.max_buckets_per_connection, @@ -65,8 +70,8 @@ export class ServiceContextContainer implements ServiceContext { return container.getOptional(routes.RouterEngine); } - get metrics(): metrics.Metrics | null { - return container.getOptional(metrics.Metrics); + get metricsEngine(): metrics.MetricsEngine { + return container.getImplementation(metrics.MetricsEngine); } get migrations(): PowerSyncMigrationManager { diff --git a/packages/service-core/src/util/config/compound-config-collector.ts b/packages/service-core/src/util/config/compound-config-collector.ts index fc9b53d76..dab753e0a 100644 --- a/packages/service-core/src/util/config/compound-config-collector.ts +++ b/packages/service-core/src/util/config/compound-config-collector.ts @@ -154,6 +154,7 @@ export class CompoundConfigCollector { metadata: baseConfig.metadata ?? {}, migrations: baseConfig.migrations, telemetry: { + prometheus_port: baseConfig.telemetry?.prometheus_port, disable_telemetry_sharing: baseConfig.telemetry?.disable_telemetry_sharing ?? false, internal_service_endpoint: baseConfig.telemetry?.internal_service_endpoint ?? 'https://pulse.journeyapps.com/v1/metrics' diff --git a/packages/service-core/src/util/config/types.ts b/packages/service-core/src/util/config/types.ts index 0781a6baa..56b834efb 100644 --- a/packages/service-core/src/util/config/types.ts +++ b/packages/service-core/src/util/config/types.ts @@ -55,6 +55,7 @@ export type ResolvedPowerSyncConfig = { }; telemetry: { + prometheus_port?: number; disable_telemetry_sharing: boolean; internal_service_endpoint: string; }; diff --git a/packages/service-core/src/util/env.ts b/packages/service-core/src/util/env.ts index c373371b0..f88267c61 100644 --- a/packages/service-core/src/util/env.ts +++ b/packages/service-core/src/util/env.ts @@ -19,10 +19,6 @@ export const env = utils.collectEnvironmentVariables({ * Runner to be started in this process */ PS_RUNNER_TYPE: utils.type.string.default(ServiceRunner.UNIFIED), - /** - * Port for metrics - */ - METRICS_PORT: utils.type.number.optional(), NODE_ENV: utils.type.string.optional() }); diff --git a/packages/types/src/config/PowerSyncConfig.ts b/packages/types/src/config/PowerSyncConfig.ts index a235ee58d..9a8c70ce3 100644 --- a/packages/types/src/config/PowerSyncConfig.ts +++ b/packages/types/src/config/PowerSyncConfig.ts @@ -104,7 +104,7 @@ export type StrictJwk = t.Decoded; export const BaseStorageConfig = t.object({ type: t.string, - // Maximum number of conncetions to the storage database, per process. + // Maximum number of connections to the storage database, per process. // Defaults to 8. max_pool_size: t.number.optional() }); @@ -200,6 +200,8 @@ export const powerSyncConfig = t.object({ telemetry: t .object({ + // When set, metrics will be available on this port for scraping by Prometheus. + prometheus_port: portCodec.optional(), disable_telemetry_sharing: t.boolean, internal_service_endpoint: t.string.optional() }) diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 03d0aaaf1..dca03e082 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -2,3 +2,5 @@ export * as configFile from './config/PowerSyncConfig.js'; export * from './definitions.js'; export * as internal_routes from './routes.js'; +export * from './metrics.js'; +export * as metric_types from './metrics.js'; diff --git a/packages/types/src/metrics.ts b/packages/types/src/metrics.ts new file mode 100644 index 000000000..360c0f961 --- /dev/null +++ b/packages/types/src/metrics.ts @@ -0,0 +1,28 @@ +export enum APIMetric { + // Uncompressed size of synced data from PowerSync to Clients + DATA_SYNCED_BYTES = 'powersync_data_synced_bytes_total', + // Number of operations synced + OPERATIONS_SYNCED = 'powersync_operations_synced_total', + // Number of concurrent sync connections + CONCURRENT_CONNECTIONS = 'powersync_concurrent_connections' +} + +export enum ReplicationMetric { + // Uncompressed size of replicated data from data source to PowerSync + DATA_REPLICATED_BYTES = 'powersync_data_replicated_bytes_total', + // Total number of replicated rows. Not used for pricing. + ROWS_REPLICATED = 'powersync_rows_replicated_total', + // Total number of replicated transactions. Not used for pricing. + TRANSACTIONS_REPLICATED = 'powersync_transactions_replicated_total', + // Total number of replication chunks. Not used for pricing. + CHUNKS_REPLICATED = 'powersync_chunks_replicated_total' +} + +export enum StorageMetric { + // Size of current replication data stored in PowerSync + REPLICATION_SIZE_BYTES = 'powersync_replication_storage_size_bytes', + // Size of operations data stored in PowerSync + OPERATION_SIZE_BYTES = 'powersync_operation_storage_size_bytes', + // Size of parameter data stored in PowerSync + PARAMETER_SIZE_BYTES = 'powersync_parameter_storage_size_bytes' +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a9a6db483..6ab9f4b22 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -464,20 +464,20 @@ importers: specifier: ^4.4.2 version: 4.4.2 '@opentelemetry/api': - specifier: ~1.8.0 - version: 1.8.0 + specifier: ~1.9.0 + version: 1.9.0 '@opentelemetry/exporter-metrics-otlp-http': - specifier: ^0.51.1 - version: 0.51.1(@opentelemetry/api@1.8.0) + specifier: ^0.57.2 + version: 0.57.2(@opentelemetry/api@1.9.0) '@opentelemetry/exporter-prometheus': - specifier: ^0.51.1 - version: 0.51.1(@opentelemetry/api@1.8.0) + specifier: ^0.57.2 + version: 0.57.2(@opentelemetry/api@1.9.0) '@opentelemetry/resources': - specifier: ^1.24.1 - version: 1.25.1(@opentelemetry/api@1.8.0) + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-metrics': - specifier: 1.24.1 - version: 1.24.1(@opentelemetry/api@1.8.0) + specifier: 1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) '@powersync/lib-services-framework': specifier: workspace:* version: link:../../libs/lib-services @@ -576,6 +576,9 @@ importers: specifier: ^3.0.5 version: 3.0.5(@types/node@22.13.1)(yaml@2.5.0) devDependencies: + '@opentelemetry/sdk-metrics': + specifier: ^1.30.1 + version: 1.30.1(@opentelemetry/api@1.9.0) typescript: specifier: ^5.7.3 version: 5.7.3 @@ -627,27 +630,12 @@ importers: '@fastify/cors': specifier: 8.4.1 version: 8.4.1 - '@opentelemetry/api': - specifier: ~1.6.0 - version: 1.6.0 - '@opentelemetry/exporter-prometheus': - specifier: ^0.43.0 - version: 0.43.0(@opentelemetry/api@1.6.0) - '@opentelemetry/sdk-metrics': - specifier: ^1.17.0 - version: 1.24.1(@opentelemetry/api@1.6.0) '@powersync/lib-services-framework': specifier: workspace:* version: link:../libs/lib-services '@powersync/service-core': specifier: workspace:* version: link:../packages/service-core - '@powersync/service-jpgwire': - specifier: workspace:* - version: link:../packages/jpgwire - '@powersync/service-jsonbig': - specifier: workspace:* - version: link:../packages/jsonbig '@powersync/service-module-mongodb': specifier: workspace:* version: link:../modules/module-mongodb @@ -666,70 +654,16 @@ importers: '@powersync/service-rsocket-router': specifier: workspace:* version: link:../packages/rsocket-router - '@powersync/service-sync-rules': - specifier: workspace:* - version: link:../packages/sync-rules - '@powersync/service-types': - specifier: workspace:* - version: link:../packages/types '@sentry/node': specifier: ^8.9.2 version: 8.17.0 - async-mutex: - specifier: ^0.5.0 - version: 0.5.0 - bson: - specifier: ^6.10.3 - version: 6.10.3 - commander: - specifier: ^12.0.0 - version: 12.1.0 - cors: - specifier: ^2.8.5 - version: 2.8.5 fastify: specifier: 4.23.2 version: 4.23.2 - ipaddr.js: - specifier: ^2.1.0 - version: 2.2.0 - ix: - specifier: ^5.0.0 - version: 5.0.0 - jose: - specifier: ^4.15.1 - version: 4.15.9 - lru-cache: - specifier: ^10.0.1 - version: 10.4.3 - mongodb: - specifier: ^6.14.1 - version: 6.14.2(socks@2.8.3) - node-fetch: - specifier: ^3.3.2 - version: 3.3.2 - pgwire: - specifier: github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 - version: https://codeload.github.com/kagis/pgwire/tar.gz/f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87 - ts-codec: - specifier: ^1.3.0 - version: 1.3.0 - uuid: - specifier: ^9.0.1 - version: 9.0.1 - winston: - specifier: ^3.13.0 - version: 3.13.1 - yaml: - specifier: ^2.3.2 - version: 2.4.5 devDependencies: '@sentry/types': specifier: ^8.9.2 version: 8.17.0 - '@types/uuid': - specifier: ^9.0.4 - version: 9.0.8 copyfiles: specifier: ^2.4.1 version: 2.4.1 @@ -1107,21 +1041,13 @@ packages: resolution: {integrity: sha512-NCcr1uQo1k5U+SYlnIrbAh3cxy+OQT1VtqiAbxdymSlptbzBb62AjH2xXgjNCoP073hoa1CfCAcwoZ8k96C4nA==} engines: {node: ^14.17.0 || ^16.13.0 || >=18.0.0} - '@opentelemetry/api-logs@0.51.1': - resolution: {integrity: sha512-E3skn949Pk1z2XtXu/lxf6QAZpawuTM/IUEXcAzpiUkTd73Hmvw26FiN3cJuTmkpM5hZzHwkomVdtrh/n/zzwA==} - engines: {node: '>=14'} - '@opentelemetry/api-logs@0.52.1': resolution: {integrity: sha512-qnSqB2DQ9TPP96dl8cDubDvrUyWc0/sK81xHTK8eSUspzDM3bsewX903qclQFvVhgStjRWdC5bLb3kQqMkfV5A==} engines: {node: '>=14'} - '@opentelemetry/api@1.6.0': - resolution: {integrity: sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==} - engines: {node: '>=8.0.0'} - - '@opentelemetry/api@1.8.0': - resolution: {integrity: sha512-I/s6F7yKUDdtMsoBWXJe8Qz40Tui5vsuKCWJEWVL+5q9sSWRzzx6v2KeNsOBEwd94j0eWkpWCH4yB6rZg9Mf0w==} - engines: {node: '>=8.0.0'} + '@opentelemetry/api-logs@0.57.2': + resolution: {integrity: sha512-uIX52NnTM0iBh84MShlpouI7UKqkZ7MrUszTmaypHBu4r7NofznSnQRfJ+uUeDtQDj6w8eFGg5KBLDAwAPz1+A==} + engines: {node: '>=14'} '@opentelemetry/api@1.9.0': resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} @@ -1133,38 +1059,26 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/core@1.17.0': - resolution: {integrity: sha512-tfnl3h+UefCgx1aeN2xtrmr6BmdWGKXypk0pflQR0urFS40aE88trnkOMc2HTJZbMrqEEl4HsaBeFhwLVXsrJg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.7.0' - - '@opentelemetry/core@1.24.1': - resolution: {integrity: sha512-wMSGfsdmibI88K9wB498zXY04yThPexo8jvwNNlm542HZB7XrrMRBbAyKJqG8qDRJwIBdBrPMi4V9ZPW/sqrcg==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' - '@opentelemetry/core@1.25.1': resolution: {integrity: sha512-GeT/l6rBYWVQ4XArluLVB6WWQ8flHbdb6r2FCHC3smtdOAbrJBIv35tpV/yp9bmYUJf+xmZpu9DRTIeJVhFbEQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/exporter-metrics-otlp-http@0.51.1': - resolution: {integrity: sha512-oFXvif9iksHUxrzG3P8ohMLt7xSrl+oDMqxD/3XXndU761RFAKSbRDpfrQs25U5D+A2aMV3qk+4kfUWdJhZ77g==} + '@opentelemetry/core@1.30.1': + resolution: {integrity: sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': ^1.3.0 + '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/exporter-prometheus@0.43.0': - resolution: {integrity: sha512-tJeZVmzzeG98BMPssrnUYZ7AdMtZEYqgOL44z/bF4YWqGePQoelmxuTn8Do0tIyBURqr0Whi/7P5/XxWMK1zTw==} + '@opentelemetry/exporter-metrics-otlp-http@0.57.2': + resolution: {integrity: sha512-ttb9+4iKw04IMubjm3t0EZsYRNWr3kg44uUuzfo9CaccYlOh8cDooe4QObDUkvx9d5qQUrbEckhrWKfJnKhemA==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/exporter-prometheus@0.51.1': - resolution: {integrity: sha512-c8TrTlLm9JJRIHW6MtFv6ESoZRgXBXD/YrTRYylWiyYBOVbYHo1c5Qaw/j/thXDhkmYOYAn4LAhJZpLl5gBFEQ==} + '@opentelemetry/exporter-prometheus@0.57.2': + resolution: {integrity: sha512-VqIqXnuxWMWE/1NatAGtB1PvsQipwxDcdG4RwA/umdBcW3/iOHp0uejvFHTRN2O78ZPged87ErJajyUBPUhlDQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': ^1.3.0 @@ -1271,64 +1185,45 @@ packages: peerDependencies: '@opentelemetry/api': ^1.3.0 - '@opentelemetry/otlp-exporter-base@0.51.1': - resolution: {integrity: sha512-UYlnOYyDdzo1Gw559EHCzru0RwhvuXCwoH8jGo9J4gO1TE58GjnEmIjomMsKBCym3qWNJfIQXw+9SZCV0DdQNg==} + '@opentelemetry/otlp-exporter-base@0.57.2': + resolution: {integrity: sha512-XdxEzL23Urhidyebg5E6jZoaiW5ygP/mRjxLHixogbqwDy2Faduzb5N0o/Oi+XTIJu+iyxXdVORjXax+Qgfxag==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': ^1.0.0 + '@opentelemetry/api': ^1.3.0 - '@opentelemetry/otlp-transformer@0.51.1': - resolution: {integrity: sha512-OppYOXwV9LQqqtYUCywqoOqX/JT9LQ5/FMuPZ//eTkvuHdUC4ZMwz2c6uSoT2R90GWvvGnF1iEqTGyTT3xAt2Q==} + '@opentelemetry/otlp-transformer@0.57.2': + resolution: {integrity: sha512-48IIRj49gbQVK52jYsw70+Jv+JbahT8BqT2Th7C4H7RCM9d0gZ5sgNPoMpWldmfjvIsSgiGJtjfk9MeZvjhoig==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.9.0' + '@opentelemetry/api': ^1.3.0 '@opentelemetry/redis-common@0.36.2': resolution: {integrity: sha512-faYX1N0gpLhej/6nyp6bgRjzAKXn5GOEMYY7YhciSfCoITAktLUtQ36d24QEWNA1/WA1y6qQunCe0OhHRkVl9g==} engines: {node: '>=14'} - '@opentelemetry/resources@1.17.0': - resolution: {integrity: sha512-+u0ciVnj8lhuL/qGRBPeVYvk7fL+H/vOddfvmOeJaA1KC+5/3UED1c9KoZQlRsNT5Kw1FaK8LkY2NVLYfOVZQw==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.7.0' - - '@opentelemetry/resources@1.24.1': - resolution: {integrity: sha512-cyv0MwAaPF7O86x5hk3NNgenMObeejZFLJJDVuSeSMIsknlsj3oOZzRv3qSzlwYomXsICfBeFFlxwHQte5mGXQ==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' - '@opentelemetry/resources@1.25.1': resolution: {integrity: sha512-pkZT+iFYIZsVn6+GzM0kSX+u3MSLCY9md+lIJOoKl/P+gJFfxJte/60Usdp8Ce4rOs8GduUpSPNe1ddGyDT1sQ==} engines: {node: '>=14'} peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-logs@0.51.1': - resolution: {integrity: sha512-ULQQtl82b673PpZc5/0EtH4V+BrwVOgKJZEB7tYZnGTG3I98tQVk89S9/JSixomDr++F4ih+LSJTCqIKBz+MQQ==} + '@opentelemetry/resources@1.30.1': + resolution: {integrity: sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.4.0 <1.9.0' - '@opentelemetry/api-logs': '>=0.39.1' - - '@opentelemetry/sdk-metrics@1.17.0': - resolution: {integrity: sha512-HlWM27yGmYuwCoVRe3yg2PqKnIsq0kEF0HQgvkeDWz2NYkq9fFaSspR6kvjxUTbghAlZrabiqbgyKoYpYaXS3w==} - engines: {node: '>=14'} - peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.7.0' + '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/sdk-metrics@1.24.1': - resolution: {integrity: sha512-FrAqCbbGao9iKI+Mgh+OsC9+U2YMoXnlDHe06yH7dvavCKzE3S892dGtX54+WhSFVxHR/TMRVJiK/CV93GR0TQ==} + '@opentelemetry/sdk-logs@0.57.2': + resolution: {integrity: sha512-TXFHJ5c+BKggWbdEQ/inpgIzEmS2BGQowLE9UhsMd7YYlUfBQJ4uax0VF/B5NYigdM/75OoJGhAV3upEhK+3gg==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.3.0 <1.9.0' + '@opentelemetry/api': '>=1.4.0 <1.10.0' - '@opentelemetry/sdk-trace-base@1.24.1': - resolution: {integrity: sha512-zz+N423IcySgjihl2NfjBf0qw1RWe11XIAWVrTNOSSI6dtSPJiVom2zipFB2AEEtJWpv0Iz6DY6+TjnyTV5pWg==} + '@opentelemetry/sdk-metrics@1.30.1': + resolution: {integrity: sha512-q9zcZ0Okl8jRgmy7eNW3Ku1XSgg3sDLa5evHZpCwjspw7E8Is4K/haRPDJrBcX3YSn/Y7gUvFnByNYEKQNbNog==} engines: {node: '>=14'} peerDependencies: - '@opentelemetry/api': '>=1.0.0 <1.9.0' + '@opentelemetry/api': '>=1.3.0 <1.10.0' '@opentelemetry/sdk-trace-base@1.25.1': resolution: {integrity: sha512-C8k4hnEbc5FamuZQ92nTOp8X/diCY56XUTnMiv9UTuJitCzaNNHAVsdm5+HLCdI8SLQsLWIrG38tddMxLVoftw==} @@ -1336,18 +1231,20 @@ packages: peerDependencies: '@opentelemetry/api': '>=1.0.0 <1.10.0' - '@opentelemetry/semantic-conventions@1.17.0': - resolution: {integrity: sha512-+fguCd2d8d2qruk0H0DsCEy2CTK3t0Tugg7MhZ/UQMvmewbZLNnJ6heSYyzIZWG5IPfAXzoj4f4F/qpM7l4VBA==} - engines: {node: '>=14'} - - '@opentelemetry/semantic-conventions@1.24.1': - resolution: {integrity: sha512-VkliWlS4/+GHLLW7J/rVBA00uXus1SWvwFvcUDxDwmFxYfg/2VI6ekwdXS28cjI8Qz2ky2BzG8OUHo+WeYIWqw==} + '@opentelemetry/sdk-trace-base@1.30.1': + resolution: {integrity: sha512-jVPgBbH1gCy2Lb7X0AVQ8XAfgg0pJ4nvl8/IiQA6nxOsPvS+0zMJaFSs2ltXe0J6C8dqjcnpyqINDJmU30+uOg==} engines: {node: '>=14'} + peerDependencies: + '@opentelemetry/api': '>=1.0.0 <1.10.0' '@opentelemetry/semantic-conventions@1.25.1': resolution: {integrity: sha512-ZDjMJJQRlyk8A1KZFCc+bCbsyrn1wTwdNt56F7twdfUfnHUZUq77/WfONCj8p72NZOyP7pNTdUWSTYC3GTbuuQ==} engines: {node: '>=14'} + '@opentelemetry/semantic-conventions@1.28.0': + resolution: {integrity: sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==} + engines: {node: '>=14'} + '@opentelemetry/sql-common@0.40.1': resolution: {integrity: sha512-nSDlnHSqzC3pXn/wZEZVLuAuJ1MYMXPBwtv2qAbCa3847SaHItdE7SzUq/Jtb0KZmh1zfAbNi3AAMjztTT4Ugg==} engines: {node: '>=14'} @@ -1380,6 +1277,36 @@ packages: '@prisma/instrumentation@5.16.1': resolution: {integrity: sha512-4m5gRFWnQb8s/yTyGbMZkL7A5uJgqOWcWJxapwcAD0T0kh5sGPEVSQl/zTQvE9aduXhFAxOtC3gO+R8Hb5xO1Q==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.4': + resolution: {integrity: sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==} + + '@protobufjs/eventemitter@1.1.0': + resolution: {integrity: sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==} + + '@protobufjs/fetch@1.1.0': + resolution: {integrity: sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/inquire@1.1.0': + resolution: {integrity: sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.0': + resolution: {integrity: sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==} + '@rollup/rollup-android-arm-eabi@4.34.8': resolution: {integrity: sha512-q217OSE8DTp8AFHuNHXo0Y86e1wtlfVrXiAlwkIvGRQv9zbc6mE3sjIVfwI8sYUyNxwOg0j/Vm1RKM04JcWLJw==} cpu: [arm] @@ -2700,9 +2627,6 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - lodash.merge@4.6.2: - resolution: {integrity: sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==} - lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} @@ -3274,6 +3198,10 @@ packages: proto-list@1.2.4: resolution: {integrity: sha512-vtK/94akxsTMhe0/cbfpR+syPuszcuwhqVjJq26CuNDgFGj682oRBXOP5MJpv2r7JtE8MsiepGIqvvOTBwn2vA==} + protobufjs@7.4.0: + resolution: {integrity: sha512-mRUWCc3KUU4w1jU8sGxICXH/gNS94DvI1gxqDvBzhj1JpcsimQkYiOJfwsPUykUI5ZaspFbSgmBLER8IrQ3tqw==} + engines: {node: '>=12.0.0'} + proxy-addr@2.0.7: resolution: {integrity: sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==} engines: {node: '>= 0.10'} @@ -4514,76 +4442,45 @@ snapshots: - bluebird - supports-color - '@opentelemetry/api-logs@0.51.1': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs@0.52.1': - dependencies: - '@opentelemetry/api': 1.6.0 - - '@opentelemetry/api@1.6.0': {} - - '@opentelemetry/api@1.8.0': {} - - '@opentelemetry/api@1.9.0': {} - - '@opentelemetry/context-async-hooks@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/semantic-conventions': 1.17.0 - - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.6.0)': + '@opentelemetry/api-logs@0.57.2': dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/semantic-conventions': 1.24.1 + '@opentelemetry/api': 1.9.0 - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/semantic-conventions': 1.24.1 + '@opentelemetry/api@1.9.0': {} - '@opentelemetry/core@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/context-async-hooks@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/core@1.25.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/semantic-conventions': 1.25.1 '@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/semantic-conventions': 1.25.1 - '@opentelemetry/exporter-metrics-otlp-http@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/core@1.30.1(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/otlp-exporter-base': 0.51.1(@opentelemetry/api@1.8.0) - '@opentelemetry/otlp-transformer': 0.51.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/exporter-prometheus@0.43.0(@opentelemetry/api@1.6.0)': + '@opentelemetry/exporter-metrics-otlp-http@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/sdk-metrics': 1.17.0(@opentelemetry/api@1.6.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-exporter-base': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) - '@opentelemetry/exporter-prometheus@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/exporter-prometheus@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-connect@0.38.0(@opentelemetry/api@1.9.0)': dependencies: @@ -4661,7 +4558,7 @@ snapshots: dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 transitivePeerDependencies: - supports-color @@ -4733,18 +4630,6 @@ snapshots: - supports-color optional: true - '@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.52.1 - '@types/shimmer': 1.2.0 - import-in-the-middle: 1.9.0 - require-in-the-middle: 7.3.0 - semver: 7.6.2 - shimmer: 1.2.1 - transitivePeerDependencies: - - supports-color - '@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4757,52 +4642,24 @@ snapshots: transitivePeerDependencies: - supports-color - '@opentelemetry/otlp-exporter-base@0.51.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/otlp-exporter-base@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - - '@opentelemetry/otlp-transformer@0.51.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.51.1 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-logs': 0.51.1(@opentelemetry/api-logs@0.51.1)(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-metrics': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-trace-base': 1.24.1(@opentelemetry/api@1.8.0) - - '@opentelemetry/redis-common@0.36.2': {} - - '@opentelemetry/resources@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/semantic-conventions': 1.17.0 - - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.6.0) - '@opentelemetry/semantic-conventions': 1.24.1 - - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.24.1 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/otlp-transformer': 0.57.2(@opentelemetry/api@1.9.0) - '@opentelemetry/resources@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/otlp-transformer@0.57.2(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.9.0) - '@opentelemetry/semantic-conventions': 1.24.1 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-logs': 0.57.2(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-metrics': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.30.1(@opentelemetry/api@1.9.0) + protobufjs: 7.4.0 - '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/redis-common@0.36.2': {} '@opentelemetry/resources@1.25.1(@opentelemetry/api@1.9.0)': dependencies: @@ -4810,54 +4667,24 @@ snapshots: '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 - '@opentelemetry/sdk-logs@0.51.1(@opentelemetry/api-logs@0.51.1)(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/api-logs': 0.51.1 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - - '@opentelemetry/sdk-metrics@1.17.0(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.17.0(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.17.0(@opentelemetry/api@1.6.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.6.0)': - dependencies: - '@opentelemetry/api': 1.6.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.6.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.6.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.8.0)': - dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - lodash.merge: 4.6.2 - - '@opentelemetry/sdk-metrics@1.24.1(@opentelemetry/api@1.9.0)': + '@opentelemetry/resources@1.30.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.9.0) - lodash.merge: 4.6.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 - '@opentelemetry/sdk-trace-base@1.24.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/sdk-logs@0.57.2(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.24.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.24.1 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/api-logs': 0.57.2 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) - '@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.8.0)': + '@opentelemetry/sdk-metrics@1.30.1(@opentelemetry/api@1.9.0)': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.8.0) - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.8.0) - '@opentelemetry/semantic-conventions': 1.25.1 + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0)': dependencies: @@ -4866,12 +4693,17 @@ snapshots: '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 - '@opentelemetry/semantic-conventions@1.17.0': {} - - '@opentelemetry/semantic-conventions@1.24.1': {} + '@opentelemetry/sdk-trace-base@1.30.1(@opentelemetry/api@1.9.0)': + dependencies: + '@opentelemetry/api': 1.9.0 + '@opentelemetry/core': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) + '@opentelemetry/semantic-conventions': 1.28.0 '@opentelemetry/semantic-conventions@1.25.1': {} + '@opentelemetry/semantic-conventions@1.28.0': {} + '@opentelemetry/sql-common@0.40.1(@opentelemetry/api@1.9.0)': dependencies: '@opentelemetry/api': 1.9.0 @@ -4904,12 +4736,35 @@ snapshots: '@prisma/instrumentation@5.16.1': dependencies: - '@opentelemetry/api': 1.8.0 - '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.8.0) - '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.8.0) + '@opentelemetry/api': 1.9.0 + '@opentelemetry/instrumentation': 0.52.1(@opentelemetry/api@1.9.0) + '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) transitivePeerDependencies: - supports-color + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.4': {} + + '@protobufjs/eventemitter@1.1.0': {} + + '@protobufjs/fetch@1.1.0': + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/inquire': 1.1.0 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/inquire@1.1.0': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.0': {} + '@rollup/rollup-android-arm-eabi@4.34.8': optional: true @@ -4993,12 +4848,12 @@ snapshots: '@opentelemetry/instrumentation-nestjs-core': 0.39.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-pg': 0.43.0(@opentelemetry/api@1.9.0) '@opentelemetry/instrumentation-redis-4': 0.41.0(@opentelemetry/api@1.9.0) - '@opentelemetry/resources': 1.25.1(@opentelemetry/api@1.9.0) + '@opentelemetry/resources': 1.30.1(@opentelemetry/api@1.9.0) '@opentelemetry/sdk-trace-base': 1.25.1(@opentelemetry/api@1.9.0) '@opentelemetry/semantic-conventions': 1.25.1 '@prisma/instrumentation': 5.16.1 '@sentry/core': 8.17.0 - '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) + '@sentry/opentelemetry': 8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1) '@sentry/types': 8.17.0 '@sentry/utils': 8.17.0 optionalDependencies: @@ -5006,7 +4861,7 @@ snapshots: transitivePeerDependencies: - supports-color - '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.6.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': + '@sentry/opentelemetry@8.17.0(@opentelemetry/api@1.9.0)(@opentelemetry/core@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/instrumentation@0.52.1(@opentelemetry/api@1.9.0))(@opentelemetry/sdk-trace-base@1.25.1(@opentelemetry/api@1.9.0))(@opentelemetry/semantic-conventions@1.25.1)': dependencies: '@opentelemetry/api': 1.9.0 '@opentelemetry/core': 1.25.1(@opentelemetry/api@1.9.0) @@ -6242,8 +6097,6 @@ snapshots: dependencies: p-locate: 5.0.0 - lodash.merge@4.6.2: {} - lodash.startcase@4.4.0: {} lodash@4.17.21: {} @@ -6880,6 +6733,21 @@ snapshots: proto-list@1.2.4: {} + protobufjs@7.4.0: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.4 + '@protobufjs/eventemitter': 1.1.0 + '@protobufjs/fetch': 1.1.0 + '@protobufjs/float': 1.0.2 + '@protobufjs/inquire': 1.1.0 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.0 + '@types/node': 22.13.1 + long: 5.2.3 + proxy-addr@2.0.7: dependencies: forwarded: 0.2.0 diff --git a/service/package.json b/service/package.json index 5420cd486..334cf2963 100644 --- a/service/package.json +++ b/service/package.json @@ -11,9 +11,6 @@ }, "dependencies": { "@fastify/cors": "8.4.1", - "@opentelemetry/api": "~1.6.0", - "@opentelemetry/exporter-prometheus": "^0.43.0", - "@opentelemetry/sdk-metrics": "^1.17.0", "@powersync/service-core": "workspace:*", "@powersync/lib-services-framework": "workspace:*", "@powersync/service-module-postgres": "workspace:*", @@ -21,32 +18,12 @@ "@powersync/service-module-mongodb": "workspace:*", "@powersync/service-module-mongodb-storage": "workspace:*", "@powersync/service-module-mysql": "workspace:*", - "@powersync/service-jpgwire": "workspace:*", - "@powersync/service-jsonbig": "workspace:*", "@powersync/service-rsocket-router": "workspace:*", - "@powersync/service-sync-rules": "workspace:*", - "@powersync/service-types": "workspace:*", "@sentry/node": "^8.9.2", - "async-mutex": "^0.5.0", - "bson": "^6.10.3", - "commander": "^12.0.0", - "cors": "^2.8.5", - "fastify": "4.23.2", - "ipaddr.js": "^2.1.0", - "ix": "^5.0.0", - "jose": "^4.15.1", - "lru-cache": "^10.0.1", - "mongodb": "^6.14.1", - "node-fetch": "^3.3.2", - "pgwire": "github:kagis/pgwire#f1cb95f9a0f42a612bb5a6b67bb2eb793fc5fc87", - "ts-codec": "^1.3.0", - "uuid": "^9.0.1", - "winston": "^3.13.0", - "yaml": "^2.3.2" + "fastify": "4.23.2" }, "devDependencies": { "@sentry/types": "^8.9.2", - "@types/uuid": "^9.0.4", "copyfiles": "^2.4.1", "nodemon": "^3.0.1", "npm-check-updates": "^16.14.4", diff --git a/service/src/metrics.ts b/service/src/metrics.ts deleted file mode 100644 index 57ef04db0..000000000 --- a/service/src/metrics.ts +++ /dev/null @@ -1,40 +0,0 @@ -import * as core from '@powersync/service-core'; - -export enum MetricModes { - API = 'api', - REPLICATION = 'replication' -} - -export type MetricsRegistrationOptions = { - service_context: core.system.ServiceContextContainer; - modes: MetricModes[]; -}; - -export const registerMetrics = async (options: MetricsRegistrationOptions) => { - const { service_context, modes } = options; - - // This requires an instantiated bucket storage, which is only created when the lifecycle starts - service_context.lifeCycleEngine.withLifecycle(null, { - start: async () => { - const instanceId = await service_context.storageEngine.activeBucketStorage.getPowerSyncInstanceId(); - await core.metrics.Metrics.initialise({ - powersync_instance_id: instanceId, - disable_telemetry_sharing: service_context.configuration.telemetry.disable_telemetry_sharing, - internal_metrics_endpoint: service_context.configuration.telemetry.internal_service_endpoint - }); - - // TODO remove singleton - const instance = core.Metrics.getInstance(); - service_context.register(core.metrics.Metrics, instance); - - if (modes.includes(MetricModes.API)) { - instance.configureApiMetrics(); - } - - if (modes.includes(MetricModes.REPLICATION)) { - instance.configureReplicationMetrics(service_context.storageEngine.activeBucketStorage); - } - }, - stop: () => service_context.metrics!.shutdown() - }); -}; diff --git a/service/src/runners/server.ts b/service/src/runners/server.ts index 1259b1e93..6eff0cfce 100644 --- a/service/src/runners/server.ts +++ b/service/src/runners/server.ts @@ -5,7 +5,7 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; import { ReactiveSocketRouter } from '@powersync/service-rsocket-router'; -import { MetricModes, registerMetrics } from '../metrics.js'; +import { logBooting } from '../util/version.js'; /** * Configures the server portion on a {@link ServiceContext} @@ -75,23 +75,23 @@ export function registerServerServices(serviceContext: core.system.ServiceContex * Starts an API server */ export async function startServer(runnerConfig: core.utils.RunnerConfig) { - logger.info('Booting'); + logBooting('API Container'); const config = await core.utils.loadConfig(runnerConfig); + core.utils.setTags(config.metadata); const serviceContext = new core.system.ServiceContextContainer(config); registerServerServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.API] + modes: [core.metrics.MetricModes.API] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); await moduleManager.initialize(serviceContext); logger.info('Starting service...'); - await serviceContext.lifeCycleEngine.start(); logger.info('Service started.'); diff --git a/service/src/runners/stream-worker.ts b/service/src/runners/stream-worker.ts index 323d71170..ed27fef71 100644 --- a/service/src/runners/stream-worker.ts +++ b/service/src/runners/stream-worker.ts @@ -1,6 +1,6 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { MetricModes, registerMetrics } from '../metrics.js'; +import { logBooting } from '../util/version.js'; /** * Configures the replication portion on a {@link serviceContext} @@ -17,18 +17,19 @@ export const registerReplicationServices = (serviceContext: core.system.ServiceC }; export const startStreamRunner = async (runnerConfig: core.utils.RunnerConfig) => { - logger.info('Booting'); + logBooting('Replication Container'); const config = await core.utils.loadConfig(runnerConfig); + core.utils.setTags(config.metadata); - // Self hosted version allows for automatic migrations + // Self-hosted version allows for automatic migrations const serviceContext = new core.system.ServiceContextContainer(config); registerReplicationServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.REPLICATION] + modes: [core.metrics.MetricModes.REPLICATION, core.metrics.MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); diff --git a/service/src/runners/unified-runner.ts b/service/src/runners/unified-runner.ts index 5e505e2f7..b5767b5a4 100644 --- a/service/src/runners/unified-runner.ts +++ b/service/src/runners/unified-runner.ts @@ -1,26 +1,26 @@ import { container, logger } from '@powersync/lib-services-framework'; import * as core from '@powersync/service-core'; -import { MetricModes, registerMetrics } from '../metrics.js'; import { registerServerServices } from './server.js'; import { registerReplicationServices } from './stream-worker.js'; +import { logBooting } from '../util/version.js'; /** * Starts an API server */ export const startUnifiedRunner = async (runnerConfig: core.utils.RunnerConfig) => { - logger.info('Booting'); + logBooting('Unified Container'); const config = await core.utils.loadConfig(runnerConfig); - + core.utils.setTags(config.metadata); const serviceContext = new core.system.ServiceContextContainer(config); registerServerServices(serviceContext); registerReplicationServices(serviceContext); - await registerMetrics({ + await core.metrics.registerMetrics({ service_context: serviceContext, - modes: [MetricModes.API, MetricModes.REPLICATION] + modes: [core.metrics.MetricModes.API, core.metrics.MetricModes.REPLICATION, core.metrics.MetricModes.STORAGE] }); const moduleManager = container.getImplementation(core.modules.ModuleManager); diff --git a/service/src/util/version.ts b/service/src/util/version.ts new file mode 100644 index 000000000..887b86921 --- /dev/null +++ b/service/src/util/version.ts @@ -0,0 +1,8 @@ +import { logger } from '@powersync/lib-services-framework'; + +export function logBooting(runner: string) { + const version = process.env.POWERSYNC_VERSION ?? '-dev'; + const isCloud = process.env.MICRO_SERVICE_NAME == 'powersync'; + const edition = isCloud ? 'Cloud Edition' : 'Enterprise Edition'; + logger.info(`Booting PowerSync Service v${version}, ${runner}, ${edition}`, { version, edition, runner }); +}