Rename post_images config option.

Author: rkistner

modules/module-mongodb/src/replication/ChangeStream.ts

@@ -12,6 +12,7 @@ import {
 } from './MongoRelation.js';
 import { escapeRegExp } from '../utils.js';
 import { CHECKPOINTS_COLLECTION } from './replication-utils.js';
+import { PostImagesOption } from '../types/types.js';
 
 export const ZERO_LSN = '0000000000000000';
 
@@ -79,8 +80,12 @@ export class ChangeStream {
     return this.abort_signal.aborted;
   }
 
-  private get postImages() {
-    return this.connections.options.postImages;
+  private get usePostImages() {
+    return this.connections.options.postImages != PostImagesOption.OFF;
+  }
+
+  private get configurePostImages() {
+    return this.connections.options.postImages == PostImagesOption.AUTO_CONFIGURE;
   }
 
   /**
@@ -228,7 +233,7 @@ export class ChangeStream {
       await this.defaultDb.createCollection(CHECKPOINTS_COLLECTION, {
         changeStreamPreAndPostImages: { enabled: true }
       });
-    } else if (this.postImages != 'updateLookup' && collection.options?.changeStreamPreAndPostImages?.enabled != true) {
+    } else if (this.usePostImages && collection.options?.changeStreamPreAndPostImages?.enabled != true) {
       await this.defaultDb.command({
         collMod: CHECKPOINTS_COLLECTION,
         changeStreamPreAndPostImages: { enabled: true }
@@ -351,14 +356,14 @@ export class ChangeStream {
   }
 
   private async checkPostImages(db: string, collectionInfo: mongo.CollectionInfo) {
-    if (this.postImages == 'updateLookup') {
+    if (!this.usePostImages) {
       // Nothing to check
       return;
     }
 
     const enabled = collectionInfo.options?.changeStreamPreAndPostImages?.enabled == true;
 
-    if (!enabled && this.postImages == 'autoConfigure') {
+    if (!enabled && this.configurePostImages) {
       await this.client.db(db).command({
         collMod: collectionInfo.name,
         changeStreamPreAndPostImages: { enabled: true }
@@ -525,12 +530,13 @@ export class ChangeStream {
 
       let fullDocument: 'required' | 'updateLookup';
 
-      if (this.connections.options.postImages == 'updateLookup') {
-        fullDocument = 'updateLookup';
-      } else {
-        // 'on' or 'autoConfigure'
-        // Configuration happens during snapshot
+      if (this.usePostImages) {
+        // 'read_only' or 'auto_configure'
+        // Configuration happens during snapshot, or when we see new
+        // collections.
         fullDocument = 'required';
+      } else {
+        fullDocument = 'updateLookup';
       }
 
       const streamOptions: mongo.ChangeStreamOptions = {

modules/module-mongodb/src/types/types.ts

@@ -4,6 +4,42 @@ import * as t from 'ts-codec';
 
 export const MONGO_CONNECTION_TYPE = 'mongodb' as const;
 
+export enum PostImagesOption {
+  /**
+   * Use fullDocument: updateLookup on the changeStream.
+   *
+   * This does not guarantee consistency - operations may
+   * arrive out of order, especially when there is replication lag.
+   *
+   * This is the default option for backwards-compatability.
+   */
+  OFF = 'off',
+
+  /**
+   * Use fullDocument: required on the changeStream.
+   *
+   * Collections are automatically configured with:
+   * `changeStreamPreAndPostImages: { enabled: true }`
+   *
+   * This is the recommended behavior for new instances.
+   */
+  AUTO_CONFIGURE = 'auto_configure',
+
+  /**
+   *
+   * Use fullDocument: required on the changeStream.
+   *
+   * Collections are not automatically configured. Each
+   * collection must be configured configured manually before
+   * replicating with:
+   *
+   * `changeStreamPreAndPostImages: { enabled: true }`
+   *
+   * Use when the collMod permission is not available.
+   */
+  READ_ONLY = 'read_only'
+}
+
 export interface NormalizedMongoConnectionConfig {
   id: string;
   tag: string;
@@ -14,7 +50,7 @@ export interface NormalizedMongoConnectionConfig {
   username?: string;
   password?: string;
 
-  postImages: 'on' | 'autoConfigure' | 'updateLookup';
+  postImages: PostImagesOption;
 }
 
 export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.and(
@@ -29,7 +65,7 @@ export const MongoConnectionConfig = service_types.configFile.DataSourceConfig.a
     password: t.string.optional(),
     database: t.string.optional(),
 
-    postImages: t.literal('on').or(t.literal('autoConfigure')).or(t.literal('updateLookup')).optional()
+    post_images: t.literal('off').or(t.literal('auto_configure')).or(t.literal('read_only')).optional()
   })
 );
 
@@ -55,7 +91,7 @@ export function normalizeConnectionConfig(options: MongoConnectionConfig): Norma
     ...base,
     id: options.id ?? 'default',
     tag: options.tag ?? 'default',
-    postImages: options.postImages ?? 'updateLookup'
+    postImages: (options.post_images as PostImagesOption | undefined) ?? PostImagesOption.OFF
   };
 }
 

modules/module-mongodb/test/src/change_stream.test.ts

@@ -6,6 +6,7 @@ import * as mongo from 'mongodb';
 import { setTimeout } from 'node:timers/promises';
 import { describe, expect, test } from 'vitest';
 import { ChangeStreamTestContext } from './change_stream_utils.js';
+import { PostImagesOption } from '@module/types/types.js';
 
 type StorageFactory = () => Promise<BucketStorageFactory>;
 
@@ -91,7 +92,7 @@ bucket_definitions:
   });
 
   test('updateLookup - no fullDocument available', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, { postImages: 'updateLookup' });
+    await using context = await ChangeStreamTestContext.open(factory, { postImages: PostImagesOption.OFF });
     const { db, client } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -135,7 +136,7 @@ bucket_definitions:
   test('postImages - autoConfigure', async () => {
     // Similar to the above test, but with postImages enabled.
     // This resolves the consistency issue.
-    await using context = await ChangeStreamTestContext.open(factory, { postImages: 'autoConfigure' });
+    await using context = await ChangeStreamTestContext.open(factory, { postImages: PostImagesOption.AUTO_CONFIGURE });
     const { db, client } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -181,7 +182,7 @@ bucket_definitions:
   test('postImages - on', async () => {
     // Similar to postImages - autoConfigure, but does not auto-configure.
     // changeStreamPreAndPostImages must be manually configured.
-    await using context = await ChangeStreamTestContext.open(factory, { postImages: 'on' });
+    await using context = await ChangeStreamTestContext.open(factory, { postImages: PostImagesOption.READ_ONLY });
     const { db, client } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -404,7 +405,7 @@ bucket_definitions:
   });
 
   test('postImages - new collection with postImages enabled', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, { postImages: 'autoConfigure' });
+    await using context = await ChangeStreamTestContext.open(factory, { postImages: PostImagesOption.AUTO_CONFIGURE });
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions:
@@ -432,8 +433,8 @@ bucket_definitions:
     ]);
   });
 
-  test.only('postImages - new collection with postImages disabled', async () => {
-    await using context = await ChangeStreamTestContext.open(factory, { postImages: 'autoConfigure' });
+  test('postImages - new collection with postImages disabled', async () => {
+    await using context = await ChangeStreamTestContext.open(factory, { postImages: PostImagesOption.AUTO_CONFIGURE });
     const { db } = context;
     await context.updateSyncRules(`
 bucket_definitions: