Refactor and move getReplacePatches() logic to fastMerge() / mergeObject()

Author: fabioh8010

lib/Onyx.ts

@@ -319,7 +319,7 @@ function merge<TKey extends OnyxKey>(key: TKey, changes: OnyxMergeInput<TKey>):
             if (!validChanges.length) {
                 return Promise.resolve();
             }
-            const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges);
+            const batchedDeltaChanges = OnyxUtils.batchMergeChanges(validChanges).result;
 
             // Case (1): When there is no existing value in storage, we want to set the value instead of merge it.
             // Case (2): The presence of a top-level `null` in the merge queue instructs us to drop the whole existing value.
@@ -391,7 +391,11 @@ function merge<TKey extends OnyxKey>(key: TKey, changes: OnyxMergeInput<TKey>):
  * @param collectionKey e.g. `ONYXKEYS.COLLECTION.REPORT`
  * @param collection Object collection keyed by individual collection member keys and values
  */
-function mergeCollection<TKey extends CollectionKeyBase, TMap>(collectionKey: TKey, collection: OnyxMergeCollectionInput<TKey, TMap>): Promise<void> {
+function mergeCollection<TKey extends CollectionKeyBase, TMap>(
+    collectionKey: TKey,
+    collection: OnyxMergeCollectionInput<TKey, TMap>,
+    mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches'],
+): Promise<void> {
     if (!OnyxUtils.isValidNonEmptyCollectionForMerge(collection)) {
         Logger.logInfo('mergeCollection() called with invalid or empty value. Skipping this update.');
         return Promise.resolve();
@@ -476,7 +480,7 @@ function mergeCollection<TKey extends CollectionKeyBase, TMap>(collectionKey: TK
             // New keys will be added via multiSet while existing keys will be updated using multiMerge
             // This is because setting a key that doesn't exist yet with multiMerge will throw errors
             if (keyValuePairsForExistingCollection.length > 0) {
-                promises.push(Storage.multiMerge(keyValuePairsForExistingCollection));
+                promises.push(Storage.multiMerge(keyValuePairsForExistingCollection, mergeReplaceNullPatches));
             }
 
             if (keyValuePairsForNewCollection.length > 0) {
@@ -773,29 +777,36 @@ function update(data: OnyxUpdate[]): Promise<void> {
                 const batchedChanges = OnyxUtils.batchMergeChanges(operations);
                 if (operations[0] === null) {
                     // eslint-disable-next-line no-param-reassign
-                    queue.set[key] = batchedChanges;
+                    queue.set[key] = batchedChanges.result;
                 } else {
                     // eslint-disable-next-line no-param-reassign
-                    queue.merge[key] = batchedChanges;
+                    queue.merge[key] = batchedChanges.result;
+                    if (batchedChanges.replaceNullPatches.length > 0) {
+                        // eslint-disable-next-line no-param-reassign
+                        queue.mergeReplaceNullPatches[key] = batchedChanges.replaceNullPatches;
+                    }
                 }
                 return queue;
             },
             {
                 merge: {},
+                mergeReplaceNullPatches: {},
                 set: {},
             },
         );
 
         if (!utils.isEmptyObject(batchedCollectionUpdates.merge)) {
-            promises.push(() => mergeCollection(collectionKey, batchedCollectionUpdates.merge as Collection<CollectionKey, unknown, unknown>));
+            promises.push(() =>
+                mergeCollection(collectionKey, batchedCollectionUpdates.merge as Collection<CollectionKey, unknown, unknown>, batchedCollectionUpdates.mergeReplaceNullPatches),
+            );
         }
         if (!utils.isEmptyObject(batchedCollectionUpdates.set)) {
             promises.push(() => multiSet(batchedCollectionUpdates.set));
         }
     });
 
     Object.entries(updateQueue).forEach(([key, operations]) => {
-        const batchedChanges = OnyxUtils.batchMergeChanges(operations);
+        const batchedChanges = OnyxUtils.batchMergeChanges(operations).result;
 
         if (operations[0] === null) {
             promises.push(() => set(key, batchedChanges));

lib/OnyxCache.ts

@@ -164,7 +164,7 @@ class OnyxCache {
             throw new Error('data passed to cache.merge() must be an Object of onyx key/value pairs');
         }
 
-        this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true)};
+        this.storageMap = {...utils.fastMerge(this.storageMap, data, true, false, true).result};
 
         Object.entries(data).forEach(([key, value]) => {
             this.addKey(key);

lib/OnyxUtils.ts

@@ -28,6 +28,7 @@ import type {
     OnyxValue,
     Selector,
 } from './types';
+import type {FastMergeResult} from './utils';
 import utils from './utils';
 import type {WithOnyxState} from './withOnyx/types';
 import type {DeferredTask} from './createDeferredTask';
@@ -1264,29 +1265,42 @@ function applyMerge<TValue extends OnyxInput<OnyxKey> | undefined, TChange exten
 
     if (changes.some((change) => change && typeof change === 'object')) {
         // Object values are then merged one after the other
-        return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true), (existingValue || {}) as TChange);
+        return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, true, false, true).result, (existingValue || {}) as TChange);
     }
 
     // If we have anything else we can't merge it so we'll
     // simply return the last value that was queued
     return lastChange as TChange;
 }
 
-function batchMergeChanges<TChange extends OnyxInput<OnyxKey> | undefined>(changes: TChange[]): TChange {
+function batchMergeChanges<TChange extends OnyxInput<OnyxKey> | undefined>(changes: TChange[]): FastMergeResult<TChange> {
     const lastChange = changes?.at(-1);
 
     if (Array.isArray(lastChange)) {
-        return lastChange;
+        return {result: lastChange, replaceNullPatches: []};
     }
 
     if (changes.some((change) => change && typeof change === 'object')) {
         // Object values are then merged one after the other
-        return changes.reduce((modifiedData, change) => utils.fastMerge(modifiedData, change, false, true, false), {} as TChange);
+        return changes.reduce<FastMergeResult<TChange>>(
+            (modifiedData, change) => {
+                const fastMergeResult = utils.fastMerge(modifiedData.result, change, false, true, false);
+                // eslint-disable-next-line no-param-reassign
+                modifiedData.result = fastMergeResult.result;
+                // eslint-disable-next-line no-param-reassign
+                modifiedData.replaceNullPatches = [...modifiedData.replaceNullPatches, ...fastMergeResult.replaceNullPatches];
+                return modifiedData;
+            },
+            {
+                result: {} as TChange,
+                replaceNullPatches: [],
+            },
+        );
     }
 
     // If we have anything else we can't merge it so we'll
     // simply return the last value that was queued
-    return lastChange as TChange;
+    return {result: lastChange as TChange, replaceNullPatches: []};
 }
 
 /**
@@ -1296,7 +1310,7 @@ function initializeWithDefaultKeyStates(): Promise<void> {
     return Storage.multiGet(Object.keys(defaultKeyStates)).then((pairs) => {
         const existingDataAsObject = Object.fromEntries(pairs);
 
-        const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false);
+        const merged = utils.fastMerge(existingDataAsObject, defaultKeyStates, true, false, false).result;
         cache.merge(merged ?? {});
 
         Object.entries(merged ?? {}).forEach(([key, value]) => keyChanged(key, value, existingDataAsObject));

lib/storage/index.ts

@@ -131,9 +131,9 @@ const storage: Storage = {
      * Multiple merging of existing and new values in a batch
      * This function also removes all nested null values from an object.
      */
-    multiMerge: (pairs) =>
+    multiMerge: (pairs, mergeReplaceNullPatches) =>
         tryOrDegradePerformance(() => {
-            const promise = provider.multiMerge(pairs);
+            const promise = provider.multiMerge(pairs, mergeReplaceNullPatches);
 
             if (shouldKeepInstancesSync) {
                 return promise.then(() => InstanceSync.multiMerge(pairs.map((pair) => pair[0])));

lib/storage/providers/IDBKeyValProvider.ts

@@ -49,7 +49,7 @@ const provider: StorageProvider = {
 
                 const upsertMany = pairsWithoutNull.map(([key, value], index) => {
                     const prev = values[index];
-                    const newValue = utils.fastMerge(prev as Record<string, unknown>, value as Record<string, unknown>, true, false, true);
+                    const newValue = utils.fastMerge(prev as Record<string, unknown>, value as Record<string, unknown>, true, false, true).result;
                     return promisifyRequest(store.put(newValue, key));
                 });
                 return Promise.all(upsertMany);

lib/storage/providers/MemoryOnlyProvider.ts

@@ -86,7 +86,7 @@ const provider: StorageProvider = {
     multiMerge(pairs) {
         _.forEach(pairs, ([key, value]) => {
             const existingValue = store[key] as Record<string, unknown>;
-            const newValue = utils.fastMerge(existingValue, value as Record<string, unknown>, true, false, true) as OnyxValue<OnyxKey>;
+            const newValue = utils.fastMerge(existingValue, value as Record<string, unknown>, true, false, true).result as OnyxValue<OnyxKey>;
 
             set(key, newValue);
         });

lib/storage/providers/SQLiteProvider.ts

@@ -2,13 +2,13 @@
  * The SQLiteStorage provider stores everything in a key/value store by
  * converting the value to a JSON string
  */
+import {getFreeDiskStorage} from 'react-native-device-info';
 import type {BatchQueryResult, QuickSQLiteConnection, SQLBatchTuple} from 'react-native-quick-sqlite';
 import {open} from 'react-native-quick-sqlite';
-import {getFreeDiskStorage} from 'react-native-device-info';
-import type StorageProvider from './types';
+import type {FastMergeReplaceNullPatch} from '../../utils';
 import utils from '../../utils';
+import type StorageProvider from './types';
 import type {KeyList, KeyValuePairList} from './types';
-import type {OnyxKey, OnyxValue} from '../../types';
 
 const DB_NAME = 'OnyxDB';
 let db: QuickSQLiteConnection;
@@ -18,47 +18,13 @@ function replacer(key: string, value: unknown) {
     return value;
 }
 
-type JSONReplacePatch = [string, string[], any];
-
-function getReplacePatches(storageKey: string, value: any): JSONReplacePatch[] {
-    const patches: JSONReplacePatch[] = [];
-
-    // eslint-disable-next-line rulesdir/prefer-early-return
-    function recurse(obj: any, path: string[] = []) {
-        if (obj && typeof obj === 'object' && !Array.isArray(obj)) {
-            if (obj.ONYX_INTERNALS__REPLACE_OBJECT_MARK) {
-                const copy = {...obj};
-                delete copy.ONYX_INTERNALS__REPLACE_OBJECT_MARK;
-
-                patches.push([storageKey, [...path], copy]);
-                return;
-            }
-
-            // eslint-disable-next-line guard-for-in, no-restricted-syntax
-            for (const key in obj) {
-                recurse(obj[key], [...path, key]);
-            }
-        }
-    }
-
-    recurse(value);
-    return patches;
-}
-
-function generateJSONReplaceSQLBatch(patches: JSONReplacePatch[]): [string, string[][]] {
-    const sql = `
-        UPDATE keyvaluepairs
-        SET valueJSON = JSON_REPLACE(valueJSON, :jsonPath, JSON(:value))
-        WHERE record_key = :key;
-    `;
-
-    const queryArguments = patches.map(([key, pathArray, value]) => {
+function generateJSONReplaceSQLQueries(key: string, patches: FastMergeReplaceNullPatch[]): string[][] {
+    const queries = patches.map(([pathArray, value]) => {
         const jsonPath = `$.${pathArray.join('.')}`;
-        // return {key, jsonPath, value: JSON.stringify(value)};
         return [jsonPath, JSON.stringify(value), key];
     });
 
-    return [sql.trim(), queryArguments];
+    return queries;
 }
 
 const provider: StorageProvider = {
@@ -108,7 +74,7 @@ const provider: StorageProvider = {
         }
         return db.executeBatchAsync([['REPLACE INTO keyvaluepairs (record_key, valueJSON) VALUES (?, json(?));', stringifiedPairs]]);
     },
-    multiMerge(pairs) {
+    multiMerge(pairs, mergeReplaceNullPatches) {
         const commands: SQLBatchTuple[] = [];
 
         const patchQuery = `INSERT INTO keyvaluepairs (record_key, valueJSON)
@@ -124,16 +90,20 @@ const provider: StorageProvider = {
         const replaceQueryArguments: string[][] = [];
 
         const nonNullishPairs = pairs.filter((pair) => pair[1] !== undefined);
+
         // eslint-disable-next-line @typescript-eslint/prefer-for-of
         for (let i = 0; i < nonNullishPairs.length; i++) {
             const pair = nonNullishPairs[i];
             const value = JSON.stringify(pair[1], replacer);
             patchQueryArguments.push([pair[0], value]);
 
-            const patches = getReplacePatches(pair[0], pair[1]);
-            const [sql, args] = generateJSONReplaceSQLBatch(patches);
-            if (args.length > 0) {
-                replaceQueryArguments.push(...args);
+            const patches = mergeReplaceNullPatches?.[pair[0]] ?? [];
+            if (patches.length > 0) {
+                const queries = generateJSONReplaceSQLQueries(pair[0], patches);
+
+                if (queries.length > 0) {
+                    replaceQueryArguments.push(...queries);
+                }
             }
         }
 

lib/storage/providers/types.ts

@@ -1,5 +1,5 @@
 import type {BatchQueryResult, QueryResult} from 'react-native-quick-sqlite';
-import type {OnyxKey, OnyxValue} from '../../types';
+import type {MixedOperationsQueue, OnyxKey, OnyxValue} from '../../types';
 
 type KeyValuePair = [OnyxKey, OnyxValue<OnyxKey>];
 type KeyList = OnyxKey[];
@@ -39,7 +39,7 @@ type StorageProvider = {
     /**
      * Multiple merging of existing and new values in a batch
      */
-    multiMerge: (pairs: KeyValuePairList) => Promise<BatchQueryResult | IDBValidKey[] | void>;
+    multiMerge: (pairs: KeyValuePairList, mergeReplaceNullPatches?: MixedOperationsQueue['mergeReplaceNullPatches']) => Promise<BatchQueryResult | IDBValidKey[] | void>;
 
     /**
      * Merges an existing value with a new one

lib/types.ts

@@ -3,6 +3,7 @@ import type {BuiltIns} from 'type-fest/source/internal';
 import type OnyxUtils from './OnyxUtils';
 import type {WithOnyxInstance, WithOnyxState} from './withOnyx/types';
 import type {OnyxMethod} from './OnyxUtils';
+import type {FastMergeReplaceNullPatch} from './utils';
 
 /**
  * Utility type that excludes `null` from the type `TValue`.
@@ -490,6 +491,7 @@ type GenericFunction = (...args: any[]) => any;
  */
 type MixedOperationsQueue = {
     merge: OnyxInputKeyValueMapping;
+    mergeReplaceNullPatches: {[TKey in OnyxKey]: FastMergeReplaceNullPatch[]};
     set: OnyxInputKeyValueMapping;
 };