Swap encoding of key safe bytes from number[] to sortable string. This improves encoding efficency by 4x.

Author: MarkDuckworth

packages/firestore/src/index/index_entry.ts

@@ -18,16 +18,16 @@
 import { isSafariOrWebkit } from '@firebase/util';
 
 import { DbIndexEntry } from '../local/indexeddb_schema';
-import { DbIndexEntryKey } from '../local/indexeddb_sentinels';
+import { DbIndexEntryKey, KeySafeBytes } from '../local/indexeddb_sentinels';
 import { DocumentKey } from '../model/document_key';
 
 /** Represents an index entry saved by the SDK in persisted storage. */
 export class IndexEntry {
   constructor(
     readonly _indexId: number,
     readonly _documentKey: DocumentKey,
-    readonly _arrayValue: Uint8Array | number[],
-    readonly _directionalValue: Uint8Array | number[]
+    readonly _arrayValue: Uint8Array,
+    readonly _directionalValue: Uint8Array
   ) {}
 
   /**
@@ -66,9 +66,9 @@ export class IndexEntry {
     return {
       indexId: this._indexId,
       uid,
-      arrayValue: indexSafeUint8Array(this._arrayValue),
-      directionalValue: indexSafeUint8Array(this._directionalValue),
-      orderedDocumentKey: indexSafeUint8Array(orderedDocumentKey),
+      arrayValue: encodeKeySafeBytes(this._arrayValue),
+      directionalValue: encodeKeySafeBytes(this._directionalValue),
+      orderedDocumentKey: encodeKeySafeBytes(orderedDocumentKey),
       documentKey: documentKey.path.toArray()
     };
   }
@@ -82,9 +82,9 @@ export class IndexEntry {
     return [
       this._indexId,
       uid,
-      indexSafeUint8Array(this._arrayValue),
-      indexSafeUint8Array(this._directionalValue),
-      indexSafeUint8Array(orderedDocumentKey),
+      encodeKeySafeBytes(this._arrayValue),
+      encodeKeySafeBytes(this._directionalValue),
+      encodeKeySafeBytes(orderedDocumentKey),
       documentKey.path.toArray()
     ];
   }
@@ -125,16 +125,56 @@ export function compareByteArrays(
   return left.length - right.length;
 }
 
-// Create an safe representation of Uint8Array values
-// If the browser is detected as Safari or WebKit, then
-// the input array will be converted to `number[]`.
-// Otherwise, the input array will be returned in its
-// original type.
-export function indexSafeUint8Array(
-  array: Uint8Array | number[]
-): Uint8Array | number[] {
+/**
+ * Workaround for WebKit bug: https://bugs.webkit.org/show_bug.cgi?id=292721
+ * Create a key safe representation of Uint8Array values.
+ * If the browser is detected as Safari or WebKit, then
+ * the input array will be converted to "sortable byte string".
+ * Otherwise, the input array will be returned in its original type.
+ */
+export function encodeKeySafeBytes(array: Uint8Array): KeySafeBytes {
   if (isSafariOrWebkit() && !Array.isArray(array)) {
-    return Array.from(array);
+    return encodeUint8ArrayToSortableString(array);
   }
   return array;
 }
+
+/**
+ * Reverts the key safe representation of Uint8Array (created by
+ * indexSafeUint8Array) to a normal Uint8Array.
+ */
+export function decodeKeySafeBytes(input: KeySafeBytes): Uint8Array {
+  if (typeof input !== 'string') {
+    return input;
+  }
+  return decodeSortableStringToUint8Array(input);
+}
+
+/**
+ * Encodes a Uint8Array into a "sortable byte string".
+ * A "sortable byte string" sorts in the same order as the Uint8Array.
+ * This works because JS string comparison sorts strings based on code points.
+ */
+function encodeUint8ArrayToSortableString(array: Uint8Array): string {
+  let byteString = '';
+  for (let i = 0; i < array.length; i++) {
+    byteString += String.fromCharCode(array[i]);
+  }
+
+  return byteString;
+}
+
+/**
+ * Decodes a "sortable byte string" back into a Uint8Array.
+ * A "sortable byte string" is assumed to be created where each character's
+ * Unicode code point directly corresponds to a single byte value (0-255).
+ */
+function decodeSortableStringToUint8Array(byteString: string): Uint8Array {
+  const uint8array = new Uint8Array(byteString.length);
+
+  for (let i = 0; i < byteString.length; i++) {
+    uint8array[i] = byteString.charCodeAt(i);
+  }
+
+  return uint8array;
+}

packages/firestore/src/local/indexeddb_index_manager.ts

@@ -42,7 +42,8 @@ import { IndexByteEncoder } from '../index/index_byte_encoder';
 import {
   IndexEntry,
   indexEntryComparator,
-  indexSafeUint8Array
+  encodeKeySafeBytes,
+  decodeKeySafeBytes
 } from '../index/index_entry';
 import { documentKeySet, DocumentMap } from '../model/collections';
 import { Document } from '../model/document';
@@ -860,7 +861,7 @@ export class IndexedDbIndexManager implements IndexManager {
           range: IDBKeyRange.only([
             fieldIndex.indexId,
             this.uid,
-            indexSafeUint8Array(
+            encodeKeySafeBytes(
               this.encodeDirectionalKey(fieldIndex, documentKey)
             )
           ])
@@ -870,8 +871,8 @@ export class IndexedDbIndexManager implements IndexManager {
             new IndexEntry(
               fieldIndex.indexId,
               documentKey,
-              entry.arrayValue,
-              entry.directionalValue
+              decodeKeySafeBytes(entry.arrayValue),
+              decodeKeySafeBytes(entry.directionalValue)
             )
           );
         }

packages/firestore/src/local/indexeddb_schema.ts

@@ -16,6 +16,7 @@
  */
 
 import { BatchId, ListenSequenceNumber, TargetId } from '../core/types';
+import { KeySafeBytes } from '../index/index_entry';
 import { IndexKind } from '../model/field_index';
 import { BundledQuery } from '../protos/firestore_bundle_proto';
 import {
@@ -52,9 +53,11 @@ import { DbTimestampKey } from './indexeddb_sentinels';
  * 14. Add overlays.
  * 15. Add indexing support.
  * 16. Parse timestamp strings before creating index entries.
+ * 17. TODO(tomandersen)
+ * 18. Encode key safe representations of IndexEntry in DbIndexEntryStore.
  */
 
-export const SCHEMA_VERSION = 17;
+export const SCHEMA_VERSION = 18;
 
 /**
  * Wrapper class to store timestamps (seconds and nanos) in IndexedDb objects.
@@ -507,14 +510,14 @@ export interface DbIndexEntry {
   /** The user id for this entry. */
   uid: string;
   /** The encoded array index value for this entry. */
-  arrayValue: Uint8Array | number[];
+  arrayValue: KeySafeBytes;
   /** The encoded directional value for equality and inequality filters. */
-  directionalValue: Uint8Array | number[];
+  directionalValue: KeySafeBytes;
   /**
    * The document key this entry points to. This entry is encoded by an ordered
    * encoder to match the key order of the index.
    */
-  orderedDocumentKey: Uint8Array | number[];
+  orderedDocumentKey: KeySafeBytes;
   /** The segments of the document key this entry points to. */
   documentKey: string[];
 }

packages/firestore/src/local/indexeddb_schema_converter.ts

@@ -15,6 +15,8 @@
  * limitations under the License.
  */
 
+import { isSafariOrWebkit } from '@firebase/util';
+
 import { User } from '../auth/user';
 import { ListenSequence } from '../core/listen_sequence';
 import { SnapshotVersion } from '../core/snapshot_version';
@@ -277,6 +279,22 @@ export class SchemaConverter implements SimpleDbSchemaConverter {
       });
     }
 
+    if (fromVersion < 18 && toVersion >= 18) {
+      // Clear the IndexEntryStores on WebKit and Safari to remove possibly
+      // corrupted index entries
+      if (isSafariOrWebkit()) {
+        p = p
+          .next(() => {
+            const indexStateStore = txn.objectStore(DbIndexStateStore);
+            indexStateStore.clear();
+          })
+          .next(() => {
+            const indexEntryStore = txn.objectStore(DbIndexEntryStore);
+            indexEntryStore.clear();
+          });
+      }
+    }
+
     return p;
   }
 

packages/firestore/src/local/indexeddb_sentinels.ts

@@ -305,6 +305,15 @@ export const DbIndexStateSequenceNumberIndex = 'sequenceNumberIndex';
 
 export const DbIndexStateSequenceNumberIndexPath = ['uid', 'sequenceNumber'];
 
+/**
+ * Representation of a byte array that is safe for
+ * use in an IndexedDb key. The value is either
+ * a "sortable byte string", which is key safe in
+ * Safari/WebKit, or the value is a Uint8Array,
+ * which is key safe in other browsers.
+ */
+export type KeySafeBytes = Uint8Array | string;
+
 /**
  * The key for each index entry consists of the index id and its user id,
  * the encoded array and directional value for the indexed fields as well as
@@ -313,9 +322,9 @@ export const DbIndexStateSequenceNumberIndexPath = ['uid', 'sequenceNumber'];
 export type DbIndexEntryKey = [
   number,
   string,
-  Uint8Array | number[],
-  Uint8Array | number[],
-  Uint8Array | number[],
+  KeySafeBytes,
+  KeySafeBytes,
+  KeySafeBytes,
   string[]
 ];
 
@@ -425,6 +434,7 @@ export const V15_STORES = [
 ];
 export const V16_STORES = V15_STORES;
 export const V17_STORES = [...V15_STORES, DbGlobalsStore];
+export const V18_STORES = V17_STORES;
 
 /**
  * The list of all default IndexedDB stores used throughout the SDK. This is
@@ -435,7 +445,9 @@ export const ALL_STORES = V12_STORES;
 
 /** Returns the object stores for the provided schema. */
 export function getObjectStores(schemaVersion: number): string[] {
-  if (schemaVersion === 17) {
+  if (schemaVersion === 18) {
+    return V18_STORES;
+  } else if (schemaVersion === 17) {
     return V17_STORES;
   } else if (schemaVersion === 16) {
     return V16_STORES;
@@ -450,6 +462,6 @@ export function getObjectStores(schemaVersion: number): string[] {
   } else if (schemaVersion === 11) {
     return V11_STORES;
   } else {
-    fail(0xeb55, 'Only schema version 11 and 12 and 13 are supported');
+    fail(0xeb55, 'Only schema versions >11 are supported');
   }
 }