⚗ [RUMF-902] implement a new mutation observer (#810)

* 🏗 [RUMF-902] introduce serializationUtils

This commit introduces a few utility functions to store and retrieve
serialized node ids stored in DOM nodes. The equivalent tools (`mirror`,
`INode`...) that are only used by the 'mutation' module are intentionally
left as-is, and will be removed in a future PR when we'll enable the new
mutation observer.

Before this commit, some 'observer' strategies didn't check if
`mirror.getId` returned a valid `id` before emitting a record. In
practice it shouldn't happen, but in unit tests it did happen because we
artificially emitted 'input' events on a DOM node that wasn't in the
document. This is why some tests were slightly adjusted to reflect the
reality more accurately.

* ⚗ [RUMF-902] implement a new mutation observer

* 🚩 [RUMF-902] use the new mutation observer via a FF

* 👌 apply some suggestions from code review

Co-authored-by: Bastien Caudan <bastien.caudan@datadoghq.com>

* rename utils.spec to serializationUtils.spec

* 👌 remove undefined properties when using existing nodes more thoroughly

* 👌 remove the "handleMutations" function

* 👌 make sure 'nodeIsIgnored' is only called with serialized nodes

* 👌 rename a few things in the mutationObserver.spec

* 👌 clarify some mutationObserver tests

* 👌 remove useless uses of [0] for unused snapshot results

* 👌 move and rename sortNodesByTopologicalOrder

* 👌 remove unneeded record.spec unit test

We chose to remove this test because its intent is a bit obscure and the
new mutation observer has a good enough test coverage. I kept the test
for the old mutation observer implementation since its coverage is still
lacking. It will be removed when we enable the new mutation observer.

* Update record spec test title

Co-authored-by: Bastien Caudan <bastien.caudan@datadoghq.com>

Co-authored-by: Bastien Caudan <bastien.caudan@datadoghq.com>

Author: BenoitZugmeyer

packages/rum-recorder/src/boot/recorder.spec.ts

@@ -17,6 +17,8 @@ describe('startRecording', () => {
     expectedCallsCount: number,
     callback: (calls: jasmine.Calls<HttpRequest['send']>) => void
   ) => void
+  let sandbox: HTMLElement
+  let textField: HTMLInputElement
   let expectNoExtraRequestSendCalls: (done: () => void) => void
 
   beforeEach(() => {
@@ -25,6 +27,12 @@ describe('startRecording', () => {
     }
     sessionId = 'session-id'
     viewId = 'view-id'
+
+    sandbox = document.createElement('div')
+    document.body.appendChild(sandbox)
+    textField = document.createElement('input')
+    sandbox.appendChild(textField)
+
     setupBuilder = setup()
       .withParentContexts({
         findView() {
@@ -57,6 +65,7 @@ describe('startRecording', () => {
   })
 
   afterEach(() => {
+    sandbox.remove()
     setMaxSegmentSize()
     setupBuilder.cleanup()
   })
@@ -85,7 +94,6 @@ describe('startRecording', () => {
   it('flushes the segment when its compressed data is getting too large', (done) => {
     setupBuilder.build()
     const inputCount = 150
-    const textField = document.createElement('input')
     const inputEvent = createNewEvent('input', { target: textField })
     for (let i = 0; i < inputCount; i += 1) {
       // Create a random value harder to deflate, so we don't have to send too many events to reach

packages/rum-recorder/src/boot/recorder.ts

@@ -27,6 +27,7 @@ export function startRecording(
 
   const { stop: stopRecording, takeFullSnapshot } = record({
     emit: addRawRecord,
+    useNewMutationObserver: configuration.isEnabled('new-mutation-observer'),
   })
 
   lifeCycle.subscribe(LifeCycleEventType.VIEW_CREATED, takeFullSnapshot)

packages/rum-recorder/src/domain/rrweb-snapshot/index.ts

@@ -1,4 +1,5 @@
-import { serializeNodeWithId, transformAttribute, IGNORED_NODE, snapshot } from './snapshot'
+import { serializeNodeWithId, transformAttribute, snapshot } from './snapshot'
 export * from './types'
+export * from './serializationUtils'
 
-export { snapshot, serializeNodeWithId, transformAttribute, IGNORED_NODE }
+export { snapshot, serializeNodeWithId, transformAttribute }

packages/rum-recorder/src/domain/rrweb-snapshot/serializationUtils.spec.ts

@@ -0,0 +1,29 @@
+import { getSerializedNodeId, hasSerializedNode, setSerializedNode } from './serializationUtils'
+
+describe('serialized Node storage in DOM Nodes', () => {
+  describe('hasSerializedNode', () => {
+    it('returns false for DOM Nodes that are not yet serialized', () => {
+      expect(hasSerializedNode(document.createElement('div'))).toBe(false)
+    })
+
+    it('returns true for DOM Nodes that have been serialized', () => {
+      const node = document.createElement('div')
+      setSerializedNode(node, {} as any)
+
+      expect(hasSerializedNode(node)).toBe(true)
+    })
+  })
+
+  describe('getSerializedNodeId', () => {
+    it('returns undefined for DOM Nodes that are not yet serialized', () => {
+      expect(getSerializedNodeId(document.createElement('div'))).toBe(undefined)
+    })
+
+    it('returns the serialized Node id', () => {
+      const node = document.createElement('div')
+      setSerializedNode(node, { id: 42 } as any)
+
+      expect(getSerializedNodeId(node)).toBe(42)
+    })
+  })
+})

packages/rum-recorder/src/domain/rrweb-snapshot/serializationUtils.ts

@@ -0,0 +1,38 @@
+import { SerializedNodeWithId } from './types'
+
+export const IGNORED_NODE_ID = -2
+
+export interface NodeWithSerializedNode extends Node {
+  __sn: SerializedNodeWithId
+}
+
+export function hasSerializedNode(n: Node): n is NodeWithSerializedNode {
+  return '__sn' in n
+}
+
+export function getSerializedNodeId(n: NodeWithSerializedNode): number
+export function getSerializedNodeId(n: Node): number | undefined
+export function getSerializedNodeId(n: Node) {
+  return hasSerializedNode(n) ? n.__sn.id : undefined
+}
+
+export function setSerializedNode(n: Node, serializeNode: SerializedNodeWithId) {
+  ;(n as Partial<NodeWithSerializedNode>).__sn = serializeNode
+}
+
+export function nodeIsIgnored(n: NodeWithSerializedNode): boolean {
+  return getSerializedNodeId(n) === IGNORED_NODE_ID
+}
+
+export function nodeOrAncestorsIsIgnored(n: NodeWithSerializedNode) {
+  let current: NodeWithSerializedNode | null = n
+  while (current) {
+    if (nodeIsIgnored(current)) {
+      return true
+    }
+    // Since we serialize the document from the root, and any new node is only serialized if they
+    // are added in a serialized node, we are guaranteed to have a serialized parent node here.
+    current = current.parentNode as NodeWithSerializedNode | null
+  }
+  return false
+}

packages/rum-recorder/src/domain/rrweb-snapshot/snapshot.spec.ts

@@ -1,6 +1,7 @@
 /* eslint-disable max-len */
 import { isIE } from '@datadog/browser-core'
-import { absoluteToStylesheet, IGNORED_NODE, serializeNodeWithId } from './snapshot'
+import { IGNORED_NODE_ID } from './serializationUtils'
+import { absoluteToStylesheet, serializeNodeWithId } from './snapshot'
 
 describe('absolute url to stylesheet', () => {
   const href = 'http://localhost/css/style.css'
@@ -95,10 +96,10 @@ describe('serializeNodeWithId', () => {
       expect(map).toEqual({})
     })
 
-    it('sets ignored serialized node id to IGNORED_NODE', () => {
+    it('sets ignored serialized node id to IGNORED_NODE_ID', () => {
       const scriptElement = document.createElement('script')
       serializeNodeWithId(scriptElement, defaultOptions)
-      expect((scriptElement as any).__sn).toEqual(jasmine.objectContaining({ id: IGNORED_NODE }))
+      expect((scriptElement as any).__sn).toEqual(jasmine.objectContaining({ id: IGNORED_NODE_ID }))
     })
 
     it('ignores script tags', () => {

packages/rum-recorder/src/domain/rrweb-snapshot/snapshot.ts

@@ -1,11 +1,10 @@
 import { nodeShouldBeHidden } from '../privacy'
 import { PRIVACY_ATTR_NAME, PRIVACY_ATTR_VALUE_HIDDEN } from '../../constants'
 import { SerializedNode, SerializedNodeWithId, NodeType, Attributes, INode, IdNodeMap } from './types'
+import { getSerializedNodeId, hasSerializedNode, IGNORED_NODE_ID, setSerializedNode } from './serializationUtils'
 
 const tagNameRegex = /[^a-z1-6-_]/
 
-export const IGNORED_NODE = -2
-
 let nextId = 1
 function genId(): number {
   return nextId++
@@ -288,7 +287,7 @@ function lowerIfExists(maybeAttr: string | number | boolean): string {
   return (maybeAttr as string).toLowerCase()
 }
 
-function isNodeIgnored(sn: SerializedNode): boolean {
+function nodeShouldBeIgnored(sn: SerializedNode): boolean {
   if (sn.type === NodeType.Comment) {
     // TODO: convert IE conditional comments to real nodes
     return true
@@ -369,33 +368,34 @@ export function serializeNodeWithId(
 ): SerializedNodeWithId | null {
   const { doc, map, skipChild = false } = options
   let { preserveWhiteSpace = true } = options
-  const _serializedNode = serializeNode(n, {
+  const serializedNode = serializeNode(n, {
     doc,
   })
-  if (!_serializedNode) {
+  if (!serializedNode) {
     // TODO: dev only
     console.warn(n, 'not serialized')
     return null
   }
 
   let id
   // Try to reuse the previous id
-  if ('__sn' in n) {
-    id = n.__sn.id
+  if (hasSerializedNode(n)) {
+    id = getSerializedNodeId(n)
   } else if (
-    isNodeIgnored(_serializedNode) ||
+    nodeShouldBeIgnored(serializedNode) ||
     (!preserveWhiteSpace &&
-      _serializedNode.type === NodeType.Text &&
-      !_serializedNode.isStyle &&
-      !_serializedNode.textContent.replace(/^\s+|\s+$/gm, '').length)
+      serializedNode.type === NodeType.Text &&
+      !serializedNode.isStyle &&
+      !serializedNode.textContent.replace(/^\s+|\s+$/gm, '').length)
   ) {
-    id = IGNORED_NODE
+    id = IGNORED_NODE_ID
   } else {
     id = genId()
   }
-  const serializedNode = Object.assign(_serializedNode, { id })
-  ;(n as INode).__sn = serializedNode
-  if (id === IGNORED_NODE) {
+  const serializedNodeWithId = serializedNode as SerializedNodeWithId
+  serializedNodeWithId.id = id
+  setSerializedNode(n, serializedNodeWithId)
+  if (id === IGNORED_NODE_ID) {
     return null
   }
   map[id] = n as INode
@@ -407,8 +407,8 @@ export function serializeNodeWithId(
   }
   if ((serializedNode.type === NodeType.Document || serializedNode.type === NodeType.Element) && recordChild) {
     if (
-      _serializedNode.type === NodeType.Element &&
-      _serializedNode.tagName === 'head'
+      serializedNode.type === NodeType.Element &&
+      serializedNode.tagName === 'head'
       // would impede performance: || getComputedStyle(n)['white-space'] === 'normal'
     ) {
       preserveWhiteSpace = false
@@ -425,7 +425,7 @@ export function serializeNodeWithId(
       }
     }
   }
-  return serializedNode
+  return serializedNodeWithId
 }
 
 export function snapshot(n: Document): [SerializedNodeWithId | null, IdNodeMap] {

packages/rum-recorder/src/domain/rrweb/mutation.ts

@@ -1,5 +1,12 @@
 import { monitor } from '@datadog/browser-core'
-import { IGNORED_NODE, INode, serializeNodeWithId, transformAttribute } from '../rrweb-snapshot'
+import {
+  hasSerializedNode,
+  IGNORED_NODE_ID,
+  INode,
+  nodeIsIgnored,
+  serializeNodeWithId,
+  transformAttribute,
+} from '../rrweb-snapshot'
 import { nodeOrAncestorsShouldBeHidden } from '../privacy'
 import {
   AddedNodeMutation,
@@ -9,7 +16,7 @@ import {
   RemovedNodeMutation,
   TextCursor,
 } from './types'
-import { forEach, isAncestorRemoved, isIgnored, mirror } from './utils'
+import { forEach, isAncestorRemoved, mirror } from './utils'
 
 interface DoubleLinkedListNode {
   previous: DoubleLinkedListNode | null
@@ -184,8 +191,8 @@ export class MutationObserverWrapper {
     const addList = new DoubleLinkedList()
     const getNextId = (n: Node): number | null => {
       let ns: Node | null = n
-      let nextId: number | null = IGNORED_NODE
-      while (nextId === IGNORED_NODE) {
+      let nextId: number | null = IGNORED_NODE_ID
+      while (nextId === IGNORED_NODE_ID) {
         ns = ns && ns.nextSibling
         nextId = ns && mirror.getId((ns as unknown) as INode)
       }
@@ -308,7 +315,7 @@ export class MutationObserverWrapper {
   }
 
   private processMutation = (m: MutationRecord) => {
-    if (isIgnored(m.target)) {
+    if (hasSerializedNode(m.target) && nodeIsIgnored(m.target)) {
       return
     }
     switch (m.type) {
@@ -344,7 +351,11 @@ export class MutationObserverWrapper {
         forEach(m.removedNodes, (n: Node) => {
           const nodeId = mirror.getId(n as INode)
           const parentId = mirror.getId(m.target as INode)
-          if (nodeOrAncestorsShouldBeHidden(n) || nodeOrAncestorsShouldBeHidden(m.target) || isIgnored(n)) {
+          if (
+            nodeOrAncestorsShouldBeHidden(n) ||
+            nodeOrAncestorsShouldBeHidden(m.target) ||
+            (hasSerializedNode(n) && nodeIsIgnored(n))
+          ) {
             return
           }
           // removed node has not been serialized yet, just remove it from the Set
@@ -388,7 +399,7 @@ export class MutationObserverWrapper {
       return
     }
     if (isINode(n)) {
-      if (isIgnored(n)) {
+      if (nodeIsIgnored(n)) {
         return
       }
       this.movedSet.add(n)