Fix sorting activities when they are grouped in either part grouping or livestreaming (#5635)

* Add sorting upsert algorithm

* With webchat:internal:position

* Update comment

* Redux store backcompat

* Add given-when-then

* Allow HowTo without position

* Upsert livestream activity based on sequence number

* Should not move livestream if timestamp is undefined

* Fix tests

* Clean up Object.freeze()

* Fix path

* Allow reuse position

* FIx update activity channel data

* No reorder when concluding livestream does not have timestamp

* Upsert as-is if no HowTo.position

* Always reuse position including livestreaming

* Fix @id

* Fix typing indicator test with duplicate activity ID

* Fix typing indicator scrolling test with duplicate activity ID

* Fix typing indicator test with duplicate livestream activity ID

* Fix collapsible test have bad livestreaming metadata

* Remove CoT

* Skip activity for finalized livestream

* Update position for plain activity

* Remove timestamp: undefined

* Update timestamp algorithm

* Sort by logical timestamp, followed by local timestamp

* Fix test: livestream finalize should move it to bottom

* Honor timestamp on finalizing livestream activity

* Fix dupe ID

* Add a clock tick before sending another message

* Patch all activities with internal channel data and fix typing entries for reproducibility

* Clean up

* Update snapshots

* Fix test: finalizing livestream will update timestamp

* Add tick between outgoing activities

* Fix unit test

* Fix typings

* Rename perma ID to local ID

* Add comment

* Fix test: part grouping will always have updated timestamp

* Rename permaId to localId

* Fix test: part grouping should use max timestamp

* Initial delete activity

* Rename activityInternalId to activityLocalId

* Add clean up after delete

* Delete livestream activities

* Delete activities in part grouping with livestream

* Reorganize tests

* Wire up delete activity

* Add clientActivityIdToLocalId index

* Update doc

* Fix ESLint

* Add *.spec.* and *.test.*

* Revert

* Clean up comment

* Use standard way to parse @id

* Remove early return

* Add multiple part grouping test

* Remove unnecessary findBeforeAfter

* Typo

* Update doc

* Fix part grouping ID

* Update test: position to undefined should be unmoved

* Refactor property types

* Use property functions

* Fix part ID

* Local ID is blank node identifier

* Use node identifier schema for local ID

* Fix import path

* Fix local ID

* Clean up

* Add typing to Map

* Clean up

* Clean up with generateLocalIdInActivity

* Remove unused BlankNodeIdentifier

* Clean up

* Use queryLocalIdFromActivity for generateLocalIdInActivity

* Patch activity on POST_ACTIVITY_PENDING

* Update PR number

* Incorporate PR changes

* Add related read

Author: compulim

CHANGELOG.md

@@ -330,7 +330,7 @@ Notes: web developers are advised to use [`~` (tilde range)](https://github.com/
 - Fixed citation links are not properly matched against markdown links, in PR [#5614](https://github.com/microsoft/BotFramework-WebChat/pull/5614), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed `botframework-webchat/decorator` import in legacy CommonJS environments, in [#5616](https://github.com/microsoft/BotFramework-WebChat/pull/5616), by [@OEvgeny](https://github.com/OEvgeny)
 - Fixed `npm start` for efficiency and reliability, in PR [#5621](https://github.com/microsoft/BotFramework-WebChat/pull/5621) and [#5629](https://github.com/microsoft/BotFramework-WebChat/pull/5629), by [@compulim](https://github.com/compulim)
-- Fixed activity sorting introduced in PR [#5622](https://github.com/microsoft/BotFramework-WebChat/pull/5622), part grouping, and livestreaming, by [@compulim](https://github.com/compulim) in PR [#XXX](https://github.com/microsoft/BotFramework-WebChat/pull/XXX)
+- Fixed activity sorting introduced in PR [#5622](https://github.com/microsoft/BotFramework-WebChat/pull/5622), part grouping, and livestreaming, by [@compulim](https://github.com/compulim) in PR [#5635](https://github.com/microsoft/BotFramework-WebChat/pull/5635)
 
 ### Removed
 

__tests__/html2/activity/collapsible.html

@@ -54,6 +54,8 @@
           )
         ).resolveAll();
 
+        clock.tick(1000);
+
         const { resolveAll: resolveAll1 } = await directLine.emulateOutgoingActivity(
           'Elit adipisicing laborum sit anim.'
         );

__tests__/html2/activityGrouping/activityGrouping.disableTimestamp.html

@@ -50,6 +50,8 @@
         await sendMessage1.echoBack();
         sendMessage1.resolvePostActivity();
 
+        clock.tick(1000);
+
         const sendMessage2 = await directLine.emulateOutgoingActivity('A retry prompt must show on this activity.');
 
         clock.tick(1000);
@@ -61,6 +63,8 @@
         await sendMessage3.echoBack();
         sendMessage3.resolvePostActivity();
 
+        clock.tick(1000);
+
         const sendMessage4 = await directLine.emulateOutgoingActivity(
           'The timestamp is shown because the next activity is not sent. When it is sent, the timestamp will be hidden.'
         );

__tests__/html2/activityGrouping/activityGrouping.groupingActivityStatus.html

@@ -0,0 +1,108 @@
+<!doctype html>
+<html lang="en-US">
+  <head>
+    <link href="/assets/index.css" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <main id="webchat"></main>
+    <script type="importmap">
+      {
+        "imports": {
+          "botframework-webchat": "/__dist__/packages/bundle/static/botframework-webchat.js",
+          "react": "/__dist__/packages/bundle/static/react.js",
+          "react-dom": "/__dist__/packages/bundle/static/react-dom.js"
+        }
+      }
+    </script>
+    <script type="module">
+      import '/test-harness.mjs';
+      import '/test-page-object.mjs';
+
+      import { createDirectLine, createStoreWithOptions, renderWebChat } from 'botframework-webchat';
+      import { version } from 'react';
+
+      run(async function () {
+        const {
+          testHelpers: { createDirectLineEmulator }
+        } = window;
+
+        // TODO: This is for `createDirectLineEmulator` only, should find ways to eliminate this line.
+        window.WebChat = { createStoreWithOptions };
+
+        const { directLine, store } = createDirectLineEmulator();
+
+        renderWebChat({ directLine, store }, document.getElementById('webchat'));
+
+        await pageConditions.uiConnected();
+
+        await directLine.emulateIncomingActivity({
+          from: { role: 'bot' },
+          id: 'a-00001',
+          text: 'Hello, World!',
+          timestamp: new Date(86_400_000).toISOString()
+        });
+
+        await directLine.emulateIncomingActivity({
+          channelData: {
+            streamSequence: 1,
+            streamType: 'streaming'
+          },
+          from: { role: 'bot' },
+          id: 'a-00002',
+          text: '`t=undefined`\n\nA quick',
+          timestamp: undefined,
+          type: 'typing'
+        });
+
+        expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          'Hello, World!',
+          't=undefined\nA quick'
+        ]);
+
+        // THEN: Screenshot should show no timestamp below the second message.
+        await host.snapshot('local');
+
+        await directLine.emulateIncomingActivity({
+          channelData: {
+            streamId: 'a-00002',
+            streamSequence: 2,
+            streamType: 'streaming'
+          },
+          from: { role: 'bot' },
+          id: 'a-00003',
+          text: '`t=0`\n\nA quick brown fox',
+          timestamp: new Date(0).toISOString(),
+          type: 'typing'
+        });
+
+        expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          'Hello, World!',
+          't=0\nA quick brown fox'
+        ]);
+
+        // THEN: Should show the livestream message on second position as it should not be moved.
+        await host.snapshot('local');
+
+        await directLine.emulateIncomingActivity({
+          channelData: {
+            streamId: 'a-00002',
+            streamType: 'final'
+          },
+          from: { role: 'bot' },
+          id: 'a-00004',
+          text: '`t=0`\n\nA quick brown fox jumped over the lazy dogs.',
+          timestamp: new Date(0).toISOString(),
+          type: 'message'
+        });
+
+        // THEN: Should have the livestream message as the first one.
+        expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          't=0\nA quick brown fox jumped over the lazy dogs.',
+          'Hello, World!'
+        ]);
+
+        await host.snapshot('local');
+      });
+    </script>
+  </body>
+</html>

__tests__/html2/activityOrdering/livestreamWithMovingTimestamp.html

@@ -42,7 +42,7 @@
               '@id': '',
               '@type': 'Message',
               type: 'https://schema.org/Message',
-              isPartOf: { '@id': 'c-00001', '@type': 'HowTo' },
+              isPartOf: { '@id': '_:c-00001', '@type': 'HowTo' },
               position: 1
             }
           ],
@@ -83,7 +83,7 @@
               '@id': '',
               '@type': 'Message',
               type: 'https://schema.org/Message',
-              isPartOf: { '@id': 'c-00001', '@type': 'HowTo' },
+              isPartOf: { '@id': '_:c-00001', '@type': 'HowTo' },
               position: 2
             }
           ],
@@ -95,9 +95,9 @@
         });
 
         expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          'a-00002: Hello, World at t = 1',
           'a-00001: Chain 1 thought 1 at t = 0',
-          'a-00003: Chain 1 thought 2 at t = 2',
-          'a-00002: Hello, World at t = 1'
+          'a-00003: Chain 1 thought 2 at t = 2'
         ]);
 
         await host.snapshot('local');
@@ -109,7 +109,7 @@
               '@id': '',
               '@type': 'Message',
               type: 'https://schema.org/Message',
-              isPartOf: { '@id': 'c-00002', '@type': 'HowTo' },
+              isPartOf: { '@id': '_:c-00002', '@type': 'HowTo' },
               position: 1
             }
           ],
@@ -121,9 +121,9 @@
         });
 
         expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          'a-00002: Hello, World at t = 1',
           'a-00001: Chain 1 thought 1 at t = 0',
           'a-00003: Chain 1 thought 2 at t = 2',
-          'a-00002: Hello, World at t = 1',
           'a-00004: Chain 2 thought 1 at t = 3'
         ]);
 
@@ -136,23 +136,24 @@
               '@id': '',
               '@type': 'Message',
               type: 'https://schema.org/Message',
-              isPartOf: { '@id': 'c-00002', '@type': 'HowTo' },
+              isPartOf: { '@id': '_:c-00002', '@type': 'HowTo' },
               position: 2
             }
           ],
           from: { role: 'bot' },
           id: 'a-00005',
           text: 'a-00005: Chain 2 thought 2 at t = 0',
           // Intentionally roll back the date.
-          // It should be grouped and not appear before "Hello, World!'
+          // It should be grouped but not appear before "Hello, World!"
+          // This is because the part grouping timestamp is max of all parts, i.e. 3 * 86_400_000.
           timestamp: new Date(0).toISOString(),
           type: 'message'
         });
 
         expect(pageElements.activityContents().map(({ textContent }) => textContent)).toEqual([
+          'a-00002: Hello, World at t = 1',
           'a-00001: Chain 1 thought 1 at t = 0',
           'a-00003: Chain 1 thought 2 at t = 2',
-          'a-00002: Hello, World at t = 1',
           'a-00004: Chain 2 thought 1 at t = 3',
           'a-00005: Chain 2 thought 2 at t = 0'
         ]);