diff --git a/CHANGELOG.md b/CHANGELOG.md index baf6ba08e0..b8f7bab26b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,7 @@ Notes: web developers are advised to use [`~` (tilde range)](https://github.com/ - Added `disableFileUpload` flag to completelly disable file upload feature, in PR [#5508](https://github.com/microsoft/BotFramework-WebChat/pull/5508), by [@JamesNewbyAtMicrosoft](https://github.com/JamesNewbyAtMicrosoft) - Deprecated `hideUploadButton` in favor of `disableFileUpload`. - Updated `BasicSendBoxToolbar` to rely solely on `disableFileUpload`. +- Added livestreaming support for livestreaming data in `entities` in PR [#5517](https://github.com/microsoft/BotFramework-WebChat/pull/5517) by [@kylerohn](https://github.com/kylerohn) ### Changed diff --git a/docs/LIVESTREAMING.md b/docs/LIVESTREAMING.md index 7917c9df1d..ccda4ea234 100644 --- a/docs/LIVESTREAMING.md +++ b/docs/LIVESTREAMING.md @@ -64,6 +64,23 @@ To simplify this documentation, we are using the term "bot" instead of "copilot" Bot developers would need to implement the livestreaming as outlined in this section. The implementation below will enable livestreaming to both Azure Bot Services and Teams. +> [!NOTE] +> In the scenarios below, the livestream metadata is inside the `channelData` field. BotFramework-WebChat checks both `channelData` and the first element of the `entities` field for livestreaming metadata. It will appear in different places depending on the platform used to communicate with BotFramework-WebChat: +> +> ```json +> { +> "entities": [ +> { +> "streamSequence": 1, +> "streamType": "streaming", +> "type": "streaminfo" +> } +> ], +> "text": "A quick", +> "type": "typing" +> } +> ``` + ### Scenario 1: Livestream from start to end > In this example, we assume the bot is livestreaming the following sentence to the user: diff --git a/packages/core/src/utils/getActivityLivestreamingMetadata.spec.ts b/packages/core/src/utils/getActivityLivestreamingMetadata.spec.ts index 4969bfdc76..df7e6caf2b 100644 --- a/packages/core/src/utils/getActivityLivestreamingMetadata.spec.ts +++ b/packages/core/src/utils/getActivityLivestreamingMetadata.spec.ts @@ -2,7 +2,7 @@ import type { WebChatActivity } from '../types/WebChatActivity'; import getActivityLivestreamingMetadata from './getActivityLivestreamingMetadata'; describe.each([['with "streamId"' as const], ['without "streamId"' as const]])('activity %s', variant => { - describe('activity with "streamType" of "streaming"', () => { + describe('activity with "streamType" of "streaming" (channelData)', () => { let activity: WebChatActivity; beforeEach(() => { @@ -32,7 +32,7 @@ describe.each([['with "streamId"' as const], ['without "streamId"' as const]])(' } }); - describe('activity with "streamType" of "informative message"', () => { + describe('activity with "streamType" of "informative message" (channelData)', () => { let activity: WebChatActivity; beforeEach(() => { @@ -62,7 +62,7 @@ describe.each([['with "streamId"' as const], ['without "streamId"' as const]])(' } }); - describe('activity with "streamType" of "final"', () => { + describe('activity with "streamType" of "final" (channelData)', () => { let activity: WebChatActivity; beforeEach(() => { @@ -91,10 +91,112 @@ describe.each([['with "streamId"' as const], ['without "streamId"' as const]])(' }); }); +describe.each([['with "streamId"' as const], ['without "streamId"' as const]])('activity %s', variant => { + describe('activity with "streamType" of "streaming" (entities)', () => { + let activity: WebChatActivity; + + beforeEach(() => { + activity = { + entities: [ + { + ...(variant === 'with "streamId"' ? { streamId: 'a-00001' } : {}), + streamSequence: 1, + streamType: 'streaming', + type: 'streaminfo' + } + ], + channelData: {}, + id: 'a-00002', + text: 'Hello, World!', + type: 'typing' + } as any; + }); + + test('should return type of "interim activity"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('type', 'interim activity')); + test('should return sequence number', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sequenceNumber', 1)); + + if (variant === 'with "streamId"') { + test('should return session ID with value from "entities.streamId"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sessionId', 'a-00001')); + } else { + test('should return session ID with value of "activity.id"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sessionId', 'a-00002')); + } + }); + + describe('activity with "streamType" of "informative message" (entities)', () => { + let activity: WebChatActivity; + + beforeEach(() => { + activity = { + entities: [ + { + ...(variant === 'with "streamId"' ? { streamId: 'a-00001' } : {}), + streamSequence: 1, + streamType: 'informative', + type: 'streaminfo' + } + ], + channelData: {}, + id: 'a-00002', + text: 'Hello, World!', + type: 'typing' + } as any; + }); + + test('should return type of "informative message"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('type', 'informative message')); + test('should return sequence number', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sequenceNumber', 1)); + + if (variant === 'with "streamId"') { + test('should return session ID with value from "entities.streamId"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sessionId', 'a-00001')); + } else { + test('should return session ID with value of "activity.id"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sessionId', 'a-00002')); + } + }); + + describe('activity with "streamType" of "final" (entities)', () => { + let activity: WebChatActivity; + + beforeEach(() => { + activity = { + entities: [ + { + ...(variant === 'with "streamId"' ? { streamId: 'a-00001' } : {}), + streamType: 'final', + type: 'streaminfo' + } + ], + channelData: {}, + id: 'a-00002', + text: 'Hello, World!', + type: 'message' + } as any; + }); + + if (variant === 'with "streamId"') { + test('should return type of "final activity"', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('type', 'final activity')); + test('should return sequence number of Infinity', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sequenceNumber', Infinity)); + test('should return session ID', () => + expect(getActivityLivestreamingMetadata(activity)).toHaveProperty('sessionId', 'a-00001')); + } else { + // Final activity must have "streamId". Final activity without "streamId" is not a valid livestream activity. + test('should return undefined', () => expect(getActivityLivestreamingMetadata(activity)).toBeUndefined()); + } + }); +}); + test('invalid activity should return undefined', () => expect(getActivityLivestreamingMetadata('invalid' as any)).toBeUndefined()); -test('activity with "streamType" of "streaming" without critical fields should return undefined', () => +test('activity with "streamType" of "streaming" without critical fields should return undefined (channelData)', () => expect( getActivityLivestreamingMetadata({ channelData: { streamType: 'streaming' }, @@ -102,11 +204,20 @@ test('activity with "streamType" of "streaming" without critical fields should r } as any) ).toBeUndefined()); +test('activity with "streamType" of "streaming" without critical fields should return undefined (entities)', () => + expect( + getActivityLivestreamingMetadata({ + entities: [{ streamType: 'streaming', type: 'streaminfo' }], + channelData: {}, + type: 'typing' + } as any) + ).toBeUndefined()); + describe.each([ ['integer', 1, true], ['zero', 0, false], ['decimal', 1.234, false] -])('activity with %s "streamSequence" should return undefined', (_, streamSequence, isValid) => { +])('activity with %s "streamSequence" should return undefined (channelData)', (_, streamSequence, isValid) => { const activity = { channelData: { streamSequence, streamType: 'streaming' }, id: 'a-00001', @@ -121,7 +232,27 @@ describe.each([ } }); -describe('"typing" activity with "streamType" of "final"', () => { +describe.each([ + ['integer', 1, true], + ['zero', 0, false], + ['decimal', 1.234, false] +])('activity with %s "streamSequence" should return undefined (entities)', (_, streamSequence, isValid) => { + const activity = { + entities: [{ streamSequence, streamType: 'streaming', type: 'streaminfo' }], + channelData: {}, + id: 'a-00001', + text: '', + type: 'typing' + } as any; + + if (isValid) { + expect(getActivityLivestreamingMetadata(activity)).toBeTruthy(); + } else { + expect(getActivityLivestreamingMetadata(activity)).toBeUndefined(); + } +}); + +describe('"typing" activity with "streamType" of "final" (channelData)', () => { test('should return undefined if "text" field is defined', () => expect( getActivityLivestreamingMetadata({ @@ -143,7 +274,31 @@ describe('"typing" activity with "streamType" of "final"', () => { ).toHaveProperty('type', 'final activity')); }); -test('activity with "streamType" of "streaming" without "content" should return type of "contentless"', () => +describe('"typing" activity with "streamType" of "final" (entities)', () => { + test('should return undefined if "text" field is defined', () => + expect( + getActivityLivestreamingMetadata({ + entities: [{ streamId: 'a-00001', streamType: 'final', type: 'streaminfo' }], + channelData: {}, + id: 'a-00002', + text: 'Final "typing" activity, must not have "text".', + type: 'typing' + } as any) + ).toBeUndefined()); + + test('should return truthy if "text" field is not defined', () => + expect( + getActivityLivestreamingMetadata({ + entities: [{ streamId: 'a-00001', streamType: 'final', type: 'streaminfo' }], + channelData: {}, + id: 'a-00002', + // Final activity can be "typing" if it does not have "text". + type: 'typing' + } as any) + ).toHaveProperty('type', 'final activity')); +}); + +test('activity with "streamType" of "streaming" without "content" should return type of "contentless" (channelData)', () => expect( getActivityLivestreamingMetadata({ channelData: { streamSequence: 1, streamType: 'streaming' }, @@ -151,3 +306,13 @@ test('activity with "streamType" of "streaming" without "content" should return type: 'typing' } as any) ).toHaveProperty('type', 'contentless')); + +test('activity with "streamType" of "streaming" without "content" should return type of "contentless" (entities)', () => + expect( + getActivityLivestreamingMetadata({ + entities: [{ streamSequence: 1, streamType: 'streaming', type: 'streaminfo' }], + channelData: {}, + id: 'a-00001', + type: 'typing' + } as any) + ).toHaveProperty('type', 'contentless')); diff --git a/packages/core/src/utils/getActivityLivestreamingMetadata.ts b/packages/core/src/utils/getActivityLivestreamingMetadata.ts index 804fed1dac..f5ce435daa 100644 --- a/packages/core/src/utils/getActivityLivestreamingMetadata.ts +++ b/packages/core/src/utils/getActivityLivestreamingMetadata.ts @@ -14,74 +14,136 @@ import { undefinedable, union } from 'valibot'; +import type { InferOutput } from 'valibot'; import { type WebChatActivity } from '../types/WebChatActivity'; import getOrgSchemaMessage from './getOrgSchemaMessage'; const EMPTY_ARRAY = Object.freeze([]); +/** + * Represents streaming data fields in a streaming Activity + */ +interface StreamingData { + /** The unique ID of the stream */ + streamId?: string; + /** The sequence number of the streaming message */ + streamSequence?: number; + /** The type of streaming message (streaming, informative, or final) */ + streamType: string; +} + const streamSequenceSchema = pipe(number(), integer(), minValue(1)); -const livestreamingActivitySchema = union([ - // Interim. - object({ - attachments: optional(array(any()), EMPTY_ARRAY), - channelData: object({ - // "streamId" is optional for the very first activity in the session. - streamId: optional(undefinedable(string())), - streamSequence: streamSequenceSchema, - streamType: literal('streaming') - }), - id: string(), - // "text" is optional. If not set or empty, it presents a contentless activity. - text: optional(undefinedable(string())), - type: literal('typing') - }), - // Informative message. +// Fields required for every activity +const activityFieldsSchema = { + // "text" is optional. If not set or empty, it presents a contentless activity. + text: optional(undefinedable(string())), + attachments: optional(array(any()), EMPTY_ARRAY), + id: string() +}; + +// Final Activities have different requirements for "text" fields +const { text: _text, ...activityFieldsFinalSchema } = activityFieldsSchema; + +const channelDataStreamingActivitySchema = union([ + // Interim or Informative message + // Informative may not have "text", but should have abstract instead (checked later) object({ - attachments: optional(array(any()), EMPTY_ARRAY), + type: literal('typing'), channelData: object({ // "streamId" is optional for the very first activity in the session. streamId: optional(undefinedable(string())), streamSequence: streamSequenceSchema, - streamType: literal('informative') + streamType: union([literal('streaming'), literal('informative')]) }), - id: string(), - // Informative may not have "text", but should have abstract instead (checked later) - text: optional(undefinedable(string())), - type: literal('typing'), - entities: optional(array(any()), EMPTY_ARRAY) + entities: optional(array(any()), EMPTY_ARRAY), + ...activityFieldsSchema }), // Conclude with a message. object({ - attachments: optional(array(any()), EMPTY_ARRAY), + // If "text" is empty, it represents "regretting" the livestream. + type: literal('message'), channelData: object({ // "streamId" is required for the final activity in the session. // The final activity must not be the sole activity in the session. streamId: pipe(string(), nonEmpty()), streamType: literal('final') }), - id: string(), - // If "text" is empty, it represents "regretting" the livestream. - text: optional(undefinedable(string())), - type: literal('message') + entities: optional(array(any()), EMPTY_ARRAY), + ...activityFieldsSchema }), // Conclude without a message. object({ - attachments: optional(array(any()), EMPTY_ARRAY), + // If "text" is not set or empty, it represents "regretting" the livestream. + type: literal('typing'), channelData: object({ // "streamId" is required for the final activity in the session. // The final activity must not be the sole activity in the session. streamId: pipe(string(), nonEmpty()), streamType: literal('final') }), - id: string(), - // If "text" is not set or empty, it represents "regretting" the livestream. + entities: optional(array(any()), EMPTY_ARRAY), text: optional(undefinedable(literal(''))), - type: literal('typing') + ...activityFieldsFinalSchema }) ]); +const entitiesStreamingActivitySchema = union([ + // Interim or Informative message + // Informative may not have "text", but should have abstract instead (checked later) + object({ + type: literal('typing'), + entities: array( + object({ + // "streamId" is optional for the very first activity in the session. + streamId: optional(undefinedable(string())), + streamSequence: streamSequenceSchema, + streamType: union([literal('streaming'), literal('informative')]), + type: literal('streaminfo') + }) + ), + channelData: optional(any()), + ...activityFieldsSchema + }), + // Conclude with a message. + object({ + // If "text" is empty, it represents "regretting" the livestream. + type: literal('message'), + entities: array( + object({ + // "streamId" is required for the final activity in the session. + // The final activity must not be the sole activity in the session. + streamId: pipe(string(), nonEmpty()), + streamType: literal('final'), + type: literal('streaminfo') + }) + ), + channelData: optional(any()), + ...activityFieldsSchema + }), + // Conclude without a message. + object({ + // If "text" is empty, it represents "regretting" the livestream. + type: literal('typing'), + entities: array( + object({ + // "streamId" is required for the final activity in the session. + // The final activity must not be the sole activity in the session. + streamId: pipe(string(), nonEmpty()), + streamType: literal('final'), + type: literal('streaminfo') + }) + ), + channelData: optional(any()), + text: optional(undefinedable(literal(''))), + ...activityFieldsFinalSchema + }) +]); + +type EntitiesStreamingActivity = InferOutput; +type ChannelDataStreamingActivity = InferOutput; + /** * Gets the livestreaming metadata of the activity, or `undefined` if the activity is not participating in a livestreaming session. * @@ -106,31 +168,43 @@ export default function getActivityLivestreamingMetadata(activity: WebChatActivi type: 'contentless' | 'final activity' | 'informative message' | 'interim activity'; }> | undefined { - const result = safeParse(livestreamingActivitySchema, activity); + let activityData: EntitiesStreamingActivity | ChannelDataStreamingActivity | undefined; + + let streamingData: StreamingData | undefined; - if (result.success) { - const { output } = result; + if (activity.entities) { + const result = safeParse(entitiesStreamingActivitySchema, activity); + activityData = result.success ? result.output : undefined; + streamingData = result.success ? activityData.entities[0] : undefined; + } + + if (!activityData && activity.channelData) { + const result = safeParse(channelDataStreamingActivitySchema, activity); + activityData = result.success ? result.output : undefined; + streamingData = result.success ? activityData.channelData : undefined; + } + if (activityData && streamingData) { // If the activity is the first in the session, session ID should be the activity ID. - const sessionId = output.channelData.streamId || output.id; + const sessionId = streamingData.streamId || activityData.id; return Object.freeze( - output.channelData.streamType === 'final' + streamingData.streamType === 'final' ? { sequenceNumber: Infinity, sessionId, type: 'final activity' } : { - sequenceNumber: output.channelData.streamSequence, + sequenceNumber: streamingData.streamSequence, sessionId, type: !( - output.text || - output.attachments?.length || - ('entities' in output && getOrgSchemaMessage(output.entities)?.abstract) + activityData.text || + activityData.attachments?.length || + ('entities' in activityData && getOrgSchemaMessage(activity.entities)?.abstract) ) ? 'contentless' - : output.channelData.streamType === 'informative' + : streamingData.streamType === 'informative' ? 'informative message' : 'interim activity' }