feat(assistant): displaying annotations

Author: sebastianmusial

apps/spa/src/app/app.config.ts

@@ -5,12 +5,14 @@ import { routes } from './app.routes';
 import { provideAnimations } from '@angular/platform-browser/animations';
 import { provideHttpClient } from '@angular/common/http';
 import { provideMarkdown } from 'ngx-markdown';
+import { AnnotationPipe } from './pipes/annotation.pipe';
 
 export const appConfig: ApplicationConfig = {
   providers: [
     provideRouter(routes),
     provideAnimations(),
     provideHttpClient(),
     provideMarkdown(),
+    AnnotationPipe,
   ],
 };

apps/spa/src/app/components/chat/chat-audio/chat-audio.component.html

@@ -1,4 +1,4 @@
-@if (getMessageText) {
+@if (isAudioEnabled && message && (message | messageText)) {
   <span class="chat-audio" [ngClass]="'chat-audio--' + message.role">
     @if (!isStarted) {
       <mat-icon (click)="speech()">play_circle</mat-icon>

apps/spa/src/app/components/chat/chat-audio/chat-audio.component.ts

@@ -2,36 +2,33 @@ import { Component, Input, OnInit } from '@angular/core';
 import { MatIconModule } from '@angular/material/icon';
 import { delay } from 'rxjs';
 import { ChatAudioResponse, PostSpeechDto } from '@boldare/openai-assistant';
-import { NgClass } from '@angular/common';
-import { getMessageText } from '../../controls/message-content/message-content.helpers';
+import { CommonModule } from '@angular/common';
 import { ChatClientService } from '../../../modules/+chat/shared/chat-client.service';
 import {
   ChatMessage,
   SpeechVoice,
 } from '../../../modules/+chat/shared/chat.model';
+import { MessageTextPipe } from '../../../pipes/message-text.pipe';
 import { environment } from '../../../../environments/environment';
 
 @Component({
   selector: 'ai-chat-audio',
   standalone: true,
-  imports: [MatIconModule, NgClass],
+  imports: [MatIconModule, CommonModule, MessageTextPipe],
+  providers: [MessageTextPipe],
   templateUrl: './chat-audio.component.html',
   styleUrl: './chat-audio.component.scss',
 })
 export class ChatAudioComponent implements OnInit {
   @Input() message!: ChatMessage;
   isStarted = false;
   audio = new Audio();
+  isAudioEnabled = environment.isAudioEnabled;
 
-  get getMessageText(): string {
-    if (!environment.isAudioEnabled || !this.message) {
-      return '';
-    }
-
-    return getMessageText(this.message);
-  }
-
-  constructor(private readonly chatService: ChatClientService) {}
+  constructor(
+    private readonly chatService: ChatClientService,
+    private readonly messageTextPipe: MessageTextPipe,
+  ) {}
 
   ngOnInit(): void {
     this.audio.onended = this.onEnded.bind(this);
@@ -50,7 +47,9 @@ export class ChatAudioComponent implements OnInit {
   }
 
   speech(): void {
-    if (!this.getMessageText) {
+    const content = this.messageTextPipe.transform(this.message);
+
+    if (!content) {
       return;
     }
 
@@ -62,7 +61,7 @@ export class ChatAudioComponent implements OnInit {
     }
 
     const payload: PostSpeechDto = {
-      content: getMessageText(this.message),
+      content,
       voice: SpeechVoice.Onyx,
     };
 

apps/spa/src/app/components/chat/chat-message/chat-message.component.html

@@ -4,17 +4,17 @@
   }
 
   <div class="chat-message">
-    @if (messageText) {
-      <span markdown [data]="messageText"></span>
+    @if (message | messageText) {
+      <span markdown [data]="message | messageText"></span>
     }
 
     @if (message.role !== chatRole.System) {
       <ai-chat-audio [message]="message"></ai-chat-audio>
     }
 
-    @if (messageImage.length) {
+    @if ((message | messageImageFile).length) {
       <div class="chat-message__file">
-        @for (image of messageImage; track messageImage) {
+        @for (image of message | messageImageFile; track image.image_file.file_id) {
           <div>File ID: {{ image.image_file.file_id }}</div>
         }
       </div>

apps/spa/src/app/components/chat/chat-message/chat-message.component.ts

@@ -7,11 +7,9 @@ import { MarkdownComponent } from 'ngx-markdown';
 import { ChatAudioComponent } from '../chat-audio/chat-audio.component';
 import { NgClass } from '@angular/common';
 import { ChatAvatarComponent } from '../chat-avatar/chat-avatar.component';
-import {
-  getMessageImage,
-  getMessageText,
-} from '../../controls/message-content/message-content.helpers';
-import { ImageFileContentBlock } from 'openai/resources/beta/threads';
+import { MessageTextPipe } from '../../../pipes/message-text.pipe';
+import { MessageImageFilePipe } from '../../../pipes/message-file.pipe';
+import { AnnotationPipe } from '../../../pipes/annotation.pipe';
 
 @Component({
   selector: 'ai-chat-message',
@@ -23,21 +21,15 @@ import { ImageFileContentBlock } from 'openai/resources/beta/threads';
     MarkdownComponent,
     ChatAudioComponent,
     ChatAvatarComponent,
+    MessageTextPipe,
+    MessageImageFilePipe,
   ],
 })
 export class ChatMessageComponent {
   @Input() message!: ChatMessage;
   @Input() class = '';
   chatRole = ChatRole;
 
-  get messageText(): string {
-    return getMessageText(this.message);
-  }
-
-  get messageImage(): ImageFileContentBlock[] {
-    return getMessageImage(this.message);
-  }
-
   @HostBinding('class') get getClasses(): string {
     return `${this.class} is-${this.message?.role || 'none'}`;
   }

apps/spa/src/app/components/controls/message-content/message-content.component.ts

@@ -1,10 +1,10 @@
 import { Component, ElementRef, Input, ViewChild } from '@angular/core';
 import { MatIcon } from '@angular/material/icon';
 import { toSignal } from '@angular/core/rxjs-interop';
-import { MessageContentService } from './message-content.service';
 import { ControlItemComponent } from '../control-item/control-item.component';
 import { ControlIconComponent } from '../control-icon/control-icon.component';
 import { AiFilesDirective } from '../files/files.directive';
+import { MessageContentService } from './message-content.service';
 
 @Component({
   selector: 'ai-message-content',

apps/spa/src/app/components/controls/message-content/message-content.helpers.ts

@@ -1,6 +1,5 @@
 import { TextContentBlock } from 'openai/resources/beta/threads/messages';
 import { ImageFileContentBlock } from 'openai/src/resources/beta/threads/messages';
-import { ChatMessage } from '../../../modules/+chat/shared/chat.model';
 
 export function isTextContentBlock(item: {
   type: string;
@@ -13,25 +12,3 @@ export function isImageFileContentBlock(item: {
 }): item is ImageFileContentBlock {
   return item.type === 'image_file';
 }
-
-export const getMessageText = (message: ChatMessage): string => {
-  if (typeof message.content === 'string') {
-    return message.content;
-  }
-
-  // @TODO: handle all types of message content
-  return message.content
-    .filter(isTextContentBlock)
-    .map(block => block.text.value)
-    .join(' ');
-};
-
-export const getMessageImage = (
-  message: ChatMessage,
-): ImageFileContentBlock[] => {
-  if (typeof message.content === 'string') {
-    return [];
-  }
-
-  return message.content.filter(isImageFileContentBlock);
-};

apps/spa/src/app/modules/+chat/shared/chat-gateway.service.ts

@@ -3,6 +3,7 @@ import { ChatEvents } from './chat.model';
 import io from 'socket.io-client';
 import {
   ChatCallDto,
+  MessageWithAnnotations,
   TextCreatedPayload,
   TextDeltaPayload,
   TextDonePayload,
@@ -37,7 +38,7 @@ export class ChatGatewayService {
     return this.watchEvent(ChatEvents.TextDelta);
   }
 
-  textDone(): Observable<TextDonePayload> {
+  textDone(): Observable<MessageWithAnnotations<TextDonePayload>> {
     return this.watchEvent(ChatEvents.TextDone);
   }
 }

apps/spa/src/app/modules/+chat/shared/chat.service.ts

@@ -1,4 +1,4 @@
-import { Injectable } from '@angular/core';
+import { Injectable, Injector } from '@angular/core';
 import {
   BehaviorSubject,
   distinctUntilChanged,
@@ -9,20 +9,21 @@ import {
   take,
   tap,
 } from 'rxjs';
-import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant';
-import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model';
-import { ChatGatewayService } from './chat-gateway.service';
-import { ChatClientService } from './chat-client.service';
-import { ThreadService } from './thread.service';
-import { ChatFilesService } from './chat-files.service';
-import { environment } from '../../../../environments/environment';
 import {
   ImageFileContentBlock,
   Message,
   MessageContent,
   Text,
 } from 'openai/resources/beta/threads/messages';
+import { OpenAiFile, GetThreadResponseDto } from '@boldare/openai-assistant';
+import { ChatRole, ChatMessage, ChatMessageStatus } from './chat.model';
+import { ChatGatewayService } from './chat-gateway.service';
+import { ChatClientService } from './chat-client.service';
+import { ThreadService } from './thread.service';
+import { ChatFilesService } from './chat-files.service';
 import { MessageContentService } from '../../../components/controls/message-content/message-content.service';
+import { environment } from '../../../../environments/environment';
+import { AnnotationPipe } from '../../../pipes/annotation.pipe';
 
 @Injectable({ providedIn: 'root' })
 export class ChatService {
@@ -38,6 +39,7 @@ export class ChatService {
     private readonly threadService: ThreadService,
     private readonly chatFilesService: ChatFilesService,
     private readonly messageContentService: MessageContentService,
+    private readonly annotationPipe: AnnotationPipe,
   ) {
     document.body.classList.add('ai-chat');
 
@@ -213,13 +215,13 @@ export class ChatService {
   }
 
   watchTextDone(): Subscription {
-    return this.chatGatewayService.textDone().subscribe(data => {
+    return this.chatGatewayService.textDone().subscribe(event => {
       this.isTyping$.next(false);
       this.isResponding$.next(false);
       this.messages$.next([
         ...this.messages$.value.slice(0, -1),
         {
-          content: data.text.value,
+          content: this.annotationPipe.transform(event),
           role: ChatRole.Assistant,
         },
       ]);

apps/spa/src/app/pipes/annotation.pipe.ts

@@ -0,0 +1,29 @@
+import { Pipe, PipeTransform } from '@angular/core';
+import { MessageWithAnnotations } from '@boldare/openai-assistant';
+import { Message } from 'openai/resources/beta/threads';
+import { isTextContentBlock } from '../components/controls/message-content/message-content.helpers';
+
+@Pipe({
+  standalone: true,
+  name: 'annotation'
+})
+export class AnnotationPipe implements PipeTransform {
+  transform(event: MessageWithAnnotations<{ message: Message }>): string {
+    if (!isTextContentBlock(event.data.message.content[0])) {
+      return '';
+    }
+
+    const { text } = event.data.message.content[0];
+    
+    if (!event.annotations?.length) {
+      return text.value;
+    }
+
+    for (const item of event.annotations) {
+      const { index, annotation } = item;
+      text.value = text.value.replace(annotation.text, `[${index}]`);
+    }
+
+    return text.value;
+  }
+}