image-description-in-replies

cmd/api/main.go

@@ -51,11 +51,14 @@ func main() {
 	// Shared channel for communication between bot and router
 	messageChannel := make(chan *bot.MessageForCompletion)
 
+	// Channel for requesting image descriptions
+	imageRequestChannel := make(chan *bot.ImageDescriptionRequest)
+
 	// tts routine
 	awsConf := tts.LoadConfig()
 
 	// Create the bot instance
-	myBot, err := bot.NewBot(settings.DiscordToken, messageChannel, awsConf)
+	myBot, err := bot.NewBot(settings.DiscordToken, messageChannel, awsConf, imageRequestChannel)
 	if err != nil {
 		log.Fatalf("Failed to create Discord bot: %v", err)
 	}
@@ -74,7 +77,7 @@ func main() {
 	go func() {
 		defer wg.Done() // Signal completion when this goroutine exits
 		// Pass the cancellable context to the loop
-		router.MessageLoop(ctx, myBot, client, messageChannel, settings.Instructions, messages, settings.ChatHistoryFilePath, settings.Instructions)
+		router.MessageLoop(ctx, myBot, client, messageChannel, settings.Instructions, messages, settings.ChatHistoryFilePath, settings.Instructions, imageRequestChannel)
 		log.Println("Router loop stopped.")
 	}()
 

internal/bot/bot.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"os"
 	"os/signal"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -26,6 +27,7 @@ type Bot struct {
 	transcribeChannel chan *transcribe.Msg           // Channel for transcription messages
 	awsConfig         aws.Config                     // AWS configuration for TTS
 	transcribeQueue   map[string]map[string][]string // Queue for transcription messages, keyed by guild ID and channel ID, value is file name
+	imageReqChan      chan *ImageDescriptionRequest
 }
 
 // MessageForCompletion represents a message that needs to be processed with a wait message.
@@ -41,12 +43,18 @@ type MessageForCompletion struct {
 	VC          *discordgo.VoiceConnection
 }
 
+type ImageDescriptionRequest struct {
+	AttachmentImg *discordgo.MessageAttachment
+	Description   *string
+}
+
 // NewBot creates a new Bot instance but doesn't connect yet
-func NewBot(token string, msgChan chan *MessageForCompletion, awsConf aws.Config) (*Bot, error) {
+func NewBot(token string, msgChan chan *MessageForCompletion, awsConf aws.Config, imageReqChan chan *ImageDescriptionRequest) (*Bot, error) {
 	b := &Bot{
 		Token:          token,
 		messageChannel: msgChan,
 		awsConfig:      awsConf,
+		imageReqChan:   imageReqChan,
 	}
 
 	b.transcribeChannel = make(chan *transcribe.Msg)
@@ -174,12 +182,26 @@ func (b *Bot) newMessage(s *discordgo.Session, m *discordgo.MessageCreate) {
 		*/
 
 		if m.ReferencedMessage != nil {
+			craftedMessage := m.ReferencedMessage.Content
+			imageDescriptions := strings.Builder{}
+			if m.ReferencedMessage.Attachments != nil {
+				for i, attachment := range m.ReferencedMessage.Attachments {
+					if strings.HasPrefix(attachment.ContentType, "image/") {
+						imageDescriptions.WriteString("{\n\"index\" : " + strconv.Itoa(i) + ",\n\"description\": " + b.getImageDescription(attachment) + "}, \n")
+					}
+				}
+			}
+
+			if imageDescriptions.Len() > 0 {
+				craftedMessage = "{\n \"images\" : " + imageDescriptions.String() + "}\n" + craftedMessage
+			}
+
 			referMsg := fmt.Sprintf(
 				"{\n"+
 					"\"referenced_message\": \"%s\", \n"+
 					"\"referenced_message_author\": \"%s\"\n"+
 					"\"message\":\"%s\"\n"+
-					"}", m.ReferencedMessage.Content, m.ReferencedMessage.Author.ID, m.Content)
+					"}", craftedMessage, m.ReferencedMessage.Author.ID, m.Content)
 			m.Content = referMsg
 		} else {
 			referMsg := fmt.Sprintf(
@@ -272,6 +294,22 @@ func (b *Bot) newMessage(s *discordgo.Session, m *discordgo.MessageCreate) {
 	}
 }
 
+func (b *Bot) getImageDescription(attachment *discordgo.MessageAttachment) string {
+	imageReq := ImageDescriptionRequest{
+		Description:   nil,
+		AttachmentImg: attachment,
+	}
+
+	b.imageReqChan <- &imageReq
+
+	for {
+		if imageReq.Description != nil {
+			return *imageReq.Description
+		}
+		time.Sleep(100 * time.Millisecond) // busy waiting, maybe not the best method
+	}
+}
+
 // addMessage adds a message to the internal queue
 func (b *Bot) addMessage(message MessageForCompletion) {
 	b.mu.Lock()

internal/bot/bot_test.go

@@ -17,7 +17,7 @@ func createTestBot(t *testing.T) (*Bot, chan *MessageForCompletion) {
 	// tts routine
 	awsConf := tts.LoadConfig()
 
-	bot, err := NewBot("test-token", msgChan, awsConf)
+	bot, err := NewBot("test-token", msgChan, awsConf, nil)
 	assert.NoError(t, err)
 
 	// Create a real session but don't open it
@@ -61,7 +61,7 @@ func createTestBotWithMockState(t *testing.T) (*Bot, chan *MessageForCompletion)
 	// tts routine
 	awsConf := tts.LoadConfig()
 
-	bot, err := NewBot("test-token", msgChan, awsConf)
+	bot, err := NewBot("test-token", msgChan, awsConf, nil)
 	assert.NoError(t, err)
 
 	// Create a session with proper state for testing
@@ -87,7 +87,7 @@ func TestNewBot(t *testing.T) {
 		// tts routine
 		awsConf := tts.LoadConfig()
 
-		bot, err := NewBot("test-token", msgChan, awsConf)
+		bot, err := NewBot("test-token", msgChan, awsConf, nil)
 
 		assert.NoError(t, err)
 		assert.NotNil(t, bot)
@@ -102,7 +102,7 @@ func TestNewBot(t *testing.T) {
 		// tts routine
 		awsConf := tts.LoadConfig()
 
-		bot, err := NewBot("", msgChan, awsConf)
+		bot, err := NewBot("", msgChan, awsConf, nil)
 
 		// Bot creation should still succeed even with empty token
 		assert.NoError(t, err)
@@ -113,7 +113,7 @@ func TestNewBot(t *testing.T) {
 		// tts routine
 		awsConf := tts.LoadConfig()
 
-		bot, err := NewBot("test-token", nil, awsConf)
+		bot, err := NewBot("test-token", nil, awsConf, nil)
 
 		assert.NoError(t, err)
 		assert.NotNil(t, bot)