[AGE-326] Update the hybrid search tool to use new searchable column data

.vscode/settings.json

@@ -1,7 +1,12 @@
 {
 	"biome.lsp.bin": "./mcp/node_modules/@biomejs/biome/bin/biome",
 	"[typescript]": {
-		"editor.defaultFormatter": "biomejs.biome"
+		"editor.defaultFormatter": "biomejs.biome",
+		"editor.formatOnSave": true,
+		"editor.codeActionsOnSave": {
+			"source.fixAll.biome": "explicit",
+			"source.organizeImports.biome": "explicit"
+		}
 	},
 	"[jsonc]": {
 		"editor.defaultFormatter": "vscode.json-language-features"

mcp/src/apis/getMessageContext.ts

@@ -1,38 +1,28 @@
 import type { ApiFactory } from '@tigerdata/mcp-boilerplate';
 import { z } from 'zod';
 import {
+  type Channel,
   type Message,
   type ServerContext,
-  type zChannel,
+  type User,
   zConversationsResults,
-  type zUser,
+  zIncludeFilters,
+  zLimitFilter,
+  zMessageFilter,
 } from '../types.js';
 import { addChannelInfo } from '../util/addChannelInfo.js';
-import { convertTsToTimestamp } from '../util/formatTs.js';
 import { getUsersMap } from '../util/getUsersMap.js';
 import { getMessageFields } from '../util/messageFields.js';
 import { messagesToTree } from '../util/messagesToTree.js';
+import { normalizeMessageFilterQueryParameters } from '../util/normalizeMessageFilterQueryParameters.js';
 import { selectExpandedMessages } from '../util/selectExpandedMessages.js';
 
 const inputSchema = {
-  ts: z
-    .string()
-    .describe(
-      'The timestamp of the target Slack message to fetch context for.',
-    ),
-  channel: z
-    .string()
-    .describe('The ID of the channel the message was posted in.'),
-  includeFiles: z
-    .boolean()
-    .describe(
-      'Specifies if file attachment metadata should be included. It is recommended to enable as it provides extra context for the thread.',
-    ),
-  limit: z.coerce
-    .number()
-    .min(1)
-    .nullable()
-    .describe('The maximum number of messages to return. Defaults to 1000.'),
+  messageFilters: z
+    .array(zMessageFilter)
+    .describe('The Slack messages to context for.'),
+  ...zLimitFilter.shape,
+  ...zIncludeFilters.shape,
   window: z.coerce
     .number()
     .min(0)
@@ -60,32 +50,45 @@ export const getMessageContextFactory: ApiFactory<
     outputSchema,
   },
   fn: async ({
-    ts,
-    channel,
     includeFiles,
+    includePermalinks,
+    messageFilters: passedMessageFilters,
     window,
     limit,
   }): Promise<{
-    channels: Record<string, z.infer<typeof zChannel>>;
-    users: Record<string, z.infer<typeof zUser>>;
+    channels: Record<string, Channel>;
+    users: Record<string, User>;
   }> => {
     const client = await pgPool.connect();
+    const messageFilters = await normalizeMessageFilterQueryParameters(
+      pgPool,
+      passedMessageFilters,
+    );
+
     try {
       const result = await client.query<Message>(
         selectExpandedMessages(
           /* sql */ `
-SELECT ${getMessageFields({ includeFiles, coerceType: false })} FROM slack.message
-WHERE ts = $1 AND channel_id = $2
-LIMIT 1
+SELECT ${getMessageFields({ includeFiles, coerceType: false, includeSearchableContent: true, messageTableAlias: 'm' })}
+FROM slack.message_vanilla m
+INNER JOIN (
+  SELECT
+    f->>'channel' AS channel_id,
+    (f->>'ts')::timestamptz AS ts
+  FROM jsonb_array_elements($1::jsonb) AS f
+) filters ON m.channel_id = filters.channel_id AND m.ts = filters.ts
 `,
+          '$2',
           '$3',
-          '$4',
           includeFiles,
         ),
-        [convertTsToTimestamp(ts), channel, window || 5, limit || 1000],
+        [JSON.stringify(messageFilters), window || 5, limit || 1000],
       );
 
-      const { channels, involvedUsers } = messagesToTree(result.rows);
+      const { channels, involvedUsers } = messagesToTree(
+        result.rows,
+        includePermalinks || false,
+      );
       await addChannelInfo(client, channels);
       const users = await getUsersMap(client, involvedUsers);
 

mcp/src/apis/getThreadMessages.ts

@@ -4,28 +4,22 @@ import {
   type Message,
   type MessageInThread,
   type ServerContext,
+  type User,
   zIncludeFilters,
+  zMessageFilter,
   zMessageInThread,
   zUser,
 } from '../types.js';
-import { convertTsToTimestamp } from '../util/formatTs.js';
 import { getUsersMap } from '../util/getUsersMap.js';
 import { getMessageFields } from '../util/messageFields.js';
 import { messagesToTree } from '../util/messagesToTree.js';
+import { normalizeMessageFilterQueryParameters } from '../util/normalizeMessageFilterQueryParameters.js';
 
 const inputSchema = {
   ...zIncludeFilters.shape,
-  channel: z
-    .string()
-    .min(1)
-    .describe('The ID of the channel to fetch messages from.'),
-
-  ts: z
-    .string()
-    .min(1)
-    .describe(
-      'The thread timestamp to fetch messages for. This is the ts of the parent message. Use the `thread_ts` field from a known message in the thread.',
-    ),
+  messageFilters: z
+    .array(zMessageFilter)
+    .describe('The messages to fetch the threads for.'),
 } as const;
 
 const outputSchema = {
@@ -52,28 +46,45 @@ export const getThreadMessagesFactory: ApiFactory<
     outputSchema,
   },
   fn: async ({
-    channel,
     includeFiles,
     includePermalinks,
-    ts,
+    messageFilters: passedMessageFilters,
   }): Promise<{
     messages: MessageInThread[];
-    users: Record<string, z.infer<typeof zUser>>;
+    users: Record<string, User>;
   }> => {
+    const messageFilters = await normalizeMessageFilterQueryParameters(
+      pgPool,
+      passedMessageFilters,
+    );
+
     const result = await pgPool.query<Message>(
       /* sql */ `
-  SELECT ${getMessageFields({ includeFiles })} FROM slack.message
-  WHERE channel_id = $1 AND (thread_ts = $2 OR ts = $2)
-  ORDER BY ts DESC`, // messagesToTree expects messages in descending order
-      [channel, convertTsToTimestamp(ts)],
+      WITH filters AS (
+        SELECT
+          f->>'channel' AS channel_id,
+          (f->>'ts')::timestamptz AS ts
+        FROM jsonb_array_elements($1::jsonb) AS f
+      )
+      SELECT ${getMessageFields({ includeFiles })}
+      FROM slack.message m
+      WHERE EXISTS (
+        SELECT 1 FROM filters f
+        WHERE m.channel_id = f.channel_id
+          AND (m.thread_ts = f.ts OR m.ts = f.ts)
+      )
+      ORDER BY ts DESC`, // messagesToTree expects messages in descending order
+      [JSON.stringify(messageFilters)],
     );
 
     const { involvedUsers, channels } = messagesToTree(
       result.rows,
       includePermalinks || false,
     );
     const users = await getUsersMap(pgPool, involvedUsers);
-    const messages = channels[channel]?.messages || [];
+
+    // Flatten messages from all channels
+    const messages = Object.values(channels).flatMap((c) => c.messages);
 
     return {
       messages,

mcp/src/apis/index.ts

@@ -13,5 +13,5 @@ export const apiFactories = [
   getConversationsWithUserFactory,
   getThreadMessagesFactory,
   getUsersFactory,
-  // searchFactory,
+  searchFactory,
 ] as const;

mcp/src/apis/search.ts

@@ -8,8 +8,8 @@ import {
   zMessage,
 } from '../types.js';
 import { generatePermalink } from '../util/addMessageLinks.js';
-import { findChannel } from '../util/findChannel.js';
 import { findUser } from '../util/findUser.js';
+import { getChannelIds } from '../util/getChannelIds.js';
 import { getMessageKey } from '../util/getMessageKey.js';
 import { getMessageFields } from '../util/messageFields.js';
 import { normalizeMessageTs } from '../util/messagesToTree.js';
@@ -32,7 +32,7 @@ const inputSchema = {
     .string()
     .array()
     .nullable()
-    .describe('Optionally filter search on channels names.'),
+    .describe('Optionally filter search on channels. Can use ids or names.'),
   users: z
     .string()
     .array()
@@ -72,9 +72,7 @@ export const searchFactory: ApiFactory<
   config: {
     title: 'Search Slack Messages',
     description:
-      'Search across Slack messages using semantic (vector) search. Returns messages organized by channel and conversation with optional filtering by users, channels, and time range.',
-    // description:
-    // 	"Hybrid search across Slack messages using semantic (vector) and keyword (BM25) search with configurable weighting. Returns messages organized by channel and conversation with optional filtering by users, channels, and time range.",
+      'Hybrid search across Slack messages using semantic (vector) and keyword (BM25) search with configurable weighting. Returns messages organized by channel and conversation with optional filtering by users, channels, and time range. Search matches the message text, as well as contents of the attachments.',
     inputSchema,
     outputSchema,
   },
@@ -113,34 +111,30 @@ export const searchFactory: ApiFactory<
       );
       userIdsToFilterOn = userObjs.map((u) => u.id);
     }
-
-    let channelIdsToFilterOn: string[] | null = channelsToFilterOn ? [] : null;
-    if (channelsToFilterOn) {
-      const channelObjs = await Promise.all(
-        channelsToFilterOn.map((channel) => findChannel(pgPool, channel)),
-      );
-      channelIdsToFilterOn = channelObjs.map((u) => u.id);
-    }
+    const channelIdsToFilterOn = await getChannelIds(
+      pgPool,
+      channelsToFilterOn,
+    );
 
     const createQuery = async (type: 'semantic' | 'keyword') =>
       pgPool.query<Message>(
         `SELECT 
-          ${getMessageFields({ includeFiles, coerceType: true })} 
+          ${getMessageFields({ includeFiles, coerceType: true, includeSearchableContent: true })} 
         FROM slack.message_vanilla
-        WHERE text != ''
+        WHERE searchable_content != ''
           AND (($1::TEXT[] IS NULL) OR (user_id = ANY($1)))
           AND (($2::TEXT[] IS NULL) OR (channel_id = ANY($2)))
-          AND (($3::TIMESTAMPTZ IS NULL AND ts >= (NOW() - interval '1 week')) OR ts >= $3::TIMESTAMPTZ)
+          AND (($3::TIMESTAMPTZ IS NULL) OR ts >= $3::TIMESTAMPTZ)
           AND ($4::TIMESTAMPTZ IS NULL OR ts <= $4::TIMESTAMPTZ)
-        ORDER BY ${type === 'semantic' ? `embedding <=> $5::vector(1536)` : `text <@> to_bm25query($5::text, 'slack.message_vanilla_text_bm25_idx')`}
+        ORDER BY ${type === 'semantic' ? `embedding <=> $5::vector(1536)` : `searchable_content <@> to_bm25query($5::text, 'slack.message_vanilla_searchable_content_bm25_idx')`}
         LIMIT $6`,
         [
           userIdsToFilterOn,
           channelIdsToFilterOn,
           timestampStart?.toISOString(),
           timestampEnd?.toISOString(),
           type === 'semantic' ? JSON.stringify(embedding?.embedding) : keyword,
-          limit * 2,
+          limit * 3,
         ],
       );
 
@@ -219,5 +213,6 @@ const getResultMessages = (
     message.permalink = includePermalinks
       ? generatePermalink(message)
       : message.permalink;
+    message.searchable_content = message.searchable_content?.trim();
     return normalizeMessageTs(message);
   }) || [];

mcp/src/types.ts

@@ -66,15 +66,30 @@ export const zFile = z
   })
   .describe('A file associated with a Slack message');
 
+export const zAttachment = z
+  .object({
+    title: z.string().nullish(),
+    text: z.string().nullish(),
+    fallback: z.string().nullish(),
+  })
+  .passthrough();
+
 export const zMessage = z.object({
   ts: z.string().describe('The timestamp of the message'),
   channel_id: z
     .string()
     .describe(
       'The Slack channel ID the message was posted in (e.g. C123ABC456)',
     ),
-  files: z.array(zFile).optional().nullable(),
-  text: z.string().describe('The text content of the message'),
+  attachments: z.array(zAttachment).nullish(),
+  files: z.array(zFile).optional().nullish(),
+  text: z.string().nullish().describe('The text content of the message'),
+  searchable_content: z
+    .string()
+    .nullish()
+    .describe(
+      'This is a concatenation of the text field and text content elements from the attachments field. This content is what is used in hybrid search matching.',
+    ),
   user_id: z
     .string()
     .nullable()
@@ -128,8 +143,23 @@ export const zIncludeFilters = z.object({
     ),
 });
 
-export const zCommonSearchFilters = z.object({
-  ...zIncludeFilters.shape,
+export const zMessageFilter = z.object({
+  channel: z
+    .string()
+    .min(1)
+    .describe('The ID of the channel to fetch messages from.'),
+
+  ts: z
+    .string()
+    .min(1)
+    .describe(
+      'The thread timestamp to fetch messages for. This is the ts of the parent message. Use the `thread_ts` field from a known message in the thread.',
+    ),
+});
+
+export type MessageFilter = z.infer<typeof zMessageFilter>;
+
+export const zTimeFilters = z.object({
   timestampStart: z.coerce
     .date()
     .nullable()
@@ -142,13 +172,22 @@ export const zCommonSearchFilters = z.object({
     .describe(
       'Optional end date for the message range. Defaults to the current time.',
     ),
+});
+
+export const zLimitFilter = z.object({
   limit: z.coerce
     .number()
     .min(1)
     .nullable()
     .describe('The maximum number of messages to return. Defaults to 1000.'),
 });
 
+export const zCommonSearchFilters = z.object({
+  ...zIncludeFilters.shape,
+  ...zTimeFilters.shape,
+  ...zLimitFilter.shape,
+});
+
 export const zUserSearchFilters = z.object({
   username: z
     .string()

mcp/src/util/getChannelIds.ts

@@ -0,0 +1,20 @@
+import type { Pool } from 'pg';
+import { findChannel } from './findChannel.js';
+
+export const getChannelIds = async (
+  pgPool: Pool,
+  channelKeywords: string[] | null,
+): Promise<string[] | null> => {
+  if (channelKeywords === null) {
+    return null;
+  }
+  let channelIds: string[] | null = channelKeywords ? [] : null;
+  if (channelKeywords) {
+    const channelObjs = await Promise.all(
+      channelKeywords.map((channel) => findChannel(pgPool, channel)),
+    );
+    channelIds = channelObjs.map((u) => u.id);
+  }
+
+  return channelIds;
+};