diff --git a/MEMORY_ENHANCEMENTS.md b/MEMORY_ENHANCEMENTS.md new file mode 100644 index 0000000..7ee2fb2 --- /dev/null +++ b/MEMORY_ENHANCEMENTS.md @@ -0,0 +1,123 @@ +# Memory Enhancement Methods + +The Langbase SDK now includes enhanced memory methods that allow for easier content upload and integration with web tools. + +## New Methods + +### 1. `memories.uploadText()` + +Upload text content directly to memory without file handling. + +```typescript +import { Langbase } from 'langbase'; + +const langbase = new Langbase({ + apiKey: 'your-langbase-api-key' +}); + +await langbase.memories.uploadText({ + memoryName: 'my-memory', + text: 'Your text content here...', + documentName: 'optional-document-name.txt', // Optional: will auto-generate if not provided + meta: { + type: 'user-content', + source: 'api-upload' + } +}); +``` + +**Parameters:** +- `memoryName`: Name of the target memory +- `text`: Text content to upload +- `documentName` (optional): Custom document name. Auto-generates if not provided +- `meta` (optional): Metadata key-value pairs + +### 2. `memories.uploadFromSearch()` + +Search the web and upload results directly to memory. + +```typescript +await langbase.memories.uploadFromSearch({ + memoryName: 'my-memory', + query: 'What is artificial intelligence?', + service: 'exa', + totalResults: 5, + domains: ['https://example.com'], // Optional: restrict to specific domains + apiKey: 'your-exa-api-key', + documentNamePrefix: 'ai-search', // Optional: prefix for generated document names + meta: { + topic: 'artificial-intelligence', + source: 'web-search' + } +}); +``` + +**Parameters:** +- `memoryName`: Name of the target memory +- `query`: Search query to execute +- `service`: Search service to use (currently supports 'exa') +- `totalResults` (optional): Number of results to retrieve +- `domains` (optional): List of domains to restrict search to +- `apiKey`: API key for the search service +- `documentNamePrefix` (optional): Prefix for generated document names +- `meta` (optional): Metadata applied to all uploaded documents + +### 3. `memories.uploadFromCrawl()` + +Crawl URLs and upload the content directly to memory. + +```typescript +await langbase.memories.uploadFromCrawl({ + memoryName: 'my-memory', + url: ['https://example.com', 'https://example.com/about'], + maxPages: 2, + apiKey: 'your-crawl-api-key', + service: 'spider', // or 'firecrawl' + documentNamePrefix: 'website-content', + meta: { + source: 'web-crawl', + website: 'example.com' + } +}); +``` + +**Parameters:** +- `memoryName`: Name of the target memory +- `url`: Array of URLs to crawl +- `maxPages` (optional): Maximum pages to crawl per URL +- `apiKey`: API key for the crawl service +- `service` (optional): Crawl service to use ('spider' or 'firecrawl') +- `documentNamePrefix` (optional): Prefix for generated document names +- `meta` (optional): Metadata applied to all uploaded documents + +## Return Values + +- `uploadText()`: Returns a `Promise` with the upload response +- `uploadFromSearch()`: Returns a `Promise` with responses for each search result +- `uploadFromCrawl()`: Returns a `Promise` with responses for each crawled page + +## Content Format + +For search and crawl methods, the uploaded content includes both the URL and the content for better context: + +``` +URL: https://example.com/page + +Content: +[Page content here...] +``` + +## Metadata + +All methods automatically add source metadata to help track the origin of content: + +- `uploadFromSearch()` adds: `source: 'web-search'`, `query`, `url`, `searchService` +- `uploadFromCrawl()` adds: `source: 'web-crawl'`, `url`, `crawlService`, `domain` + +## Examples + +See the `/examples/nodejs/memory/` directory for complete working examples: + +- `memory.upload-text.ts`: Basic text upload example +- `memory.upload-from-search.ts`: Web search and upload example +- `memory.upload-from-crawl.ts`: Web crawl and upload example \ No newline at end of file diff --git a/examples/nodejs/memory/memory.upload-from-crawl.ts b/examples/nodejs/memory/memory.upload-from-crawl.ts new file mode 100644 index 0000000..3be09b1 --- /dev/null +++ b/examples/nodejs/memory/memory.upload-from-crawl.ts @@ -0,0 +1,30 @@ +import 'dotenv/config'; +import {Langbase} from 'langbase'; + +const langbase = new Langbase({ + apiKey: process.env.LANGBASE_API_KEY!, +}); + +async function main() { + const responses = await langbase.memories.uploadFromCrawl({ + memoryName: 'memory-sdk', + url: ['https://langbase.com', 'https://langbase.com/about'], + maxPages: 1, + apiKey: process.env.CRAWL_KEY!, + service: 'spider', // or 'firecrawl' + documentNamePrefix: 'langbase-crawl', + meta: { + type: 'web-crawl', + source: 'spider-api', + topic: 'langbase-website', + }, + }); + + console.log(`Uploaded ${responses.length} crawled pages to memory`); + responses.forEach((response, index) => { + console.log(`Document ${index + 1} upload status:`, response.status); + console.log(`Document ${index + 1} upload successful:`, response.ok); + }); +} + +main().catch(console.error); \ No newline at end of file diff --git a/examples/nodejs/memory/memory.upload-from-search.ts b/examples/nodejs/memory/memory.upload-from-search.ts new file mode 100644 index 0000000..3642668 --- /dev/null +++ b/examples/nodejs/memory/memory.upload-from-search.ts @@ -0,0 +1,31 @@ +import 'dotenv/config'; +import {Langbase} from 'langbase'; + +const langbase = new Langbase({ + apiKey: process.env.LANGBASE_API_KEY!, +}); + +async function main() { + const responses = await langbase.memories.uploadFromSearch({ + memoryName: 'memory-sdk', + query: 'What is Langbase?', + service: 'exa', + totalResults: 2, + domains: ['https://langbase.com'], + apiKey: process.env.EXA_API_KEY!, // Find Exa key: https://dashboard.exa.ai/api-keys + documentNamePrefix: 'langbase-info', + meta: { + type: 'web-search', + source: 'exa-api', + topic: 'langbase-info', + }, + }); + + console.log(`Uploaded ${responses.length} search results to memory`); + responses.forEach((response, index) => { + console.log(`Document ${index + 1} upload status:`, response.status); + console.log(`Document ${index + 1} upload successful:`, response.ok); + }); +} + +main().catch(console.error); \ No newline at end of file diff --git a/examples/nodejs/memory/memory.upload-text.ts b/examples/nodejs/memory/memory.upload-text.ts new file mode 100644 index 0000000..56d1c4c --- /dev/null +++ b/examples/nodejs/memory/memory.upload-text.ts @@ -0,0 +1,37 @@ +import 'dotenv/config'; +import {Langbase} from 'langbase'; + +const langbase = new Langbase({ + apiKey: process.env.LANGBASE_API_KEY!, +}); + +async function main() { + const textContent = ` +# Langbase SDK Memory Upload Example + +This is a sample text that demonstrates how to upload text directly to memory +without needing to handle files. This feature allows you to: + +- Upload plain text content to memory +- Add metadata to the uploaded text +- Automatically generate document names or provide custom ones + +This makes it easy to add content from variables, API responses, or any other text source. + `.trim(); + + const response = await langbase.memories.uploadText({ + memoryName: 'memory-sdk', + text: textContent, + documentName: 'sdk-demo-text.txt', + meta: { + type: 'demo', + source: 'sdk-example', + description: 'Example of uploadText functionality', + }, + }); + + console.log('Text upload response status:', response.status); + console.log('Upload successful:', response.ok); +} + +main().catch(console.error); \ No newline at end of file diff --git a/packages/langbase/src/langbase/langbase.ts b/packages/langbase/src/langbase/langbase.ts index b6e886f..c635651 100644 --- a/packages/langbase/src/langbase/langbase.ts +++ b/packages/langbase/src/langbase/langbase.ts @@ -366,6 +366,34 @@ export interface MemoryRetryDocEmbedOptions { documentName: string; } +export interface MemoryUploadTextOptions { + memoryName: string; + text: string; + documentName?: string; + meta?: Record; +} + +export interface MemoryUploadFromSearchOptions { + memoryName: string; + query: string; + service: 'exa'; + totalResults?: number; + domains?: string[]; + apiKey: string; + documentNamePrefix?: string; + meta?: Record; +} + +export interface MemoryUploadFromCrawlOptions { + memoryName: string; + url: string[]; + maxPages?: number; + apiKey: string; + service?: 'spider' | 'firecrawl'; + documentNamePrefix?: string; + meta?: Record; +} + export interface MemoryCreateResponse extends MemoryBaseResponse { chunk_size: number; chunk_overlap: number; @@ -549,6 +577,9 @@ export class Langbase { options: MemoryRetrieveOptions, ) => Promise; list: () => Promise; + uploadText: (options: MemoryUploadTextOptions) => Promise; + uploadFromSearch: (options: MemoryUploadFromSearchOptions) => Promise; + uploadFromCrawl: (options: MemoryUploadFromCrawlOptions) => Promise; documents: { list: ( options: MemoryListDocOptions, @@ -691,6 +722,9 @@ export class Langbase { delete: this.deleteMemory.bind(this), retrieve: this.retrieveMemory.bind(this), list: this.listMemory.bind(this), + uploadText: this.uploadText.bind(this), + uploadFromSearch: this.uploadFromSearch.bind(this), + uploadFromCrawl: this.uploadFromCrawl.bind(this), documents: { list: this.listDocs.bind(this), delete: this.deleteDoc.bind(this), @@ -949,6 +983,157 @@ export class Langbase { } } + /** + * Uploads text content directly to memory without file handling. + * + * @param {MemoryUploadTextOptions} options - The options for uploading text content. + * @param {string} options.memoryName - The name of the memory to upload the text to. + * @param {string} options.text - The text content to upload. + * @param {string} [options.documentName] - Optional name for the document. If not provided, generates from timestamp. + * @param {Record} [options.meta] - Optional metadata associated with the text. + * @returns {Promise} The response from the upload request. + * @throws Will throw an error if the upload fails. + */ + private async uploadText( + options: MemoryUploadTextOptions, + ): Promise { + try { + // Generate document name if not provided + const documentName = options.documentName || `text-${Date.now()}.txt`; + + const response = (await this.request.post({ + endpoint: `/v1/memory/documents`, + body: { + memoryName: options.memoryName, + fileName: documentName, + meta: options.meta, + }, + })) as unknown as {signedUrl: string}; + + const uploadUrl = response.signedUrl; + + return await fetch(uploadUrl, { + method: 'PUT', + headers: { + Authorization: `Bearer ${this.apiKey}`, + 'Content-Type': 'text/plain', + }, + body: options.text, + }); + } catch (error) { + throw error; + } + } + + /** + * Searches the web and uploads results to memory. + * + * @param {MemoryUploadFromSearchOptions} options - The options for searching and uploading to memory. + * @param {string} options.memoryName - The name of the memory to upload the search results to. + * @param {string} options.query - The search query to perform. + * @param {string} options.service - The search service to use (currently only 'exa'). + * @param {number} [options.totalResults] - The number of search results to retrieve. + * @param {string[]} [options.domains] - Optional list of domains to restrict search to. + * @param {string} options.apiKey - API key for the search service. + * @param {string} [options.documentNamePrefix] - Optional prefix for document names. + * @param {Record} [options.meta] - Optional metadata for all uploaded documents. + * @returns {Promise} Array of responses from the upload requests. + * @throws Will throw an error if the search or upload fails. + */ + private async uploadFromSearch( + options: MemoryUploadFromSearchOptions, + ): Promise { + try { + // Perform web search + const searchResults = await this.webSearch({ + query: options.query, + service: options.service, + totalResults: options.totalResults, + domains: options.domains, + apiKey: options.apiKey, + }); + + // Upload each search result to memory + const uploadPromises = searchResults.map(async (result, index) => { + const documentName = `${options.documentNamePrefix || 'search'}-${index + 1}-${Date.now()}.txt`; + + // Combine URL and content for better context + const content = `URL: ${result.url}\n\nContent:\n${result.content}`; + + return this.uploadText({ + memoryName: options.memoryName, + text: content, + documentName: documentName, + meta: { + ...options.meta, + source: 'web-search', + query: options.query, + url: result.url, + searchService: options.service, + }, + }); + }); + + return await Promise.all(uploadPromises); + } catch (error) { + throw error; + } + } + + /** + * Crawls URLs and uploads the content to memory. + * + * @param {MemoryUploadFromCrawlOptions} options - The options for crawling and uploading to memory. + * @param {string} options.memoryName - The name of the memory to upload the crawled content to. + * @param {string[]} options.url - Array of URLs to crawl. + * @param {number} [options.maxPages] - Maximum number of pages to crawl per URL. + * @param {string} options.apiKey - API key for the crawl service. + * @param {'spider' | 'firecrawl'} [options.service] - The crawl service to use. + * @param {string} [options.documentNamePrefix] - Optional prefix for document names. + * @param {Record} [options.meta] - Optional metadata for all uploaded documents. + * @returns {Promise} Array of responses from the upload requests. + * @throws Will throw an error if the crawl or upload fails. + */ + private async uploadFromCrawl( + options: MemoryUploadFromCrawlOptions, + ): Promise { + try { + // Perform web crawl + const crawlResults = await this.webCrawl({ + url: options.url, + maxPages: options.maxPages, + apiKey: options.apiKey, + service: options.service, + }); + + // Upload each crawl result to memory + const uploadPromises = crawlResults.map(async (result, index) => { + const urlDomain = new URL(result.url).hostname.replace('www.', ''); + const documentName = `${options.documentNamePrefix || 'crawl'}-${urlDomain}-${index + 1}-${Date.now()}.txt`; + + // Combine URL and content for better context + const content = `URL: ${result.url}\n\nContent:\n${result.content}`; + + return this.uploadText({ + memoryName: options.memoryName, + text: content, + documentName: documentName, + meta: { + ...options.meta, + source: 'web-crawl', + url: result.url, + crawlService: options.service || 'spider', + domain: urlDomain, + }, + }); + }); + + return await Promise.all(uploadPromises); + } catch (error) { + throw error; + } + } + /** * Retries the embedding process for a specific document in memory. * diff --git a/packages/langbase/src/langbase/memory-new-methods.test.ts b/packages/langbase/src/langbase/memory-new-methods.test.ts new file mode 100644 index 0000000..19be9ae --- /dev/null +++ b/packages/langbase/src/langbase/memory-new-methods.test.ts @@ -0,0 +1,223 @@ +import {beforeEach, describe, expect, it, vi} from 'vitest'; +import {Langbase} from '../langbase/langbase'; + +// Mock fetch globally +const mockFetch = vi.fn(); +global.fetch = mockFetch; + +// Mock the Request class +vi.mock('../common/request'); + +describe('Memory - New Methods', () => { + let langbase: Langbase; + const mockApiKey = 'test-api-key'; + + beforeEach(() => { + langbase = new Langbase({apiKey: mockApiKey}); + vi.resetAllMocks(); + }); + + describe('uploadText', () => { + it('should upload text content to memory', async () => { + const mockSignedUrl = 'https://test-upload-url.com'; + const mockResponse = {signedUrl: mockSignedUrl}; + const mockUploadResponse = new Response(null, {status: 200}); + + // Mock the request to get signed URL + const mockPost = vi.fn().mockResolvedValue(mockResponse); + (langbase as any).request = {post: mockPost}; + + // Mock fetch for the actual upload + mockFetch.mockResolvedValue(mockUploadResponse); + + const options = { + memoryName: 'test-memory', + text: 'Hello, this is test content', + documentName: 'test-doc.txt', + meta: {type: 'test'}, + }; + + const result = await langbase.memories.uploadText(options); + + // Check that signed URL request was made correctly + expect(mockPost).toHaveBeenCalledWith({ + endpoint: '/v1/memory/documents', + body: { + memoryName: 'test-memory', + fileName: 'test-doc.txt', + meta: {type: 'test'}, + }, + }); + + // Check that fetch was called with correct parameters + expect(mockFetch).toHaveBeenCalledWith(mockSignedUrl, { + method: 'PUT', + headers: { + Authorization: `Bearer ${mockApiKey}`, + 'Content-Type': 'text/plain', + }, + body: 'Hello, this is test content', + }); + + expect(result).toBe(mockUploadResponse); + }); + + it('should generate document name if not provided', async () => { + const mockSignedUrl = 'https://test-upload-url.com'; + const mockResponse = {signedUrl: mockSignedUrl}; + const mockUploadResponse = new Response(null, {status: 200}); + + const mockPost = vi.fn().mockResolvedValue(mockResponse); + (langbase as any).request = {post: mockPost}; + mockFetch.mockResolvedValue(mockUploadResponse); + + const options = { + memoryName: 'test-memory', + text: 'Hello, this is test content', + }; + + await langbase.memories.uploadText(options); + + // Check that a document name was generated + const callArgs = mockPost.mock.calls[0][0]; + expect(callArgs.body.fileName).toMatch(/^text-\d+\.txt$/); + }); + }); + + describe('uploadFromSearch', () => { + it('should search and upload results to memory', async () => { + const mockSearchResults = [ + {url: 'https://example.com/1', content: 'First result content'}, + {url: 'https://example.com/2', content: 'Second result content'}, + ]; + + const mockSignedUrl = 'https://test-upload-url.com'; + const mockResponse = {signedUrl: mockSignedUrl}; + const mockUploadResponse = new Response(null, {status: 200}); + + // Mock webSearch method + vi.spyOn(langbase as any, 'webSearch').mockResolvedValue(mockSearchResults); + + // Mock request for signed URLs + const mockPost = vi.fn().mockResolvedValue(mockResponse); + (langbase as any).request = {post: mockPost}; + + // Mock fetch for uploads + mockFetch.mockResolvedValue(mockUploadResponse); + + const options = { + memoryName: 'test-memory', + query: 'test query', + service: 'exa' as const, + totalResults: 2, + apiKey: 'test-search-key', + documentNamePrefix: 'search-result', + meta: {type: 'search'}, + }; + + const results = await langbase.memories.uploadFromSearch(options); + + // Check that webSearch was called correctly + expect((langbase as any).webSearch).toHaveBeenCalledWith({ + query: 'test query', + service: 'exa', + totalResults: 2, + domains: undefined, + apiKey: 'test-search-key', + }); + + // Check that we got results for each search result + expect(results).toHaveLength(2); + expect(results[0]).toBe(mockUploadResponse); + expect(results[1]).toBe(mockUploadResponse); + + // Check that signed URL requests were made for each result + expect(mockPost).toHaveBeenCalledTimes(2); + + // Check that fetch was called for each result + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + }); + + describe('uploadFromCrawl', () => { + it('should crawl URLs and upload content to memory', async () => { + const mockCrawlResults = [ + {url: 'https://example.com/', content: 'Homepage content'}, + {url: 'https://example.com/about', content: 'About page content'}, + ]; + + const mockSignedUrl = 'https://test-upload-url.com'; + const mockResponse = {signedUrl: mockSignedUrl}; + const mockUploadResponse = new Response(null, {status: 200}); + + // Mock webCrawl method + vi.spyOn(langbase as any, 'webCrawl').mockResolvedValue(mockCrawlResults); + + // Mock request for signed URLs + const mockPost = vi.fn().mockResolvedValue(mockResponse); + (langbase as any).request = {post: mockPost}; + + // Mock fetch for uploads + mockFetch.mockResolvedValue(mockUploadResponse); + + const options = { + memoryName: 'test-memory', + url: ['https://example.com/', 'https://example.com/about'], + maxPages: 1, + apiKey: 'test-crawl-key', + service: 'spider' as const, + documentNamePrefix: 'crawl-result', + meta: {type: 'crawl'}, + }; + + const results = await langbase.memories.uploadFromCrawl(options); + + // Check that webCrawl was called correctly + expect((langbase as any).webCrawl).toHaveBeenCalledWith({ + url: ['https://example.com/', 'https://example.com/about'], + maxPages: 1, + apiKey: 'test-crawl-key', + service: 'spider', + }); + + // Check that we got results for each crawl result + expect(results).toHaveLength(2); + expect(results[0]).toBe(mockUploadResponse); + expect(results[1]).toBe(mockUploadResponse); + + // Check that signed URL requests were made for each result + expect(mockPost).toHaveBeenCalledTimes(2); + + // Check that fetch was called for each result + expect(mockFetch).toHaveBeenCalledTimes(2); + }); + + it('should handle URL domain extraction correctly', async () => { + const mockCrawlResults = [ + {url: 'https://www.example.com/page', content: 'Page content'}, + ]; + + const mockSignedUrl = 'https://test-upload-url.com'; + const mockResponse = {signedUrl: mockSignedUrl}; + const mockUploadResponse = new Response(null, {status: 200}); + + vi.spyOn(langbase as any, 'webCrawl').mockResolvedValue(mockCrawlResults); + + const mockPost = vi.fn().mockResolvedValue(mockResponse); + (langbase as any).request = {post: mockPost}; + mockFetch.mockResolvedValue(mockUploadResponse); + + const options = { + memoryName: 'test-memory', + url: ['https://www.example.com/page'], + apiKey: 'test-crawl-key', + }; + + await langbase.memories.uploadFromCrawl(options); + + // Check that the document name includes the domain without 'www.' + const callArgs = mockPost.mock.calls[0][0]; + expect(callArgs.body.fileName).toMatch(/^crawl-example\.com-\d+-\d+\.txt$/); + }); + }); +}); \ No newline at end of file diff --git a/packages/langbase/vitest.node.config.js b/packages/langbase/vitest.node.config.js index d2f853c..5857ae1 100644 --- a/packages/langbase/vitest.node.config.js +++ b/packages/langbase/vitest.node.config.js @@ -1,7 +1,13 @@ import {defineConfig} from 'vite'; +import path from 'path'; // https://vitejs.dev/config/ export default defineConfig({ + resolve: { + alias: { + '@': path.resolve(__dirname, './src'), + }, + }, test: { environment: 'node', globals: true,