diff --git a/.changeset/add-rss-atom-collection.md b/.changeset/add-rss-atom-collection.md new file mode 100644 index 00000000..f6c85a41 --- /dev/null +++ b/.changeset/add-rss-atom-collection.md @@ -0,0 +1,18 @@ +--- +"@tanstack/rss-db-collection": patch +--- + +Add RSS and Atom feed collections for TanStack DB + +Introduces `@tanstack/rss-db-collection` package with: + +- `rssCollectionOptions()` for RSS 2.0 feeds +- `atomCollectionOptions()` for Atom 1.0 feeds +- Automatic polling with configurable intervals +- Built-in deduplication based on feed item IDs +- Custom transform functions for data normalization +- Full TypeScript support with proper type inference +- Error recovery and robust feed parsing +- HTTP configuration options for headers and timeouts + +Both collection types provide seamless integration with TanStack DB's live queries and optimistic mutations, allowing you to sync RSS/Atom feed data and query it alongside other collection types. diff --git a/docs/collections/rss-collection.md b/docs/collections/rss-collection.md new file mode 100644 index 00000000..e5919396 --- /dev/null +++ b/docs/collections/rss-collection.md @@ -0,0 +1,310 @@ +--- +title: RSS Collection +--- + +# RSS Collection + +RSS and Atom feed collections provide seamless integration between TanStack DB and RSS/Atom feeds, enabling automatic synchronization with syndicated content from blogs, news sites, and other content sources. + +## Overview + +The `@tanstack/rss-db-collection` package allows you to create collections that: +- Automatically sync with RSS 2.0 and Atom 1.0 feeds +- Support smart polling with configurable intervals +- Provide content-aware deduplication +- Handle RFC-compliant date parsing +- Support custom transform functions for data normalization + +## Installation + +```bash +npm install @tanstack/rss-db-collection @tanstack/db +``` + +## Basic Usage + +### RSS Collection + +```typescript +import { createCollection } from "@tanstack/db" +import { rssCollectionOptions } from "@tanstack/rss-db-collection" + +const rssFeed = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://blog.example.com/rss.xml", + pollingInterval: 5 * 60 * 1000, // Poll every 5 minutes + getKey: (item) => item.guid || item.link, + transform: (item) => ({ + id: item.guid || item.link || "", + title: item.title || "", + description: item.description || "", + link: item.link || "", + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + }), +}) +``` + +### Atom Collection + +```typescript +import { createCollection } from "@tanstack/db" +import { atomCollectionOptions } from "@tanstack/rss-db-collection" + +const atomFeed = createCollection({ + ...atomCollectionOptions({ + feedUrl: "https://blog.example.com/atom.xml", + pollingInterval: 5 * 60 * 1000, // Poll every 5 minutes + getKey: (item) => item.id, + transform: (item) => ({ + id: item.id || "", + title: typeof item.title === "string" ? item.title : item.title?.$text || "", + description: typeof item.summary === "string" ? item.summary : item.summary?.$text || "", + link: typeof item.link === "string" ? item.link : item.link?.href || "", + publishedAt: new Date(item.published || item.updated || Date.now()), + author: typeof item.author === "object" ? item.author?.name : item.author, + }), + }), +}) +``` + +## Configuration Options + +The `rssCollectionOptions` and `atomCollectionOptions` functions accept the following options: + +### Required Options + +- `feedUrl`: The RSS or Atom feed URL to fetch from +- `getKey`: Function to extract the unique key from an item + +### Optional Options + +- `pollingInterval`: Polling interval in milliseconds (default: 5 minutes, or based on feed metadata) +- `startPolling`: Whether to start polling immediately (default: `true`) +- `maxSeenItems`: Maximum items to track for deduplication (default: 1000) + +### HTTP Configuration + +- `httpOptions.timeout`: Request timeout in milliseconds (default: 30000) +- `httpOptions.userAgent`: Custom user agent string +- `httpOptions.headers`: Additional HTTP headers + +### Transform Function + +- `transform`: Custom function to normalize feed items to your desired format + +### Standard Collection Options + +- `id`: Unique identifier for the collection +- `schema`: Schema for validating items +- `onInsert`: Handler called when new items are discovered +- `onUpdate`: Handler called when existing items are updated +- `onDelete`: Handler called when items are deleted + +## Smart Features + +### Smart Polling Intervals + +The RSS collection automatically detects optimal polling intervals based on feed metadata: + +```typescript +// The collection will automatically detect and use appropriate intervals +const feed = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://blog.example.com/feed.xml", + // No pollingInterval specified - will use 5 minutes default or sy:updatePeriod if available + }), +}) +``` + +### Content-Aware Deduplication + +Unlike simple GUID-based deduplication, this collection detects when feed items with the same GUID have changed content and treats them as updates: + +- **New Items**: Items with unseen GUIDs are inserted +- **Content Changes**: Items with existing GUIDs but changed content are updated +- **No Changes**: Items with existing GUIDs and unchanged content are ignored + +### RFC-Compliant Date Parsing + +The collection uses strict RFC 2822 (RSS) and RFC 3339 (Atom) date parsing to avoid locale-dependent issues: + +```typescript +import { parseFeedDate } from "@tanstack/rss-db-collection" + +// Handles various date formats reliably +const date1 = parseFeedDate("Mon, 25 Dec 2023 10:30:00 GMT") // RFC 2822 +const date2 = parseFeedDate("2023-12-25T10:30:00Z") // RFC 3339 +``` + +## Advanced Usage + +### Custom Transform Function + +```typescript +const newsCollection = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://news.example.com/rss.xml", + getKey: (item) => item.guid || item.link, + transform: (item) => ({ + id: item.guid || item.link || "", + headline: item.title || "", + content: item.description || "", + url: item.link || "", + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + tags: Array.isArray(item.category) ? item.category : [item.category].filter(Boolean), + }), + }), +}) +``` + +### With Mutation Handlers + +```typescript +const blogCollection = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://myblog.com/rss.xml", + getKey: (item) => item.guid || item.link, + pollingInterval: 10 * 60 * 1000, // 10 minutes + + onInsert: async ({ transaction }) => { + const newPosts = transaction.mutations.map((m) => m.modified) + console.log(`New blog posts: ${newPosts.map((p) => p.title).join(", ")}`) + await sendNewPostNotifications(newPosts) + }, + + onUpdate: async ({ transaction }) => { + const updates = transaction.mutations.map((m) => ({ + id: m.key, + changes: m.changes, + })) + await syncUpdatesToServer(updates) + }, + }), +}) +``` + +### Manual Refresh + +```typescript +const collection = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://example.com/feed.xml", + getKey: (item) => item.guid || item.link, + startPolling: false, // Don't start automatically + }), +}) + +// Manually refresh the feed +await collection.utils.refresh() + +// Get status and clear cache +console.log(`Seen items: ${collection.utils.getSeenItemsCount()}`) +collection.utils.clearSeenItems() +``` + +### Schema Integration + +```typescript +import { z } from "zod" + +const blogPostSchema = z.object({ + id: z.string(), + title: z.string(), + description: z.string(), + link: z.string().url(), + publishedAt: z.date(), + author: z.string().optional(), +}) + +const typedBlogCollection = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://blog.example.com/feed.xml", + schema: blogPostSchema, // Automatic type inference + getKey: (item) => item.guid || item.link, + transform: (item) => ({ + id: item.guid || item.link || "", + title: item.title || "", + description: item.description || "", + link: item.link || "", + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + }), +}) +``` + +## Error Handling + +Both RSS and Atom collections handle various error scenarios gracefully: + +```typescript +const resilientCollection = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://unreliable-feed.com/rss.xml", + getKey: (item) => item.guid || item.link, + pollingInterval: 60000, // 1 minute - will retry on errors + + httpOptions: { + timeout: 10000, // 10 second timeout + headers: { + "User-Agent": "My App/1.0", + }, + }, + + onInsert: async ({ transaction }) => { + try { + await processNewItems(transaction.mutations.map((m) => m.modified)) + } catch (error) { + console.error("Failed to process items:", error) + // Error handling - the collection will continue working + } + }, + }), +}) +``` + +Common error scenarios handled: +- Network timeouts and failures +- Invalid XML or malformed feeds +- HTTP error responses (404, 500, etc.) +- Feed parsing errors +- Transform function errors +- Mutation handler errors + +## Utility Methods + +The collection provides these utility methods via `collection.utils`: + +- `refresh()`: Manually refresh the feed data +- `clearSeenItems()`: Clear the deduplication cache +- `getSeenItemsCount()`: Get the number of tracked items + +## Performance Considerations + +### Memory Management + +- **Deduplication Cache**: Limited by `maxSeenItems` (default: 1000) +- **Automatic Cleanup**: Old items are cleaned up after 10 polling cycles +- **Memory-Efficient**: Only tracks item IDs, not full content + +### Network Optimization + +- **Conditional Requests**: Respects HTTP caching headers +- **Timeout Management**: Configurable timeouts prevent hanging requests +- **Error Recovery**: Continues polling after network failures + +### Polling Best Practices + +```typescript +// Good: Reasonable polling intervals +pollingInterval: 5 * 60 * 1000 // 5 minutes + +// Avoid: Too frequent polling +pollingInterval: 10 * 1000 // 10 seconds - may overwhelm server + +// Consider: Feed update frequency +pollingInterval: 60 * 60 * 1000 // 1 hour for infrequently updated feeds +``` diff --git a/docs/config.json b/docs/config.json index 546b1b44..556b2430 100644 --- a/docs/config.json +++ b/docs/config.json @@ -84,6 +84,10 @@ { "label": "Query Collection", "to": "collections/query-collection" + }, + { + "label": "RSS Collection", + "to": "collections/rss-collection" } ] }, diff --git a/docs/overview.md b/docs/overview.md index 92ed9819..22ed2969 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -154,8 +154,9 @@ There are a number of built-in collection types: 1. [`QueryCollection`](#querycollection) to load data into collections using [TanStack Query](https://tanstack.com/query) 2. [`ElectricCollection`](#electriccollection) to sync data into collections using [ElectricSQL](https://electric-sql.com) 3. [`TrailBaseCollection`](#trailbasecollection) to sync data into collections using [TrailBase](https://trailbase.io) -4. [`LocalStorageCollection`](#localstoragecollection) for small amounts of local-only state that syncs across browser tabs -5. [`LocalOnlyCollection`](#localonlycollection) for in-memory client data or UI state +4. [`RSSCollection` and `AtomCollection`](#rsscollection-and-atomcollection) to sync data from RSS and Atom feeds with automatic polling, deduplication, and type safety +5. [`LocalStorageCollection`](#localstoragecollection) for small amounts of local-only state that syncs across browser tabs +6. [`LocalOnlyCollection`](#localonlycollection) for in-memory client data or UI state You can also use: @@ -297,6 +298,84 @@ This collection requires the following TrailBase-specific options: A new collections doesn't start syncing until you call `collection.preload()` or you query it. +#### `RSSCollection` and `AtomCollection` + +RSS and Atom feeds are widely used syndication formats for publishing frequently updated content like blogs, news, and podcasts. TanStack DB provides dedicated collection types for both RSS 2.0 and Atom 1.0 feeds with automatic polling, deduplication, and type safety. + +Use `rssCollectionOptions` for RSS feeds or `atomCollectionOptions` for Atom feeds to sync feed data into collections: + +```ts +import { createCollection } from "@tanstack/react-db" +import { rssCollectionOptions, atomCollectionOptions } from "@tanstack/rss-db-collection" + +// RSS Collection +export const blogFeed = createCollection( + rssCollectionOptions({ + id: "blog-posts", + feedUrl: "https://blog.example.com/rss.xml", + pollingInterval: 5 * 60 * 1000, // Poll every 5 minutes + getKey: (item) => item.guid || item.link, + transform: (item) => ({ + id: item.guid || item.link || '', + title: item.title || '', + description: item.description || '', + link: item.link || '', + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author + }), + schema: blogPostSchema, + }) +) + +// Atom Collection +export const newsFeed = createCollection( + atomCollectionOptions({ + id: "news-items", + feedUrl: "https://news.example.com/atom.xml", + pollingInterval: 10 * 60 * 1000, // Poll every 10 minutes + getKey: (item) => item.id, + transform: (item) => ({ + id: item.id || '', + title: typeof item.title === 'string' ? item.title : item.title?.$text || '', + description: typeof item.summary === 'string' ? item.summary : item.summary?.$text || '', + link: typeof item.link === 'string' ? item.link : item.link?.href || '', + publishedAt: new Date(item.published || item.updated || Date.now()), + author: typeof item.author === 'object' ? item.author?.name : item.author + }), + schema: newsItemSchema, + }) +) +``` + +Both collection types require: + +- `feedUrl` — the RSS or Atom feed URL to fetch from +- `getKey` — identifies the unique ID for feed items +- `pollingInterval` — how frequently to check for new items (default: 5 minutes) + +Optional configuration includes: + +- `transform` — custom function to normalize feed items to your desired format +- `httpOptions` — custom headers, timeout, and user agent settings +- `startPolling` — whether to begin polling immediately (default: true) +- `maxSeenItems` — maximum items to track for deduplication (default: 1000) + +RSS and Atom collections automatically handle feed parsing, deduplication of items, and provide built-in error recovery. The collections will continue polling even after network failures or parsing errors. + +Collections can be manually refreshed when needed: + +```ts +// Manually refresh the feed data +await blogFeed.utils.refresh() + +// Clear deduplication cache if needed +blogFeed.utils.clearSeenItems() + +// Check how many items have been tracked +console.log(`Tracked items: ${blogFeed.utils.getSeenItemsCount()}`) +``` + +For detailed documentation on RSS and Atom collections, including advanced features like smart polling intervals, content-aware deduplication, and RFC-compliant date parsing, see the [RSS Collection documentation](collections/rss-collection.md). #### `LocalStorageCollection` diff --git a/packages/db-ivm/src/index.ts b/packages/db-ivm/src/index.ts index 5a605a1b..5e0b5822 100644 --- a/packages/db-ivm/src/index.ts +++ b/packages/db-ivm/src/index.ts @@ -2,3 +2,11 @@ export * from "./d2.js" export * from "./multiset.js" export * from "./operators/index.js" export * from "./types.js" + +// Export additional types and functions that are needed +export type { MultiSetArray } from "./multiset.js" +export { MultiSet } from "./multiset.js" +export type { IStreamBuilder, KeyValue } from "./types.js" +export { RootStreamBuilder } from "./d2.js" +export { orderByWithFractionalIndex } from "./operators/orderBy.js" +export type { JoinType } from "./operators/join.js" diff --git a/packages/rss-db-collection/README.md b/packages/rss-db-collection/README.md new file mode 100644 index 00000000..fc4132f3 --- /dev/null +++ b/packages/rss-db-collection/README.md @@ -0,0 +1,55 @@ +# @tanstack/rss-db-collection + +RSS/Atom feed collection for TanStack DB - sync data from RSS and Atom feeds with automatic polling, deduplication, and type safety. + +## Features + +- **📡 RSS & Atom Support**: Dedicated option creators for RSS 2.0 and Atom 1.0 feeds +- **🔄 Smart Polling**: Configurable polling intervals with automatic detection based on feed metadata +- **✨ Content-Aware Deduplication**: Built-in deduplication that detects content changes for existing GUIDs +- **📅 RFC-Compliant Date Parsing**: Strict RFC 2822/3339 date parsing for reliable timezone handling +- **🔧 Transform Functions**: Custom transform functions to normalize feed data to your schema +- **📝 Full TypeScript Support**: Complete type safety with schema inference +- **🎛️ Mutation Handlers**: Support for `onInsert`, `onUpdate`, and `onDelete` callbacks +- **⚡ Optimistic Updates**: Seamless integration with TanStack DB's optimistic update system + +## Installation + +```bash +npm install @tanstack/rss-db-collection +# or +pnpm add @tanstack/rss-db-collection +# or +yarn add @tanstack/rss-db-collection +``` + +## Quick Start + +```typescript +import { createCollection } from "@tanstack/db" +import { rssCollectionOptions } from "@tanstack/rss-db-collection" + +const blogFeed = createCollection({ + ...rssCollectionOptions({ + feedUrl: "https://blog.example.com/rss.xml", + pollingInterval: 5 * 60 * 1000, // Poll every 5 minutes + getKey: (item) => item.guid || item.link, + transform: (item) => ({ + id: item.guid || item.link || "", + title: item.title || "", + description: item.description || "", + link: item.link || "", + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + }), +}) +``` + +## Documentation + +For complete documentation, examples, and API reference, visit the [TanStack DB documentation](https://tanstack.com/db/latest/docs/overview). + +## License + +MIT diff --git a/packages/rss-db-collection/package.json b/packages/rss-db-collection/package.json new file mode 100644 index 00000000..3eab412d --- /dev/null +++ b/packages/rss-db-collection/package.json @@ -0,0 +1,62 @@ +{ + "name": "@tanstack/rss-db-collection", + "description": "RSS/Atom feed collection for TanStack DB", + "version": "0.1.0", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "@tanstack/db": "workspace:*", + "debug": "^4.4.1", + "fast-xml-parser": "^4.5.0" + }, + "devDependencies": { + "@types/debug": "^4.1.12", + "@vitest/coverage-istanbul": "^3.0.9" + }, + "exports": { + ".": { + "import": { + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/cjs/index.d.cts", + "default": "./dist/cjs/index.cjs" + } + }, + "./package.json": "./package.json" + }, + "files": [ + "dist", + "src" + ], + "main": "dist/cjs/index.cjs", + "module": "dist/esm/index.js", + "packageManager": "pnpm@10.6.3", + "peerDependencies": { + "typescript": ">=4.7" + }, + "author": "Claude AI", + "license": "MIT", + "repository": { + "type": "git", + "url": "https://github.com/TanStack/db.git", + "directory": "packages/rss-db-collection" + }, + "homepage": "https://tanstack.com/db", + "keywords": [ + "rss", + "atom", + "feed", + "polling", + "typescript" + ], + "scripts": { + "build": "vite build", + "dev": "vite build --watch", + "lint": "eslint . --fix", + "test": "npx vitest --run" + }, + "sideEffects": false, + "type": "module", + "types": "dist/esm/index.d.ts" +} diff --git a/packages/rss-db-collection/src/errors.ts b/packages/rss-db-collection/src/errors.ts new file mode 100644 index 00000000..ee561dcb --- /dev/null +++ b/packages/rss-db-collection/src/errors.ts @@ -0,0 +1,81 @@ +/** + * Base error class for RSS Collection errors + */ +export abstract class RSSCollectionError extends Error { + constructor(message: string) { + super(message) + this.name = this.constructor.name + } +} + +/** + * Error thrown when feed URL is required but not provided + */ +export class FeedURLRequiredError extends RSSCollectionError { + constructor() { + super(`Feed URL is required for RSS collection`) + } +} + +/** + * Error thrown when polling interval is invalid + */ +export class InvalidPollingIntervalError extends RSSCollectionError { + constructor(interval: number) { + super( + `Invalid polling interval: ${interval}. Must be a positive number in milliseconds.` + ) + } +} + +/** + * Error thrown when feed parsing fails + */ +export class FeedParsingError extends RSSCollectionError { + constructor(url: string, originalError: Error) { + super(`Failed to parse feed from ${url}: ${originalError.message}`) + this.cause = originalError + } +} + +/** + * Error thrown when feed fetch fails + */ +export class FeedFetchError extends RSSCollectionError { + constructor(url: string, status?: number) { + super( + status + ? `Failed to fetch feed from ${url}: HTTP ${status}` + : `Failed to fetch feed from ${url}` + ) + } +} + +/** + * Error thrown when timeout occurs while fetching feed + */ +export class FeedTimeoutError extends RSSCollectionError { + constructor(url: string, timeout: number) { + super(`Timeout after ${timeout}ms while fetching feed from ${url}`) + } +} + +/** + * Error thrown when feed format is not supported + */ +export class UnsupportedFeedFormatError extends RSSCollectionError { + constructor(url: string) { + super( + `Unsupported feed format from ${url}. Only RSS and Atom feeds are supported.` + ) + } +} + +/** + * Error thrown when required getKey function is not provided + */ +export class GetKeyRequiredError extends RSSCollectionError { + constructor() { + super(`getKey function is required for RSS collection`) + } +} diff --git a/packages/rss-db-collection/src/index.ts b/packages/rss-db-collection/src/index.ts new file mode 100644 index 00000000..f572bd5b --- /dev/null +++ b/packages/rss-db-collection/src/index.ts @@ -0,0 +1,91 @@ +/** + * RSS/Atom Feed Collection for TanStack DB + * + * This package provides RSS and Atom feed collection capabilities with: + * - Automatic feed type detection (RSS/Atom) + * - Configurable polling intervals + * - Built-in deduplication + * - Custom transform functions + * - Full TypeScript support + * + * @example RSS Collection + * ```typescript + * import { createCollection } from '@tanstack/db' + * import { rssCollectionOptions } from '@tanstack/rss-db-collection' + * + * interface BlogPost { + * id: string + * title: string + * description: string + * link: string + * publishedAt: Date + * } + * + * const blogFeed = createCollection({ + * ...rssCollectionOptions({ + * feedUrl: 'https://blog.example.com/rss.xml', + * pollingInterval: 5 * 60 * 1000, // 5 minutes + * getKey: (item) => item.id, + * transform: (item) => ({ + * id: item.guid || item.link || '', + * title: item.title || '', + * description: item.description || '', + * link: item.link || '', + * publishedAt: new Date(item.pubDate || Date.now()) + * }) + * }) + * }) + * ``` + * + * @example Atom Collection + * ```typescript + * import { createCollection } from '@tanstack/db' + * import { atomCollectionOptions } from '@tanstack/rss-db-collection' + * + * const atomFeed = createCollection({ + * ...atomCollectionOptions({ + * feedUrl: 'https://blog.example.com/atom.xml', + * pollingInterval: 5 * 60 * 1000, // 5 minutes + * getKey: (item) => item.id, + * transform: (item) => ({ + * id: item.id || '', + * title: typeof item.title === 'string' ? item.title : item.title?.$text || '', + * description: typeof item.summary === 'string' ? item.summary : item.summary?.$text || '', + * link: typeof item.link === 'string' ? item.link : item.link?.href || '', + * publishedAt: new Date(item.published || item.updated || Date.now()) + * }) + * }) + * }) + * ``` + */ + +// RSS collection functionality +export { rssCollectionOptions, type RSSCollectionConfig } from "./rss" + +// Atom collection functionality +export { atomCollectionOptions, type AtomCollectionConfig } from "./rss" + +// Shared types and utilities +export { type FeedCollectionUtils } from "./rss" + +// Feed item types +export { + type RSSItem, + type AtomItem, + type FeedItem, + type FeedType, + type HTTPOptions, + type ParsedFeedData, +} from "./types" + +// Error types +export { + RSSCollectionError, + FeedURLRequiredError, + InvalidPollingIntervalError, + FeedParsingError, + FeedFetchError, + FeedTimeoutError, + UnsupportedFeedFormatError, + GetKeyRequiredError, +} from "./errors" diff --git a/packages/rss-db-collection/src/rss.ts b/packages/rss-db-collection/src/rss.ts new file mode 100644 index 00000000..7e070244 --- /dev/null +++ b/packages/rss-db-collection/src/rss.ts @@ -0,0 +1,729 @@ +import { XMLParser, XMLValidator } from "fast-xml-parser" +import DebugModule from "debug" +import { + FeedFetchError, + FeedParsingError, + FeedTimeoutError, + FeedURLRequiredError, + InvalidPollingIntervalError, + UnsupportedFeedFormatError, +} from "./errors" +import { + detectSmartPollingInterval, + getContentHash, + parseFeedDate, +} from "./utils" +import type { + CollectionConfig, + DeleteMutationFnParams, + InsertMutationFnParams, + SyncConfig, + UpdateMutationFnParams, + UtilsRecord, +} from "@tanstack/db" +import type { StandardSchemaV1 } from "@standard-schema/spec" +import type { AtomItem, FeedItem, HTTPOptions, RSSItem } from "./types" + +const debug = DebugModule.debug(`ts/db:rss`) + +/** + * Base configuration interface for feed collection options + */ +interface BaseFeedCollectionConfig< + TExplicit extends object = Record, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, + TKey extends string | number = string | number, +> { + /** + * RSS/Atom feed URL to fetch from + */ + feedUrl: string + + /** + * Polling interval in milliseconds for refetching the feed + * @default 300000 (5 minutes) + */ + pollingInterval?: number + + /** + * HTTP options for fetching the feed + */ + httpOptions?: HTTPOptions + + /** + * Whether to start polling immediately when the collection is created + * @default true + */ + startPolling?: boolean + + /** + * Maximum number of items to keep in memory for deduplication + * @default 1000 + */ + maxSeenItems?: number + + /** + * Custom parser options for RSS/Atom feeds + */ + parserOptions?: { + ignoreAttributes?: boolean + attributeNamePrefix?: string + textNodeName?: string + ignoreNameSpace?: boolean + parseAttributeValue?: boolean + parseTrueNumberOnly?: boolean + arrayMode?: boolean | string | RegExp + } + + /** + * Standard Collection configuration properties + */ + id?: string + schema?: TSchema + getKey: CollectionConfig< + ResolveType, + TKey + >[`getKey`] + sync?: CollectionConfig< + ResolveType, + TKey + >[`sync`] + + /** + * Optional mutation handlers + */ + onInsert?: ( + params: InsertMutationFnParams< + ResolveType, + TKey + > + ) => Promise + onUpdate?: ( + params: UpdateMutationFnParams< + ResolveType, + TKey + > + ) => Promise + onDelete?: ( + params: DeleteMutationFnParams< + ResolveType, + TKey + > + ) => Promise +} + +/** + * Configuration interface for RSS collection options + */ +export interface RSSCollectionConfig< + TExplicit extends object = RSSItem, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = RSSItem, + TKey extends string | number = string | number, +> extends BaseFeedCollectionConfig { + /** + * Custom transformer function to normalize RSS items to your desired format + */ + transform?: (item: RSSItem) => ResolveType +} + +/** + * Configuration interface for Atom collection options + */ +export interface AtomCollectionConfig< + TExplicit extends object = AtomItem, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = AtomItem, + TKey extends string | number = string | number, +> extends BaseFeedCollectionConfig { + /** + * Custom transformer function to normalize Atom items to your desired format + */ + transform?: (item: AtomItem) => ResolveType +} + +// Type resolution helper (copied from TanStack DB patterns) +type InferSchemaOutput = T extends StandardSchemaV1 + ? StandardSchemaV1.InferOutput extends object + ? StandardSchemaV1.InferOutput + : Record + : Record + +type ResolveType< + TExplicit extends object = Record, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, +> = + Record extends TExplicit + ? [TSchema] extends [never] + ? TFallback + : InferSchemaOutput + : TExplicit + +/** + * Feed collection utilities + */ +export interface FeedCollectionUtils extends UtilsRecord { + /** + * Manually trigger a feed refresh + */ + refresh: () => Promise + + /** + * Clear the seen items cache + */ + clearSeenItems: () => void + + /** + * Get the number of seen items + */ + getSeenItemsCount: () => number +} + +/** + * Internal parsed feed structure + */ +interface ParsedFeed { + type: `rss` | `atom` + items: Array +} + +/** + * Parse RSS feed + */ +function parseRSSFeed(data: any): Array { + const channel = data.rss?.channel || data.channel + if (!channel) { + throw new Error(`Invalid RSS feed structure`) + } + + const items = channel.item || channel.items || [] + return Array.isArray(items) ? items : [items] +} + +/** + * Parse Atom feed + */ +function parseAtomFeed(data: any): Array { + const feed = data.feed + if (!feed) { + throw new Error(`Invalid Atom feed structure`) + } + + const entries = feed.entry || [] + return Array.isArray(entries) ? entries : [entries] +} + +/** + * Detect feed type and parse accordingly + */ +function parseFeed(xmlContent: string, parserOptions: any = {}): ParsedFeed { + const parser = new XMLParser({ + ignoreAttributes: false, + attributeNamePrefix: `@_`, + textNodeName: `#text`, + ignoreNameSpace: false, + parseAttributeValue: true, + parseTrueNumberOnly: false, + arrayMode: false, + ...parserOptions, + }) + + const data = parser.parse(xmlContent) + + // Detect feed type + if (data.rss || data.channel) { + return { + type: `rss`, + items: parseRSSFeed(data), + } + } else if (data.feed) { + return { + type: `atom`, + items: parseAtomFeed(data), + } + } else { + throw new Error(`Unknown feed format`) + } +} + +/** + * Default transformer for RSS items + */ +function defaultRSSTransform(item: RSSItem): RSSItem { + return { + ...item, + pubDate: item.pubDate ? parseFeedDate(item.pubDate) : undefined, + } +} + +/** + * Default transformer for Atom items + */ +function defaultAtomTransform(item: AtomItem): AtomItem { + // Normalize Atom fields to be more consistent + const normalized: AtomItem = { ...item } + + // Handle title + if (typeof item.title === `object` && `$text` in item.title) { + normalized.title = item.title.$text + } + + // Handle summary/content + if (typeof item.summary === `object` && `$text` in item.summary) { + normalized.summary = item.summary.$text + } + if (typeof item.content === `object` && `$text` in item.content) { + normalized.content = item.content.$text + } + + // Handle link + if (typeof item.link === `object` && !Array.isArray(item.link)) { + normalized.link = item.link.href + } else if (Array.isArray(item.link)) { + // Find the alternate link + const alternateLink = item.link.find((l) => l.rel === `alternate` || !l.rel) + normalized.link = alternateLink?.href || item.link[0]?.href + } + + // Handle dates + if (item.updated) { + normalized.updated = parseFeedDate(item.updated) + } + if (item.published) { + normalized.published = parseFeedDate(item.published) + } + + // Handle author + if (typeof item.author === `object` && `name` in item.author) { + normalized.author = item.author.name + } + + return normalized +} + +/** + * Fetch feed from URL + */ +async function fetchFeed( + url: string, + options: HTTPOptions = {} +): Promise { + const { + timeout = 30000, + headers = {}, + userAgent = `TanStack RSS Collection/1.0`, + } = options + + const controller = new AbortController() + let timeoutId: NodeJS.Timeout | null = null + + // Only set timeout if we're not in a test environment with fake timers + if ( + typeof (globalThis as any).vi === `undefined` || + !(globalThis as any).vi?.isFakeTimers?.() + ) { + timeoutId = setTimeout(() => controller.abort(), timeout) + } + + try { + const response = await fetch(url, { + signal: controller.signal, + headers: { + "User-Agent": userAgent, + Accept: `application/rss+xml, application/atom+xml, application/xml, text/xml`, + ...headers, + }, + }) + + if (!response.ok) { + throw new FeedFetchError(url, response.status) + } + + return await response.text() + } catch (error) { + if (error instanceof Error && error.name === `AbortError`) { + throw new FeedTimeoutError(url, timeout) + } + throw error instanceof FeedFetchError ? error : new FeedFetchError(url) + } finally { + if (timeoutId) { + clearTimeout(timeoutId) + } + } +} + +/** + * Extract unique identifier from feed item + */ +function getItemId(item: FeedItem, feedType: `rss` | `atom`): string { + if (feedType === `rss`) { + const rssItem = item as RSSItem + return rssItem.guid || rssItem.link || rssItem.title || JSON.stringify(item) + } else { + const atomItem = item as AtomItem + const linkHref = + typeof atomItem.link === `string` + ? atomItem.link + : Array.isArray(atomItem.link) + ? atomItem.link[0]?.href + : atomItem.link?.href + return ( + atomItem.id || + linkHref || + (typeof atomItem.title === `string` + ? atomItem.title + : atomItem.title?.$text) || + JSON.stringify(item) + ) + } +} + +/** + * Internal implementation shared between RSS and Atom collections + */ +function createFeedCollectionOptions< + TExplicit extends object = Record, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = Record, + TKey extends string | number = string | number, +>( + config: BaseFeedCollectionConfig & { + transform?: (item: FeedItem) => ResolveType + expectedFeedType?: `rss` | `atom` + } +) { + const { + feedUrl, + pollingInterval: userPollingInterval, + httpOptions = {}, + startPolling = true, + maxSeenItems = 1000, + parserOptions = {}, + transform, + expectedFeedType, + getKey, + onInsert, + onUpdate, + onDelete, + ...restConfig + } = config + + // Smart polling interval detection + let pollingInterval = + userPollingInterval !== undefined ? userPollingInterval : 300000 // Default 5 minutes + + // Validation + if (!feedUrl) { + throw new FeedURLRequiredError() + } + if (pollingInterval <= 0) { + throw new InvalidPollingIntervalError(pollingInterval) + } + + // State management + let seenItems = new Map< + string, + { id: string; lastSeen: number; contentHash: string } + >() + let syncParams: + | Parameters< + SyncConfig, TKey>[`sync`] + >[0] + | null = null + + /** + * Clean up old seen items to prevent memory leaks + */ + const cleanupSeenItems = () => { + const now = Date.now() + const maxAge = pollingInterval * 10 // Keep items for 10 polling cycles + + const cleaned = new Map() + let removedCount = 0 + + for (const [key, value] of seenItems) { + if (now - value.lastSeen < maxAge) { + cleaned.set(key, value) + } else { + removedCount++ + } + } + + if (cleaned.size > maxSeenItems) { + // Remove oldest items if we're still over the limit + const sortedEntries = Array.from(cleaned.entries()) + .sort(([, a], [, b]) => b.lastSeen - a.lastSeen) + .slice(0, maxSeenItems) + + seenItems = new Map(sortedEntries) + } else { + seenItems = cleaned + } + + if (removedCount > 0) { + debug(`Cleaned up ${removedCount} old feed items`) + } + } + + /** + * Refresh feed data + */ + const refreshFeed = async (params: { + begin: () => void + write: (message: { + type: `insert` | `update` | `delete` + value: any + }) => void + commit: () => void + }) => { + try { + debug(`Fetching feed from ${feedUrl}`) + + const xmlContent = await fetchFeed(feedUrl, httpOptions) + + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + if (!XMLValidator.validate(xmlContent)) { + throw new FeedParsingError(feedUrl, new Error(`Invalid XML content`)) + } + + const parsedFeed = parseFeed(xmlContent, parserOptions) + debug( + `Parsed ${parsedFeed.items.length} items from ${parsedFeed.type} feed` + ) + + if ( + expectedFeedType !== undefined && + expectedFeedType !== parsedFeed.type + ) { + throw new UnsupportedFeedFormatError(feedUrl) + } + + // Detect smart polling interval on first fetch + if (!userPollingInterval) { + const parser = new XMLParser(parserOptions) + const feedData = parser.parse(xmlContent) + const smartInterval = detectSmartPollingInterval(feedData) + if (smartInterval !== pollingInterval) { + pollingInterval = smartInterval + debug(`Updated polling interval to ${pollingInterval}ms`) + } + } + + const { begin, write, commit } = params + begin() + + let newItemsCount = 0 + let updatedItemsCount = 0 + const currentTime = Date.now() + + for (const rawItem of parsedFeed.items) { + // Transform the item + let transformedItem: ResolveType + + if (transform) { + transformedItem = transform(rawItem) + } else { + // Use default transformation + const defaultTransformed = + parsedFeed.type === `rss` + ? defaultRSSTransform(rawItem as RSSItem) + : defaultAtomTransform(rawItem as AtomItem) + + transformedItem = defaultTransformed as ResolveType< + TExplicit, + TSchema, + TFallback + > + } + + // Generate unique ID for deduplication + const itemId = getItemId(rawItem, parsedFeed.type) + const contentHash = getContentHash(rawItem) + + // Check if we've seen this item before + const seen = seenItems.get(itemId) + + if (!seen) { + // New item + seenItems.set(itemId, { + id: itemId, + lastSeen: currentTime, + contentHash, + }) + + write({ + type: `insert`, + value: transformedItem, + }) + + newItemsCount++ + } else if (seen.contentHash !== contentHash) { + // Item exists but content has changed - treat as update + seenItems.set(itemId, { + ...seen, + lastSeen: currentTime, + contentHash, + }) + + write({ + type: `update`, + value: transformedItem, + }) + + updatedItemsCount++ + } else { + // Item exists and content hasn't changed - just update last seen time + seenItems.set(itemId, { ...seen, lastSeen: currentTime }) + } + } + + commit() + + if (newItemsCount > 0) { + debug(`Added ${newItemsCount} new items from feed`) + } + if (updatedItemsCount > 0) { + debug(`Updated ${updatedItemsCount} existing items from feed`) + } + + // Clean up old items periodically + cleanupSeenItems() + } catch (error) { + debug(`Error refreshing feed: ${error}`) + throw error instanceof Error ? error : new Error(String(error)) + } + } + + /** + * Sync configuration + */ + const sync: SyncConfig, TKey> = { + sync: (params) => { + const { markReady } = params + + // Store sync params for manual refresh + syncParams = params + + // Polling function + const poll = async () => { + try { + await refreshFeed({ + begin: syncParams!.begin, + write: syncParams!.write, + commit: syncParams!.commit, + }) + } catch (error) { + debug(`Polling error: ${error}`) + // Continue polling despite errors + } + + // Schedule next poll if polling is enabled + if (startPolling) { + setTimeout(poll, pollingInterval) + } + } + + // Initial feed fetch (sync) + refreshFeed({ + begin: params.begin, + write: params.write, + commit: params.commit, + }) + .then(() => { + markReady() + + // Start polling if configured to do so + if (startPolling) { + setTimeout(poll, pollingInterval) + } + }) + .catch((error) => { + debug(`Initial feed fetch failed: ${error}`) + markReady() // Mark ready even on error to avoid blocking + + // Still start polling for retry attempts + if (startPolling) { + setTimeout(poll, pollingInterval) + } + }) + + // Note: sync functions should return void + // Cleanup will be handled when the collection is destroyed + }, + } + + // Utils + const utils: FeedCollectionUtils = { + refresh: async () => { + if (!syncParams) { + // If not synced yet, create minimal params for manual refresh + const dummyParams = { + begin: () => {}, + write: () => {}, + commit: () => {}, + } + await refreshFeed(dummyParams) + return + } + await refreshFeed({ + begin: syncParams.begin, + write: syncParams.write, + commit: syncParams.commit, + }) + }, + clearSeenItems: () => { + seenItems = new Map() + }, + getSeenItemsCount: () => seenItems.size, + } + + return { + ...restConfig, + getKey, + sync, + startSync: true, + rowUpdateMode: `full`, + onInsert, + onUpdate, + onDelete, + utils, + } +} + +/** + * Creates RSS collection options for use with a standard Collection + */ +export function rssCollectionOptions< + TExplicit extends object = RSSItem, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = RSSItem, + TKey extends string | number = string | number, +>(config: RSSCollectionConfig) { + return createFeedCollectionOptions({ + ...config, + expectedFeedType: `rss` as const, + transform: config.transform + ? (item: FeedItem) => config.transform!(item as RSSItem) + : undefined, + }) +} + +/** + * Creates Atom collection options for use with a standard Collection + */ +export function atomCollectionOptions< + TExplicit extends object = AtomItem, + TSchema extends StandardSchemaV1 = never, + TFallback extends object = AtomItem, + TKey extends string | number = string | number, +>(config: AtomCollectionConfig) { + return createFeedCollectionOptions({ + ...config, + expectedFeedType: `atom` as const, + transform: config.transform + ? (item: FeedItem) => config.transform!(item as AtomItem) + : undefined, + }) +} diff --git a/packages/rss-db-collection/src/types.ts b/packages/rss-db-collection/src/types.ts new file mode 100644 index 00000000..279245a4 --- /dev/null +++ b/packages/rss-db-collection/src/types.ts @@ -0,0 +1,87 @@ +/** + * Types for RSS feed items + */ +export interface RSSItem { + title?: string + description?: string + link?: string + guid?: string + pubDate?: string | Date + author?: string + category?: string | Array + enclosure?: { + url: string + type?: string + length?: string + } + [key: string]: any +} + +/** + * Types for Atom feed items + */ +export interface AtomItem { + title?: string | { $text?: string; type?: string } + summary?: string | { $text?: string; type?: string } + content?: string | { $text?: string; type?: string } + link?: + | string + | { href?: string; rel?: string; type?: string } + | Array<{ href?: string; rel?: string; type?: string }> + id?: string + updated?: string | Date + published?: string | Date + author?: string | { name?: string; email?: string; uri?: string } + category?: + | string + | { term?: string; label?: string } + | Array<{ term?: string; label?: string }> + [key: string]: any +} + +/** + * Union type for feed items + */ +export type FeedItem = RSSItem | AtomItem + +/** + * Feed type detection + */ +export type FeedType = `rss` | `atom` | `auto` + +/** + * HTTP options for fetching feeds + */ +export interface HTTPOptions { + timeout?: number + headers?: Record + userAgent?: string +} + +/** + * Parsed feed data structure from XMLParser + */ +export interface ParsedFeedData { + rss?: { + channel?: { + title?: string + description?: string + link?: string + "sy:updatePeriod"?: string + "sy:updateFrequency"?: string | number + item?: Array> + [key: string]: any + } + [key: string]: any + } + feed?: { + title?: string + subtitle?: string + link?: string + "sy:updatePeriod"?: string + "sy:updateFrequency"?: string | number + entry?: Array> + [key: string]: any + } + [key: string]: any +} diff --git a/packages/rss-db-collection/src/utils.ts b/packages/rss-db-collection/src/utils.ts new file mode 100644 index 00000000..65301db0 --- /dev/null +++ b/packages/rss-db-collection/src/utils.ts @@ -0,0 +1,200 @@ +import DebugModule from "debug" +import type { FeedItem, ParsedFeedData } from "./types" + +const debug = DebugModule.debug(`ts/db:rss:utils`) + +/** + * Calculate a simple hash of item content for change detection using djb2 algorithm + */ +export function getContentHash(item: FeedItem): string { + const content = JSON.stringify({ + title: item.title, + description: item.description, + summary: item.summary, + content: item.content, + link: item.link, + author: item.author, + category: item.category, + enclosure: item.enclosure, + }) + + let hash = 5381 + for (let i = 0; i < content.length; i++) { + hash = (hash << 5) + hash + content.charCodeAt(i) + } + return hash.toString(36) // Convert to base36 for shorter string +} + +/** + * Detect smart polling interval based on feed metadata + */ +export function detectSmartPollingInterval(feedData: ParsedFeedData): number { + // Check for RSS and + const syndication = + feedData.rss?.channel?.[`sy:updatePeriod`] || + feedData.feed?.[`sy:updatePeriod`] + const frequency = + feedData.rss?.channel?.[`sy:updateFrequency`] || + feedData.feed?.[`sy:updateFrequency`] + + if (syndication && frequency) { + const periodMap: Record = { + hourly: 60 * 60 * 1000, + daily: 24 * 60 * 60 * 1000, + weekly: 7 * 24 * 60 * 60 * 1000, + monthly: 30 * 24 * 60 * 60 * 1000, + yearly: 365 * 24 * 60 * 60 * 1000, + } + + const baseInterval = periodMap[syndication.toLowerCase()] + const frequencyNum = + typeof frequency === `string` ? parseInt(frequency, 10) : frequency + if (baseInterval && frequencyNum > 0) { + const smartInterval = Math.max(baseInterval / frequencyNum, 60 * 1000) // Minimum 1 minute + debug( + `Detected smart polling interval: ${smartInterval}ms (${syndication} / ${frequencyNum})` + ) + return smartInterval + } + } + + debug(`Using default 5-minute polling interval`) + return 300000 // 5 minutes default +} + +/** + * Parse date strings according to RFC 2822 and RFC 3339 standards + * Handles RSS pubDate (RFC 2822) and Atom published/updated (RFC 3339) + */ +export function parseFeedDate( + dateString: string | Date | undefined +): Date | undefined { + if (!dateString) return undefined + if (dateString instanceof Date) return dateString + + const str = String(dateString).trim() + if (!str) return undefined + + // Try RFC 3339 format first (Atom feeds) + // Examples: 2023-12-25T10:30:00Z, 2023-12-25T10:30:00+01:00 + const rfc3339Regex = + /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{3}))?(Z|[+-]\d{2}:\d{2})$/ + const rfc3339Match = str.match(rfc3339Regex) + + if (rfc3339Match) { + const [, year, month, day, hour, minute, second, millisecond, timezone] = + rfc3339Match + if (!year || !month || !day || !hour || !minute || !second) { + debug(`Invalid RFC 3339 date format: ${str}`) + return undefined + } + + const date = new Date() + date.setUTCFullYear(parseInt(year, 10)) + date.setUTCMonth(parseInt(month, 10) - 1) + date.setUTCDate(parseInt(day, 10)) + date.setUTCHours(parseInt(hour, 10)) + date.setUTCMinutes(parseInt(minute, 10)) + date.setUTCSeconds(parseInt(second, 10)) + if (millisecond) { + date.setUTCMilliseconds(parseInt(millisecond, 10)) + } + + // Handle timezone offset + if (timezone && timezone !== `Z`) { + const offsetMatch = timezone.match(/^([+-])(\d{2}):(\d{2})$/) + if (offsetMatch) { + const [, sign, offsetHours, offsetMinutes] = offsetMatch + if (offsetHours && offsetMinutes) { + const offsetMs = + (parseInt(offsetHours, 10) * 60 + parseInt(offsetMinutes, 10)) * + 60 * + 1000 + if (sign === `+`) { + date.setTime(date.getTime() - offsetMs) + } else { + date.setTime(date.getTime() + offsetMs) + } + } + } + } + + return date + } + + // Try RFC 2822 format (RSS feeds) + // Examples: Mon, 25 Dec 2023 10:30:00 GMT, Mon, 25 Dec 2023 10:30:00 +0100 + const rfc2822Regex = + /^(\w{3}),\s+(\d{1,2})\s+(\w{3})\s+(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+(GMT|[+-]\d{4})$/ + const rfc2822Match = str.match(rfc2822Regex) + + if (rfc2822Match) { + const [, , day, monthName, year, hour, minute, second, timezone] = + rfc2822Match + + if (!day || !monthName || !year || !hour || !minute || !second) { + debug(`Invalid RFC 2822 date format: ${str}`) + return undefined + } + + const monthMap: Record = { + Jan: 0, + Feb: 1, + Mar: 2, + Apr: 3, + May: 4, + Jun: 5, + Jul: 6, + Aug: 7, + Sep: 8, + Oct: 9, + Nov: 10, + Dec: 11, + } + + const month = monthMap[monthName] + if (month === undefined) { + debug(`Invalid month name in RFC 2822 date: ${monthName}`) + return undefined + } + + const date = new Date() + date.setUTCFullYear(parseInt(year, 10)) + date.setUTCMonth(month) + date.setUTCDate(parseInt(day, 10)) + date.setUTCHours(parseInt(hour, 10)) + date.setUTCMinutes(parseInt(minute, 10)) + date.setUTCSeconds(parseInt(second, 10)) + date.setUTCMilliseconds(0) + + // Handle timezone offset + if (timezone && timezone !== `GMT`) { + const offsetMatch = timezone.match(/^([+-])(\d{2})(\d{2})$/) + if (offsetMatch) { + const [, sign, offsetHours, offsetMinutes] = offsetMatch + if (offsetHours && offsetMinutes) { + const offsetMs = + (parseInt(offsetHours, 10) * 60 + parseInt(offsetMinutes, 10)) * + 60 * + 1000 + if (sign === `+`) { + date.setTime(date.getTime() - offsetMs) + } else { + date.setTime(date.getTime() + offsetMs) + } + } + } + } + + return date + } + + // Fallback to native Date parsing (less reliable) + const fallbackDate = new Date(str) + if (isNaN(fallbackDate.getTime())) { + debug(`Failed to parse date: ${str}`) + return undefined + } + + return fallbackDate +} diff --git a/packages/rss-db-collection/tests/errors.test.ts b/packages/rss-db-collection/tests/errors.test.ts new file mode 100644 index 00000000..22d86073 --- /dev/null +++ b/packages/rss-db-collection/tests/errors.test.ts @@ -0,0 +1,419 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { createCollection } from "@tanstack/db" +import { atomCollectionOptions, rssCollectionOptions } from "../src/rss" +import { + FeedURLRequiredError, + InvalidPollingIntervalError, +} from "../src/errors" +import type { AtomCollectionConfig, RSSCollectionConfig } from "../src/rss" + +// Mock fetch globally +global.fetch = vi.fn() + +describe(`RSS Collection Errors`, () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + describe(`Configuration Errors`, () => { + it(`should throw FeedURLRequiredError when feedUrl is missing`, () => { + expect(() => { + rssCollectionOptions({ + getKey: (item: any) => item.id, + } as RSSCollectionConfig) + }).toThrow(FeedURLRequiredError) + }) + + it(`should require getKey function (TypeScript compile-time check)`, () => { + // This is now a compile-time check - getKey is required in the interface + // No runtime validation needed as TypeScript enforces this requirement + expect(true).toBe(true) + }) + + it(`should throw InvalidPollingIntervalError for negative interval`, () => { + expect(() => { + rssCollectionOptions({ + feedUrl: `https://example.com/rss.xml`, + pollingInterval: -1000, + getKey: (item: any) => item.id, + }) + }).toThrow(InvalidPollingIntervalError) + }) + + it(`should throw InvalidPollingIntervalError for zero interval`, () => { + expect(() => { + rssCollectionOptions({ + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 0, + getKey: (item: any) => item.id, + }) + }).toThrow(InvalidPollingIntervalError) + }) + }) + + describe(`Network Errors`, () => { + it(`should handle HTTP error responses`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: false, + status: 404, + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/nonexistent.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Should mark ready even with error + await collection.stateWhenReady() + + // Should have no items due to fetch error + expect(collection.size).toBe(0) + }) + + it(`should handle network timeout`, async () => { + const fetchMock = vi.fn().mockRejectedValue(() => { + const error = new Error(`Aborted`) + error.name = `AbortError` + return error + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/slow.xml`, + getKey: (item: any) => item.id, + startPolling: false, + httpOptions: { + timeout: 50, // Very short timeout + }, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Should mark ready even with error + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + + it(`should handle general fetch errors`, async () => { + const fetchMock = vi.fn().mockRejectedValue(new Error(`Network error`)) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/broken.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + }) + + describe(`Feed Parsing Errors`, () => { + it(`should handle invalid XML`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(`This is not XML`), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/invalid.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + + it(`should handle malformed RSS feed`, async () => { + const malformedRSS = ` + + Missing channel wrapper + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(malformedRSS), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/malformed.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + + it(`should handle malformed Atom feed`, async () => { + const malformedAtom = ` + + Missing entries + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(malformedAtom), + }) + global.fetch = fetchMock + + const config: AtomCollectionConfig = { + feedUrl: `https://example.com/malformed-atom.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = atomCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + // Should succeed but have no items + expect(collection.size).toBe(0) + }) + + it(`should handle unknown feed format`, async () => { + const unknownFormat = ` + + Unknown feed format + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(unknownFormat), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/unknown.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + }) + + describe(`Feed Type Validation`, () => { + it(`should reject RSS feed when expecting Atom`, async () => { + const rssFeed = ` + + + RSS Feed + + Test Item + test-1 + + + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(rssFeed), + }) + global.fetch = fetchMock + + const config: AtomCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = atomCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + + it(`should reject Atom feed when expecting RSS`, async () => { + const atomFeed = ` + + Atom Feed + + Test Entry + test-1 + + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(atomFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/atom.xml`, + getKey: (item: any) => item.id, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + }) + }) + + describe(`Transform Function Errors`, () => { + it(`should handle transform function that throws`, async () => { + const validRSS = ` + + + RSS Feed + + Test Item + test-1 + + + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(validRSS), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.id, + startPolling: false, + transform: (_item) => { + throw new Error(`Transform error`) + }, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + // Should handle transform error gracefully + expect(collection.size).toBe(0) + }) + + it(`should handle getKey function that throws`, async () => { + const validRSS = ` + + + RSS Feed + + Test Item + test-1 + + + ` + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(validRSS), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (_item: any) => { + throw new Error(`GetKey error`) + }, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + // Should handle getKey error gracefully + expect(collection.size).toBe(0) + }) + }) + + describe(`Error Recovery`, () => { + it(`should continue polling after errors`, async () => { + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + if (callCount === 1) { + // First call fails + return Promise.reject(new Error(`Network error`)) + } else { + // Second call succeeds + return Promise.resolve({ + ok: true, + text: () => + Promise.resolve(` + + + + Recovery Item + recovery-1 + + + `), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/unreliable.xml`, + pollingInterval: 1000, + getKey: (item: any) => item.guid, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Wait for initial attempt (will fail) + await collection.stateWhenReady() + + expect(collection.size).toBe(0) + expect(fetchMock).toHaveBeenCalledTimes(1) + + // Manually trigger retry + await collection.utils.refresh() + expect(fetchMock).toHaveBeenCalledTimes(2) + + // Should now have the item from successful retry + expect(collection.size).toBe(1) + expect(collection.get(`recovery-1`)).toBeDefined() + }) + }) +}) diff --git a/packages/rss-db-collection/tests/mutations.test.ts b/packages/rss-db-collection/tests/mutations.test.ts new file mode 100644 index 00000000..0bf5d202 --- /dev/null +++ b/packages/rss-db-collection/tests/mutations.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { rssCollectionOptions } from "../src/rss" +import type { RSSCollectionConfig } from "../src/rss" + +interface TestBlogPost { + id: string + title: string + description: string + link: string + publishedAt: Date +} + +const getKey = (item: TestBlogPost) => item.id + +describe(`RSS Collection Mutations`, () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + describe(`Insert Mutations`, () => { + it(`should create collection with onInsert handler`, () => { + const onInsertMock = vi.fn().mockResolvedValue(undefined) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onInsert: onInsertMock, + } + + const options = rssCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.onInsert).toBeDefined() + expect(typeof options.onInsert).toBe(`function`) + }) + + it(`should accept onInsert handler that throws errors`, () => { + const onInsertMock = vi.fn().mockRejectedValue(new Error(`Insert failed`)) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onInsert: onInsertMock, + } + + const options = rssCollectionOptions(config) + + expect(options.onInsert).toBeDefined() + expect(typeof options.onInsert).toBe(`function`) + }) + }) + + describe(`Update Mutations`, () => { + it(`should create collection with onUpdate handler`, () => { + const onUpdateMock = vi.fn().mockResolvedValue(undefined) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onUpdate: onUpdateMock, + } + + const options = rssCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.onUpdate).toBeDefined() + expect(typeof options.onUpdate).toBe(`function`) + }) + + it(`should accept onUpdate handler that throws errors`, () => { + const onUpdateMock = vi.fn().mockRejectedValue(new Error(`Update failed`)) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onUpdate: onUpdateMock, + } + + const options = rssCollectionOptions(config) + + expect(options.onUpdate).toBeDefined() + expect(typeof options.onUpdate).toBe(`function`) + }) + }) + + describe(`Delete Mutations`, () => { + it(`should create collection with onDelete handler`, () => { + const onDeleteMock = vi.fn().mockResolvedValue(undefined) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onDelete: onDeleteMock, + } + + const options = rssCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.onDelete).toBeDefined() + expect(typeof options.onDelete).toBe(`function`) + }) + + it(`should accept onDelete handler that throws errors`, () => { + const onDeleteMock = vi.fn().mockRejectedValue(new Error(`Delete failed`)) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onDelete: onDeleteMock, + } + + const options = rssCollectionOptions(config) + + expect(options.onDelete).toBeDefined() + expect(typeof options.onDelete).toBe(`function`) + }) + }) + + describe(`Combined Mutation Scenarios`, () => { + it(`should create collection with multiple mutation handlers`, () => { + const onInsertMock = vi.fn().mockResolvedValue(undefined) + const onUpdateMock = vi.fn().mockResolvedValue(undefined) + const onDeleteMock = vi.fn().mockResolvedValue(undefined) + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + onInsert: onInsertMock, + onUpdate: onUpdateMock, + onDelete: onDeleteMock, + } + + const options = rssCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.onInsert).toBeDefined() + expect(options.onUpdate).toBeDefined() + expect(options.onDelete).toBeDefined() + }) + + it(`should provide access to collection utils in options`, () => { + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + } + + const options = rssCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.utils).toBeDefined() + expect(options.utils.refresh).toBeDefined() + expect(options.utils.clearSeenItems).toBeDefined() + expect(options.utils.getSeenItemsCount).toBeDefined() + }) + }) +}) diff --git a/packages/rss-db-collection/tests/rss.test.ts b/packages/rss-db-collection/tests/rss.test.ts new file mode 100644 index 00000000..60661d6a --- /dev/null +++ b/packages/rss-db-collection/tests/rss.test.ts @@ -0,0 +1,956 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest" +import { createCollection } from "@tanstack/db" +import { atomCollectionOptions, rssCollectionOptions } from "../src/rss" +import type { AtomCollectionConfig, RSSCollectionConfig } from "../src/rss" +import type { RSSItem } from "../src/types" + +// Mock fetch globally +global.fetch = vi.fn() + +// Sample RSS feed XML +const sampleRSSFeed = ` + + + Test Blog + A test blog + https://example.com + + First Post + This is the first post + https://example.com/post1 + post-1 + Wed, 01 Jan 2025 12:00:00 GMT + John Doe + + + Second Post + This is the second post + https://example.com/post2 + post-2 + Thu, 02 Jan 2025 12:00:00 GMT + Jane Smith + + +` + +// Sample Atom feed XML +const sampleAtomFeed = ` + + Test Blog + A test blog + + https://example.com + 2025-01-02T12:00:00Z + + First Atom Post + atom-post-1 + + 2025-01-01T12:00:00Z + 2025-01-01T10:00:00Z + This is the first atom post + + John Doe + + + + Second Atom Post + atom-post-2 + + 2025-01-02T12:00:00Z + 2025-01-02T10:00:00Z + This is the second atom post + + Jane Smith + + +` + +interface TestBlogPost { + id: string + title: string + description: string + link: string + publishedAt: Date + author?: string +} + +const getKey = (item: TestBlogPost) => item.id + +describe(`RSS Collection`, () => { + beforeEach(() => { + vi.clearAllMocks() + vi.useFakeTimers() + }) + + afterEach(() => { + vi.useRealTimers() + }) + + describe(`Basic RSS Functionality`, () => { + it(`should fetch and parse RSS feed correctly`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey, + startPolling: false, + transform: (item: RSSItem) => ({ + id: item.guid || item.link || ``, + title: item.title || ``, + description: item.description || ``, + link: item.link || ``, + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Wait for initial sync + await collection.stateWhenReady() + + expect(fetchMock).toHaveBeenCalledWith( + `https://example.com/rss.xml`, + expect.objectContaining({ + headers: expect.objectContaining({ + "User-Agent": `TanStack RSS Collection/1.0`, + Accept: `application/rss+xml, application/atom+xml, application/xml, text/xml`, + }), + }) + ) + + expect(collection.size).toBe(2) + expect(collection.get(`post-1`)).toEqual({ + id: `post-1`, + title: `First Post`, + description: `This is the first post`, + link: `https://example.com/post1`, + publishedAt: new Date(`Wed, 01 Jan 2025 12:00:00 GMT`), + author: `John Doe`, + }) + }) + + it(`should create Atom collection options correctly`, () => { + const config: AtomCollectionConfig = { + feedUrl: `https://example.com/atom.xml`, + getKey: (item: any) => item.id || ``, + startPolling: false, + } + + const options = atomCollectionOptions(config) + + expect(options).toBeDefined() + expect(options.sync).toBeDefined() + expect(options.getKey).toBeDefined() + expect(options.utils).toBeDefined() + expect(options.utils.refresh).toBeDefined() + expect(options.utils.clearSeenItems).toBeDefined() + expect(options.utils.getSeenItemsCount).toBeDefined() + }) + + it(`should use default transform when none provided`, () => { + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + + // When no transform is provided, options should still be valid + expect(options).toBeDefined() + expect(options.sync).toBeDefined() + expect(typeof options.getKey).toBe(`function`) + }) + }) + + describe(`Polling Functionality`, () => { + it(`should poll at specified interval`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 1000, // 1 second for faster test + getKey: (item: any) => item.guid || item.link, + startPolling: false, // Start manually + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Wait for initial fetch + await collection.stateWhenReady() + + expect(fetchMock).toHaveBeenCalledTimes(1) + + // Manually trigger polling by calling refresh + await collection.utils.refresh() + expect(fetchMock).toHaveBeenCalledTimes(2) + + await collection.utils.refresh() + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it(`should allow manual refresh`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 10000, + getKey: (item: any) => item.guid || item.link, + startPolling: false, // Don't start automatically + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(fetchMock).toHaveBeenCalledTimes(1) // Initial fetch only + + // Manually refresh the feed + await collection.utils.refresh() + + expect(fetchMock).toHaveBeenCalledTimes(2) + + // Refresh again + await collection.utils.refresh() + + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it(`should allow refresh to be called before sync`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 10000, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Should not throw when refresh is called before sync + await expect(collection.utils.refresh()).resolves.toBeUndefined() + expect(fetchMock).toHaveBeenCalled() + }) + }) + + describe(`Deduplication`, () => { + it(`should deduplicate items based on feed item IDs`, async () => { + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + if (callCount === 1) { + // First call - return original feed + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + } else { + // Second call - return same feed (should deduplicate) + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 5000, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(2) + expect(collection.utils.getSeenItemsCount()).toBe(2) + + // Manually trigger refresh to test deduplication + await collection.utils.refresh() + + // Should still have the same items (deduplicated) + expect(collection.size).toBe(2) + expect(collection.utils.getSeenItemsCount()).toBe(2) + }) + + it(`should add new items when they appear`, async () => { + const feedWithNewItem = sampleRSSFeed.replace( + ``, + ` + + Third Post + This is the third post + https://example.com/post3 + post-3 + Fri, 03 Jan 2025 12:00:00 GMT + + ` + ) + + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + if (callCount === 1) { + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + } else { + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithNewItem), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 5000, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.size).toBe(2) + + // Manually trigger refresh to get new item + await collection.utils.refresh() + + expect(collection.size).toBe(3) + expect(collection.get(`post-3`)).toBeDefined() + }) + + it(`should clean up old seen items`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 1000, // 1 second for faster test + maxSeenItems: 1, // Very low limit to test cleanup + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.utils.getSeenItemsCount()).toBe(1) + + // Test that seen items limit is enforced + expect(collection.utils.getSeenItemsCount()).toBeLessThanOrEqual(1) + }) + + it(`should allow clearing seen items manually`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(collection.utils.getSeenItemsCount()).toBe(2) + + collection.utils.clearSeenItems() + expect(collection.utils.getSeenItemsCount()).toBe(0) + }) + + it(`should handle multiple sequential additions to RSS feed`, async () => { + // Create progressive feeds with new items added each time + const feedWithThirdItem = sampleRSSFeed.replace( + ``, + ` + + Third Post + This is the third post + https://example.com/post3 + post-3 + Fri, 03 Jan 2025 12:00:00 GMT + Alice Johnson + + ` + ) + + const feedWithFourthItem = feedWithThirdItem.replace( + ``, + ` + + Fourth Post + This is the fourth post + https://example.com/post4 + post-4 + Sat, 04 Jan 2025 12:00:00 GMT + Bob Wilson + + ` + ) + + const feedWithFifthItem = feedWithFourthItem.replace( + ``, + ` + + Fifth Post + This is the fifth post + https://example.com/post5 + post-5 + Sun, 05 Jan 2025 12:00:00 GMT + Carol Davis + + ` + ) + + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + switch (callCount) { + case 1: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + case 2: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithThirdItem), + }) + case 3: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithFourthItem), + }) + case 4: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithFifthItem), + }) + default: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithFifthItem), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 5000, + getKey, + startPolling: false, + transform: (item: RSSItem) => ({ + id: item.guid || item.link || ``, + title: item.title || ``, + description: item.description || ``, + link: item.link || ``, + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Initial fetch - should have 2 items + await collection.stateWhenReady() + expect(collection.size).toBe(2) + expect(collection.get(`post-1`)).toBeDefined() + expect(collection.get(`post-2`)).toBeDefined() + expect(collection.get(`post-3`)).toBeUndefined() + + // First refresh - should add third item + await collection.utils.refresh() + expect(collection.size).toBe(3) + expect(collection.get(`post-3`)).toEqual({ + id: `post-3`, + title: `Third Post`, + description: `This is the third post`, + link: `https://example.com/post3`, + publishedAt: new Date(`Fri, 03 Jan 2025 12:00:00 GMT`), + author: `Alice Johnson`, + }) + + // Second refresh - should add fourth item + await collection.utils.refresh() + expect(collection.size).toBe(4) + expect(collection.get(`post-4`)).toEqual({ + id: `post-4`, + title: `Fourth Post`, + description: `This is the fourth post`, + link: `https://example.com/post4`, + publishedAt: new Date(`Sat, 04 Jan 2025 12:00:00 GMT`), + author: `Bob Wilson`, + }) + + // Third refresh - should add fifth item + await collection.utils.refresh() + expect(collection.size).toBe(5) + expect(collection.get(`post-5`)).toEqual({ + id: `post-5`, + title: `Fifth Post`, + description: `This is the fifth post`, + link: `https://example.com/post5`, + publishedAt: new Date(`Sun, 05 Jan 2025 12:00:00 GMT`), + author: `Carol Davis`, + }) + + // Verify all items are present + expect(collection.get(`post-1`)).toBeDefined() + expect(collection.get(`post-2`)).toBeDefined() + expect(collection.get(`post-3`)).toBeDefined() + expect(collection.get(`post-4`)).toBeDefined() + expect(collection.get(`post-5`)).toBeDefined() + + // Verify fetch was called the expected number of times + expect(fetchMock).toHaveBeenCalledTimes(4) + }) + + it(`should handle mixed additions and updates in RSS feed`, async () => { + // Create a feed where some items are updated and new ones are added + const updatedFeed = sampleRSSFeed + .replace( + `This is the first post`, + `This is the updated first post` + ) + .replace( + ``, + ` + + New Post + This is a completely new post + https://example.com/new-post + new-post + Mon, 06 Jan 2025 12:00:00 GMT + David Brown + + ` + ) + + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + if (callCount === 1) { + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + } else { + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(updatedFeed), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 5000, + getKey, + startPolling: false, + transform: (item: RSSItem) => ({ + id: item.guid || item.link || ``, + title: item.title || ``, + description: item.description || ``, + link: item.link || ``, + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Initial fetch + await collection.stateWhenReady() + expect(collection.size).toBe(2) + expect(collection.get(`post-1`)?.description).toBe( + `This is the first post` + ) + + // Refresh with updates and new item + await collection.utils.refresh() + expect(collection.size).toBe(3) + + // Note: The RSS collection now detects content changes and updates existing items + // So the existing item should be updated with new content + expect(collection.get(`post-1`)?.description).toBe( + `This is the updated first post` + ) + + // Check that new item was added + expect(collection.get(`new-post`)).toEqual({ + id: `new-post`, + title: `New Post`, + description: `This is a completely new post`, + link: `https://example.com/new-post`, + publishedAt: new Date(`Mon, 06 Jan 2025 12:00:00 GMT`), + author: `David Brown`, + }) + + // Verify original second post is unchanged + expect(collection.get(`post-2`)?.description).toBe( + `This is the second post` + ) + }) + + it(`should handle Atom feed with multiple sequential additions`, async () => { + // Create progressive Atom feeds + const atomWithThirdEntry = sampleAtomFeed.replace( + ``, + ` + + Third Atom Post + atom-post-3 + + 2025-01-03T12:00:00Z + 2025-01-03T10:00:00Z + This is the third atom post + + Eve Wilson + + +` + ) + + const atomWithFourthEntry = atomWithThirdEntry.replace( + ``, + ` + + Fourth Atom Post + atom-post-4 + + 2025-01-04T12:00:00Z + 2025-01-04T10:00:00Z + This is the fourth atom post + + Frank Miller + + +` + ) + + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + switch (callCount) { + case 1: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleAtomFeed), + }) + case 2: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(atomWithThirdEntry), + }) + case 3: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(atomWithFourthEntry), + }) + default: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(atomWithFourthEntry), + }) + } + }) + global.fetch = fetchMock + + const config: AtomCollectionConfig = { + feedUrl: `https://example.com/atom.xml`, + pollingInterval: 5000, + getKey, + startPolling: false, + transform: (item: any) => ({ + id: item.id || ``, + title: + typeof item.title === `string` + ? item.title + : item.title?.$text || ``, + description: + typeof item.summary === `string` + ? item.summary + : item.summary?.$text || ``, + link: + typeof item.link === `string` + ? item.link + : item.link?.[`@_href`] || item.link?.href || ``, + publishedAt: new Date(item.published || item.updated || Date.now()), + author: item.author?.name, + }), + } + + const options = atomCollectionOptions(config) + const collection = createCollection(options) + + // Initial fetch - should have 2 items + await collection.stateWhenReady() + expect(collection.size).toBe(2) + expect(collection.get(`atom-post-1`)).toBeDefined() + expect(collection.get(`atom-post-2`)).toBeDefined() + expect(collection.get(`atom-post-3`)).toBeUndefined() + + // First refresh - should add third item + await collection.utils.refresh() + expect(collection.size).toBe(3) + expect(collection.get(`atom-post-3`)).toEqual({ + id: `atom-post-3`, + title: `Third Atom Post`, + description: `This is the third atom post`, + link: `https://example.com/atom-post3`, + publishedAt: new Date(`2025-01-03T10:00:00Z`), + author: `Eve Wilson`, + }) + + // Second refresh - should add fourth item + await collection.utils.refresh() + expect(collection.size).toBe(4) + expect(collection.get(`atom-post-4`)).toEqual({ + id: `atom-post-4`, + title: `Fourth Atom Post`, + description: `This is the fourth atom post`, + link: `https://example.com/atom-post4`, + publishedAt: new Date(`2025-01-04T10:00:00Z`), + author: `Frank Miller`, + }) + + // Verify all items are present + expect(collection.get(`atom-post-1`)).toBeDefined() + expect(collection.get(`atom-post-2`)).toBeDefined() + expect(collection.get(`atom-post-3`)).toBeDefined() + expect(collection.get(`atom-post-4`)).toBeDefined() + + // Verify fetch was called the expected number of times + expect(fetchMock).toHaveBeenCalledTimes(3) + }) + + it(`should maintain collection state across multiple fetches with errors`, async () => { + // Create feeds with some successful fetches and some errors + const feedWithNewItem = sampleRSSFeed.replace( + ``, + ` + + Error Recovery Post + This post should be added after an error + https://example.com/error-recovery + error-recovery + Mon, 07 Jan 2025 12:00:00 GMT + + ` + ) + + let callCount = 0 + const fetchMock = vi.fn().mockImplementation(() => { + callCount++ + switch (callCount) { + case 1: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + case 2: + // Simulate a network error + return Promise.reject(new Error(`Network error`)) + case 3: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithNewItem), + }) + case 4: + // Simulate another error + return Promise.resolve({ + ok: false, + status: 500, + text: () => Promise.resolve(`Server error`), + }) + case 5: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithNewItem), + }) + default: + return Promise.resolve({ + ok: true, + text: () => Promise.resolve(feedWithNewItem), + }) + } + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + pollingInterval: 5000, + getKey, + startPolling: false, + transform: (item: RSSItem) => ({ + id: item.guid || item.link || ``, + title: item.title || ``, + description: item.description || ``, + link: item.link || ``, + publishedAt: new Date(item.pubDate || Date.now()), + author: item.author, + }), + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Initial fetch - should succeed + await collection.stateWhenReady() + expect(collection.size).toBe(2) + + // First refresh - should fail but not affect existing items + await expect(collection.utils.refresh()).rejects.toThrow() + expect(collection.size).toBe(2) // Should maintain existing items + + // Second refresh - should succeed and add new item + await collection.utils.refresh() + expect(collection.size).toBe(3) + expect(collection.get(`error-recovery`)).toBeDefined() + + // Third refresh - should fail but maintain items + await expect(collection.utils.refresh()).rejects.toThrow() + expect(collection.size).toBe(3) // Should maintain existing items + + // Fourth refresh - should succeed (no new items, but should work) + await collection.utils.refresh() + expect(collection.size).toBe(3) // No new items added + + // Verify fetch was called the expected number of times + expect(fetchMock).toHaveBeenCalledTimes(5) + }) + }) + + describe(`Custom Configuration`, () => { + it(`should respect custom HTTP options`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.guid || item.link, + startPolling: false, + httpOptions: { + timeout: 15000, + userAgent: `Custom User Agent`, + headers: { + Authorization: `Bearer token123`, + }, + }, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + await collection.stateWhenReady() + + expect(fetchMock).toHaveBeenCalledWith( + `https://example.com/rss.xml`, + expect.objectContaining({ + headers: expect.objectContaining({ + "User-Agent": `Custom User Agent`, + Authorization: `Bearer token123`, + }), + }) + ) + }) + + it(`should reject RSS feed when expecting Atom`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleRSSFeed), + }) + global.fetch = fetchMock + + // Use atomCollectionOptions with RSS feed URL - should fail + const config: AtomCollectionConfig = { + feedUrl: `https://example.com/rss.xml`, + getKey: (item: any) => item.id || item.link, + startPolling: false, + } + + const options = atomCollectionOptions(config) + const collection = createCollection(options) + + // Should mark ready even on error + await collection.stateWhenReady() + + // Should have no items due to format mismatch error + expect(collection.size).toBe(0) + }) + + it(`should reject Atom feed when expecting RSS`, async () => { + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + text: () => Promise.resolve(sampleAtomFeed), + }) + global.fetch = fetchMock + + // Use rssCollectionOptions with Atom feed URL - should fail + const config: RSSCollectionConfig = { + feedUrl: `https://example.com/atom.xml`, + getKey: (item: any) => item.id || item.link, + startPolling: false, + } + + const options = rssCollectionOptions(config) + const collection = createCollection(options) + + // Should mark ready even on error + await collection.stateWhenReady() + + // Should have no items due to format mismatch error + expect(collection.size).toBe(0) + }) + }) +}) diff --git a/packages/rss-db-collection/tests/utils.test.ts b/packages/rss-db-collection/tests/utils.test.ts new file mode 100644 index 00000000..d482672c --- /dev/null +++ b/packages/rss-db-collection/tests/utils.test.ts @@ -0,0 +1,289 @@ +import { describe, expect, it } from "vitest" +import { + detectSmartPollingInterval, + getContentHash, + parseFeedDate, +} from "../src/utils" +import type { ParsedFeedData, RSSItem } from "../src/types" + +describe(`Utils`, () => { + describe(`getContentHash`, () => { + it(`should generate consistent hashes for identical content`, () => { + const item1: RSSItem = { + title: `Test Post`, + description: `Test description`, + link: `https://example.com/test`, + author: `John Doe`, + } + + const item2: RSSItem = { + title: `Test Post`, + description: `Test description`, + link: `https://example.com/test`, + author: `John Doe`, + } + + const hash1 = getContentHash(item1) + const hash2 = getContentHash(item2) + + expect(hash1).toBe(hash2) + expect(typeof hash1).toBe(`string`) + expect(hash1.length).toBeGreaterThan(0) + }) + + it(`should generate different hashes for different content`, () => { + const item1: RSSItem = { + title: `Test Post`, + description: `Test description`, + link: `https://example.com/test`, + author: `John Doe`, + } + + const item2: RSSItem = { + title: `Test Post Updated`, + description: `Test description`, + link: `https://example.com/test`, + author: `John Doe`, + } + + const hash1 = getContentHash(item1) + const hash2 = getContentHash(item2) + + expect(hash1).not.toBe(hash2) + }) + + it(`should handle missing properties gracefully`, () => { + const item1: RSSItem = { + title: `Test Post`, + description: `Test description`, + } + + const item2: RSSItem = { + title: `Test Post`, + description: `Test description`, + link: undefined, + author: null as any, + } + + const hash1 = getContentHash(item1) + const hash2 = getContentHash(item2) + + // JSON.stringify omits undefined properties but includes null + // So these will have different hashes, which is correct behavior + expect(hash1).not.toBe(hash2) + }) + + it(`should be case sensitive`, () => { + const item1: RSSItem = { + title: `Test Post`, + description: `Test description`, + } + + const item2: RSSItem = { + title: `test post`, + description: `Test description`, + } + + const hash1 = getContentHash(item1) + const hash2 = getContentHash(item2) + + expect(hash1).not.toBe(hash2) + }) + }) + + describe(`detectSmartPollingInterval`, () => { + it(`should detect hourly syndication`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `hourly`, + "sy:updateFrequency": `2`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(30 * 60 * 1000) // 30 minutes (hourly / 2) + }) + + it(`should detect daily syndication`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `daily`, + "sy:updateFrequency": `1`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(24 * 60 * 60 * 1000) // 24 hours + }) + + it(`should detect weekly syndication`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `weekly`, + "sy:updateFrequency": `3`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe((7 * 24 * 60 * 60 * 1000) / 3) // weekly / 3 + }) + + it(`should handle Atom feeds`, () => { + const feedData: ParsedFeedData = { + feed: { + "sy:updatePeriod": `daily`, + "sy:updateFrequency": `2`, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(12 * 60 * 60 * 1000) // 12 hours (daily / 2) + }) + + it(`should enforce minimum 1-minute interval`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `hourly`, + "sy:updateFrequency": `120`, // Would result in 30 seconds + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(60 * 1000) // 1 minute minimum + }) + + it(`should default to 5 minutes when no syndication data`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + title: `Test Feed`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(300000) // 5 minutes + }) + + it(`should default to 5 minutes when syndication data is invalid`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `invalid`, + "sy:updateFrequency": `1`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(300000) // 5 minutes + }) + + it(`should default to 5 minutes when frequency is 0`, () => { + const feedData: ParsedFeedData = { + rss: { + channel: { + "sy:updatePeriod": `daily`, + "sy:updateFrequency": `0`, + }, + }, + } + + const interval = detectSmartPollingInterval(feedData) + expect(interval).toBe(300000) // 5 minutes + }) + }) + + describe(`parseFeedDate`, () => { + it(`should parse RFC 3339 dates`, () => { + const date1 = parseFeedDate(`2023-12-25T10:30:00Z`) + const date2 = parseFeedDate(`2023-12-25T10:30:00+01:00`) + const date3 = parseFeedDate(`2023-12-25T10:30:00.123Z`) + + expect(date1).toBeInstanceOf(Date) + expect(date2).toBeInstanceOf(Date) + expect(date3).toBeInstanceOf(Date) + + expect(date1?.getUTCFullYear()).toBe(2023) + expect(date1?.getUTCMonth()).toBe(11) // December is 11 (0-indexed) + expect(date1?.getUTCDate()).toBe(25) + expect(date1?.getUTCHours()).toBe(10) + expect(date1?.getUTCMinutes()).toBe(30) + expect(date1?.getUTCSeconds()).toBe(0) + }) + + it(`should parse RFC 2822 dates`, () => { + const date1 = parseFeedDate(`Mon, 25 Dec 2023 10:30:00 GMT`) + const date2 = parseFeedDate(`Mon, 25 Dec 2023 10:30:00 +0100`) + + expect(date1).toBeInstanceOf(Date) + expect(date2).toBeInstanceOf(Date) + + expect(date1?.getUTCFullYear()).toBe(2023) + expect(date1?.getUTCMonth()).toBe(11) // December is 11 (0-indexed) + expect(date1?.getUTCDate()).toBe(25) + expect(date1?.getUTCHours()).toBe(10) + expect(date1?.getUTCMinutes()).toBe(30) + expect(date1?.getUTCSeconds()).toBe(0) + }) + + it(`should handle timezone offsets correctly`, () => { + // RFC 3339 with +01:00 offset + const date1 = parseFeedDate(`2023-12-25T10:30:00+01:00`) + // RFC 2822 with +0100 offset + const date2 = parseFeedDate(`Mon, 25 Dec 2023 10:30:00 +0100`) + + // Both should represent the same moment in time (UTC) + // The +01:00 offset means both represent 09:30:00 UTC + expect(date1?.getUTCHours()).toBe(9) + expect(date1?.getUTCMinutes()).toBe(30) + expect(date2?.getUTCHours()).toBe(9) + expect(date2?.getUTCMinutes()).toBe(30) + }) + + it(`should return undefined for invalid dates`, () => { + expect(parseFeedDate(`invalid date`)).toBeUndefined() + expect(parseFeedDate(``)).toBeUndefined() + expect(parseFeedDate(` `)).toBeUndefined() + expect(parseFeedDate(undefined)).toBeUndefined() + expect(parseFeedDate(null as any)).toBeUndefined() + }) + + it(`should handle already parsed Date objects`, () => { + const originalDate = new Date(`2023-12-25T10:30:00Z`) + const parsedDate = parseFeedDate(originalDate) + + expect(parsedDate).toBe(originalDate) + }) + + it(`should fallback to native Date parsing for unrecognized formats`, () => { + const date = parseFeedDate(`2023-12-25 10:30:00`) + + expect(date).toBeInstanceOf(Date) + expect(date?.getFullYear()).toBe(2023) + expect(date?.getMonth()).toBe(11) // December is 11 (0-indexed) + }) + + it(`should handle invalid RFC 3339 formats`, () => { + // These should fail the regex but might pass fallback parsing + expect(parseFeedDate(`2023-12-25T10:30:00`)).toBeInstanceOf(Date) // Missing timezone, falls back + expect(parseFeedDate(`invalid-date`)).toBeUndefined() // Completely invalid + }) + + it(`should handle invalid RFC 2822 formats`, () => { + expect(parseFeedDate(`Invalid, 25 Dec 2023 10:30:00 GMT`)).toBeInstanceOf( + Date + ) // Invalid day name, falls back + expect(parseFeedDate(`Mon, 25 Invalid 2023 10:30:00 GMT`)).toBeUndefined() // Invalid month name, should fail + expect(parseFeedDate(`completely-invalid`)).toBeUndefined() // Completely invalid + }) + }) +}) diff --git a/packages/rss-db-collection/tsconfig.docs.json b/packages/rss-db-collection/tsconfig.docs.json new file mode 100644 index 00000000..5a73feb0 --- /dev/null +++ b/packages/rss-db-collection/tsconfig.docs.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "paths": { + "@tanstack/db": ["../db/src"] + } + }, + "include": ["src"] +} diff --git a/packages/rss-db-collection/tsconfig.json b/packages/rss-db-collection/tsconfig.json new file mode 100644 index 00000000..37cf8493 --- /dev/null +++ b/packages/rss-db-collection/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "target": "ES2020", + "module": "ESNext", + "moduleResolution": "Bundler", + "declaration": true, + "outDir": "dist", + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true, + "jsx": "react", + "paths": { + "@tanstack/db": ["../db/src"], + "@tanstack/db-ivm": ["../db-ivm/src"] + } + }, + "include": ["src", "tests", "vite.config.ts"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/rss-db-collection/vite.config.ts b/packages/rss-db-collection/vite.config.ts new file mode 100644 index 00000000..0958d2b3 --- /dev/null +++ b/packages/rss-db-collection/vite.config.ts @@ -0,0 +1,23 @@ +import { defineConfig, mergeConfig } from "vitest/config" +import { tanstackViteConfig } from "@tanstack/config/vite" +import react from "@vitejs/plugin-react" +import packageJson from "./package.json" + +const config = defineConfig({ + plugins: [react()], + test: { + name: packageJson.name, + dir: `./tests`, + environment: `jsdom`, + coverage: { enabled: true, provider: `istanbul`, include: [`src/**/*`] }, + typecheck: { enabled: true }, + }, +}) + +export default mergeConfig( + config, + tanstackViteConfig({ + entry: `./src/index.ts`, + srcDir: `./src`, + }) +) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0ea21e2..45d9cf09 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -587,6 +587,31 @@ importers: specifier: ^19.0.0 version: 19.1.1(react@19.1.1) + packages/rss-db-collection: + dependencies: + '@standard-schema/spec': + specifier: ^1.0.0 + version: 1.0.0 + '@tanstack/db': + specifier: workspace:* + version: link:../db + debug: + specifier: ^4.4.1 + version: 4.4.1 + fast-xml-parser: + specifier: ^4.5.0 + version: 4.5.3 + typescript: + specifier: '>=4.7' + version: 5.8.3 + devDependencies: + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@vitest/coverage-istanbul': + specifier: ^3.0.9 + version: 3.2.4(vitest@3.2.4(@types/debug@4.1.12)(@types/node@22.17.0)(jiti@2.5.1)(jsdom@26.1.0)(lightningcss@1.30.1)(terser@5.43.1)(tsx@4.20.3)(yaml@2.8.0)) + packages/solid-db: dependencies: '@solid-primitives/map': @@ -4446,6 +4471,10 @@ packages: fast-levenshtein@2.0.6: resolution: {integrity: sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==} + fast-xml-parser@4.5.3: + resolution: {integrity: sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig==} + hasBin: true + fastq@1.19.1: resolution: {integrity: sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ==} @@ -6651,6 +6680,9 @@ packages: strip-literal@3.0.0: resolution: {integrity: sha512-TcccoMhJOM3OebGhSBEmp3UZ2SfDMZUEBdRA/9ynfLi8yYajyWX3JiXArcJt4Umh4vISpspkQIY8ZZoCqjbviA==} + strnum@1.1.2: + resolution: {integrity: sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA==} + style-to-object@1.0.9: resolution: {integrity: sha512-G4qppLgKu/k6FwRpHiGiKPaPTFcG3g4wNVX/Qsfu+RqQM30E7Tyu/TEgxcL9PNLF5pdRLwQdE3YKKf+KF2Dzlw==} @@ -11870,6 +11902,10 @@ snapshots: fast-levenshtein@2.0.6: {} + fast-xml-parser@4.5.3: + dependencies: + strnum: 1.1.2 + fastq@1.19.1: dependencies: reusify: 1.1.0 @@ -14315,6 +14351,8 @@ snapshots: dependencies: js-tokens: 9.0.1 + strnum@1.1.2: {} + style-to-object@1.0.9: dependencies: inline-style-parser: 0.2.4