Cleans up rss feed and service, sets config during initialize (#22)

* clean up rss plugin and service

* working rss feed and service with config set http

* handle edge cases

* nitpick and update docs

Author: elliotBraem

apps/example/package.json

@@ -18,6 +18,8 @@
   "dependencies": {
     "@module-federation/node": "^2.6.22",
     "@types/express": "^5.0.0",
-    "express": "^4.21.2"
+    "@types/lodash": "^4.17.16",
+    "express": "^4.21.2",
+    "lodash": "^4.17.21"
   }
 }

apps/example/src/frontend/frontend.js

@@ -403,52 +403,45 @@ async function transformContent() {
       throw new Error("No transform plugins configured");
     }
 
-    let currentContent = content;
-    updateTransformStatus("Starting transformations...", "info");
-
-    for (const [index, transformConfig] of config.transform.entries()) {
-      updateTransformStatus(
-        `Transforming with ${transformConfig.plugin} (${index + 1}/${config.transform.length})...`,
-        "info",
-      );
-
-      // Try to parse current content as JSON if it's a string that looks like JSON
-      let parsedContent = currentContent;
-      if (typeof currentContent === "string") {
-        try {
-          parsedContent = JSON.parse(currentContent);
-        } catch (e) {
-          // Not JSON, use as-is
-          parsedContent = { content: currentContent };
-        }
+    // Try to parse content as JSON if it's a string that looks like JSON
+    let parsedContent = content;
+    if (typeof content === "string") {
+      try {
+        parsedContent = JSON.parse(content);
+      } catch (e) {
+        // Not JSON, use as-is
+        parsedContent = { content };
       }
+    }
 
-      const response = await fetch("/api/transform", {
-        method: "POST",
-        headers: {
-          "Content-Type": "application/json",
-        },
-        body: JSON.stringify({
-          plugin: transformConfig.plugin,
-          config: transformConfig.config,
-          content: parsedContent,
-        }),
-      });
+    updateTransformStatus("Applying transformations...", "info");
 
-      if (!response.ok) {
-        const error = await response.json();
-        throw new Error(error.message || "Transform failed");
-      }
+    // Send all transforms at once
+    const response = await fetch("/api/transform", {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        transform: config.transform,
+        content: parsedContent,
+      }),
+    });
 
-      const result = await response.json();
-      currentContent = result.output;
-      // Format the output for display
-      contentEditor.value =
-        typeof currentContent === "object"
-          ? JSON.stringify(currentContent, null, 2)
-          : currentContent;
+    if (!response.ok) {
+      const error = await response.json();
+      throw new Error(error.message || "Transform failed");
     }
 
+    const result = await response.json();
+    const transformedContent = result.output;
+
+    // Format the output for display
+    contentEditor.value =
+      typeof transformedContent === "object"
+        ? JSON.stringify(transformedContent, null, 2)
+        : transformedContent;
+
     updateTransformStatus(
       "All transformations completed successfully",
       "success",

apps/example/src/index.ts

@@ -9,6 +9,7 @@ import {
   setPluginRegistry,
 } from "./plugin-service/plugin-registry";
 import { hydrateConfigValues } from "./utils";
+import { merge } from "lodash";
 
 async function main() {
   const app = express();
@@ -92,26 +93,96 @@ async function main() {
     }
   });
 
+  /**
+   * Combines transform results, merging objects or returning the new result
+   */
+  function combineResults(prevResult: unknown, newResult: unknown): unknown {
+    // If both are objects (not arrays), merge them with new values taking precedence
+    if (
+      typeof prevResult === "object" &&
+      prevResult !== null &&
+      !Array.isArray(prevResult) &&
+      typeof newResult === "object" &&
+      newResult !== null &&
+      !Array.isArray(newResult)
+    ) {
+      return merge({}, prevResult, newResult);
+    }
+
+    // Otherwise return the new result (string will just return)
+    return newResult;
+  }
+
+  /**
+   * Apply a series of transformations to content
+   */
+  async function applyTransforms(
+    content: any,
+    transforms: Array<{ plugin: string; config: any }> = [],
+    stage: string = "global",
+  ) {
+    let result = content;
+
+    for (let i = 0; i < transforms.length; i++) {
+      const transform = transforms[i];
+      try {
+        // Hydrate config with environment variables
+        const hydratedConfig = hydrateConfigValues(transform.config);
+
+        // Load and configure transform plugin
+        const plugin = await pluginService.getPlugin<"transformer">(
+          transform.plugin,
+          {
+            type: "transformer",
+            config: hydratedConfig,
+          },
+        );
+
+        console.log(
+          `Applying ${stage} transform #${i + 1} (${transform.plugin})`,
+        );
+        const transformResult = await plugin.transform({
+          input: result,
+          config: hydratedConfig,
+        });
+
+        // Validate transform output
+        if (transformResult === undefined || transformResult === null) {
+          throw new Error(
+            `Transform ${transform.plugin} returned null or undefined`,
+          );
+        }
+
+        // Combine results, either merging objects or using new result
+        result = combineResults(result, transformResult);
+      } catch (error) {
+        console.error(`Transform error (${transform.plugin}):`, error);
+        throw new Error(
+          `Transform failed at ${stage} stage, plugin ${transform.plugin}, index ${i}: ${
+            error instanceof Error ? error.message : "Unknown error"
+          }`,
+        );
+      }
+    }
+
+    return result;
+  }
+
   // Transform endpoint
   app.post("/api/transform", async (req, res) => {
     try {
-      const { plugin: pluginName, config: pluginConfig, content } = req.body;
+      const { transform, content } = req.body;
 
       if (!content) {
         throw new Error("No content provided for transformation");
       }
 
-      // Hydrate config with environment variables
-      const hydratedConfig = hydrateConfigValues(pluginConfig);
-
-      // Load and configure transform plugin
-      const plugin = await pluginService.getPlugin<"transformer">(pluginName, {
-        type: "transformer",
-        config: hydratedConfig,
-      });
+      if (!Array.isArray(transform) || transform.length === 0) {
+        throw new Error("No transforms specified");
+      }
 
-      // Transform content
-      const result = await plugin.transform({ input: content });
+      // Apply all transforms in sequence
+      const result = await applyTransforms(content, transform);
       res.json({ success: true, output: result });
     } catch (error) {
       const errorMessage =

bun.lockb

@@ -60,11 +60,42 @@ Full configuration options for the RSS plugin:
   "plugin": "@curatedotfun/rss",
   "config": {
     "serviceUrl": "https://your-rss-service-url.com", // URL of your deployed RSS service
-    "apiSecret": "{API_SECRET}" // Automatically injected from environment
+    "apiSecret": "{API_SECRET}", // Automatically injected from environment
+    "feedConfig": { // Optional feed configuration
+      "title": "My Custom RSS Feed",
+      "description": "A feed of curated content",
+      "siteUrl": "https://example.com",
+      "language": "en",
+      "copyright": "© 2025",
+      "maxItems": 50,
+      "image": "https://example.com/logo.png",
+      "author": {
+        "name": "Feed Author",
+        "email": "author@example.com",
+        "link": "https://author.example.com"
+      }
+    }
   }
 }
 ```
 
+### Feed Configuration
+
+The `feedConfig` property allows you to customize your RSS feed's metadata during plugin initialization. This configuration is sent to the RSS service to be available when feed data is queried.
+
+Key configuration options:
+
+- **title**: The title of your RSS feed
+- **description**: A brief description of your feed's content
+- **siteUrl**: The URL of your website (used for feed links)
+- **language**: The language code for your feed (e.g., "en", "es", "fr")
+- **copyright**: Copyright notice for your feed content
+- **maxItems**: Maximum number of items to keep in the feed (older items are removed)
+- **image**: URL to an image representing your feed
+- **author**: Information about the feed author (name, email, link)
+
+If you don't provide a `feedConfig`, the RSS service will use default values.
+
 ## 🔄 Data Transformation
 
 The RSS plugin accepts structured input data that maps to RSS item fields. You can provide as much or as little data as you want, and the plugin will fill in defaults for missing fields.

docs/docs/plugins/distributors/rss.md

@@ -70,11 +70,42 @@ Full configuration options for the RSS plugin:
   "plugin": "@curatedotfun/rss",
   "config": {
     "serviceUrl": "https://your-rss-service-url.com", // URL of your deployed RSS service
-    "apiSecret": "{API_SECRET}" // Automatically injected from environment
+    "apiSecret": "{API_SECRET}", // Automatically injected from environment
+    "feedConfig": { // Optional feed configuration
+      "title": "My Custom RSS Feed",
+      "description": "A feed of curated content",
+      "siteUrl": "https://example.com",
+      "language": "en",
+      "copyright": "© 2025",
+      "maxItems": 50,
+      "image": "https://example.com/logo.png",
+      "author": {
+        "name": "Feed Author",
+        "email": "author@example.com",
+        "link": "https://author.example.com"
+      }
+    }
   }
 }
 ```
 
+### Feed Configuration
+
+The `feedConfig` property allows you to customize your RSS feed's metadata during plugin initialization. This configuration is sent to the RSS service to be available when feed data is queried.
+
+Key configuration options:
+
+- **title**: The title of your RSS feed
+- **description**: A brief description of your feed's content
+- **siteUrl**: The URL of your website (used for feed links)
+- **language**: The language code for your feed (e.g., "en", "es", "fr")
+- **copyright**: Copyright notice for your feed content
+- **maxItems**: Maximum number of items to keep in the feed (older items are removed)
+- **image**: URL to an image representing your feed
+- **author**: Information about the feed author (name, email, link)
+
+If you don't provide a `feedConfig`, the RSS service will use default values.
+
 ## 🔄 Data Transformation
 
 The RSS plugin accepts structured input data that maps to RSS item fields. You can provide as much or as little data as you want, and the plugin will fill in defaults for missing fields.

packages/rss/README.md

@@ -1,6 +1,6 @@
 {
   "name": "@curatedotfun/rss",
-  "version": "0.0.9",
+  "version": "0.0.10",
   "description": "RSS plugin for curatedotfun",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",

packages/rss/package.json

@@ -1,6 +1,4 @@
 import "dotenv/config";
-import fs from "fs";
-import path from "path";
 import { FeedConfig } from "./types.js";
 
 // Environment variables validation
@@ -65,23 +63,32 @@ const DEFAULT_CONFIG: FeedConfig = {
   author: { name: "Feed Author", email: "author@example.com" },
 };
 
-// Load feed configuration from JSON file
-export function loadConfig(): FeedConfig {
-  const CONFIG_FILE_PATH = path.join(process.cwd(), "feed-config.json");
-
-  try {
-    const configFile = fs.readFileSync(CONFIG_FILE_PATH, "utf8");
-    const config = JSON.parse(configFile) as FeedConfig;
-    console.log("Loaded feed configuration from feed-config.json");
-    return config;
-  } catch (error) {
-    console.warn(
-      "Could not load feed-config.json, using default configuration",
-    );
-    return DEFAULT_CONFIG;
-  }
+let currentConfig: FeedConfig | null = null;
+
+// Set feed configuration
+export function setFeedConfig(config: FeedConfig): void {
+  config.id = DEFAULT_FEED_ID;
+
+  // Set default values for optional fields if not provided
+  config.title = config.title || DEFAULT_CONFIG.title;
+  config.description = config.description || DEFAULT_CONFIG.description;
+  config.siteUrl = config.siteUrl || DEFAULT_CONFIG.siteUrl;
+
+  // Ensure maxItems is always a positive number
+  config.maxItems =
+    typeof config.maxItems === "number" && config.maxItems > 0
+      ? config.maxItems
+      : DEFAULT_CONFIG.maxItems;
+
+  config.language = config.language || DEFAULT_CONFIG.language;
+
+  // Update the in-memory configuration
+  currentConfig = config;
+  console.log("Updated feed configuration");
 }
 
+// Get the current feed configuration
 export const getFeedConfig = (): FeedConfig => {
-  return loadConfig();
+  const config = currentConfig || DEFAULT_CONFIG;
+  return JSON.parse(JSON.stringify(config)) as FeedConfig;
 };

packages/rss/service/src/config.ts

@@ -11,6 +11,8 @@ import {
   handleGetItems,
   handleAddItem,
   handleHealth,
+  handleUpdateConfig,
+  handleGetConfig,
 } from "./routes.js";
 import { authenticate } from "./middleware.js";
 import { initializeFeed } from "./storage.js";
@@ -56,6 +58,8 @@ app.get("/feed.json", handleJsonFeed);
 app.get("/raw.json", handleRawJson);
 app.get("/api/items", handleGetItems);
 app.post("/api/items", handleAddItem);
+app.get("/api/config", handleGetConfig);
+app.put("/api/config", handleUpdateConfig);
 
 // Initialize feed
 await initializeFeed();