Update Sendbird migration script

Author: keerlu

rest-api/migrate-data-from-sendbird-to-talkjs/config.js

@@ -0,0 +1,4 @@
+const appID = "<APP_ID>"; // Add your app ID from the TalkJS dashboard
+exports.appID = appID;
+const secretKey = "<SECRET_KEY>"; // Add your secret API key from the TalkJS dashboard
+exports.secretKey = secretKey;

rest-api/migrate-data-from-sendbird-to-talkjs/importers/channelImporter.js

@@ -0,0 +1,65 @@
+const fs = require("fs").promises;
+const axios = require("axios");
+const { secretKey, appID } = require("../config.js");
+
+const instance = axios.create({
+  baseURL: "https://api.talkjs.com",
+  headers: {
+    Authorization: `Bearer ${secretKey}`,
+    "Content-Type": "application/json",
+  },
+});
+
+async function importChannels(filePath) {
+  try {
+    // Read and parse file
+    const data = await fs.readFile(filePath, "utf8");
+    console.log(`Read file: ${filePath}`);
+
+    const channelsJson = JSON.parse(data);
+    console.log(
+      `Parsed JSON data with ${
+        channelsJson.group_channels?.length || 0
+      } channels`
+    );
+
+    if (!Array.isArray(channelsJson.group_channels)) {
+      throw new Error("Invalid data structure: group_channels is not an array");
+    }
+
+    for (let channel of channelsJson.group_channels) {
+      try {
+        const channelParticipants = [];
+        for (let member of channel.members) {
+          channelParticipants.push(member.user_id);
+        }
+
+        // Create the TalkJS conversation data
+        const conversation = {
+          participants: channelParticipants,
+        };
+
+        // Only add subject if channel name exists
+        if (channel.name) {
+          conversation.subject = channel.name;
+        }
+
+        await instance.put(
+          `/v1/${appID}/conversations/${channel.channel_url}`,
+          conversation
+        );
+        console.log(`Imported conversation: ${channel.channel_url}`);
+      } catch (err) {
+        console.error(
+          `Failed to import conversation ${channel.channel_url}:`,
+          err.response?.data || err.message
+        );
+      }
+    }
+  } catch (err) {
+    console.error(`Failed to process channel file ${filePath}:`, err.message);
+    throw err;
+  }
+}
+
+module.exports = { importChannels };

rest-api/migrate-data-from-sendbird-to-talkjs/importers/messageImporter.js

@@ -0,0 +1,121 @@
+const fs = require("fs").promises;
+const axios = require("axios");
+const { secretKey, appID } = require("../config.js");
+
+const instance = axios.create({
+  baseURL: "https://api.talkjs.com",
+  headers: {
+    Authorization: `Bearer ${secretKey}`,
+    "Content-Type": "application/json",
+  },
+});
+
+// Batch size of messages to import in one go
+// The TalkJS import endpoint has a limit of 10 messages per request
+const batchSize = 10;
+
+async function processMessages(messages) {
+  const uniqueConversations = [
+    ...new Set(messages.map((item) => item.channel_url)),
+  ];
+  console.log(`Found ${uniqueConversations.length} unique conversations`);
+
+  let conversationJsonArray = [];
+  let skippedCount = 0;
+
+  // Initialize conversation objects
+  for (let conversation of uniqueConversations) {
+    let conversationJsonObj = {};
+    conversationJsonObj[conversation] = [];
+    conversationJsonArray.push(conversationJsonObj);
+  }
+
+  // Group messages by conversation
+  for (let conversation of conversationJsonArray) {
+    for (let message of messages) {
+      // Skip messages with removed flag
+      if (message.is_removed === true) {
+        console.log(`Skipping removed message: ${message.message_id}`);
+        skippedCount++;
+        continue;
+      }
+      // Skip messages with a non-MESG Sendbird message type
+      if (message.type !== "MESG") {
+        console.log(`Skipping non-MESG type message: ${message.message_id}`);
+        skippedCount++;
+        continue;
+      }
+
+      if (Object.keys(conversation)[0] === message.channel_url) {
+        let messageObj = {
+          text: message.message,
+          sender: message.user.user_id,
+          type: "UserMessage",
+          timestamp: message.created_at,
+          readBy: [],
+        };
+        conversation[message.channel_url].push(messageObj);
+      }
+    }
+  }
+
+  console.log(`Skipped ${skippedCount} messages`);
+  return conversationJsonArray;
+}
+
+async function importBatch(conversationId, messages) {
+  await instance.post(
+    `/v1/${appID}/import/conversations/${conversationId}/messages`,
+    messages
+  );
+  console.log(
+    `Imported batch of ${messages.length} messages for conversation: ${conversationId}`
+  );
+}
+
+async function importMessages(filePath) {
+  try {
+    // Read and parse file
+    const data = await fs.readFile(filePath, "utf8");
+    console.log(`Read file: ${filePath}`);
+
+    const messagesJson = JSON.parse(data);
+    console.log(
+      `Parsed JSON data with ${messagesJson.messages?.length || 0} messages`
+    );
+
+    if (!Array.isArray(messagesJson.messages)) {
+      throw new Error("Invalid data structure: messages is not an array");
+    }
+
+    const conversationJsonArray = await processMessages(messagesJson.messages);
+
+    // Import messages for each conversation in batches
+    for (let conversation of conversationJsonArray) {
+      const conversationId = Object.keys(conversation)[0];
+      const messages = conversation[conversationId];
+
+      console.log(
+        `Processing ${messages.length} messages for conversation: ${conversationId}`
+      );
+
+      // Split messages into batches
+      for (let i = 0; i < messages.length; i += batchSize) {
+        const batch = messages.slice(i, i + batchSize);
+        try {
+          await importBatch(conversationId, batch);
+        } catch (err) {
+          console.error(
+            `Failed to import batch for conversation ${conversationId}:`,
+            err.response?.data || err.message
+          );
+        }
+      }
+    }
+  } catch (err) {
+    console.error(`Failed to process message file ${filePath}:`, err.message);
+    throw err;
+  }
+}
+
+module.exports = { importMessages };

rest-api/migrate-data-from-sendbird-to-talkjs/importers/userImporter.js

@@ -0,0 +1,60 @@
+const fs = require("fs").promises;
+const axios = require("axios");
+const { secretKey, appID } = require("../config.js");
+
+const instance = axios.create({
+  baseURL: "https://api.talkjs.com",
+  headers: {
+    Authorization: `Bearer ${secretKey}`,
+    "Content-Type": "application/json",
+  },
+});
+
+async function importUsers(filePath) {
+  try {
+    // Read and parse file
+    const data = await fs.readFile(filePath, "utf8");
+    console.log(`Read file: ${filePath}`);
+
+    const userData = JSON.parse(data);
+    console.log(
+      `Parsed JSON data with ${userData.active_users?.length || 0} users`
+    );
+
+    if (!Array.isArray(userData.active_users)) {
+      throw new Error("Invalid data structure: active_users is not an array");
+    }
+
+    for (let sendbirdUser of userData.active_users) {
+      try {
+        // Build user with required fields
+        const user = {
+          id: sendbirdUser.user_id,
+          name: sendbirdUser.nickname,
+        };
+
+        // Add optional fields if they exist
+        if (sendbirdUser.metadata?.emailId) {
+          user.email = [sendbirdUser.metadata.emailId];
+        }
+
+        if (sendbirdUser.profile_url) {
+          user.photoUrl = sendbirdUser.profile_url;
+        }
+
+        await instance.put(`/v1/${appID}/users/${sendbirdUser.user_id}`, user);
+        console.log(`Imported user: ${sendbirdUser.user_id}`);
+      } catch (err) {
+        console.error(
+          `Failed to import user ${sendbirdUser.user_id}:`,
+          err.response?.data || err.message
+        );
+      }
+    }
+  } catch (err) {
+    console.error(`Failed to process user file ${filePath}:`, err.message);
+    throw err;
+  }
+}
+
+module.exports = { importUsers };

rest-api/migrate-data-from-sendbird-to-talkjs/package-lock.json

@@ -2,9 +2,10 @@
   "name": "migrate-data-from-sendbird-to-talkjs",
   "version": "1.0.0",
   "description": "",
-  "main": "talkjs-migration-users.js",
+  "main": "runImport.js",
   "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node runImport.js"
   },
   "author": "",
   "license": "ISC",

rest-api/migrate-data-from-sendbird-to-talkjs/package.json

@@ -0,0 +1,72 @@
+const fs = require("fs").promises;
+const path = require("path");
+const { importUsers } = require("./importers/userImporter");
+const { importChannels } = require("./importers/channelImporter");
+const { importMessages } = require("./importers/messageImporter");
+
+async function runImport() {
+  try {
+    console.log("Starting import process...");
+
+    // Import users
+    const userFiles = await fs.readdir(
+      path.join(__dirname, "sendbird-data/users")
+    );
+    console.log(`\nFound ${userFiles.length} user files to process...`);
+
+    for (const filename of userFiles) {
+      try {
+        const filePath = path.join(__dirname, "sendbird-data/users", filename);
+        await importUsers(filePath);
+      } catch (err) {
+        console.error(`Failed to import users from ${filename}`);
+      }
+    }
+
+    // Import channels
+    const channelFiles = await fs.readdir(
+      path.join(__dirname, "sendbird-data/channels")
+    );
+    console.log(`\nFound ${channelFiles.length} channel files to process...`);
+
+    for (const filename of channelFiles) {
+      try {
+        const filePath = path.join(
+          __dirname,
+          "sendbird-data/channels",
+          filename
+        );
+        await importChannels(filePath);
+      } catch (err) {
+        console.error(`Failed to import channels from ${filename}`);
+      }
+    }
+
+    // Import messages
+    const messageFiles = await fs.readdir(
+      path.join(__dirname, "sendbird-data/messages")
+    );
+    console.log(`\nFound ${messageFiles.length} message files to process...`);
+
+    for (const filename of messageFiles) {
+      try {
+        const filePath = path.join(
+          __dirname,
+          "sendbird-data/messages",
+          filename
+        );
+        await importMessages(filePath);
+      } catch (err) {
+        console.error(`Failed to import messages from ${filename}`);
+      }
+    }
+  } catch (err) {
+    console.error("Import process failed:", err.message);
+    process.exit(1);
+  }
+}
+
+runImport().catch((err) => {
+  console.error("Fatal error during import:", err.message);
+  process.exit(1);
+});