[refactor] simplify Data Import script with MobX-RESTful-migrator

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "typescript.tsdk": "node_modules\\typescript\\lib"
+}

package.json

@@ -9,13 +9,14 @@
   "dependencies": {
     "@arshiash80/strapi-plugin-iconhub": "^1.1.1",
     "@ckeditor/strapi-plugin-ckeditor": "^1.1.1",
-    "@strapi/plugin-cloud": "~5.23.5",
-    "@strapi/plugin-color-picker": "~5.23.5",
-    "@strapi/plugin-documentation": "~5.23.5",
-    "@strapi/plugin-users-permissions": "~5.23.5",
-    "@strapi/strapi": "~5.23.5",
-    "better-sqlite3": "^12.2.0",
+    "@strapi/plugin-cloud": "~5.24.1",
+    "@strapi/plugin-color-picker": "~5.24.1",
+    "@strapi/plugin-documentation": "~5.24.1",
+    "@strapi/plugin-users-permissions": "~5.24.1",
+    "@strapi/strapi": "~5.24.1",
+    "better-sqlite3": "^12.4.1",
     "cross-env": "^10.0.0",
+    "mobx-restful-migrator": "^0.1.1",
     "pg": "^8.16.3",
     "react": "^18.3.1",
     "react-dom": "^18.3.1",
@@ -32,15 +33,16 @@
     "@types/react-dom": "^18.3.7",
     "husky": "^9.1.7",
     "koajax": "^3.1.2",
-    "lint-staged": "^16.1.6",
+    "lint-staged": "^16.2.1",
+    "mobx-restful": "^2.1.3",
     "mobx-strapi": "^0.8.1",
     "prettier": "^3.6.2",
-    "swagger-typescript-api": "^13.2.11",
-    "tsx": "^4.20.5",
+    "swagger-typescript-api": "13.2.7",
+    "tsx": "^4.20.6",
     "typescript": "~5.9.2",
     "web-utility": "^4.6.1",
     "xlsx": "^0.18.5",
-    "zx": "^8.8.1"
+    "zx": "^8.8.3"
   },
   "pnpm": {
     "onlyBuiltDependencies": [

scripts/README.md

@@ -128,7 +128,6 @@ EXCEL_FILE=my-data.xlsx STRAPI_TOKEN=your_api_token tsx import-data.ts
 | `STRAPI_URL`   | `http://localhost:1337`     | Strapi 服务器地址                      |
 | `STRAPI_TOKEN` | -                           | Strapi API Token（必填，除非 DRY_RUN） |
 | `EXCEL_FILE`   | `教育公益开放式数据库.xlsx` | Excel 文件路径                         |
-| `BATCH_SIZE`   | `10`                        | 批量处理大小                           |
 | `DRY_RUN`      | `false`                     | 是否为模拟模式                         |
 
 ### 命令行选项
@@ -207,13 +206,7 @@ tsx import-data.ts --dry-run
    DRY_RUN=true tsx import-data.ts
    ```
 
-2. **减少批次大小**
-
-   ```bash
-   BATCH_SIZE=1 STRAPI_TOKEN=your_token tsx import-data.ts
-   ```
-
-3. **查看详细错误信息**
+2. **查看详细错误信息**
    - 脚本会输出详细的错误信息
    - 检查 Strapi 服务器日志
    - 查看 `logs/` 目录下自动生成的错误日志文件
@@ -234,9 +227,7 @@ tsx import-data.ts --dry-run
 
 ### 性能优化
 
-1. 调整 `BATCH_SIZE` 参数
-2. 实现并行处理
-3. 添加数据缓存机制
+1. 添加数据缓存机制
 
 ## 日志文件
 

scripts/import-data.ts

@@ -1,50 +1,40 @@
 #!/usr/bin/env tsx
 
 /**
- * Strapi database import script (Refactored)
+ * Strapi database import script using MobX-RESTful-migrator
  * Support import NGO organization data from Excel file to Strapi database
  */
 
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import { Config, OrganizationData, ExtendedUserData } from './types';
+import { RestMigrator } from 'mobx-restful-migrator';
 
-// Create WeakMap to store user data for organizations
-const userWeakMap = new WeakMap<OrganizationData, ExtendedUserData>();
-
-// Import refactored modules
-import { DataTransformer } from './transformers/data-transformer';
-import { UserTransformer } from './transformers/user-transformer';
+import { migrationMapping } from './transformers/data-transformer';
+import { Config } from './types';
 import { ExcelReader } from './utils/excel-reader';
-import { StrapiAPI } from './utils/strapi-api';
-import { DataImporter } from './utils/data-importer';
+import { ImportLogger } from './utils/import-logger';
+import { TargetOrganizationModel } from './utils/strapi-api';
 
 // Configuration
 const CONFIG: Config = {
   STRAPI_URL: process.env.STRAPI_URL || 'http://localhost:1337',
   STRAPI_TOKEN: process.env.STRAPI_TOKEN || '',
   EXCEL_FILE: process.env.EXCEL_FILE || '教育公益开放式数据库.xlsx',
   SHEET_NAME: process.env.SHEET_NAME || null,
-  BATCH_SIZE: parseInt(process.env.BATCH_SIZE || '10'), // Default batch size
-  BATCH_DELAY: parseInt(process.env.BATCH_DELAY || '0'), // Default no delay
   DRY_RUN: process.env.DRY_RUN === 'true',
   MAX_ROWS: parseInt(process.env.MAX_ROWS || '0'),
 };
 
-// Main function
-async function main(): Promise<void> {
-  let importer: DataImporter | null = null;
+async function main() {
+  const logger = new ImportLogger();
 
   // Handle process signals to ensure logs are saved on forced exit
   const handleExit = (signal: string) => {
     console.log(`\n收到 ${signal} 信号，正在保存日志...`);
-    if (importer?.logger) {
-      importer.logger.saveToFiles();
+    if (logger) {
+      logger.saveToFiles();
       console.log('日志已保存，程序退出。');
     }
     process.exit(0);
   };
-
   process.on('SIGINT', () => handleExit('SIGINT'));
   process.on('SIGTERM', () => handleExit('SIGTERM'));
   process.on('SIGQUIT', () => handleExit('SIGQUIT'));
@@ -53,92 +43,49 @@ async function main(): Promise<void> {
     console.log('=== Strapi 数据导入工具 ===\n');
 
     // Validate configuration
-    if (!CONFIG.STRAPI_TOKEN && !CONFIG.DRY_RUN) {
+    if (!CONFIG.STRAPI_TOKEN && !CONFIG.DRY_RUN)
       throw new Error('请设置 STRAPI_TOKEN 环境变量或使用 DRY_RUN=true');
-    }
 
-    const excelPath = path.join(process.cwd(), CONFIG.EXCEL_FILE);
-    if (!fs.existsSync(excelPath)) {
-      throw new Error(`Excel 文件不存在: ${excelPath}`);
-    }
+    if (CONFIG.DRY_RUN) console.log('🔥 DRY RUN 模式 - 不会实际创建数据\n');
 
-    // Read Excel data
-    console.log('读取 Excel 数据...');
-    const rawData = ExcelReader.readExcelFile(excelPath, CONFIG.SHEET_NAME);
-
-    // Limit data for testing
-    const limitedData =
-      CONFIG.MAX_ROWS > 0 ? rawData.slice(0, CONFIG.MAX_ROWS) : rawData;
-    if (CONFIG.MAX_ROWS > 0) {
-      console.log(
-        `限制导入数据量: ${limitedData.length} 行 (总共 ${rawData.length} 行)`,
-      );
-    }
+    const migrator = new RestMigrator(
+      () => ExcelReader.readExcelFile(CONFIG.EXCEL_FILE, CONFIG.SHEET_NAME),
+      TargetOrganizationModel,
+      migrationMapping,
+      logger,
+    );
+    console.log('开始数据迁移...\n');
 
-    // Transform data format with user support
-    console.log('转换数据格式...');
-    const organizations = limitedData
-      .map((row) => {
-        try {
-          const organization = DataTransformer.transformOrganization(row);
-
-          // Extract user data from the same row
-          const userData = UserTransformer.transformUser(row);
-
-          // Attach user data for later processing using WeakMap
-          if (userData) {
-            userWeakMap.set(organization, userData);
-          }
-
-          return organization;
-        } catch (error: any) {
-          const orgName = row['常用名称'] || row.name || 'Unknown';
-          console.warn(`转换数据失败，跳过行: ${orgName}`, error.message);
-          return null;
-        }
-      })
-      .filter((org): org is OrganizationData => org !== null && !!org.name);
-
-    console.log(`转换完成，准备导入 ${organizations.length} 个组织\n`);
-
-    // Show examples in dry run mode
-    if (CONFIG.DRY_RUN) {
-      console.log('=== DRY RUN 模式 ===');
-      for (const [index, org] of organizations.slice(0, 3).entries()) {
-        console.log(`示例 ${index + 1}:`, JSON.stringify(org, null, 2));
-      }
-      console.log('==================\n');
-    }
+    let count = 0;
 
-    // Initialize API client and importer
-    const api = new StrapiAPI(CONFIG.STRAPI_URL, CONFIG.STRAPI_TOKEN);
-    importer = new DataImporter(
-      api,
-      userWeakMap,
-      CONFIG.BATCH_SIZE,
-      CONFIG.BATCH_DELAY,
-      CONFIG.DRY_RUN,
-    );
+    for await (const organization of migrator.boot({
+      dryRun: CONFIG.DRY_RUN,
+    }))
+      if (++count === CONFIG.MAX_ROWS && CONFIG.MAX_ROWS > 0) break;
 
-    // Start import
-    await importer.importOrganizations(organizations);
+    logger.printStats();
 
     console.log('导入完成！');
+
+    await logger.saveToFiles();
   } catch (error: any) {
     console.error('导入失败:', error.message);
+    console.error('错误堆栈:', error.stack);
+
+    await logger?.saveToFiles();
+
     process.exit(1);
   }
 }
 
-// Handle command line arguments
-function parseArgs(): void {
+function parseArgs() {
   const args = process.argv.slice(2);
 
   if (args.includes('--help') || args.includes('-h')) {
     console.log(`
-Strapi 数据导入工具 (增强版)
+Strapi 数据导入工具
 
-支持同时导入组织信息和联系人用户，并自动建立关联关系。
+支持从 Excel 文件导入 NGO 组织数据到 Strapi 数据库。
 
 用法:
   tsx scripts/import-data.ts [选项]
@@ -152,17 +99,9 @@ Strapi 数据导入工具 (增强版)
   STRAPI_TOKEN      Strapi API Token
   EXCEL_FILE        Excel 文件路径 (默认: 教育公益开放式数据库.xlsx)
   SHEET_NAME        工作表名称 (默认: 使用第一个工作表)
-  BATCH_SIZE        批次大小 (默认: 10)
-  BATCH_DELAY       批次间延迟秒数 (默认: 0, 表示无延迟)
-  MAX_ROWS          最大导入行数 (默认: 0, 表示导入所有行)
-  DRY_RUN           模拟模式 (true/false)
-
-功能特性:
-  - 导入组织基本信息
-  - 自动创建联系人用户账户
-  - 建立组织与用户的关联关系
-  - 支持用户名冲突自动处理
-  - 重复检查和错误处理
+  MAX_ROWS          最大处理行数 (默认: 0，表示全部)
+  DRY_RUN           模拟运行 (true/false, 默认: false)
+  VERBOSE_LOGGING   详细日志 (true/false, 默认: false)
 
 示例:
   # 正常导入
@@ -177,21 +116,14 @@ Strapi 数据导入工具 (增强版)
   # 仅测试前10行
   MAX_ROWS=10 DRY_RUN=true tsx import-data.ts
 
-  # 设置批次间延迟
-  BATCH_DELAY=2 STRAPI_TOKEN=your_token tsx import-data.ts
+  # 设置详细日志
+  VERBOSE_LOGGING=true STRAPI_TOKEN=your_token tsx import-data.ts
 `);
     process.exit(0);
   }
 
-  if (args.includes('--dry-run') || args.includes('-d')) {
-    CONFIG.DRY_RUN = true;
-  }
-}
-
-// Entry point
-if (require.main === module) {
-  parseArgs();
-  main();
+  if (args.includes('--dry-run') || args.includes('-d')) CONFIG.DRY_RUN = true;
 }
 
-export { DataTransformer, ExcelReader, DataImporter, StrapiAPI };
+parseArgs();
+main();