[add] File Crawler API based on Web-file-cache template repository

.env

@@ -9,3 +9,6 @@ SENTRY_PROJECT =
 NEXT_PUBLIC_LARK_API_HOST = https://open.feishu.cn/open-apis/
 NEXT_PUBLIC_LARK_APP_ID = cli_a2c7771153f8900c
 NEXT_PUBLIC_LARK_WIKI_URL = https://idea2app.feishu.cn/wiki/space/7318346900506181660
+
+NEXT_PUBLIC_CACHE_HOST = https://cache.example.com
+CACHE_REPOSITORY = your-namespace/Web-file-cache

README.md

@@ -14,29 +14,30 @@
 - Component suite: [Bootstrap v5][4]
 - PWA framework: [Workbox v6][5]
 - State management: [MobX v6][11]
-- API router: [Koa 2][12]
+- API router: [Koa 3][12]
 - CI / CD: GitHub [Actions][13] + [Vercel][14]
 - Monitor service: [Sentry][15]
 
 ## Major examples
 
 1. [Markdown articles](pages/article/)
-2. [Editor components](pages/component.tsx)
-3. [Pagination table](pages/pagination.tsx)
-4. [Scroll list](pages/scroll-list.tsx)
-5. [Not Found page (NGO)](pages/_error.tsx)
+2. [Lark wiki](pages/wiki/)
+3. [Editor components](pages/component.tsx)
+4. [Pagination table](pages/pagination.tsx)
+5. [Scroll list](pages/scroll-list.tsx)
+6. [Not Found page (NGO)](pages/_error.tsx)
    - Global: https://notfound.org/
    - Chinese: https://www.dnpw.org/cn/pa-notfound.html
 
 ## User cases
 
 1. https://github.com/kaiyuanshe/kaiyuanshe.github.io
 2. https://github.com/idea2app/idea2app.github.io
+3. https://github.com/Open-Source-Bazaar/Open-Source-Bazaar.github.io
 
 ## Best practice
 
 1.  Install GitHub apps in your organization or account:
-
     1.  [Probot settings][16]: set up Issue labels & Pull Request rules
     2.  [PR badge][17]: set up Online [VS Code][18] editor entries in Pull Request description
 
@@ -80,21 +81,24 @@ You can check out [the Next.js GitHub repository][28] - your feedback and contri
 
 ### Environment variables
 
-|           name            |     file     |        description        |
-| :-----------------------: | :----------: | :-----------------------: |
-|    `SENTRY_AUTH_TOKEN`    | `.env.local` |  [Official document][29]  |
-|       `SENTRY_ORG`        |    `.env`    |  [Official document][30]  |
-|     `SENTRY_PROJECT`      |    `.env`    |  [Official document][30]  |
-| `NEXT_PUBLIC_SENTRY_DSN`  |    `.env`    |  [Official document][31]  |
-| `NEXT_PUBLIC_LARK_APP_ID` | `.env.local` |  [Official document][32]  |
-|     `LARK_APP_SECRET`     | `.env.local` |  [Official document][32]  |
-| `NEXT_PUBLIC_CACHE_HOST`  |    `.env`    | Static files CDN for Lark |
+|            name             |     file     |                   description                   |
+| :-------------------------: | :----------: | :---------------------------------------------: |
+|        `JWT_SECRET`         | `.env.local` | Random String for **JSON Web Token** encryption |
+|     `SENTRY_AUTH_TOKEN`     | `.env.local` |             [Official document][29]             |
+|        `SENTRY_ORG`         |    `.env`    |             [Official document][30]             |
+|      `SENTRY_PROJECT`       |    `.env`    |             [Official document][30]             |
+|  `NEXT_PUBLIC_SENTRY_DSN`   |    `.env`    |             [Official document][31]             |
+|  `NEXT_PUBLIC_LARK_APP_ID`  | `.env.local` |             [Official document][32]             |
+|      `LARK_APP_SECRET`      | `.env.local` |             [Official document][32]             |
+|  `NEXT_PUBLIC_CACHE_HOST`   |    `.env`    |            Static files CDN for Lark            |
+|     `CACHE_REPOSITORY`      |    `.env`    |   [GitHub repository for Lark file cache][33]   |
+| `NEXT_PUBLIC_LARK_WIKI_URL` |    `.env`    |            Entry URL of a Lark wiki             |
 
 ### Vercel
 
 The easiest way to deploy your Next.js app is to use the [Vercel Platform][14] from the creators of Next.js.
 
-Check out our [Next.js deployment documentation][33] for more details.
+Check out our [Next.js deployment documentation][34] for more details.
 
 ### Docker
 
@@ -103,6 +107,14 @@ pnpm pack-image
 pnpm container
 ```
 
+### CDN
+
+1. create & configure your own `CACHE_REPOSITORY` with https://github.com/idea2app/Web-file-cache
+2. set the `NEXT_PUBLIC_CACHE_HOST` environment variable to your CDN URL
+3. start your Lark-Next.js service, then copy the Crawler JWT from CLI console
+4. create an Automation Flow in your Lark BI Table to watch Attachment fields changing, then send an HTTP `POST` request to `https://your.next.js/api/file/crawler/task` with the Crawler JWT in the `Authorization` header
+5. your Lark file attachments will be cached in your CDN and loaded by your Lark-Next.js service automatically
+
 [0]: https://www.larksuite.com/
 [1]: https://react.dev/
 [2]: https://www.typescriptlang.org/
@@ -136,4 +148,5 @@ pnpm container
 [30]: https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/#use-environment-variables
 [31]: https://docs.sentry.io/platforms/javascript/guides/nextjs/manual-setup/#create-initialization-config-files
 [32]: https://open.larksuite.com/document/server-docs/getting-started/api-access-token/app-access-token-development-guide#1f8b587c
-[33]: https://nextjs.org/docs/deployment
+[33]: https://github.com/idea2app/Web-file-cache?tab=readme-ov-file
+[34]: https://nextjs.org/docs/deployment

models/Base.ts

@@ -23,6 +23,8 @@ githubClient.use(({ request }, next) => {
   return next();
 });
 
+export { githubClient };
+
 export const repositoryStore = new RepositoryModel('idea2app');
 
 type UploadedFile = Record<'originalname' | 'filename' | 'location', string>;

models/configuration.ts

@@ -4,15 +4,17 @@ export const Name = process.env.NEXT_PUBLIC_SITE_NAME,
   Summary = process.env.NEXT_PUBLIC_SITE_SUMMARY,
   DefaultImage = process.env.NEXT_PUBLIC_LOGO!;
 
-export const { VERCEL_URL, GITHUB_TOKEN } = process.env;
+export const { VERCEL_URL, JWT_SECRET, GITHUB_TOKEN, CACHE_REPOSITORY } =
+  process.env;
 
 export const API_Host = isServer()
   ? VERCEL_URL
     ? `https://${VERCEL_URL}`
     : 'http://localhost:3000'
   : globalThis.location.origin;
 
-export const CACHE_HOST = process.env.NEXT_PUBLIC_CACHE_HOST!;
+export const CACHE_HOST = process.env.NEXT_PUBLIC_CACHE_HOST!,
+  CrawlerEmail = `crawler@idea2.app`;
 
 export const LARK_API_HOST = `${API_Host}/api/Lark/`;
 

next-env.d.ts

@@ -1,5 +1,6 @@
 /// <reference types="next" />
 /// <reference types="next/image-types/global" />
+/// <reference path="./.next/types/routes.d.ts" />
 
 // NOTE: This file should not be edited
 // see https://nextjs.org/docs/pages/api-reference/config/typescript for more information.

package.json

@@ -15,36 +15,36 @@
     "@editorjs/list": "^2.0.8",
     "@editorjs/paragraph": "^2.11.7",
     "@editorjs/quote": "^2.7.6",
-    "@koa/router": "^14.0.0",
-    "@mdx-js/loader": "^3.1.0",
-    "@mdx-js/react": "^3.1.0",
-    "@next/mdx": "^15.4.5",
-    "@sentry/nextjs": "^10.1.0",
-    "copy-webpack-plugin": "^13.0.0",
-    "core-js": "^3.45.0",
+    "@mdx-js/loader": "^3.1.1",
+    "@mdx-js/react": "^3.1.1",
+    "@next/mdx": "^15.5.2",
+    "@sentry/nextjs": "^10.9.0",
+    "copy-webpack-plugin": "^13.0.1",
+    "core-js": "^3.45.1",
     "editorjs-html": "^4.0.5",
     "file-type": "^21.0.0",
     "formidable": "^3.5.4",
     "idea-react": "^2.0.0-rc.13",
     "jsonwebtoken": "^9.0.2",
     "koa": "^3.0.1",
+    "koa-jwt": "^4.0.4",
     "koajax": "^3.1.2",
-    "less": "^4.4.0",
+    "less": "^4.4.1",
     "less-loader": "^12.3.0",
     "lodash": "^4.17.21",
-    "marked": "^16.1.2",
+    "marked": "^16.2.1",
     "mime": "^4.0.7",
     "mobx": "^6.13.7",
-    "mobx-github": "^0.3.11",
+    "mobx-github": "^0.4.0",
     "mobx-i18n": "^0.7.1",
-    "mobx-lark": "^2.4.0",
+    "mobx-lark": "^2.4.1",
     "mobx-react": "^9.2.0",
     "mobx-react-helper": "^0.5.1",
     "mobx-restful": "^2.1.0",
-    "mobx-restful-table": "^2.5.2",
-    "next": "^15.4.5",
+    "mobx-restful-table": "^2.5.3",
+    "next": "^15.5.2",
     "next-pwa": "~5.6.0",
-    "next-ssr-middleware": "^1.0.2",
+    "next-ssr-middleware": "^1.0.3",
     "next-with-less": "^3.0.1",
     "prismjs": "^1.30.0",
     "react": "^19.1.1",
@@ -55,43 +55,44 @@
     "remark-frontmatter": "^5.0.0",
     "remark-gfm": "^4.0.1",
     "remark-mdx-frontmatter": "^5.2.0",
-    "undici": "^7.13.0",
-    "web-utility": "^4.5.1",
-    "webpack": "^5.101.0",
-    "yaml": "^2.8.0"
+    "undici": "^7.15.0",
+    "web-utility": "^4.5.3",
+    "webpack": "^5.101.3",
+    "yaml": "^2.8.1"
   },
   "devDependencies": {
     "@babel/plugin-proposal-decorators": "^7.28.0",
     "@babel/plugin-transform-typescript": "^7.28.0",
     "@babel/preset-react": "^7.27.1",
-    "@cspell/eslint-plugin": "^9.2.0",
-    "@eslint/compat": "^1.3.1",
+    "@cspell/eslint-plugin": "^9.2.1",
+    "@eslint/compat": "^1.3.2",
     "@eslint/eslintrc": "^3.3.1",
-    "@eslint/js": "^9.32.0",
-    "@next/eslint-plugin-next": "^15.4.5",
+    "@eslint/js": "^9.34.0",
+    "@next/eslint-plugin-next": "^15.5.2",
     "@softonus/prettier-plugin-duplicate-remover": "^1.1.2",
-    "@stylistic/eslint-plugin": "^5.2.2",
+    "@stylistic/eslint-plugin": "^5.3.1",
     "@types/eslint-config-prettier": "^6.11.3",
     "@types/formidable": "^3.4.5",
+    "@types/jsonwebtoken": "^9.0.10",
     "@types/koa": "^3.0.0",
     "@types/koa__router": "^12.0.4",
     "@types/lodash": "^4.17.20",
     "@types/next-pwa": "^5.6.9",
-    "@types/node": "^22.17.0",
-    "@types/react": "^19.1.9",
-    "eslint": "^9.32.0",
-    "eslint-config-next": "^15.4.5",
+    "@types/node": "^22.18.0",
+    "@types/react": "^19.1.12",
+    "eslint": "^9.34.0",
+    "eslint-config-next": "^15.5.2",
     "eslint-config-prettier": "^10.1.8",
     "eslint-plugin-react": "^7.37.5",
     "eslint-plugin-simple-import-sort": "^12.1.1",
     "globals": "^16.3.0",
     "husky": "^9.1.7",
     "jiti": "^2.5.1",
-    "lint-staged": "^16.1.4",
+    "lint-staged": "^16.1.6",
     "prettier": "^3.6.2",
     "prettier-plugin-css-order": "^2.1.2",
     "typescript": "~5.9.2",
-    "typescript-eslint": "^8.39.0"
+    "typescript-eslint": "^8.42.0"
   },
   "resolutions": {
     "next": "$next"

pages/api/Lark/bitable/v1/[...slug].ts

@@ -1,8 +1,9 @@
+import { Context } from 'koa';
 import { LarkPageData, TableRecord, TableRecordData } from 'mobx-lark';
 import { DataObject } from 'mobx-restful';
-import { createKoaRouter } from 'next-ssr-middleware';
+import { createKoaRouter, withKoaRouter } from 'next-ssr-middleware';
 
-import { withSafeKoaRouter } from '../../../core';
+import { safeAPI } from '../../../core';
 import { proxyLark, proxyLarkAll } from '../../core';
 
 export const config = { api: { bodyParser: false } };
@@ -14,30 +15,38 @@ function filterData(fields: DataObject) {
     if (!/^\w+$/.test(key)) delete fields[key];
 }
 
-router.get('/apps/:app/tables/:table/records/:record', async context => {
-  const { status, body } =
-    await proxyLark<TableRecordData<DataObject>>(context);
+router.get(
+  '/apps/:app/tables/:table/records/:record',
+  safeAPI,
+  async (context: Context) => {
+    const { status, body } =
+      await proxyLark<TableRecordData<DataObject>>(context);
 
-  const { fields } = body!.data!.record;
+    const { fields } = body!.data!.record;
 
-  filterData(fields);
+    filterData(fields);
 
-  context.status = status;
-  context.body = body;
-});
+    context.status = status;
+    context.body = body;
+  },
+);
 
-router.get('/apps/:app/tables/:table/records', async context => {
-  const { status, body } =
-    await proxyLark<LarkPageData<TableRecord<DataObject>>>(context);
+router.get(
+  '/apps/:app/tables/:table/records',
+  safeAPI,
+  async (context: Context) => {
+    const { status, body } =
+      await proxyLark<LarkPageData<TableRecord<DataObject>>>(context);
 
-  const list = body!.data!.items || [];
+    const list = body!.data!.items || [];
 
-  for (const { fields } of list) filterData(fields);
+    for (const { fields } of list) filterData(fields);
 
-  context.status = status;
-  context.body = body;
-});
+    context.status = status;
+    context.body = body;
+  },
+);
 
-router.all('/(.*)', proxyLarkAll);
+router.all('/(.*)', safeAPI, proxyLarkAll);
 
-export default withSafeKoaRouter(router);
+export default withKoaRouter(router);

pages/api/Lark/document/markdown/[...slug].ts

@@ -1,11 +1,11 @@
-import { createKoaRouter } from 'next-ssr-middleware';
+import { createKoaRouter, withKoaRouter } from 'next-ssr-middleware';
 
-import { withSafeKoaRouter } from '../../../core';
+import { safeAPI } from '../../../core';
 import { lark } from '../../core';
 
 const router = createKoaRouter(import.meta.url);
 
-router.get('/:type/:id', async context => {
+router.get('/:type/:id', safeAPI, async context => {
   const { type, id } = context.params;
 
   const markdown = await lark.downloadMarkdown(`${type}/${id}`);
@@ -14,4 +14,4 @@ router.get('/:type/:id', async context => {
   context.body = markdown;
 });
 
-export default withSafeKoaRouter(router);
+export default withKoaRouter(router);

pages/api/Lark/file/[id]/[name].ts

@@ -1,15 +1,16 @@
 import { fileTypeFromStream } from 'file-type';
+import { Middleware } from 'koa';
 import MIME from 'mime';
-import { createKoaRouter } from 'next-ssr-middleware';
+import { createKoaRouter, withKoaRouter } from 'next-ssr-middleware';
 import { Readable } from 'stream';
 
 import { CACHE_HOST } from '../../../../../models/configuration';
-import { withSafeKoaRouter } from '../../../core';
+import { safeAPI } from '../../../core';
 import { lark } from '../../core';
 
 const router = createKoaRouter(import.meta.url);
 
-router.all('/:id/:name', async context => {
+const downloader: Middleware = async context => {
   const { method, url, params, query } = context;
   const { id, name } = params;
 
@@ -47,6 +48,10 @@ router.all('/:id/:name', async context => {
   if (method === 'GET')
     // @ts-expect-error Web type compatibility
     context.body = Readable.fromWeb(stream2);
-});
+};
 
-export default withSafeKoaRouter(router);
+router
+  .head('/:id/:name', safeAPI, downloader)
+  .get('/:id/:name', safeAPI, downloader);
+
+export default withKoaRouter(router);

pages/api/Lark/file/index.ts

@@ -3,18 +3,18 @@ import formidable from 'formidable';
 import { readFile } from 'fs/promises';
 import MIME from 'mime';
 import { UploadTargetType } from 'mobx-lark';
-import { createKoaRouter } from 'next-ssr-middleware';
+import { createKoaRouter, withKoaRouter } from 'next-ssr-middleware';
 import { parse } from 'path';
 
 import { LARK_API_HOST } from '../../../../models/configuration';
-import { withSafeKoaRouter } from '../../core';
+import { safeAPI } from '../../core';
 import { lark } from '../core';
 
 export const config = { api: { bodyParser: false } };
 
 const router = createKoaRouter(import.meta.url);
 
-router.post('/', async context => {
+router.post('/', safeAPI, async context => {
   const form = formidable();
 
   const [{ parent_type, parent_node }, { file }] = await form.parse(
@@ -45,4 +45,4 @@ router.post('/', async context => {
   return (context.body = { link });
 });
 
-export default withSafeKoaRouter(router);
+export default withKoaRouter(router);