add markdown support

Author: joyqi

README.md

@@ -94,4 +94,5 @@ Sample input file(json format):
 
 * `auto`: Automatically detect the file format.
 * `json`: JSON format.
-* `yaml`: YAML format.
+* `yaml`: YAML format.
+* `markdown`: Markdown format.

package.json

@@ -40,6 +40,7 @@
   "devDependencies": {
     "@types/flat": "^5.0.2",
     "@types/js-yaml": "^4.0.5",
+    "@types/marked": "^5.0.0",
     "@types/mocha": "latest",
     "@types/node": "latest",
     "@types/yargs": "^17.0.24",
@@ -57,6 +58,7 @@
     "flat": "^5.0.2",
     "js-yaml": "^4.1.0",
     "langchain": "^0.0.70",
+    "marked": "^5.1.0",
     "ora": "^5.0.0",
     "yargs": "^17.7.2"
   }

pnpm-lock.yaml

@@ -1,13 +1,17 @@
 import { extname } from 'path';
 import json from './json';
 import yaml from './yaml';
+import markdown from './markdown';
 
 export type FormatterType = {
     type: 'json',
     formatter: json
 } | {
     type: 'yaml',
     formatter: yaml
+} | {
+    type: 'markdown',
+    formatter: markdown
 };
 
 export interface Formatter {
@@ -21,6 +25,8 @@ export function getFormatter<T extends FormatterType>(type: T['type']): T['forma
             return new json();
         case 'yaml':
             return new yaml();
+        case 'markdown':
+            return new markdown();
         default:
             throw new Error(`Unknown formatter type: ${type}`);
     }
@@ -35,6 +41,8 @@ export function detectFormatterType(fileName: string): FormatterType['type'] {
         case '.yaml':
         case '.yml':
             return 'yaml';
+        case '.md':
+            return 'markdown';
         default:
             throw new Error(`Unknown file extension: ${ext}`);
     }

src/formatter/index.ts

@@ -0,0 +1,40 @@
+import { Formatter } from ".";
+import { marked } from 'marked';
+
+const PREFIX = 'p_';
+
+type Links = {
+    [key: string]: { href: string | null; title: string | null };
+};
+
+export default class implements Formatter {
+    private links: Links = {};
+
+    unmarshal(text: string): any {
+        const tokens = marked.lexer(text);
+        const result: Record<string, string> = {};
+
+        if (tokens.links) {
+            this.links = tokens.links;
+        }
+
+        for (const [i, token] of tokens.entries()) {
+            result[PREFIX + i] = token.raw;
+        }
+
+        return result;
+    }
+
+    marshal(data: any, indent: number): string {
+        let links = '';
+
+        if (this.links) {
+            for (const [key, link] of Object.entries(this.links)) {
+                const suffix = link.title ? ` "${link.title}"` : '';
+                links += `[${key}]: ${link.href}${suffix}\n`;
+            }
+        }
+
+        return Object.values(data).join('') + links;
+    }
+}

src/formatter/markdown.ts

@@ -1,5 +1,3 @@
-import yargs from 'yargs';
-import { hideBin } from 'yargs/helpers';
 import { ChatOpenAI } from 'langchain/chat_models/openai';
 import {
     ChatPromptTemplate,
@@ -15,20 +13,6 @@ import ora from 'ora';
 import languageEncoding from 'detect-file-encoding-and-language';
 import detectIndent from 'detect-indent';
 
-yargs(hideBin(process.argv))
-    .command('tc [file]', 'Translate formated file', (yargs) => {
-        return yargs.positional('file', {
-            describe: 'File to translate',
-            type: 'string',
-            default: 'lang.json',
-        });
-    })
-    .option('key', {
-        alias: 'k',
-        type: 'string',
-        description: 'OpenAI API key',
-    }).parse();
-
 // Build the chat model
 function buildChain(chat: ChatOpenAI, systemMessage: string): LLMChain {
     const chatPrompt = ChatPromptTemplate.fromPromptMessages([