refactor: Use tokenize instead of getTokenizer (#15)

* Use tokenize instead of getTokenizer

* Reduce diff

Author: k-tahiro

src/no-doubled-conjunction.js

@@ -1,7 +1,7 @@
 // LICENSE : MIT
 "use strict";
 import { RuleHelper } from "textlint-rule-helper";
-import { getTokenizer } from "kuromojin";
+import { tokenize } from "kuromojin";
 import { split as splitSentences, Syntax as SentenceSyntax } from "sentence-splitter";
 import { StringSource } from "textlint-util-to-string";
 
@@ -42,46 +42,44 @@ export default function (context, options = {}) {
             if (sentences.length === 0) {
                 return;
             }
-            return getTokenizer().then(tokenizer => {
-                const selectConjenction = (sentence) => {
-                    const tokens = tokenizer.tokenizeForSentence(sentence.raw);
-                    const conjunctionTokens = tokens.filter((token, index) => {
-                        const prevToken = tokens[index - 1];
-                        // スペースが切れ目として認識されてしまう問題を回避
-                        // https://github.com/textlint-ja/textlint-rule-no-doubled-conjunction/issues/14
-                        if (prevToken && prevToken.pos_detail_1 === "空白" && token.pos === "接続詞") {
-                            return false;
-                        }
-                        return token.pos === "接続詞"
-                    });
-                    return [sentence, conjunctionTokens];
-                };
-                let prev_token = null;
-                sentences.map(selectConjenction).reduce((prev, current) => {
-                    const [sentence, current_tokens] = current;
-                    const [prev_sentence, prev_tokens] = prev;
-                    let token = prev_token;
-                    if (prev_tokens && prev_tokens.length > 0) {
-                        token = prev_tokens[0];
+            const selectConjenction = async (sentence) => {
+                const tokens = await tokenize(sentence.raw);
+                const conjunctionTokens = tokens.filter((token, index) => {
+                    const prevToken = tokens[index - 1];
+                    // スペースが切れ目として認識されてしまう問題を回避
+                    // https://github.com/textlint-ja/textlint-rule-no-doubled-conjunction/issues/14
+                    if (prevToken && prevToken.pos_detail_1 === "空白" && token.pos === "接続詞") {
+                        return false;
                     }
-                    if (current_tokens.length > 0) {
-                        if (token && current_tokens[0].surface_form === token.surface_form) {
-                            const conjunctionSurface = token.surface_form;
-                            const originalIndex = source.originalIndexFromPosition({
-                                line: sentence.loc.start.line,
-                                column: sentence.loc.start.column + (current_tokens[0].word_position - 1)
-                            });
-                            // padding position
-                            const padding = {
-                                index: originalIndex
-                            };
-                            report(node, new RuleError(`同じ接続詞（${conjunctionSurface}）が連続して使われています。`, padding));
-                        }
-                    }
-                    prev_token = token;
-                    return current;
+                    return token.pos === "接続詞"
                 });
-            });
+                return [sentence, conjunctionTokens];
+            }
+            let prev_token = null;
+            return Promise.all(sentences.map(selectConjenction)).then((result) => result.reduce((prev, current) => {
+                const [sentence, current_tokens] = current;
+                const [prev_sentence, prev_tokens] = prev;
+                let token = prev_token;
+                if (prev_tokens && prev_tokens.length > 0) {
+                    token = prev_tokens[0];
+                }
+                if (current_tokens.length > 0) {
+                    if (token && current_tokens[0].surface_form === token.surface_form) {
+                        const conjunctionSurface = token.surface_form;
+                        const originalIndex = source.originalIndexFromPosition({
+                            line: sentence.loc.start.line,
+                            column: sentence.loc.start.column + (current_tokens[0].word_position - 1)
+                        });
+                        // padding position
+                        const padding = {
+                            index: originalIndex
+                        };
+                        report(node, new RuleError(`同じ接続詞（${conjunctionSurface}）が連続して使われています。`, padding));
+                    }
+                }
+                prev_token = token;
+                return current;
+            }));
         }
     }
 };