Refactored/Restructured the whole project

Author: gignupg

.gitignore

@@ -0,0 +1,2 @@
+umd/
+lib/

bin/cli.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-const languageEncoding = require("../index.js");
+const languageEncoding = require("../src/index.js");
 
 const path = process.argv[2];
 

package-lock.json

@@ -2,18 +2,20 @@
   "name": "detect-file-encoding-and-language",
   "version": "1.6.1",
   "description": "An NPM package to detect the encoding and language of a file",
-  "main": "index.js",
+  "main": "src/server/index-node.js",
   "scripts": {
-    "regextest": "node regexTester.test.js",
-    "test": "node index.test.js",
+    "regextest": "node ./src/testing/regexTester.test.js",
+    "test": "node ./src/testing/index.test.js",
     "prepublishOnly": "npm test"
   },
-  "browser": {
-    "fs": false
-  },
+  "browser": "umd/browser-bundle.js",
   "bin": {
-    "dfeal": "./bin/cli.js"
+    "dfeal": "bin/cli.js"
   },
+  "files": [
+    "umd"
+  ],
+  "unpkg": "umd/browser-bundle.js",
   "repository": {
     "type": "git",
     "url": "git+https://github.com/gignupg/Detect-File-Encoding-and-Language.git"
@@ -33,6 +35,5 @@
   "bugs": {
     "url": "https://github.com/gignupg/Detect-File-Encoding-and-Language/issues"
   },
-  "homepage": "https://github.com/gignupg/Detect-File-Encoding-and-Language#readme",
-  "dependencies": {}
+  "homepage": "https://github.com/gignupg/Detect-File-Encoding-and-Language#readme"
 }

package.json

@@ -0,0 +1,9 @@
+module.exports = (content) => {
+    for (let b = 0; b < content.length; b++) {
+        // If ? is encountered it's definitely not utf8!
+        if (content[b] === "�") {
+            return false;
+        }
+    }
+    return true;
+}

src/components/checkUTF.js

@@ -0,0 +1,34 @@
+const countAllMatches = require('./processing-content/countAllMatches.js');
+const calculateConfidenceScore = require('./processing-content/calculateConfidenceScore.js');
+
+module.exports = (data) => {
+    const fileInfo = {};
+    const languageArr = require('../language-config/languageObject.js');
+
+    data.languageArr = countAllMatches(data, languageArr);
+
+    fileInfo.language = data.languageArr.reduce((acc, val) => acc.count > val.count ? acc : val).name;
+
+    // "pos" gives us the position in the language array that has the most matches
+    data.pos = data.languageArr.findIndex(elem => elem.name === fileInfo.language);
+
+    // Determine the encoding
+    fileInfo.encoding = data.utf8 ? "UTF-8" : data.languageArr[data.pos].encoding;
+
+    const calculations = calculateConfidenceScore(data, fileInfo);
+
+    if (data.testFilePath) {
+        return calculations;
+    }
+
+    fileInfo.confidence = calculations;
+
+    // Edge case, when no matches were found
+    if (!data.languageArr[data.pos].count) {
+        fileInfo.language = null;
+        fileInfo.encoding = data.utf8 ? "UTF-8" : null;
+        fileInfo.confidence = data.utf8 ? 1 : null;
+    }
+
+    return fileInfo;
+};

src/components/processContent.js

@@ -0,0 +1,70 @@
+module.exports = (data, fileInfo) => {
+    const charRegex = new RegExp(/\d|\n|\s|\-|\.|\,|\:|\;|\?|\!|\<|\>|\[|\]|\{|\}|\&|\=|\|/, "g");
+    const totalCharacters = data.content.replace(charRegex, "").length;
+    const langArr = data.languageArr;
+    const pos = data.pos;
+    const testFilePath = data.testFilePath;
+
+    const secondLanguage = langArr.reduce((acc, val) => {
+        if (acc.name === fileInfo.language) return val;
+        if (val.name === fileInfo.language) return acc;
+
+        return acc.count >= val.count ? acc : val;
+    });
+
+    const languageRatio = langArr[pos].count / (secondLanguage.count + langArr[pos].count);
+    const characterWordRatio = langArr[pos].count / totalCharacters;
+
+    let lowerLimit = null;
+    let upperLimit = null;
+    const multiplier = 0.8;
+
+    if (data.utf8) {
+        lowerLimit = langArr[pos].utfFrequency ? langArr[pos].utfFrequency.low * multiplier : null;
+        upperLimit = langArr[pos].utfFrequency ? (langArr[pos].utfFrequency.low + langArr[pos].utfFrequency.high) / 2 : null;
+
+    } else {
+        lowerLimit = langArr[pos].isoFrequency ? langArr[pos].isoFrequency.low * multiplier : null;
+        upperLimit = langArr[pos].isoFrequency ? (langArr[pos].isoFrequency.low + langArr[pos].isoFrequency.high) / 2 : null;
+    }
+
+    let confidenceScore;
+
+    if (!lowerLimit || !upperLimit) {
+        confidenceScore = null;
+
+    } else if (characterWordRatio >= upperLimit) {
+        confidenceScore = 1;
+
+    } else if (characterWordRatio > lowerLimit) {
+        const range = upperLimit - lowerLimit;
+        const surplus = characterWordRatio - lowerLimit;
+        const confidenceRaisePercentage = surplus / range;
+        const confidenceRaise = (1 - languageRatio) * confidenceRaisePercentage;
+        confidenceScore = Number((languageRatio + confidenceRaise).toFixed(2));
+
+    } else {
+        confidenceScore = Number((languageRatio * (characterWordRatio / lowerLimit)).toFixed(2));
+    }
+
+    // If the test script is running
+    if (testFilePath) {
+        return {
+            name: testFilePath.substr(testFilePath.lastIndexOf('/') + 1),
+            path: testFilePath,
+            language: fileInfo.language,
+            utf8: data.utf8,
+            confidence: confidenceScore,
+            ratio: Number(languageRatio.toFixed(2)),
+            count: langArr[pos].count,
+            totalCharacters: totalCharacters,
+            characterWordRatio: characterWordRatio.toFixed(6),
+            secondLanguage: {
+                name: secondLanguage.name,
+                count: secondLanguage.count
+            }
+        };
+    }
+
+    return confidenceScore;
+};

src/components/processing-content/calculateConfidenceScore.js

@@ -0,0 +1,29 @@
+module.exports = (data, languageArr) => {
+    const newLanguageArr = [];
+
+    // Cloning the language array and making sure that "count" has no reference to "languageArr"!
+    languageArr.forEach((obj) => {
+        const updatedLangObj = {};
+        Object.keys(obj).forEach(key => {
+            if (key !== "count") {
+                updatedLangObj[key] = obj[key];
+            } else {
+                updatedLangObj.count = 0;
+            }
+        });
+        newLanguageArr.push(updatedLangObj);
+    });
+
+    const regex = data.utf8 ? "utfRegex" : "isoRegex";
+
+    // Populate the count property of our language array!
+    newLanguageArr.forEach(lang => {
+        if (lang[regex]) {
+            const matches = data.content.match(lang[regex]);
+
+            if (matches) lang.count = matches.length;
+        }
+    });
+
+    return newLanguageArr;
+}

src/components/processing-content/countAllMatches.js

@@ -0,0 +1,35 @@
+const checkUTF = require('./components/checkUTF.js');
+const processContent = require('./components/processContent.js');
+
+module.exports = (file, test) => {
+    return new Promise((resolve, reject) => {
+        const input = {};
+        const utfReader = new FileReader();
+
+        utfReader.onerror = (err) => {
+            reject(err);
+        };
+
+        utfReader.onload = () => {
+            const utfContent = utfReader.result;
+
+            input.utf8 = checkUTF(utfContent);
+
+            if (utf8) {
+                input.content = utfContent;
+                resolve(processContent(input));
+
+            } else {
+                const isoReader = new FileReader();
+
+                isoReader.onload = () => {
+                    input.content = isoReader.result;
+                    resolve(processContent(input));
+                };
+
+                isoReader.readAsText(file, "ISO-8859-1");
+            }
+        };
+        utfReader.readAsText(file, "UTF-8");
+    });
+};

src/index-browser.js

@@ -0,0 +1,29 @@
+const checkUTF = require('./components/checkUTF.js');
+const processContent = require('./components/processContent.js');
+
+module.exports = (file, test) => {
+    return new Promise((resolve) => {
+        const data = {};
+        const fs = require('fs');
+        data.testFilePath = test ? file : null;
+
+        fs.readFile(file, "UTF-8", (err, utfContent) => {
+            if (err) reject(err);
+
+            data.utf8 = checkUTF(utfContent);
+
+            if (data.utf8) {
+                data.content = utfContent;
+                resolve(processContent(data));
+
+            } else {
+                fs.readFile(file, "latin1", (err, isoContent) => {
+                    if (err) reject(err);
+
+                    data.content = isoContent;
+                    resolve(processContent(data));
+                });
+            }
+        });
+    });
+};