Updates for v0.0.3

Author: michelcaradec

CHANGELOG.md

@@ -11,15 +11,52 @@ Here are a few common issues.
 
 ## [Unreleased]
 
-## [0.0.2]
+## [0.0.3] - 2019-11-01
+
+### Added
+
+- Code provider:
+  - Strings starting with a digit detection.
+  - DateTime patterns (eg.: `dd/MM/yyyy hh:mm`) detection.
+  - `img` HTML expression detection.
+  - Mathematical symbols detection (eg.: `this > that`).
+  - Strings with more than two contiguous spaces.
+- Class provider:
+  - `calc()` JavaScript expression detection.
+  - GraphQL expressions (`query`, `mutation`) detection.
+- New configuration setting `string-checker-js.file-extension-exclude` to specify file extensions to exclude from scan (default: `.d.ts`, `.min.js`).
+- New configuration setting `string-checker-js.variable.non-alpha-ratio-threshold` to specify the amount of non-alphabetical characters allowed in a string (default: `0.2`).
+
+### Changed
+
+- Entropy provider:
+  - Entropy is now computed from a string cleaned of its non-alphabetical characters.  
+    Example:  
+    `'"{field}" is invalid!'` has an entropy of 3.63.  
+    `'field is invalid'` has an entropy of = 3.12.
+- File extension filtering now uses a suffix-compare method to support multiple-dots extension (eg.: `.d.ts`).
+- Configuration setting `string-checker-js.workspace.file-max` maximum value set to 1000 (500 before).
+- Tokens are all collapsed by default when switching to token/file view.
+- Code refactoring.
+
+## [0.0.2] - 2019-10-30
+
+### Added
+
+- Code provider:
+  - Environment variable detection.
+- Class provider:
+  - `rgb()` JavaScript expression detection.
+
+### Changed
 
 - Formatting in README file.
 - "Release Notes" section removed from README file.
 - "Known Issues" section mode from README to CHANGELOG file.
-- Code provider improvement (environment variable detection).
-- Class provider improvement (`rgb()` javascript statement detection).
 - Code refactoring.
 
-## [0.0.1]
+## [0.0.1] - 2019-10-28
+
+### Added
 
 - Initial release.

README.md

@@ -30,12 +30,14 @@ Strings are evaluated by different providers, each being dedicated to a specific
 | Provider | Description | Example |
 |---|---|---|
 | Keywords provider | Detects strings from **user list**. | "far fa-smile" will be detected as a [Font Awesome smile icon](https://fontawesome.com/icons/smile?style=regular). |
-| Class provider | Detects strings as **class names**. |  |
+| Class provider | Detects strings as **class names** or **expressions**. | "use strict" will be detected as JavaScript expression. |
 | Code provider | Detects strings as **code** (variable names). | "../path/to/my/file" will be detected as a path.<br>"someVariable" will be detected as a camel case variable. |
 | Natural language provider | Detects strings as **natural language**. | ["Ceci n'est pas une pipe"](https://en.wikipedia.org/wiki/Ren%C3%A9_Magritte) will be detected as french language. |
-| Entropy provider | Detects string as **[Gibberish](https://en.wikipedia.org/wiki/Gibberish)**.<br>String [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) threshold can be configured in settings (`entropy.threshold`, default = 3). | "abbcccddddeeeee" has an entropy of 2.15.<br>"dd/MM/yyyy hh:mm:ss" has an entropy of 2.88.<br>["Gloubi-boulga"](https://fr.wikipedia.org/wiki/Gloubi-boulga) has an entropy of 2.93. |
+| Entropy provider | Detects string as **[Gibberish](https://en.wikipedia.org/wiki/Gibberish)**.<br>String [entropy](https://en.wikipedia.org/wiki/Entropy_(information_theory)) [1] threshold can be configured in settings (`entropy.threshold`, default = 3). | "abbcccddddeeeee" has an entropy of 2.44.<br>"dd/MM/yyyy hh:mm:ss" has an entropy of 2.88.<br>["Gloubi-boulga"](https://fr.wikipedia.org/wiki/Gloubi-boulga) has an entropy of 2.75. |
 | String provider | **Pass-through** detection. | *Any string will be detected as such.* |
 
+- [1] Starting at version v0.0.3, **string entropy** is computed after removing non-alphabetical characters.
+
 The `string.checker.js.testString` [command](#extension-settings) brings a convenient way to test all providers for a given string.
 
 ![demo-test-string](https://raw.githubusercontent.com/michelcaradec/string-checker-js/master/readme_assets/demo-test-string.gif)
@@ -71,4 +73,5 @@ This extension contributes the following settings:
 - [Font Awesome](https://fontawesome.com/icons/) icon is used for tokens activity bar.
 - [freeicons.io](https://www.freeicons.io/) icons are used for tokens view.
 - [franc](https://github.com/wooorm/franc) library is used for natural language detection.
+- [escape-string-regexp](https://github.com/sindresorhus/escape-string-regexp) library is used for regular expressions strings escape.
 - [TypeScript-Node-Starter](https://github.com/Microsoft/TypeScript-Node-Starter) Microsoft sample project is used for demonstration purpose.

package-lock.json

@@ -9,7 +9,7 @@
 		"url": "https://github.com/michelcaradec/string-checker-js"
 	},
 	"license": "MIT",
-	"version": "0.0.2",
+	"version": "0.0.3",
 	"engines": {
 		"vscode": "^1.38.0"
 	},
@@ -212,18 +212,26 @@
 					],
 					"description": "Set workspace file extensions to include in document scan."
 				},
+				"string-checker-js.file-extension-exclude": {
+					"type": "[string]",
+					"default": [
+						".d.ts",
+						".min.js"
+					],
+					"description": "Set workspace file extensions to exclude from document scan."
+				},
 				"string-checker-js.folder-name-exclude": {
 					"type": "[string]",
 					"default": [
 						"node_modules"
 					],
-					"description": "Set workspace folder names to exclude in document scan."
+					"description": "Set workspace folder names to exclude from document scan."
 				},
 				"string-checker-js.workspace.file-max": {
 					"type": "number",
 					"default": 100,
 					"minimum": 0,
-					"maximum": 500,
+					"maximum": 1000,
 					"description": "Set maximum number of files to process while executing workspace documents scan."
 				},
 				"string-checker-js.entropy.threshold": {
@@ -235,8 +243,12 @@
 				"string-checker-js.language.languages-check": {
 					"type": "[string]",
 					"default": [
-						"fra", "eng", "spa", "ita"
+						"fra",
+						"eng",
+						"spa",
+						"ita"
 					],
+					"description": "Set languages to detect (see franc (https://github.com/wooorm/franc) library).",
 					"markdownDescription": "Set languages to detect (see [franc](https://github.com/wooorm/franc) library)."
 				},
 				"string-checker-js.variable.word-min-length": {
@@ -251,6 +263,13 @@
 					"minimum": 10,
 					"description": "Set variable detection maximum word length. All words above this length will be detected as technical items."
 				},
+				"string-checker-js.variable.non-alpha-ratio-threshold": {
+					"type": "number",
+					"default": 0.2,
+					"minimum": 0,
+					"maximum": 1,
+					"description": "Set non-alphabetical characters threshold."
+				},
 				"string-checker-js.parser.jquery-exclude": {
 					"type": "boolean",
 					"default": true,
@@ -280,6 +299,7 @@
 		"typescript": "^3.6.4"
 	},
 	"dependencies": {
+		"escape-string-regexp": "^2.0.0",
 		"franc": "^4.1.0",
 		"typescript": "^3.6.4"
 	}

package.json

@@ -15,9 +15,29 @@ export class ClassNameDetect implements IDetectProvider {
             return [ConfidenceLevel.Technical, 'javascript'];
         }
 
+        if (/^calc\([^\)]+\)$/m.test(text)) {
+            return [ConfidenceLevel.Technical, 'javascript'];
+        }
+
         if (text.search(/fa[rs]? fa(-[^-]*)*/g) >= 0) {
             return [ConfidenceLevel.Technical, 'font awesome'];
         }
+
+        if (/^([a-z]+)\s\1-.+$/.test(text)) {
+            // `table table-*`.
+            return [ConfidenceLevel.Technical, 'class'];
+        }
+
+        if (/^query +\{/.test(text)
+            || /^query\(/.test(text)
+            || /^query +[a-zA-Z_][a-zA-Z\d_]+\(/.test(text)
+            || /^mutation\(/.test(text)) {
+            // `query {...}`
+            // `query(...)`
+            // `query myFunction(...)`
+            // `mutation(...)`
+            return [ConfidenceLevel.Technical, 'graphql'];
+        }
 
         if (text.search(/btn btn(-[^-]*)*/g) >= 0) {
             return [ConfidenceLevel.Technical, 'button'];

src/checker/providers/classNameProvider.ts

@@ -2,17 +2,20 @@ import * as vscode from 'vscode';
 import { IDetectProvider } from "./detectProvider";
 import { ConfidenceLevel } from "../../enumerations";
 import { Constants } from '../../constants';
+import { isCamelCase, isPascalCase, getNonAlphaRatio } from '../../helpers/utils';
 
 export class CodeDetect implements IDetectProvider {
     private _minWordLength: number;
     private _maxWordLength: number;
+    private _nonAlphaThreshold: number;
     readonly name: string = 'Code provider';
 
     readonly isStopOnEval: boolean = true;
 
     constructor() {
         this._minWordLength = vscode.workspace.getConfiguration(Constants.ExtensionID).get<number>('variable.word-min-length')!;
         this._maxWordLength = vscode.workspace.getConfiguration(Constants.ExtensionID).get<number>('variable.word-max-length')!;
+        this._nonAlphaThreshold = vscode.workspace.getConfiguration(Constants.ExtensionID).get<number>('variable.non-alpha-ratio-threshold')!;
     }
 
     private static readonly _startChars = [
@@ -62,6 +65,31 @@ export class CodeDetect implements IDetectProvider {
             return [ConfidenceLevel.Technical, 'env'];
         }
 
+        if (/^\d.*$/.test(text)) {
+            // Starting with a digit.
+            return [ConfidenceLevel.Technical, 'digit'];
+        }
+
+        if (/^((d{2,3}|m{2,3}|yyyy|yy|hh|s{2,3})(\/|\-|:|\.|,| )?)+$/i.test(text)) {
+            // DateTime patterns.
+            return [ConfidenceLevel.Technical, 'datetime'];
+        }
+
+        if (/^img\d+x\d+\s/.test(text)) {
+            // img.
+            return [ConfidenceLevel.Technical, 'img'];
+        }
+
+        if (/ ?(>|<|\+|\*|=|\/) ?/.test(text)) {
+            // Text with mathematical symbols.
+            return [ConfidenceLevel.Technical, 'formula'];
+        }
+
+        if (/\s{3,}/.test(text)) {
+            // Many contiguous spaces (2 contiguous might me a typo error, but not 3 or more).
+            return [ConfidenceLevel.Technical, 'spaces'];
+        }
+
         const posSpace = text.search(/\s/g);
         if (posSpace < 0) {
             // No white space...
@@ -75,11 +103,11 @@ export class CodeDetect implements IDetectProvider {
                 return [ConfidenceLevel.Technical, 'length'];
             }
 
-            if (this.isCamelCase(text)) {
+            if (isCamelCase(text)) {
                 return [ConfidenceLevel.Technical, 'camelCase'];
             }
 
-            if (this.isPascalCase(text)) {
+            if (isPascalCase(text)) {
                 return [ConfidenceLevel.Technical, 'PascalCase'];
             }
         } else {
@@ -89,14 +117,11 @@ export class CodeDetect implements IDetectProvider {
             }
         }
 
-        return [ConfidenceLevel.Unknown, ''];
-    }
-
-    private isCamelCase(text: string): boolean {
-        return /^[a-z][^\s]+$/.test(text);
-    }
+        const nonAlphaRatio = getNonAlphaRatio(text);
+        if (nonAlphaRatio > this._nonAlphaThreshold) {
+            return [ConfidenceLevel.Technical, `non-alpha=${nonAlphaRatio.toFixed(2)}`];
+        }
 
-    private isPascalCase(text: string): boolean {
-        return /^([A-Z][^\sA-Z]+)+$/.test(text);
+        return [ConfidenceLevel.Unknown, ''];
     }
 }

src/checker/providers/codeDetect.ts

@@ -2,6 +2,7 @@ import * as vscode from 'vscode';
 import { IDetectProvider } from "./detectProvider";
 import { ConfidenceLevel } from "../../enumerations";
 import { Constants } from '../../constants';
+import { stripNonAlphaCharacters } from '../../helpers/utils';
 
 export class EntropyDetect implements IDetectProvider {
     private _threshold: number;
@@ -15,7 +16,7 @@ export class EntropyDetect implements IDetectProvider {
     readonly isStopOnEval: boolean = false;
 
     check(text: string): [ConfidenceLevel, string] {
-        var entropy = this.getShannonEntropy(text.toLowerCase());
+        var entropy = this.getShannonEntropy(stripNonAlphaCharacters(text.toLowerCase())!);
 
         return [entropy > this._threshold ? ConfidenceLevel.Message : ConfidenceLevel.Unknown, `entropy=${entropy.toFixed(2)}`];
     }

src/checker/providers/entropyDetect.ts

@@ -194,7 +194,7 @@ export class Commands {
         UserDictionaryPersist.add(DictionaryType.ExcludeFolder, () => {
             const fullPath = path.parse((<TreeItemFile>node).uri.fsPath).dir;
 
-            return nameOnly ? ItemRegex.fromValue(`\\${path.sep}${path.basename(fullPath)}$`).rawValue : fullPath;
+            return nameOnly ? ItemRegex.fromValue(`\\${path.sep}${ItemRegex.escape(path.basename(fullPath))}$`).rawValue : fullPath;
         });
     }
 
@@ -203,7 +203,12 @@ export class Commands {
             return;
         }
 
-        UserDictionaryPersist.add(DictionaryType.ExcludeFile, () => nameOnly ? ItemRegex.fromValue(`\\${path.sep}${path.basename((<TreeItemFile>node).uri.fsPath)}$`).rawValue : (<TreeItemFile>node).uri.fsPath);
+        UserDictionaryPersist.add(
+            DictionaryType.ExcludeFile,
+            () =>
+                nameOnly
+                ? ItemRegex.fromValue(`\\${path.sep}${ItemRegex.escape(path.basename((<TreeItemFile>node).uri.fsPath))}$`).rawValue
+                : (<TreeItemFile>node).uri.fsPath);
     }
 
     static addTokenDictionary(type: DictionaryType, node: vscode.TreeItem): void {

src/commands.ts

@@ -1,6 +1,6 @@
 export class Constants {
     static readonly ExtensionName = 'string-checker-js';
-    static readonly ExtensionVersion = 'v0.0.2';
+    static readonly ExtensionVersion = 'v0.0.3';
     static readonly ExtensionID = 'string-checker-js';
     static readonly ItemStringPrefix = 'string:';
     static readonly ItemRegexPrefix = 'regex:';
@@ -31,3 +31,9 @@ export class Messages {
     static readonly EnterString = 'Type a string and press [Enter]';
     static readonly PressEscapeToExit = 'Press [Escape] or [Enter] to close';
 }
+
+export class ConfidenceLevelStr {
+    static readonly Unknown = '?';
+    static readonly Technical = 'Technical';
+    static readonly Message = 'Message';
+}