SONARPY-888 Rule S5868 Unicode Grapheme Clusters should be avoided inside regex character classes (#967)

Author: nils-werner-sonarsource

python-checks/src/main/java/org/sonar/python/checks/CheckList.java

@@ -51,6 +51,7 @@
 import org.sonar.python.checks.hotspots.UnverifiedHostnameCheck;
 import org.sonar.python.checks.regex.AnchorPrecedenceCheck;
 import org.sonar.python.checks.regex.EmptyStringRepetitionCheck;
+import org.sonar.python.checks.regex.GraphemeClustersInClassesCheck;
 import org.sonar.python.checks.regex.SingleCharacterAlternationCheck;
 import org.sonar.python.checks.regex.RedundantRegexAlternativesCheck;
 import org.sonar.python.checks.regex.RegexLookaheadCheck;
@@ -130,6 +131,7 @@ public static Iterable<Class> getChecks() {
       FunctionReturnTypeCheck.class,
       FunctionUsingLoopVariableCheck.class,
       GenericExceptionRaisedCheck.class,
+      GraphemeClustersInClassesCheck.class,
       HardCodedCredentialsCheck.class,
       HardcodedIPCheck.class,
       HashingDataCheck.class,

python-checks/src/main/java/org/sonar/python/checks/regex/GraphemeClustersInClassesCheck.java

@@ -0,0 +1,34 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2021 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.checks.regex;
+
+import org.sonar.check.Rule;
+import org.sonar.plugins.python.api.tree.CallExpression;
+import org.sonarsource.analyzer.commons.regex.RegexParseResult;
+import org.sonarsource.analyzer.commons.regex.finders.GraphemeInClassFinder;
+
+@Rule(key = "S5868")
+public class GraphemeClustersInClassesCheck extends AbstractRegexCheck {
+
+  @Override
+  public void checkRegex(RegexParseResult regexParseResult, CallExpression regexFunctionCall) {
+    new GraphemeInClassFinder(this::addIssue).visit(regexParseResult);
+  }
+}

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/S5868.html

@@ -0,0 +1,17 @@
+<p>When placing Unicode <a href="https://unicode.org/glossary/#grapheme_cluster">Grapheme Clusters</a> (characters which require to be encoded in
+multiple <a href="https://unicode.org/glossary/#code_point">Code Points</a>) inside a character class of a regular expression, this will likely lead
+to unintended behavior.</p>
+<p>For instance, the grapheme cluster <code>c̈</code> requires two code points: one for <code>'c'</code>, followed by one for the <em>umlaut</em>
+modifier <code>'\u{0308}'</code>. If placed within a character class, such as <code>[c̈]</code>, the regex will consider the character class being the
+enumeration <code>[c\u{0308}]</code> instead. It will, therefore, match every <code>'c'</code> and every <em>umlaut</em> that isn’t expressed as a
+single codepoint, which is extremely unlikely to be the intended behavior.</p>
+<p>This rule raises an issue every time Unicode Grapheme Clusters are used within a character class of a regular expression.</p>
+<h2>Noncompliant Code Example</h2>
+<pre>
+re.sub(r"[c̈d̈]", "X", "cc̈d̈d"); # Noncompliant, print "XXXXXX" instead of expected "cXXd".
+</pre>
+<h2>Compliant Solution</h2>
+<pre>
+re.sub(r"c̈|d̈", "X", "cc̈d̈d"); # print "cXXd"
+</pre>
+

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/S5868.json

@@ -0,0 +1,17 @@
+{
+  "title": "Unicode Grapheme Clusters should be avoided inside regex character classes",
+  "type": "BUG",
+  "status": "ready",
+  "remediation": {
+    "func": "Constant\/Issue",
+    "constantCost": "5min"
+  },
+  "tags": [
+    "regex"
+  ],
+  "defaultSeverity": "Major",
+  "ruleSpecification": "RSPEC-5868",
+  "sqKey": "S5868",
+  "scope": "Main",
+  "quickfix": "unknown"
+}

python-checks/src/main/resources/org/sonar/l10n/py/rules/python/Sonar_way_profile.json

@@ -137,6 +137,7 @@
     "S5850",
     "S5855",
     "S5864",
+    "S5868",
     "S5886",
     "S5890",
     "S6002",

python-checks/src/test/java/org/sonar/python/checks/regex/GraphemeClustersInClassesCheckTest.java

@@ -0,0 +1,32 @@
+/*
+ * SonarQube Python Plugin
+ * Copyright (C) 2011-2021 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.python.checks.regex;
+
+import org.junit.Test;
+import org.sonar.python.checks.utils.PythonCheckVerifier;
+
+public class GraphemeClustersInClassesCheckTest {
+
+  @Test
+  public void test() {
+    PythonCheckVerifier.verify("src/test/resources/checks/regex/graphemeClustersInClassesCheck.py", new GraphemeClustersInClassesCheck());
+  }
+
+}

python-checks/src/test/resources/checks/regex/graphemeClustersInClassesCheck.py

@@ -0,0 +1,27 @@
+import re
+
+
+def non_compliant(input):
+    re.match(r'[aaaèaaa]', input)  # Noncompliant {{Extract 1 Grapheme Cluster(s) from this character class.}}
+    #          ^^^^^^^^^^
+    re.match(r'[0Ṩ0]', input)  # Noncompliant {{Extract 1 Grapheme Cluster(s) from this character class.}}
+    re.match(r'aaa[è]aaa', input)  # Noncompliant
+    # two secondary per line: one for the regex location, and one for the cluster location
+    re.match(r'[èaaèaaè]', input)  # Noncompliant {{Extract 3 Grapheme Cluster(s) from this character class.}}
+    re.match(r'[èa-dä]', input)  # Noncompliant
+    re.match(r'[èa]aaa[dè]', input)     # Noncompliant 2
+    re.match(r'[ä]', input)  # Noncompliant
+    re.match(r'[c̈]', input)  # Noncompliant
+    re.match(r'[e⃝]', input)  # Noncompliant
+
+
+def compliant(input):
+    re.match(r'[é]', input)  # Compliant, a single char
+    re.match(r'[e\u0300]', input)  # Compliant, escaped unicode
+    re.match(r'[e\x{0300}]', input)  # Compliant, escaped unicode
+    re.match(r'[e\u20DD̀]', input)  # Compliant, (letter, escaped unicode, mark) can not be combined
+    re.match(r'[\u0300e]', input)  # Compliant, escaped unicode, letter
+    re.match(r'[̀̀]', input)  # Compliant, two marks
+    re.match(r'[̀̀]', input)  # Compliant, one mark
+
+    re.match(r'/ä/', input) # Compliant, not in a class