Merge pull request github#3464 from yoff/UnicodeEscape

Python: Handle more escapes in regexes

Author: tausbn

python/ql/src/semmle/python/regex.qll

@@ -124,16 +124,40 @@ abstract class RegexString extends Expr {
         )
     }
 
+    /** Named unicode characters, eg \N{degree sign} */
+    private predicate escapedName(int start, int end) {
+        this.escapingChar(start) and
+        this.getChar(start + 1) = "N" and
+        this.getChar(start + 2) = "{" and
+        this.getChar(end - 1) = "}" and
+        end > start and
+        not exists(int i | start + 2 < i and i < end - 1 |
+            this.getChar(i) = "}"
+        )
+    }
+
     private predicate escapedCharacter(int start, int end) {
         this.escapingChar(start) and
         not exists(this.getText().substring(start + 1, end + 1).toInt()) and
         (
+            // hex value \xhh
             this.getChar(start + 1) = "x" and end = start + 4
             or
+            // octal value \ooo
             end in [start + 2 .. start + 4] and
             exists(this.getText().substring(start + 1, end).toInt())
             or
-            this.getChar(start + 1) != "x" and end = start + 2
+            // 16-bit hex value \uhhhh
+            this.getChar(start + 1) = "u" and end = start + 6
+            or
+            // 32-bit hex value \Uhhhhhhhh
+            this.getChar(start + 1) = "U" and end = start + 10
+            or
+            escapedName(start, end)
+            or
+            // escape not handled above, update when adding a new case
+            not this.getChar(start + 1) in ["x", "u", "U", "N"] and
+            end = start + 2
         )
     }
 

python/ql/test/query-tests/Expressions/Regex/test.py

@@ -139,3 +139,10 @@
 
 #Potentially mis-parsed character set
 re.compile(r"\[(?P<txt>[^[]*)\]\((?P<uri>[^)]*)")
+
+#Allow unicode in raw strings
+re.compile(r"[\U00010000-\U0010FFFF]")
+re.compile(r"[\u0000-\uFFFF]")
+
+#Allow unicode names
+re.compile(r"[\N{degree sign}\N{EM DASH}]")