Skip to content

Commit 3ccd75a

Browse files
authored
[pysrc2cpg] Fix tab handling in parser grammar. (#5703)
Old implementation was not able to handle more than 2 tabs.
1 parent af41a7e commit 3ccd75a

File tree

2 files changed

+8
-1
lines changed

2 files changed

+8
-1
lines changed

joern-cli/frontends/pysrc2cpg/pythonGrammar.jj

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -822,7 +822,13 @@ SPECIAL_TOKEN: {
822822

823823
<INDENT_CHECK> SKIP: {
824824
<INDENT_CHECK_SPACE: " "> { currentIndent += 1; }
825-
| <INDENT_CHECK_TAB: "\t"> { currentIndent = currentIndent / 8 + 8; }
825+
| <INDENT_CHECK_TAB: "\t"> {
826+
// The "/ 8 * 8" rounds down to the closest multiple of 8.
827+
// This is needed to handle python 2 mixed space and tab use cases
828+
// where tab is not strictly counted as 8 spaces but rather adds the
829+
// amount of spaces to get to the next multiple of 8.
830+
currentIndent = currentIndent / 8 * 8 + 8;
831+
}
826832
| <INDENT_CHECK_NEWLINE: "\n" | "\r"> { currentIndent = 0; }
827833
| <INDENT_CHECK_END_OF_MEDIUM: "\u0019">
828834
| <INDENT_CHECK_END: ~[]> {

joern-cli/frontends/pysrc2cpg/src/test/scala/io/joern/pythonparser/ParserTests.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -350,6 +350,7 @@ class ParserTests extends AnyFreeSpec with Matchers {
350350
| z = (x,y)
351351
|a""".stripMargin
352352
)
353+
testT("def init():\n\tmatch x:\n\t\tcase 'C':\n\t\t\tz = 1")
353354
}
354355

355356
"explicit line joining tests" in {

0 commit comments

Comments
 (0)