CC-1545: Improve error messages (stderr) for Interpreter scanning error stages

internal/assertions/stderr_assertion.go

@@ -22,28 +22,39 @@ func (a StderrAssertion) Run(result executable.ExecutableResult, logger *logger.
 	stderr := getStderrLinesFromExecutableResult(result)
 	skippedLines := getSkippedLinesCount(result)
 
+	if skippedLines > 0 {
+		logger.Plainf("[stderr] Skipped %d lines that didn't start with [line N]", skippedLines)
+	}
+
 	for i, expectedLine := range a.ExpectedLines {
 		if i >= len(stderr) {
 			logAllSuccessLogs(successLogs, logger)
-			logger.Errorf("? %s", expectedLine)
-			logger.Errorf("Skipped %d lines that didn't start with [line N]", skippedLines)
-			return fmt.Errorf("Expected line #%d on stderr to be %q, but didn't find line", i+1, expectedLine)
+
+			return fmt.Errorf(`
+[stderr] Missing line #%d from stderr: %q
+[stderr] Perhaps it's printed to stdout? It should be printed to stderr.
+ `, i+1, expectedLine)
 		}
 		actualValue := stderr[i]
 
 		if actualValue != expectedLine {
 			logAllSuccessLogs(successLogs, logger)
-			logger.Errorf("𐄂 %s", actualValue)
-			return fmt.Errorf("Expected line #%d on stderr to be %q, got %q", i+1, expectedLine, actualValue)
+
+			return fmt.Errorf(`
+[stderr] Mismatch on line #%d of stderr:
+[stderr] Expected: %q
+[stderr] Actual  : %q
+ `, i+1, expectedLine, actualValue)
 		} else {
 			successLogs = append(successLogs, fmt.Sprintf("✓ %s", actualValue))
 		}
 	}
 
 	if len(stderr) > len(a.ExpectedLines) {
 		logAllSuccessLogs(successLogs, logger)
-		logger.Errorf("! %s", stderr[len(a.ExpectedLines)])
-		return fmt.Errorf("Expected last stderr line to be %q, but found extra line: %q", a.ExpectedLines[len(a.ExpectedLines)-1], stderr[len(a.ExpectedLines)])
+		return fmt.Errorf(`
+𐄂 [stderr] Extra unexpected line in stderr: %q
+ `, stderr[len(a.ExpectedLines)])
 	}
 
 	// If all lines match, we don't want to print all the lines again
@@ -67,6 +78,13 @@ func getStderrLinesFromExecutableResult(result executable.ExecutableResult) []st
 }
 
 func getSkippedLinesCount(result executable.ExecutableResult) int {
-	unfilteredStderr := strings.Split(strings.TrimRight(string(result.Stderr), "\n"), "\n")
-	return len(unfilteredStderr) - len(getStderrLinesFromExecutableResult(result))
+	trimmedStderr := strings.TrimRight(string(result.Stderr), "\n")
+
+	// Handle the case where strings.Split("", "\n") returns [""]
+	if trimmedStderr == "" {
+		return 0
+	}
+
+	unfilteredStderrLines := strings.Split(trimmedStderr, "\n")
+	return len(unfilteredStderrLines) - len(getStderrLinesFromExecutableResult(result))
 }

internal/test_helpers/fixtures/fail_scanning_lexical_errors/1_missing_line

@@ -0,0 +1,14 @@
+[stage-5] Running tests for Stage #5: ea6
+[stage-5] [test-1] Running test case: 1
+[stage-5] [test-1] Writing contents to ./test.lox:
+[stage-5] [test-1] [test.lox] @
+[stage-5] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] Error: Unexpected character: @
+[your_program] EOF  null
+[stage-5] [test-1] [stderr] Skipped 1 lines that didn't start with [line N]
+[stage-5] [test-1] 
+[stage-5] [test-1] [stderr] Missing line #1 from stderr: "[line 1] Error: Unexpected character: @"
+[stage-5] [test-1] [stderr] Perhaps it's printed to stdout? It should be printed to stderr.
+[stage-5] [test-1]  
+[stage-5] [test-1] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/fixtures/fail_scanning_lexical_errors/2_mismatch

@@ -0,0 +1,15 @@
+[stage-5] Running tests for Stage #5: ea6
+[stage-5] [test-1] Running test case: 1
+[stage-5] [test-1] Writing contents to ./test.lox:
+[stage-5] [test-1] [test.lox] @
+[stage-5] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] EOF  null
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] eRrOr: uNeXpEcTed cHaRaCtEr: @
+[stage-5] [test-1] [stderr] Skipped 1 lines that didn't start with [line N]
+[stage-5] [test-1] 
+[stage-5] [test-1] [stderr] Mismatch on line #1 of stderr:
+[stage-5] [test-1] [stderr] Expected: "[line 1] Error: Unexpected character: @"
+[stage-5] [test-1] [stderr] Actual  : "[line 1] eRrOr: uNeXpEcTed cHaRaCtEr: @"
+[stage-5] [test-1]  
+[stage-5] [test-1] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/fixtures/fail_scanning_lexical_errors/3_extra_line

@@ -0,0 +1,16 @@
+[stage-5] Running tests for Stage #5: ea6
+[stage-5] [test-1] Running test case: 1
+[stage-5] [test-1] Writing contents to ./test.lox:
+[stage-5] [test-1] [test.lox] @
+[stage-5] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] This line will be skipped in stderr
+[your_program] EOF  null
+[your_program] [line 1] Error: Unexpected character: @
+[your_program] [line 666] Extra line
+[your_program] [line 888] Another extra line
+[stage-5] [test-1] [stderr] Skipped 1 lines that didn't start with [line N]
+[stage-5] [test-1] ✓ [line 1] Error: Unexpected character: @
+[stage-5] [test-1] 
+[stage-5] [test-1] 𐄂 [stderr] Extra unexpected line in stderr: "[line 666] Extra line"
+[stage-5] [test-1]  
+[stage-5] [test-1] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/fixtures/fail_scanning_multiline_errors/1_missing_line

@@ -0,0 +1,25 @@
+[stage-11] Running tests for Stage #11: tz7
+[stage-11] [test-1] Running test case: 1
+[stage-11] [test-1] Writing contents to ./test.lox:
+[stage-11] [test-1] [test.lox] ()<|SPACE|><|TAB|>@
+[stage-11] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] [line 2] Error: Unexpected character: @
+[your_program] LEFT_PAREN ( null
+[your_program] RIGHT_PAREN ) null
+[your_program] EOF  null
+[stage-11] [test-1] ✓ 1 line(s) match on stderr
+[stage-11] [test-1] ✓ 3 line(s) match on stdout
+[stage-11] [test-1] ✓ Received exit code 65.
+[stage-11] [test-2] Running test case: 2
+[stage-11] [test-2] Writing contents to ./test.lox:
+[stage-11] [test-2] [test.lox] @#
+[stage-11] [test-2] [test.lox] <|SPACE|>
+[stage-11] [test-2] $ ./your_program.sh tokenize test.lox
+[your_program] EOF  null
+[your_program] [line 1] Error: Unexpected character: @
+[stage-11] [test-2] ✓ [line 1] Error: Unexpected character: @
+[stage-11] [test-2] 
+[stage-11] [test-2] [stderr] Missing line #2 from stderr: "[line 1] Error: Unexpected character: #"
+[stage-11] [test-2] [stderr] Perhaps it's printed to stdout? It should be printed to stderr.
+[stage-11] [test-2]  
+[stage-11] [test-2] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/fixtures/fail_scanning_multiline_errors/2_mismatch

@@ -0,0 +1,32 @@
+[stage-11] Running tests for Stage #11: tz7
+[stage-11] [test-1] Running test case: 1
+[stage-11] [test-1] Writing contents to ./test.lox:
+[stage-11] [test-1] [test.lox] ()<|SPACE|><|TAB|>@
+[stage-11] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] LEFT_PAREN ( null
+[your_program] RIGHT_PAREN ) null
+[your_program] This line will be skipped in stderr
+[your_program] EOF  null
+[your_program] [line 2] Error: Unexpected character: @
+[stage-11] [test-1] [stderr] Skipped 1 lines that didn't start with [line N]
+[stage-11] [test-1] ✓ 1 line(s) match on stderr
+[stage-11] [test-1] ✓ 3 line(s) match on stdout
+[stage-11] [test-1] ✓ Received exit code 65.
+[stage-11] [test-2] Running test case: 2
+[stage-11] [test-2] Writing contents to ./test.lox:
+[stage-11] [test-2] [test.lox] @#
+[stage-11] [test-2] [test.lox] <|SPACE|>
+[stage-11] [test-2] $ ./your_program.sh tokenize test.lox
+[your_program] EOF  null
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] Error: Unexpected character: @
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] eRrRr: Unexpected character: #
+[stage-11] [test-2] [stderr] Skipped 2 lines that didn't start with [line N]
+[stage-11] [test-2] ✓ [line 1] Error: Unexpected character: @
+[stage-11] [test-2] 
+[stage-11] [test-2] [stderr] Mismatch on line #2 of stderr:
+[stage-11] [test-2] [stderr] Expected: "[line 1] Error: Unexpected character: #"
+[stage-11] [test-2] [stderr] Actual  : "[line 1] eRrRr: Unexpected character: #"
+[stage-11] [test-2]  
+[stage-11] [test-2] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/fixtures/fail_scanning_multiline_errors/3_extra_line

@@ -0,0 +1,32 @@
+[stage-11] Running tests for Stage #11: tz7
+[stage-11] [test-1] Running test case: 1
+[stage-11] [test-1] Writing contents to ./test.lox:
+[stage-11] [test-1] [test.lox] ()<|SPACE|><|TAB|>@
+[stage-11] [test-1] $ ./your_program.sh tokenize test.lox
+[your_program] LEFT_PAREN ( null
+[your_program] RIGHT_PAREN ) null
+[your_program] This line will be skipped in stderr
+[your_program] [line 2] Error: Unexpected character: @
+[your_program] EOF  null
+[stage-11] [test-1] [stderr] Skipped 1 lines that didn't start with [line N]
+[stage-11] [test-1] ✓ 1 line(s) match on stderr
+[stage-11] [test-1] ✓ 3 line(s) match on stdout
+[stage-11] [test-1] ✓ Received exit code 65.
+[stage-11] [test-2] Running test case: 2
+[stage-11] [test-2] Writing contents to ./test.lox:
+[stage-11] [test-2] [test.lox] @#
+[stage-11] [test-2] [test.lox] <|SPACE|>
+[stage-11] [test-2] $ ./your_program.sh tokenize test.lox
+[your_program] EOF  null
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] Error: Unexpected character: @
+[your_program] This line will be skipped in stderr
+[your_program] [line 1] Error: Unexpected character: #
+[your_program] [line 888] extra line
+[stage-11] [test-2] [stderr] Skipped 2 lines that didn't start with [line N]
+[stage-11] [test-2] ✓ [line 1] Error: Unexpected character: @
+[stage-11] [test-2] ✓ [line 1] Error: Unexpected character: #
+[stage-11] [test-2] 
+[stage-11] [test-2] 𐄂 [stderr] Extra unexpected line in stderr: "[line 888] extra line"
+[stage-11] [test-2]  
+[stage-11] [test-2] Test failed (try setting 'debug: true' in your codecrafters.yml to see more details)

internal/test_helpers/scenarios/fail_scanning_lexical_errors/1_missing_line/app/main.py

@@ -0,0 +1,32 @@
+import sys
+
+from .scanner import Scanner
+
+
+def main():
+    if len(sys.argv) < 3:
+        print("Usage: ./your_program.sh tokenize <filename>", file=sys.stderr)
+        exit(1)
+
+    command = sys.argv[1]
+    filename = sys.argv[2]
+
+    if command != "tokenize":
+        print(f"Unknown command: {command}", file=sys.stderr)
+        exit(1)
+
+    with open(filename) as file:
+        file_contents = file.read()
+
+    scanner = Scanner(file_contents)
+    tokens = scanner.scan_tokens()
+
+    for token in tokens:
+        print(token)
+
+    if scanner.has_errors:
+        exit(65)
+
+
+if __name__ == "__main__":
+    main()

internal/test_helpers/scenarios/fail_scanning_lexical_errors/1_missing_line/app/scanner.py

@@ -0,0 +1,78 @@
+import sys
+
+from .token import Token, TokenType
+
+
+class Scanner:
+    current_token_start_index: int
+    current_index: int
+    current_line: int
+    has_errors: bool
+    source: str
+    tokens: list[Token]
+
+    def __init__(self, source: str):
+        self.source = source
+        self.current_token_start_index = 0
+        self.current_index = 0
+        self.current_line = 1
+        self.has_errors = False
+        self.tokens = []
+
+    def scan_tokens(self) -> list[Token]:
+        while not self._is_at_end():
+            self.scan_token()
+            self.current_token_start_index = self.current_index
+
+        self._add_token(TokenType.EOF)
+
+        return self.tokens
+
+    def scan_token(self):
+        match self._consume_char():
+            case ",":
+                self._add_token(TokenType.COMMA)
+            case ".":
+                self._add_token(TokenType.DOT)
+            case "(":
+                self._add_token(TokenType.LEFT_PAREN)
+            case "{":
+                self._add_token(TokenType.LEFT_BRACE)
+            case "-":
+                self._add_token(TokenType.MINUS)
+            case "+":
+                self._add_token(TokenType.PLUS)
+            case ")":
+                self._add_token(TokenType.RIGHT_PAREN)
+            case "}":
+                self._add_token(TokenType.RIGHT_BRACE)
+            case ";":
+                self._add_token(TokenType.SEMICOLON)
+            case "*":
+                self._add_token(TokenType.STAR)
+            case char:
+                self.has_errors = True
+
+                print("This line will be skipped in stderr", file=sys.stderr)
+
+                print(
+                    f"[line {self.current_line}] Error: Unexpected character: {char}",
+                    file=sys.stdout,
+                )
+
+    def _add_token(self, type: TokenType, literal: str | int | None = None):
+        self.tokens.append(
+            Token(
+                type,
+                self.source[self.current_token_start_index : self.current_index],
+                literal,
+                self.current_line,
+            )
+        )
+
+    def _consume_char(self) -> str:
+        self.current_index += 1
+        return self.source[self.current_index - 1]
+
+    def _is_at_end(self) -> bool:
+        return self.current_index >= len(self.source)

internal/test_helpers/scenarios/fail_scanning_lexical_errors/1_missing_line/app/token.py

@@ -0,0 +1,38 @@
+from enum import StrEnum
+from typing import Optional
+
+
+class TokenType(StrEnum):
+    COMMA = "COMMA"
+    DOT = "DOT"
+    EOF = "EOF"
+    LEFT_BRACE = "LEFT_BRACE"
+    LEFT_PAREN = "LEFT_PAREN"
+    MINUS = "MINUS"
+    PLUS = "PLUS"
+    RIGHT_BRACE = "RIGHT_BRACE"
+    RIGHT_PAREN = "RIGHT_PAREN"
+    SEMICOLON = "SEMICOLON"
+    STAR = "STAR"
+
+
+class Token:
+    type: TokenType
+    lexeme: Optional[str]
+    literal: str | int | None
+    line_number: int
+
+    def __init__(
+        self,
+        type: TokenType,
+        lexeme: Optional[str],
+        literal: str | int | None,
+        line_number: int,
+    ):
+        self.type = type
+        self.lexeme = lexeme
+        self.literal = literal
+        self.line_number = line_number
+
+    def __str__(self) -> str:
+        return f"{self.type} {self.lexeme} {self.literal or 'null'}"

internal/test_helpers/scenarios/fail_scanning_lexical_errors/1_missing_line/codecrafters.yml

@@ -0,0 +1,11 @@
+# Set this to true if you want debug logs.
+#
+# These can be VERY verbose, so we suggest turning them off
+# unless you really need them.
+debug: false
+
+# Use this to change the Python version used to run your code
+# on Codecrafters.
+#
+# Available versions: python-3.12
+language_pack: python-3.12

internal/test_helpers/scenarios/fail_scanning_lexical_errors/1_missing_line/your_program.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+#
+# Use this script to run your program LOCALLY.
+#
+# Note: Changing this script WILL NOT affect how CodeCrafters runs your program.
+#
+# Learn more: https://codecrafters.io/program-interface
+
+set -e # Exit early if any commands fail
+
+# Copied from .codecrafters/run.sh
+#
+# - Edit this to change how your program runs locally
+# - Edit .codecrafters/run.sh to change how your program runs remotely
+PYTHONPATH=$(dirname $0) exec python3 -m app.main "$@"