Fix parser validate UTF-8 on first input byte (#14750)

Author: straight-shoota

spec/compiler/crystal/tools/format_spec.cr

@@ -57,7 +57,7 @@ describe Crystal::Command::FormatCommand do
     format_command.run
     format_command.status_code.should eq(1)
     stdout.to_s.should be_empty
-    stderr.to_s.should contain("file 'STDIN' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8")
+    stderr.to_s.should contain("file 'STDIN' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8")
   end
 
   it "formats stdin (bug)" do
@@ -162,7 +162,7 @@ describe Crystal::Command::FormatCommand do
         format_command.status_code.should eq(1)
         stdout.to_s.should contain("Format #{Path[".", "format.cr"]}")
         stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF")
-        stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8")
+        stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8")
 
         File.read(File.join(path, "format.cr")).should eq("if true\n  1\nend\n")
       end
@@ -226,7 +226,7 @@ describe Crystal::Command::FormatCommand do
         stderr.to_s.should_not contain("not_format.cr")
         stderr.to_s.should contain("formatting '#{Path[".", "format.cr"]}' produced changes")
         stderr.to_s.should contain("syntax error in '#{Path[".", "syntax_error.cr"]}:1:3': unexpected token: EOF")
-        stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xff at position 1, malformed UTF-8")
+        stderr.to_s.should contain("file '#{Path[".", "invalid_byte_sequence_error.cr"]}' is not a valid Crystal source file: Unexpected byte 0xfe at position 0, malformed UTF-8")
       end
     end
   end

spec/compiler/lexer/lexer_spec.cr

@@ -657,6 +657,15 @@ describe "Lexer" do
   assert_syntax_error "'\\u{DFFF}'", "invalid unicode codepoint (surrogate half)"
   assert_syntax_error ":+1", "unexpected token"
 
+  it "invalid byte sequence" do
+    expect_raises(InvalidByteSequenceError, "Unexpected byte 0xff at position 0, malformed UTF-8") do
+      parse "\xFF"
+    end
+    expect_raises(InvalidByteSequenceError, "Unexpected byte 0xff at position 1, malformed UTF-8") do
+      parse " \xFF"
+    end
+  end
+
   assert_syntax_error "'\\1'", "invalid char escape sequence"
 
   it_lexes_string %("\\1"), String.new(Bytes[1])

src/compiler/crystal/syntax/lexer.cr

@@ -59,6 +59,7 @@ module Crystal
     def initialize(string, string_pool : StringPool? = nil, warnings : WarningCollection? = nil)
       @warnings = warnings || WarningCollection.new
       @reader = Char::Reader.new(string)
+      check_reader_error
       @token = Token.new
       @temp_token = Token.new
       @line_number = 1
@@ -2754,11 +2755,13 @@ module Crystal
     end
 
     def next_char_no_column_increment
-      char = @reader.next_char
+      @reader.next_char.tap { check_reader_error }
+    end
+
+    private def check_reader_error
       if error = @reader.error
         ::raise InvalidByteSequenceError.new("Unexpected byte 0x#{error.to_s(16)} at position #{@reader.pos}, malformed UTF-8")
       end
-      char
     end
 
     def next_char