Fix encoding handling in C lexer

Author: rmosolgo

graphql-c_parser/ext/graphql_c_parser_ext/graphql_c_parser_ext.c

@@ -1,6 +1,6 @@
 #include "graphql_c_parser_ext.h"
 
-VALUE GraphQL_CParser_Lexer_tokenize(VALUE self, VALUE query_string) {
+VALUE GraphQL_CParser_Lexer_tokenize_with_c(VALUE self, VALUE query_string) {
   return tokenize(query_string);
 }
 
@@ -13,7 +13,7 @@ void Init_graphql_c_parser_ext() {
   VALUE GraphQL = rb_define_module("GraphQL");
   VALUE CParser = rb_define_module_under(GraphQL, "CParser");
   VALUE Lexer = rb_define_module_under(CParser, "Lexer");
-  rb_define_singleton_method(Lexer, "tokenize", GraphQL_CParser_Lexer_tokenize, 1);
+  rb_define_singleton_method(Lexer, "tokenize_with_c", GraphQL_CParser_Lexer_tokenize_with_c, 1);
   setup_static_token_variables();
 
   VALUE Parser = rb_define_class_under(CParser, "Parser", rb_cObject);

graphql-c_parser/lib/graphql/c_parser.rb

@@ -40,6 +40,26 @@ def self.prepare_parse_error(message, parser)
       GraphQL::ParseError.new(message, line, col, parser.query_string, filename: parser.filename)
     end
 
+    module Lexer
+      def self.tokenize(graphql_string)
+        if !(graphql_string.encoding == Encoding::UTF_8 || graphql_string.ascii_only?)
+          graphql_string = graphql_string.dup.force_encoding(Encoding::UTF_8)
+        end
+        if !graphql_string.valid_encoding?
+          return [
+            [
+              :BAD_UNICODE_ESCAPE,
+              1,
+              1,
+              graphql_string,
+              nil
+            ]
+          ]
+        end
+        tokenize_with_c(graphql_string)
+      end
+    end
+
     class Parser
       def initialize(query_string, filename, trace)
         if query_string.nil?
@@ -56,7 +76,7 @@ def initialize(query_string, filename, trace)
       def result
         if @result.nil?
           @tokens = @trace.lex(query_string: @query_string) do
-            GraphQL::CParser::Lexer.tokenize(@query_string)
+            GraphQL::CParser.scan_with_c(@query_string)
           end
           @trace.parse(query_string: @query_string) do
             c_parse