Change LexBuffer type from char to uint16

Author: ncave

src/absil/illex.fsl

@@ -16,7 +16,7 @@ open FSharp.Compiler.AbstractIL.Internal.AsciiParser
 open FSharp.Compiler.AbstractIL.Internal.AsciiConstants 
 
 
-let lexeme (lexbuf : LexBuffer<char>) = new System.String(lexbuf.Lexeme)
+let lexeme (lexbuf : LexBuffer<_>) = LexBuffer<_>.LexemeString (lexbuf)
 
 let unexpectedChar _lexbuf =
   raise Parsing.RecoverableParseError ;;

src/absil/illib.fs

@@ -571,6 +571,9 @@ module String =
         else None
 
     let getLines (str: string) =
+#if FABLE_COMPILER
+        System.Text.RegularExpressions.Regex.Split(str, "\r\n|\r|\n");
+#else
         use reader = new StringReader(str)
         [|
             let line = ref (reader.ReadLine())

src/fsharp/UnicodeLexing.fs

@@ -13,12 +13,16 @@ open System.IO
 
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf = LexBuffer<LexBufferChar>
 
 let StringAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, s:string) : Lexbuf =
+#if FABLE_COMPILER
+    LexBuffer<_>.FromString (supportsFeature, s)
+#else
     LexBuffer<_>.FromChars (supportsFeature, s.ToCharArray())
+#endif
 
-let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferFiller: char[] * int * int -> int) : Lexbuf =
+let FunctionAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, bufferFiller: LexBufferChar[] * int * int -> int) : Lexbuf =
     LexBuffer<_>.FromFunction(supportsFeature, bufferFiller)
 
 let SourceTextAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, sourceText) =
@@ -69,5 +73,9 @@ let UnicodeFileAsLexbuf (supportsFeature: Features.LanguageFeature -> bool, file
                else 
                    reraise()
     let source = getSource 0
+#if FABLE_COMPILER
+    let lexbuf = LexBuffer<_>.FromString (supportsFeature, source)
+#else
     let lexbuf = LexBuffer<_>.FromChars(supportsFeature, source.ToCharArray())  
+#endif
     lexbuf

src/fsharp/UnicodeLexing.fsi

@@ -7,8 +7,8 @@ open FSharp.Compiler.Text
 open Microsoft.FSharp.Text
 open Internal.Utilities.Text.Lexing
 
-type Lexbuf =  LexBuffer<char>
+type Lexbuf =  LexBuffer<LexBufferChar>
 val internal StringAsLexbuf: (Features.LanguageFeature -> bool) * string -> Lexbuf
-val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (char [] * int * int -> int) -> Lexbuf
+val public FunctionAsLexbuf: (Features.LanguageFeature -> bool) * (LexBufferChar[] * int * int -> int) -> Lexbuf
 val public UnicodeFileAsLexbuf: (Features.LanguageFeature -> bool) * string * int option * (*retryLocked*) bool -> Lexbuf
 val public SourceTextAsLexbuf: (Features.LanguageFeature -> bool) * ISourceText -> Lexbuf

src/fsharp/fsi/fsi.fs

@@ -1619,7 +1619,7 @@ type internal FsiStdinLexerProvider
 
     let LexbufFromLineReader (fsiStdinSyphon: FsiStdinSyphon) readf = 
         UnicodeLexing.FunctionAsLexbuf
-          (isFeatureSupported, (fun (buf: char[], start, len) ->
+          (isFeatureSupported, (fun (buf, start, len) ->
             //fprintf fsiConsoleOutput.Out "Calling ReadLine\n"
             let inputOption = try Some(readf()) with :? EndOfStreamException -> None
             inputOption |> Option.iter (fun t -> fsiStdinSyphon.Add (t + "\n"))
@@ -1633,7 +1633,11 @@ type internal FsiStdinLexerProvider
                 if ninput > len then fprintf fsiConsoleOutput.Error  "%s" (FSIstrings.SR.fsiLineTooLong())
                 let ntrimmed = min len ninput 
                 for i = 0 to ntrimmed-1 do
+#if FABLE_COMPILER
+                    buf.[i+start] <- uint16 input.[i]
+#else
                     buf.[i+start] <- input.[i]
+#endif
                 ntrimmed
           ))
 

src/fsharp/lex.fsl

@@ -147,7 +147,11 @@ let shouldStartFile args lexbuf (m:range) err tok =
     else tok
 
 let evalIfDefExpression startPos isFeatureSupported args (lookup:string->bool) (lexed:string) =
+#if FABLE_COMPILER
+    let lexbuf          = LexBuffer<_>.FromString (isFeatureSupported, lexed)
+#else
     let lexbuf          = LexBuffer<char>.FromChars (isFeatureSupported, lexed.ToCharArray ())
+#endif
     lexbuf.StartPos     <- startPos
     lexbuf.EndPos       <- startPos
     let tokenStream     = FSharp.Compiler.PPLexer.tokenstream args

src/fsharp/service/ServiceLexing.fsi

@@ -3,6 +3,7 @@
 namespace FSharp.Compiler.SourceCodeServices
 
 open FSharp.Compiler 
+open Internal.Utilities.Text.Lexing
 
 type Position = int * int
 type Range = Position * Position
@@ -237,7 +238,7 @@ type FSharpLineTokenizer =
 type FSharpSourceTokenizer =
     new : conditionalDefines:string list * fileName:string option -> FSharpSourceTokenizer
     member CreateLineTokenizer : lineText:string -> FSharpLineTokenizer
-    member CreateBufferTokenizer : bufferFiller:(char[] * int * int -> int) -> FSharpLineTokenizer
+    member CreateBufferTokenizer : bufferFiller:(LexBufferChar[] * int * int -> int) -> FSharpLineTokenizer
 
 module internal TestExpose =     
     val TokenInfo : Parser.token -> (FSharpTokenColorKind * FSharpTokenCharKind * FSharpTokenTriggerClass)

src/utils/prim-lexing.fs

@@ -31,6 +31,9 @@ type ISourceText =
 type StringText(str: string) =
 
     let getLines (str: string) =
+#if FABLE_COMPILER
+        System.Text.RegularExpressions.Regex.Split(str, "\r\n|\r|\n");
+#else
         use reader = new StringReader(str)
         [|
         let mutable line = reader.ReadLine()
@@ -42,6 +45,7 @@ type StringText(str: string) =
             // http://stackoverflow.com/questions/19365404/stringreader-omits-trailing-linebreak
             yield String.Empty
         |]
+#endif
 
     let getLines =
         // This requires allocating and getting all the lines.
@@ -169,6 +173,12 @@ namespace Internal.Utilities.Text.Lexing
                       0,
                       0)
 
+#if FABLE_COMPILER
+    type internal LexBufferChar = uint16
+#else
+    type internal LexBufferChar = char
+#endif
+
     type internal LexBufferFiller<'Char> = (LexBuffer<'Char> -> unit)
 
     and [<Sealed>]
@@ -217,6 +227,8 @@ namespace Internal.Utilities.Text.Lexing
            and  set b =  endPos <- b
 
         member lexbuf.Lexeme         = Array.sub buffer bufferScanStart lexemeLength
+        member lexbuf.LexemeChar(n)  = buffer.[n+bufferScanStart]
+
         member lexbuf.BufferLocalStore = (context :> IDictionary<_,_>)
         member lexbuf.LexemeLength        with get() : int = lexemeLength    and set v = lexemeLength <- v
         member lexbuf.Buffer              with get() : 'Char[] = buffer              and set v = buffer <- v
@@ -225,8 +237,14 @@ namespace Internal.Utilities.Text.Lexing
         member lexbuf.BufferScanStart     with get() : int = bufferScanStart     and set v = bufferScanStart <- v
         member lexbuf.BufferAcceptAction  with get() = bufferAcceptAction  and set v = bufferAcceptAction <- v
         member lexbuf.RefillBuffer () = filler lexbuf
-        static member LexemeString(lexbuf:LexBuffer<char>) = 
-            new System.String(lexbuf.Buffer,lexbuf.BufferScanStart,lexbuf.LexemeLength)
+
+        static member LexemeString (lexbuf: LexBuffer<LexBufferChar>) =
+#if FABLE_COMPILER
+            let chars = Array.init lexbuf.LexemeLength (lexbuf.LexemeChar >> char)
+            new System.String(chars)
+#else
+            new System.String(lexbuf.Buffer, lexbuf.BufferScanStart, lexbuf.LexemeLength)
+#endif
 
         member lexbuf.IsPastEndOfStream 
            with get() = eof
@@ -266,9 +284,13 @@ namespace Internal.Utilities.Text.Lexing
             LexBuffer<'Char>.FromArrayNoCopy(supportsFeature, buffer)
 
         // Important: This method takes ownership of the array
-        static member FromChars (supportsFeature:LanguageFeature -> bool, arr:char[]) = LexBuffer.FromArrayNoCopy (supportsFeature, arr)
+        static member FromChars (supportsFeature:LanguageFeature -> bool, arr:LexBufferChar[]) = LexBuffer.FromArrayNoCopy (supportsFeature, arr)
 
         static member FromSourceText (supportsFeature: LanguageFeature -> bool, sourceText: ISourceText) =
+#if FABLE_COMPILER
+            let arr = Array.init sourceText.Length (fun i -> uint16 (sourceText.Item i))
+            LexBuffer.FromArrayNoCopy (supportsFeature, arr)
+#else
             let mutable currentSourceIndex = 0
             LexBuffer<char>.FromFunction(supportsFeature, fun (chars, start, length) ->
                 let lengthToCopy = 
@@ -283,16 +305,25 @@ namespace Internal.Utilities.Text.Lexing
                     currentSourceIndex <- currentSourceIndex + lengthToCopy
                     lengthToCopy
             )
+#endif
+
+        static member FromString (supportsFeature: LanguageFeature -> bool, s: string) =
+#if FABLE_COMPILER
+            let arr = Array.init s.Length (fun i -> uint16 s.[i])
+            LexBuffer.FromArrayNoCopy (supportsFeature, arr)
+#else
+            LexBuffer.FromArrayNoCopy (supportsFeature, s.ToCharArray())
+#endif
 
     module GenericImplFragments = 
-        let startInterpret(lexBuffer:LexBuffer<char>) = 
+        let startInterpret(lexBuffer:LexBuffer<LexBufferChar>) = 
             lexBuffer.BufferScanStart <- lexBuffer.BufferScanStart + lexBuffer.LexemeLength;
             lexBuffer.BufferMaxScanLength <- lexBuffer.BufferMaxScanLength - lexBuffer.LexemeLength;
             lexBuffer.BufferScanLength <- 0;
             lexBuffer.LexemeLength <- 0;
             lexBuffer.BufferAcceptAction <- -1;
 
-        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<char>,scanUntilSentinel,endOfScan,state,eofPos) = 
+        let afterRefill (trans: uint16[][],sentinel,lexBuffer:LexBuffer<LexBufferChar>,scanUntilSentinel,endOfScan,state,eofPos) = 
             // end of file occurs if we couldn't extend the buffer 
             if lexBuffer.BufferScanLength = lexBuffer.BufferMaxScanLength then  
                 let snew = int trans.[state].[eofPos] // == EOF 
@@ -306,9 +337,9 @@ namespace Internal.Utilities.Text.Lexing
             else 
                 scanUntilSentinel lexBuffer state
 
-        let onAccept (lexBuffer:LexBuffer<char>,a) = 
-            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength;
-            lexBuffer.BufferAcceptAction <- a;
+        let onAccept (lexBuffer:LexBuffer<LexBufferChar>, a) = 
+            lexBuffer.LexemeLength <- lexBuffer.BufferScanLength
+            lexBuffer.BufferAcceptAction <- a
 
     open GenericImplFragments
 
@@ -333,12 +364,16 @@ namespace Internal.Utilities.Text.Lexing
                         // ways
                         let baseForUnicodeCategories = numLowUnicodeChars+numSpecificUnicodeChars*2
                         let unicodeCategory = 
+#if FABLE_COMPILER
+                            System.Char.GetUnicodeCategory(char inp)
+#else
                             System.Char.GetUnicodeCategory(inp)
+#endif
                         //System.Console.WriteLine("inp = {0}, unicodeCategory = {1}", [| box inp; box unicodeCategory |]);
                         int trans.[state].[baseForUnicodeCategories + int32 unicodeCategory]
                     else 
                         // This is the specific unicode character
-                        let c = char (int trans.[state].[baseForSpecificUnicodeChars+i*2])
+                        let c = trans.[state].[baseForSpecificUnicodeChars+i*2]
                         //System.Console.WriteLine("c = {0}, inp = {1}, i = {2}", [| box c; box inp; box i |]);
                         // OK, have we found the entry for a specific unicode character?
                         if c = inp
@@ -360,7 +395,7 @@ namespace Internal.Utilities.Text.Lexing
                 afterRefill (trans,sentinel,lexBuffer,scanUntilSentinel,lexBuffer.EndOfScan,state,eofPos)
             else
                 // read a character - end the scan if there are no further transitions 
-                let inp = lexBuffer.Buffer.[lexBuffer.BufferScanPos]
+                let inp = uint16 lexBuffer.Buffer.[lexBuffer.BufferScanPos]
 
                 // Find the new state
                 let snew = lookupUnicodeCharacters state inp
@@ -378,7 +413,7 @@ namespace Internal.Utilities.Text.Lexing
         //      30 entries, one for each UnicodeCategory
         //      1 entry for EOF
 
-        member tables.Interpret(initialState,lexBuffer : LexBuffer<char>) = 
+        member tables.Interpret(initialState, lexBuffer: LexBuffer<LexBufferChar>) = 
             startInterpret(lexBuffer)
             scanUntilSentinel lexBuffer initialState
 

src/utils/prim-lexing.fsi

@@ -85,6 +85,12 @@ type internal Position =
 
      static member FirstLine : fileIdx:int -> Position
 
+#if FABLE_COMPILER
+type internal LexBufferChar = uint16
+#else
+type internal LexBufferChar = char
+#endif
+
 [<Sealed>]
 /// Input buffers consumed by lexers generated by <c>fslex.exe</c>.
 /// The type must be generic to match the code generated by FsLex and FsYacc (if you would like to
@@ -100,7 +106,7 @@ type internal LexBuffer<'Char> =
     member Lexeme: 'Char []
 
     /// Fast helper to turn the matched characters into a string, avoiding an intermediate array.
-    static member LexemeString : LexBuffer<char> -> string
+    static member LexemeString : LexBuffer<LexBufferChar> -> string
 
     /// Dynamically typed, non-lexically scoped parameter table.
     member BufferLocalStore : IDictionary<string,obj>
@@ -113,13 +119,16 @@ type internal LexBuffer<'Char> =
 
     /// Create a lex buffer suitable for Unicode lexing that reads characters from the given array.
     /// Important: does take ownership of the array.
-    static member FromChars: (LanguageFeature -> bool) * char[] -> LexBuffer<char>
+    static member FromChars: (LanguageFeature -> bool) * LexBufferChar[] -> LexBuffer<LexBufferChar>
+
+    /// Create a lex buffer suitable for Unicode lexing that reads characters from the given string.
+    static member FromString: (LanguageFeature -> bool) * string -> LexBuffer<LexBufferChar>
 
     /// Create a lex buffer that reads character or byte inputs by using the given function.
     static member FromFunction: (LanguageFeature -> bool) * ('Char[] * int * int -> int) -> LexBuffer<'Char>
 
     /// Create a lex buffer backed by source text.
-    static member FromSourceText : (LanguageFeature -> bool) * ISourceText -> LexBuffer<char>
+    static member FromSourceText : (LanguageFeature -> bool) * ISourceText -> LexBuffer<LexBufferChar>
 
 /// The type of tables for an unicode lexer generated by <c>fslex.exe</c>. 
 [<Sealed>]
@@ -129,5 +138,5 @@ type internal UnicodeTables =
     static member Create : uint16[][] * uint16[] -> UnicodeTables
 
     /// Interpret tables for a unicode lexer generated by <c>fslex.exe</c>. 
-    member Interpret:  initialState:int * LexBuffer<char> -> int
+    member Interpret:  initialState:int * LexBuffer<LexBufferChar> -> int
 

src/utils/prim-parsing.fs

@@ -14,7 +14,7 @@ exception RecoverableParseError
 exception Accept of obj
 
 [<Sealed>]
-type internal IParseState(ruleStartPoss:Position[], ruleEndPoss:Position[], lhsPos:Position[], ruleValues:obj[], lexbuf:LexBuffer<char>) = 
+type internal IParseState(ruleStartPoss:Position[], ruleEndPoss:Position[], lhsPos:Position[], ruleValues:obj[], lexbuf:LexBuffer<LexBufferChar>) = 
     member p.LexBuffer = lexbuf
 
     member p.InputRange n = ruleStartPoss.[n-1], ruleEndPoss.[n-1]