@@ -7,26 +7,23 @@ module SqlSquared.Parser.Tokenizer
77import Prelude
88
99import Control.Alt ((<|>))
10-
1110import Data.Array as A
12- import Data.Int as Int
11+ import Data.Char as Ch
1312import Data.Either (Either )
14- import Data.Maybe (Maybe (..), isJust )
15- import Data.Foldable as F
13+ import Data.HugeInt as HI
1614import Data.HugeNum as HN
17- import Data.Char as Ch
15+ import Data.Json.Extended.Signature.Parse as EJP
16+ import Data.Maybe (isJust )
1817import Data.String as S
19-
2018import SqlSquared.Utils ((∘))
21-
2219import Text.Parsing.Parser as P
2320import Text.Parsing.Parser.Combinators as PC
24- import Text.Parsing.Parser.Token as PT
2521import Text.Parsing.Parser.String as PS
22+ import Text.Parsing.Parser.Token as PT
2623
2724data Literal
2825 = String String
29- | Integer Int
26+ | Integer HI.HugeInt
3027 | Decimal HN.HugeNum
3128
3229derive instance eqTokenLit ∷ Eq Literal
@@ -202,121 +199,13 @@ notQuotedIdent = do
202199 else pure str
203200
204201stringLit ∷ ∀ m . Monad m ⇒ P.ParserT String m Token
205- stringLit =
206- map (Lit ∘ String )
207- $ PC .between (PS .string " \" " ) (PS .string " \" " )
208- $ map S .fromCharArray
209- $ A .many stringChar
210- where
211- stringChar = PC .try stringEscape <|> stringLetter
212- stringLetter = PS .satisfy (not ∘ eq ' "' )
213- stringEscape = PS .string " \\\" " $> ' "'
202+ stringLit = Lit ∘ String <$> EJP .parseStringLiteral
214203
215204numLit ∷ ∀ m . Monad m ⇒ P.ParserT String m Token
216- numLit = map ( Lit ∘ Decimal ) parseDecimal
205+ numLit = Lit ∘ Decimal <$> EJP .parseDecimalLiteral
217206
218207intLit ∷ ∀ m . Monad m ⇒ P.ParserT String m Token
219- intLit = map (Lit ∘ Integer ) parseIntLiteral
220-
221- parseIntLiteral ∷ ∀ m . Monad m ⇒ P.ParserT String m Int
222- parseIntLiteral = parseSigned parseNat
223-
224- parseDecimal ∷ ∀ m . Monad m ⇒ P.ParserT String m HN.HugeNum
225- parseDecimal = parseHugeNum <|> parseScientific
226-
227- parseHugeNum ∷ ∀ m . Monad m ⇒ P.ParserT String m HN.HugeNum
228- parseHugeNum = do
229- chars ←
230- map S .fromCharArray
231- $ A .many
232- $ PS .oneOf
233- $ digits
234- <> [ ' -' , ' .' ]
235- case HN .fromString chars of
236- Just num → pure num
237- Nothing → P .fail $ " Failed to parse decimal: " <> chars
238-
239- parseDigit ∷ ∀ m . Monad m ⇒ P.ParserT String m Int
240- parseDigit =
241- PC .choice
242- [ 0 <$ PS .string " 0"
243- , 1 <$ PS .string " 1"
244- , 2 <$ PS .string " 2"
245- , 3 <$ PS .string " 3"
246- , 4 <$ PS .string " 4"
247- , 5 <$ PS .string " 5"
248- , 6 <$ PS .string " 6"
249- , 7 <$ PS .string " 7"
250- , 8 <$ PS .string " 8"
251- , 9 <$ PS .string " 9"
252- ]
253-
254- parseScientific ∷ ∀ m . Monad m ⇒ P.ParserT String m HN.HugeNum
255- parseScientific =
256- parseSigned parsePositiveScientific
257-
258- parseNat
259- ∷ ∀ m
260- . Monad m
261- ⇒ P.ParserT String m Int
262- parseNat =
263- A .some parseDigit
264- <#> F .foldl (\a i → a * 10 + i) 0
265-
266- parseNegative
267- ∷ ∀ m a
268- . Monad m
269- ⇒ Ring a
270- ⇒ P.ParserT String m a
271- → P.ParserT String m a
272- parseNegative p =
273- PS .string " -"
274- *> PS .skipSpaces
275- *> p
276- <#> negate
277-
278- parsePositive
279- ∷ ∀ m a
280- . Monad m
281- ⇒ Ring a
282- ⇒ P.ParserT String m a
283- → P.ParserT String m a
284- parsePositive p =
285- PC .optional (PS .string " +" *> PS .skipSpaces)
286- *> p
287-
288- parseSigned
289- ∷ ∀ m a
290- . Monad m
291- ⇒ Ring a
292- ⇒ P.ParserT String m a
293- → P.ParserT String m a
294- parseSigned p = parseNegative p <|> parsePositive p
295-
296- parseExponent
297- ∷ ∀ m
298- . Monad m
299- ⇒ P.ParserT String m Int
300- parseExponent =
301- (PS .string " e" <|> PS .string " E" )
302- *> parseIntLiteral
303-
304- parsePositiveScientific ∷ ∀ m . Monad m ⇒ P.ParserT String m HN.HugeNum
305- parsePositiveScientific = do
306- let ten = HN .fromNumber 10.0
307- lhs ← PC .try $ fromInt <$> parseNat <* PS .string " ."
308- rhs ← A .many parseDigit <#> F .foldr (\d f → divNum (f + fromInt d) ten) zero
309- exp ← parseExponent
310- pure $ (lhs + rhs) * HN .pow ten exp
311-
312- where
313- fromInt = HN .fromNumber <<< Int .toNumber
314-
315- -- TODO: remove when HugeNum adds division
316- divNum a b =
317- HN .fromNumber $
318- HN .toNumber a / HN .toNumber b
319-
208+ intLit = Lit ∘ Integer <$> EJP .parseHugeIntLiteral
320209
321210keyword ∷ ∀ m . Monad m ⇒ P.ParserT String m Token
322211keyword = map Kw $ PC .choice $ map (PC .try ∘ parseKeyWord) keywords
@@ -329,7 +218,6 @@ parseKeyWord s =
329218 c ← PC .try $ PS .oneOf [ Ch .toUpper ch, Ch .toLower ch ]
330219 pure $ A .snoc acc c
331220
332-
333221tokens ∷ ∀ m . Monad m ⇒ P.ParserT String m (Array Token )
334222tokens = do
335223 PS .skipSpaces
0 commit comments