Skip to content

Commit 74a9b2b

Browse files
PiotrJustynaPiotrJustyna
authored
3 new lexer examples (#284)
* 3 new lexer examples * included wyvern lexers in the examples makefile * whitespace correction
1 parent a148cae commit 74a9b2b

File tree

10 files changed

+352
-9
lines changed

10 files changed

+352
-9
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
/examples/*.happy.hs
1313
/examples/*.bin
1414
/examples/*.exe
15+
/examples/wyvern/*.alex.hs
1516
/old-*/
1617
/tests/*.[dign].hs
1718
/tests/*.[dign].bin

‎examples/Makefile‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,11 @@
1+
# NOTE: This logic follows the tests/Makefile for consistency.
2+
ifndef ALEX
3+
ALEX=$(shell which alex)
4+
ifeq "$(filter $(dir $(shell pwd))%,$(ALEX))" ""
15
ALEX=../dist/build/alex/alex
6+
endif
7+
endif
8+
29
HC=ghc -Wall -fno-warn-unused-binds -fno-warn-missing-signatures -fno-warn-unused-matches -fno-warn-name-shadowing -fno-warn-unused-imports -fno-warn-tabs
310

411
HAPPY=happy
@@ -10,7 +17,7 @@ else
1017
exeext=.bin
1118
endif
1219

13-
PROGS = lit Tokens Tokens_gscan words words_posn words_monad tiny haskell tiger
20+
PROGS = lit Tokens Tokens_gscan words words_posn words_monad tiny haskell tiger WyvernLexerV1 WyvernLexerV2 WyvernLexerV3
1421

1522
ALEX_OPTS = --template=../data/ -g
1623

@@ -54,6 +61,15 @@ haskell$(exeext) : haskell.alex.hs
5461
tiger$(exeext) : tiger.alex.hs
5562
$(HC) $(HC_OPTS) -main-is TigerLexer -o $@ $^
5663

64+
WyvernLexerV1$(exeext) : wyvern/WyvernLexerV1.alex.hs
65+
$(HC) $(HC_OPTS) -main-is WyvernLexerV1 -o $@ $^
66+
67+
WyvernLexerV2$(exeext) : wyvern/WyvernLexerV2.alex.hs
68+
$(HC) $(HC_OPTS) -main-is WyvernLexerV2 -o $@ $^
69+
70+
WyvernLexerV3$(exeext) : wyvern/WyvernLexerV3.alex.hs
71+
$(HC) $(HC_OPTS) -main-is WyvernLexerV3 -o $@ $^
72+
5773
.PHONY: clean
5874
clean:
5975
rm -f *.o *.hi $(addsuffix $(exeext),$(PROGS)) \

‎examples/haskell.x‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212
{
1313
module Main (main) where
14-
import Data.Char (chr)
14+
import Data.Char (chr, ord)
1515
}
1616

1717
%wrapper "monad"
@@ -39,7 +39,7 @@ $idchar = [$alpha $digit \']
3939
$symchar = [$symbol \:]
4040
$nl = [\n\r]
4141
42-
@reservedid =
42+
@reservedid =
4343
as|case|class|data|default|deriving|do|else|hiding|if|
4444
import|in|infix|infixl|infixr|instance|let|module|newtype|
4545
of|qualified|then|type|where
@@ -88,7 +88,7 @@ haskell :-
8888
<0> @varsym { mkL LVarSym }
8989
<0> @consym { mkL LConSym }
9090
91-
<0> @decimal
91+
<0> @decimal
9292
| 0[oO] @octal
9393
| 0[xX] @hexadecimal { mkL LInteger }
9494
@@ -121,7 +121,7 @@ data LexemeClass
121121
| LQConSym
122122
| LEOF
123123
deriving Eq
124-
124+
125125
mkL :: LexemeClass -> AlexInput -> Int -> Alex Lexeme
126126
mkL c (p,_,_,str) len = return (L p c (take len str))
127127
@@ -149,17 +149,17 @@ nested_comment _ _ = do
149149
Just (c,input) -> go n input
150150
c -> go n input
151151
152-
err input = do alexSetInput input; lexError "error in nested comment"
152+
err input = do alexSetInput input; lexError "error in nested comment"
153153
154154
lexError s = do
155155
(p,c,_,input) <- alexGetInput
156-
alexError (showPosn p ++ ": " ++ s ++
156+
alexError (showPosn p ++ ": " ++ s ++
157157
(if (not (null input))
158158
then " before " ++ show (head input)
159159
else " at end of file"))
160160
161161
scanner str = runAlex str $ do
162-
let loop i = do tok@(L _ cl _) <- alexMonadScan;
162+
let loop i = do tok@(L _ cl _) <- alexMonadScan;
163163
if cl == LEOF
164164
then return i
165165
else do loop $! (i+1)

‎examples/tiger.x‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ import System.Directory ( doesFileExist )
2121
import Control.Monad
2222
import Data.Maybe
2323
import Numeric ( readDec )
24-
import Data.Char ( chr )
24+
import Data.Char ( chr, ord )
2525
import Data.Map ( Map )
2626
import qualified Data.Map as Map ( empty )
2727
}

‎examples/wyvern/WyvernLexerV1.x‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
{
2+
module WyvernLexerV1
3+
(main,
4+
alexScanTokens,
5+
Token(TokenAction,
6+
TokenSoloIdentifier,
7+
TokenOCB,
8+
TokenCCB)) where
9+
}
10+
11+
%wrapper "basic"
12+
13+
$digit = 0-9
14+
$alpha = [a-zA-Z]
15+
16+
$idChar = [$alpha $digit \']
17+
$contentChar = [$alpha $digit $white \' \, \! \- \. \/ \? \= \< \> \[ \] \+ \( \)]
18+
19+
@id = $idChar+
20+
@content = $contentChar+
21+
22+
tokens :-
23+
24+
$white+ ;
25+
@id [$white]+ \"@content\" { \s -> TokenAction s }
26+
\"@content\" { \s -> TokenAction ("# " <> s) -- # is a placeholder id that will later be replaced by a unique identifier }
27+
@id { \s -> TokenSoloIdentifier s }
28+
\{ { \_ -> TokenOCB }
29+
\} { \_ -> TokenCCB }
30+
31+
{
32+
data Token
33+
= TokenAction String
34+
| TokenSoloIdentifier String
35+
| TokenOCB
36+
| TokenCCB
37+
deriving Show
38+
39+
main = do
40+
putStrLn "Wyvern lexer v1 correct example: "
41+
correctFileContent <- readFile "./wyvern/correct-input.txt"
42+
let correctTokens = alexScanTokens correctFileContent
43+
print correctTokens
44+
45+
putStrLn "Wyvern lexer v1 incorrect example: "
46+
incorrectFileContent <- readFile "./wyvern/incorrect-input.txt"
47+
let incorrectTokens = alexScanTokens incorrectFileContent
48+
print incorrectTokens
49+
}

‎examples/wyvern/WyvernLexerV2.x‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
module WyvernLexerV2
3+
(main,
4+
alexScanTokens,
5+
Token(TokenAction,
6+
TokenSoloIdentifier,
7+
TokenOCB,
8+
TokenCCB),
9+
AlexPosn(..)) where
10+
}
11+
12+
%wrapper "posn"
13+
14+
$digit = 0-9
15+
$alpha = [a-zA-Z]
16+
17+
$idChar = [$alpha $digit \']
18+
$contentChar = [$alpha $digit $white \' \, \! \- \. \/ \? \= \< \> \[ \] \+ \( \)]
19+
20+
@id = $idChar+
21+
@content = $contentChar+
22+
23+
tokens :-
24+
25+
$white+ ;
26+
@id [$white]+ \"@content\" { (\position input -> TokenAction position input) }
27+
\"@content\" { (\position input -> TokenAction position ("# " <> input)) -- # is a placeholder id that will later be replaced by a unique identifier }
28+
@id { (\position input -> TokenSoloIdentifier position input) }
29+
\{ { (\position _ -> TokenOCB position) }
30+
\} { (\position _ -> TokenCCB position) }
31+
32+
{
33+
-- Each token action (the right hand side function) is of type :: AlexPosn -> String -> Token
34+
35+
data Token
36+
= TokenAction AlexPosn String
37+
| TokenSoloIdentifier AlexPosn String
38+
| TokenOCB AlexPosn
39+
| TokenCCB AlexPosn
40+
deriving (Eq, Show)
41+
42+
main = do
43+
putStrLn "Wyvern lexer v2 correct example: "
44+
correctFileContent <- readFile "./wyvern/correct-input.txt"
45+
let correctTokens = alexScanTokens correctFileContent
46+
print correctTokens
47+
48+
putStrLn "Wyvern lexer v2 incorrect example: "
49+
incorrectFileContent <- readFile "./wyvern/incorrect-input.txt"
50+
let incorrectTokens = alexScanTokens incorrectFileContent
51+
print incorrectTokens
52+
}

‎examples/wyvern/WyvernLexerV3.x‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{
2+
module WyvernLexerV3
3+
(main,
4+
lexAll,
5+
runAlex,
6+
Token(TokenAction,
7+
TokenSoloIdentifier,
8+
TokenOCB,
9+
TokenCCB),
10+
AlexPosn(..)) where
11+
}
12+
13+
%wrapper "monad"
14+
15+
$digit = 0-9
16+
$alpha = [a-zA-Z]
17+
18+
$idChar = [$alpha $digit \']
19+
$contentChar = [$alpha $digit $white \' \, \! \- \. \/ \? \= \< \> \[ \] \+ \( \)]
20+
21+
@id = $idChar+
22+
@content = $contentChar+
23+
24+
tokens :-
25+
26+
$white+ ;
27+
@id [$white]+ \"@content\" { (\(position, _previousCharacter, _bytes, inputString) len -> return $ TokenAction position (take len inputString)) }
28+
\"@content\" { (\(position, _previousCharacter, _bytes, inputString) len -> return $ TokenAction position ("# " <> take len inputString)) -- # is a placeholder id that will later be replaced by a unique identifier }
29+
@id { (\(position, _previousCharacter, _bytes, inputString) len -> return $ TokenSoloIdentifier position (take len inputString)) }
30+
\{ { (\(position, _previousCharacter, _bytes, _inputString) len -> return $ TokenOCB position) }
31+
\} { (\(position, _previousCharacter, _bytes, _inputString) len -> return $ TokenCCB position) }
32+
33+
{
34+
-- Each token action (the right hand side function) is of type :: AlexInput -> Int -> Alex Token
35+
36+
data Token
37+
= TokenAction AlexPosn String
38+
| TokenSoloIdentifier AlexPosn String
39+
| TokenOCB AlexPosn
40+
| TokenCCB AlexPosn
41+
| TokenEOF
42+
deriving (Eq, Show)
43+
44+
alexEOF :: Alex Token
45+
alexEOF = return TokenEOF
46+
47+
lexAll :: Alex [Token]
48+
lexAll = go
49+
where
50+
go = do
51+
t <- alexMonadScan
52+
case t of
53+
TokenEOF -> return []
54+
_ -> (t:) <$> go
55+
56+
main = do
57+
putStrLn "Wyvern lexer v3 correct example: "
58+
correctFileContent <- readFile "./wyvern/correct-input.txt"
59+
let correctTokens = runAlex correctFileContent lexAll
60+
print correctTokens
61+
62+
putStrLn "Wyvern lexer v3 incorrect example: "
63+
incorrectFileContent <- readFile "./wyvern/incorrect-input.txt"
64+
let incorrectTokens = runAlex incorrectFileContent lexAll
65+
print incorrectTokens
66+
}

‎examples/wyvern/correct-input.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
"action"
2+
"question"
3+
{
4+
"action"
5+
}
6+
{
7+
"action"
8+
}
9+
"action"

‎examples/wyvern/incorrect-input.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
"action"
2+
"question"
3+
{
4+
"action"|
5+
}
6+
{
7+
"action"
8+
}
9+
"action"

0 commit comments

Comments
 (0)