Skip to content

Commit 34a0498

Browse files
ReFreezedReFreezed
committed
Added preprocessor symbols ($identifier).
Detecting errors such as !(x,y).
1 parent 540e946 commit 34a0498

File tree

4 files changed

+126
-2
lines changed

4 files changed

+126
-2
lines changed

‎preprocess.lua‎

Lines changed: 60 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,7 +123,7 @@
123123

124124

125125

126-
local PP_VERSION = "1.15.0"
126+
local PP_VERSION = "1.15.0-dev"
127127

128128
local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString().
129129

@@ -733,6 +733,21 @@ function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax)
733733
tok = {type="pp_keyword", representation=repr, value=word}
734734
end
735735

736+
-- Preprocessor symbol.
737+
elseif s:find("^%$", ptr) then
738+
if not allowPpTokens then
739+
errorInFile(s, path, ptr, "Tokenizer", "Encountered preprocessor symbol. (Feature not enabled.)")
740+
end
741+
742+
local i1, i2, repr, word = s:find("^(%$([%a_][%w_]*))", ptr)
743+
if not i1 then
744+
errorInFile(s, path, ptr+1, "Tokenizer", "Expected an identifier.")
745+
elseif KEYWORDS[word] then
746+
errorInFile(s, path, ptr+1, "Tokenizer", "Invalid preprocessor symbol '%s'. (Must not be a Lua keyword.)", word)
747+
end
748+
ptr = i2+1
749+
tok = {type="pp_symbol", representation=repr, value=word}
750+
736751
else
737752
errorInFile(s, path, ptr, "Tokenizer", "Unknown character.")
738753
end
@@ -1629,6 +1644,7 @@ local numberFormatters = {
16291644
-- whitespaceToken = newToken( "whitespace", contents )
16301645
-- ppEntryToken = newToken( "pp_entry", isDouble )
16311646
-- ppKeywordToken = newToken( "pp_keyword", ppKeyword ) -- ppKeyword can be "@".
1647+
-- ppKeywordToken = newToken( "pp_symbol", identifier )
16321648
--
16331649
-- commentToken = { type="comment", representation=string, value=string, long=isLongForm }
16341650
-- identifierToken = { type="identifier", representation=string, value=string }
@@ -1639,6 +1655,7 @@ local numberFormatters = {
16391655
-- whitespaceToken = { type="whitespace", representation=string, value=string }
16401656
-- ppEntryToken = { type="pp_entry", representation=string, value=string, double=isDouble }
16411657
-- ppKeywordToken = { type="pp_keyword", representation=string, value=string }
1658+
-- ppSymbolToken = { type="pp_symbol", representation=string, value=string }
16421659
--
16431660
-- Number formats:
16441661
-- "integer" E.g. 42
@@ -1684,6 +1701,8 @@ function metaFuncs.newToken(tokType, ...)
16841701
error("Identifier length is 0.", 2)
16851702
elseif not ident:find"^[%a_][%w_]*$" then
16861703
errorf(2, "Bad identifier format: '%s'", ident)
1704+
elseif KEYWORDS[ident] then
1705+
errorf(2, "Identifier must not be a keyword: '%s'", ident)
16871706
end
16881707

16891708
return {type="identifier", representation=ident, value=ident}
@@ -1780,6 +1799,20 @@ function metaFuncs.newToken(tokType, ...)
17801799
return {type="pp_keyword", representation="@"..keyword, value=keyword}
17811800
end
17821801

1802+
elseif tokType == "pp_symbol" then
1803+
local ident = ...
1804+
assertarg(2, ident, "string")
1805+
1806+
if ident == "" then
1807+
error("Identifier length is 0.", 2)
1808+
elseif not ident:find"^[%a_][%w_]*$" then
1809+
errorf(2, "Bad identifier format: '%s'", ident)
1810+
elseif KEYWORDS[ident] then
1811+
errorf(2, "Identifier must not be a keyword: '%s'", ident)
1812+
else
1813+
return {type="pp_symbol", representation="$"..ident, value=ident}
1814+
end
1815+
17831816
else
17841817
errorf(2, "Invalid token type '%s'.", tostring(tokType))
17851818
end
@@ -1812,6 +1845,13 @@ function metaEnv.__ASSERTLUA(lua)
18121845
return lua
18131846
end
18141847

1848+
function metaEnv.__EVALSYMBOL(v)
1849+
if type(v) == "function" then
1850+
v = v()
1851+
end
1852+
return v
1853+
end
1854+
18151855

18161856

18171857
local function getLineCountWithCode(tokens)
@@ -1861,6 +1901,7 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats)
18611901
-- @file
18621902
-- @line
18631903
-- ` ... `
1904+
-- $symbol
18641905
--
18651906
if not stats.hasPreprocessorCode then
18661907
return tokensToExpand
@@ -1903,6 +1944,20 @@ local function doEarlyExpansions(tokensToExpand, fileBuffers, params, stats)
19031944
tableInsert(tokens, stringTok)
19041945
tableRemove(tokenStack) -- the string
19051946

1947+
-- Symbol. (Should this expand later? Does it matter?)
1948+
elseif isToken(tok, "pp_symbol") then
1949+
local ppSymbolTok = tok
1950+
1951+
-- $symbol
1952+
tableRemove(tokenStack) -- '$symbol'
1953+
tableInsert(tokens, newTokenAt({type="pp_entry", value="!!", representation="!!", double=true}, ppSymbolTok))
1954+
tableInsert(tokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok))
1955+
tableInsert(tokens, newTokenAt({type="identifier", value="__EVALSYMBOL", representation="__EVALSYMBOL" }, ppSymbolTok))
1956+
tableInsert(tokens, newTokenAt({type="punctuation", value="(", representation="(" }, ppSymbolTok))
1957+
tableInsert(tokens, newTokenAt({type="identifier", value=ppSymbolTok.value, representation=ppSymbolTok.value}, ppSymbolTok))
1958+
tableInsert(tokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok))
1959+
tableInsert(tokens, newTokenAt({type="punctuation", value=")", representation=")" }, ppSymbolTok))
1960+
19061961
-- Anything else.
19071962
else
19081963
tableInsert(tokens, tok)
@@ -2549,7 +2604,7 @@ local function _processFileOrString(params, isFile)
25492604
}
25502605

25512606
for _, tok in ipairs(tokens) do
2552-
if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") then
2607+
if isToken(tok, "pp_entry") or isToken(tok, "pp_keyword", "insert") or isToken(tok, "pp_symbol") then
25532608
stats.hasPreprocessorCode = true
25542609
stats.hasMetaprogram = true
25552610
break
@@ -2857,6 +2912,9 @@ local function _processFileOrString(params, isFile)
28572912
tableInsert(metaParts, metaBlock)
28582913
tableInsert(metaParts, "))\n")
28592914

2915+
elseif metaBlock:find(",", 1, true) and loadLuaString("return'',"..metaBlock) then
2916+
errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Ambiguous preprocessor block contents. (Comma-separated lists are not supported here.)")
2917+
28602918
elseif doOutputLua then
28612919
-- We could do something other than error here. Room for more functionality.
28622920
errorAfterToken(fileBuffers, tokens[startIndex+1], "Parser", "Preprocessor block does not contain a valid value expression.")

‎tests/quickTest.lua2p‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,17 @@ local n2 = @@ADD1!!("43-2")
144144

145145

146146

147+
-- Symbols.
148+
!local RANDOM = "math.random()"
149+
local rand = $RANDOM
150+
151+
!local function EQUATION() return "x*3+y" end
152+
local x = 5
153+
local y = 89
154+
local z = $EQUATION
155+
156+
157+
147158
-- Misc.
148159
print(!("dataFromCommandLine: "..tostring(dataFromCommandLine)))
149160
print(!(("This file and line: %s:%d"):format(@file, @line)))

‎tests/quickTest.output.lua‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,19 @@ local d = {1+2}
9999

100100
local n = 58
101101

102+
local n1 = 41+1
103+
local n2 = 43-2+1
104+
105+
106+
107+
-- Symbols
108+
109+
local rand = math.random()
110+
111+
local x = 5
112+
local y = 89
113+
local z = x*3+y
114+
102115

103116

104117
-- Misc.

‎tests/suite.lua‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,13 @@ doTest("Expression or not?", function()
176176
!( x = math.floor(1.5) )
177177
]]})
178178
assertCodeOutput(luaOut, [[]])
179+
180+
-- Invalid: Comma-separated expressions are ambiguous.
181+
assert(not pp.processString{ code=[[ x = !(1, 2) ]]})
182+
assert(not pp.processString{ code=[[ x = !!("a", "b") ]]})
183+
184+
-- Invalid: !!() must always have an expression.
185+
assert(not pp.processString{ code=[[ !!( x = y ) ]]})
179186
end)
180187

181188
doTest("Output values of different types", function()
@@ -364,6 +371,32 @@ doTest("Macros", function()
364371
assert(not pp.processString{ code=[[ !(function ECHO(v) return v end) v = @@ECHO( !!( !(1) ) ) ]]})
365372
end)
366373

374+
doTest("Preprocessor symbols", function()
375+
local pp = ppChunk()
376+
377+
local luaOut = assert(pp.processString{ code=[[
378+
!local FOO = "y"
379+
x = $FOO
380+
]]})
381+
assertCodeOutput(luaOut, [[x = y]])
382+
383+
local luaOut = assert(pp.processString{ code=[[
384+
!local function FOO() return "y" end
385+
x = $FOO
386+
]]})
387+
assertCodeOutput(luaOut, [[x = y]])
388+
389+
-- Invalid: Symbols must result in strings.
390+
assert(not pp.processString{ code=[[
391+
!local BAD = 840
392+
v = $BAD
393+
]]})
394+
assert(not pp.processString{ code=[[
395+
!local function BAD() return 840 end
396+
v = $BAD
397+
]]})
398+
end)
399+
367400

368401

369402
addLabel("Library API")
@@ -389,6 +422,8 @@ doTest("Create tokens", function()
389422
-- Identifier.
390423
assertToken(pp.newToken("identifier", "foo"), "identifier", "foo", "foo", nil, nil)
391424

425+
assert(not pcall(pp.newToken, "identifier", "if"))
426+
392427
-- Keyword.
393428
assertToken(pp.newToken("keyword", "if"), "keyword", "if", "if", nil, nil)
394429

@@ -440,6 +475,13 @@ doTest("Create tokens", function()
440475
assertToken(pp.newToken("pp_keyword", "@" ), "pp_keyword", "insert", "@@", nil, nil)
441476

442477
assert(not pcall(pp.newToken, "pp_keyword", "bad"))
478+
479+
-- Preprocessor symbol.
480+
assertToken(pp.newToken("pp_symbol", "foo"), "pp_symbol", "foo", "$foo", nil, nil)
481+
482+
assert(not pcall(pp.newToken, "pp_symbol", ""))
483+
assert(not pcall(pp.newToken, "pp_symbol", "if"))
484+
assert(not pcall(pp.newToken, "pp_symbol", "$foo"))
443485
end)
444486

445487
doTest("Get useful tokens", function()

0 commit comments

Comments
 (0)