Skip to content

Commit 4742f22

Browse files
ReFreezedReFreezed
committed
Added functions: updateToken, cloneToken.
Fixed error for newToken("string").
1 parent 99c01b3 commit 4742f22

File tree

2 files changed

+124
-17
lines changed

2 files changed

+124
-17
lines changed

‎dumbParser.lua‎

Lines changed: 111 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ print(lua)
5252
----------------------------------------------------------------
5353
5454
tokenize, tokenizeFile
55-
newToken, concatTokens
55+
newToken, updateToken, cloneToken, concatTokens
5656
parse, parseExpression, parseFile
5757
newNode, newNodeFast, valueToAst, cloneNode, cloneTree, getChild, setChild, addChild, removeChild
5858
validateTree
@@ -78,8 +78,16 @@ newToken()
7878
token = parser.newToken( tokenType, tokenValue )
7979
Create a new token. (See below or search for 'TokenCreation' for more info.)
8080
81+
updateToken()
82+
parser.updateToken( token, tokenValue )
83+
Update the value and representation of an existing token. (Search for 'TokenModification' for more info.)
84+
85+
cloneToken()
86+
tokenClone = parser.cloneToken( token )
87+
Clone an existing token.
88+
8189
concatTokens()
82-
parser.concatTokens( tokens )
90+
luaString = parser.concatTokens( tokens )
8391
Concatenate tokens. Whitespace is added between tokens when necessary.
8492
8593
parse()
@@ -1618,13 +1626,13 @@ end
16181626
--
16191627
-- :TokenCreation
16201628
--
1621-
-- token = newToken( "comment", contents )
1622-
-- token = newToken( "identifier", name )
1623-
-- token = newToken( "keyword", name )
1624-
-- token = newToken( "number", number )
1625-
-- token = newToken( "punctuation", punctuationString )
1626-
-- token = newToken( "string", stringValue )
1627-
-- token = newToken( "whitespace", contents )
1629+
-- commentToken = newToken( "comment", contents )
1630+
-- identifierToken = newToken( "identifier", name )
1631+
-- keywordToken = newToken( "keyword", name )
1632+
-- numberToken = newToken( "number", number )
1633+
-- punctuationToken = newToken( "punctuation", punctuationString )
1634+
-- stringToken = newToken( "string", stringValue )
1635+
-- whitespaceToken = newToken( "whitespace", contents )
16281636
--
16291637
local function newToken(tokType, tokValue)
16301638
local tokRepr
@@ -1654,7 +1662,7 @@ local function newToken(tokType, tokValue)
16541662

16551663
elseif tokType == "string" then
16561664
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end
1657-
tokRepr = stringGsub(F("%q", tokRepr), "\n", "n")
1665+
tokRepr = stringGsub(F("%q", tokValue), "\n", "n")
16581666

16591667
elseif tokType == "punctuation" then
16601668
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end
@@ -1702,6 +1710,95 @@ local function newToken(tokType, tokValue)
17021710
}
17031711
end
17041712

1713+
--
1714+
-- :TokenModification
1715+
--
1716+
-- updateToken( commentToken, contents )
1717+
-- updateToken( identifierToken, name )
1718+
-- updateToken( keywordToken, name )
1719+
-- updateToken( numberToken, number )
1720+
-- updateToken( punctuationToken, punctuationString )
1721+
-- updateToken( stringToken, stringValue )
1722+
-- updateToken( whitespaceToken, contents )
1723+
--
1724+
local function updateToken(tok, tokValue)
1725+
if tok.type == "keyword" then
1726+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'keyword' token. (Got %s)", type(tokValue)) end
1727+
if not KEYWORDS[tokValue] then errorf(2, "Invalid keyword '%s'.", tokValue) end
1728+
tok.representation = tokValue
1729+
1730+
elseif tok.type == "identifier" then
1731+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'identifier' token. (Got %s)", type(tokValue)) end
1732+
if not stringFind(tokValue, "^[%a_][%w_]*$") then errorf(2, "Invalid identifier '%s'.", tokValue) end
1733+
if KEYWORDS[tokValue] then errorf(2, "Invalid identifier '%s'.", tokValue) end
1734+
tok.representation = tokValue
1735+
1736+
elseif tok.type == "number" then
1737+
if type(tokValue) ~= "number" then
1738+
errorf(2, "Expected number value for 'number' token. (Got %s)", type(tokValue))
1739+
end
1740+
tok.representation = (
1741+
tokValue == 0 and NORMALIZE_MINUS_ZERO and "0" or -- Avoid '-0' sometimes.
1742+
tokValue == 1/0 and "(1/0)" or
1743+
tokValue == -1/0 and "(-1/0)" or
1744+
tokValue ~= tokValue and "(0/0)" or
1745+
formatNumber(tokValue)
1746+
)
1747+
1748+
elseif tok.type == "string" then
1749+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'string' token. (Got %s)", type(tokValue)) end
1750+
tok.representation = stringGsub(F("%q", tokValue), "\n", "n")
1751+
1752+
elseif tok.type == "punctuation" then
1753+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'punctuation' token. (Got %s)", type(tokValue)) end
1754+
if not PUNCTUATION[tokValue] then errorf(2, "Invalid punctuation '%s'.", tokValue) end
1755+
tok.representation = tokValue
1756+
1757+
elseif tok.type == "comment" then
1758+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'comment' token. (Got %s)", type(tokValue)) end
1759+
1760+
if stringFind(tokValue, "\n") then
1761+
local equalSigns = stringFind(tokValue, "[[", 1, true) and "=" or ""
1762+
1763+
while stringFind(tokValue, "]"..equalSigns.."]", 1, true) do
1764+
equalSigns = equalSigns.."="
1765+
end
1766+
1767+
tok.representation = F("--[%s[%s]%s]", equalSigns, tokValue, equalSigns)
1768+
1769+
else
1770+
tok.representation = F("--%s\n", tokValue)
1771+
end
1772+
1773+
elseif tok.type == "whitespace" then
1774+
if type(tokValue) ~= "string" then errorf(2, "Expected string value for 'whitespace' token. (Got %s)", type(tokValue)) end
1775+
if tokValue == "" then errorf(2, "Value is empty.") end -- Having a token that is zero characters long would be weird, so we disallow it.
1776+
if stringFind(tokValue, "[^ \t\n]") then errorf(2, "Value has non-whitespace characters.") end
1777+
tok.representation = tokValue
1778+
1779+
else
1780+
errorf(2, "Internal error: Invalid token type '%s'.", tostring(tok.type))
1781+
end
1782+
1783+
tok.value = tokValue
1784+
end
1785+
1786+
local function cloneToken(tok)
1787+
return {
1788+
type = tok.type,
1789+
value = tok.value,
1790+
representation = tok.representation,
1791+
1792+
sourceString = tok.sourceString,
1793+
sourcePath = tok.sourcePath,
1794+
1795+
lineStart = tok.lineStart,
1796+
lineEnd = tok.lineEnd,
1797+
positionStart = tok.positionStart,
1798+
positionEnd = tok.positionEnd,
1799+
}
1800+
end
1801+
17051802
local function concatTokens(tokens)
17061803
local parts = {}
17071804

@@ -6634,6 +6731,8 @@ parser = {
66346731

66356732
-- Token actions.
66366733
newToken = newToken,
6734+
updateToken = updateToken,
6735+
cloneToken = cloneToken,
66376736
concatTokens = concatTokens,
66386737

66396738
-- AST parsing.
@@ -6698,7 +6797,7 @@ return parser
66986797

66996798

67006799

6701-
--[=[===========================================================
6800+
--[[!===========================================================
67026801
67036802
Copyright © 2020-2022 Marcus 'ReFreezed' Thunström
67046803
@@ -6720,4 +6819,4 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
67206819
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
67216820
SOFTWARE.
67226821
6723-
=============================================================]=]
6822+
==============================================================]]

‎testsuite.lua‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -284,28 +284,36 @@ test("Tokens", function()
284284
local tokens = {}
285285

286286
-- Construct a call.
287-
table.insert(tokens, parser.newToken("identifier", "math"))
287+
table.insert(tokens, parser.newToken("identifier", "lib"))
288288
table.insert(tokens, parser.newToken("punctuation", "."))
289-
table.insert(tokens, parser.newToken("identifier", "abs"))
289+
table.insert(tokens, parser.newToken("identifier", "doThing"))
290290
table.insert(tokens, parser.newToken("whitespace", " "))
291291
table.insert(tokens, parser.newToken("punctuation", "("))
292292
table.insert(tokens, parser.newToken("punctuation", "-"))
293293
table.insert(tokens, parser.newToken("number", 1.75))
294+
table.insert(tokens, parser.newToken("punctuation", ","))
295+
table.insert(tokens, parser.newToken("string", "foo"))
294296
table.insert(tokens, parser.newToken("punctuation", ")"))
297+
table.insert(tokens, parser.newToken("comment", "Blah!"))
295298

296-
-- Add the call to a declaration with an error.
299+
-- Add the call to a declaration, but with an error.
297300
table.insert(tokens, 1, parser.newToken("keyword", "local"))
298-
table.insert(tokens, 2, parser.newToken("identifier", "n"))
301+
table.insert(tokens, 2, parser.newToken("identifier", "x"))
299302
table.insert(tokens, 3, parser.newToken("punctuation", "="))
300303
table.insert(tokens, 4, parser.newToken("punctuation", "/")) -- Error!
301304

302305
-- Remove the error.
303306
table.remove(tokens, 4)
304307

308+
-- Update all tokens (with no modifications).
309+
for _, tok in ipairs(tokens) do
310+
parser.updateToken(tok, tok.value)
311+
end
312+
305313
local ast = assert(parser.parse(tokens))
306314
local lua = assert(parser.toLua(ast))
307315
-- print(lua)
308-
assertLua(lua, [[ local n=math.abs(-1.75); ]])
316+
assertLua(lua, [[ local x=lib.doThing(-1.75,"foo"); ]])
309317
end
310318

311319
do

0 commit comments

Comments
 (0)