Fixed issue when files don't end in a newline.

CL: Many more tests!

Author: ReFreezed

preprocess-cl.lua

@@ -11,8 +11,9 @@ exec lua "$0" "$@"
 --=
 --=  License: MIT (see the bottom of this file)
 --=  Website: http://refreezed.com/luapreprocess/
+--=  Documentation: http://refreezed.com/luapreprocess/docs/command-line/
 --=
---=  Tested with Lua 5.1, 5.2, 5.3 and 5.4.
+--=  Tested with Lua 5.1, 5.2, 5.3, 5.4 and LuaJIT.
 --=
 --==============================================================
 
@@ -111,7 +112,7 @@ exec lua "$0" "$@"
 			enabled if an output path is stdout.)
 
 		--version
-			Print the version of LuaPreprocess and exit.
+			Print the version of LuaPreprocess to stdout and exit.
 
 		--debug
 			Enable some preprocessing debug features. Useful if you want
@@ -335,7 +336,7 @@ for _, arg in ipairs(args) do
 		if hasOutputExtension then
 			errorLine("Cannot specify both --outputpaths and --outputextension")
 		elseif pathsIn[1] then
-			errorLine(arg.." must appear before any path.")
+			errorLine(arg.." must appear before any input path.")
 		end
 		hasOutputPaths = true
 
@@ -364,7 +365,7 @@ for _, arg in ipairs(args) do
 		maxLogLevel = arg:gsub("^.-=", "")
 
 	elseif arg == "--version" then
-		print(pp.VERSION)
+		io.stdout:write(pp.VERSION)
 		os.exit()
 
 	-- elseif arg == "/?" or arg:find"^%-%-?help" or arg:lower() == "/help" then

preprocess.lua

@@ -432,36 +432,36 @@ local function parseStringlikeToken(s, ptr)
 	local valueEnd
 
 	local longEqualSigns = s:match("^%[(=*)%[", ptr)
-	local isLong = (longEqualSigns ~= nil)
+	local isLong         = longEqualSigns ~= nil
 
 	-- Single line.
 	if not isLong then
 		valueStart = ptr
 
-		local i1, i2 = s:find("\r?\n", ptr)
-		if not i1 then
+		local i = s:find("\n", ptr, true)
+		if not i then
 			reprEnd  = #s
 			valueEnd = #s
-			ptr      = reprEnd+1
+			ptr      = reprEnd + 1
 		else
-			reprEnd  = i2
-			valueEnd = i1-1
-			ptr      = reprEnd+1
+			reprEnd  = i
+			valueEnd = i - 1
+			ptr      = reprEnd + 1
 		end
 
 	-- Multiline.
 	else
-		ptr        = ptr+1+#longEqualSigns+1
+		ptr        = ptr + 1 + #longEqualSigns + 1
 		valueStart = ptr
 
-		local i1, i2 = s:find("%]"..longEqualSigns.."%]", ptr)
+		local i1, i2 = s:find("]"..longEqualSigns.."]", ptr, true)
 		if not i1 then
 			return nil, ERROR_UNFINISHED_STRINGLIKE
 		end
 
 		reprEnd  = i2
-		valueEnd = i1-1
-		ptr      = reprEnd+1
+		valueEnd = i1 - 1
+		ptr      = reprEnd + 1
 	end
 
 	local repr = s:sub(reprStart,  reprEnd)
@@ -484,6 +484,8 @@ local NUM_DEC          = ("^(        %d+  %.?                    )"):gsub(" +",
 
 -- tokens = _tokenize( luaString, path, allowPreprocessorTokens, allowBacktickStrings, allowJitSyntax )
 local function _tokenize(s, path, allowPpTokens, allowBacktickStrings, allowJitSyntax)
+	s = s:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
+
 	local tokens = {}
 	local ptr    = 1
 	local ln     = 1
@@ -2400,8 +2402,8 @@ local function doEarlyExpansions(tokensToExpand, stats)
 
 		-- Backtick string.
 		elseif isToken(tok, "string") and tok.representation:find"^`" then
-			local stringTok = tok
-			stringTok.representation = F("%q", stringTok.value)
+			local stringTok          = tok
+			stringTok.representation = toLua(stringTok.value)--F("%q", stringTok.value)
 
 			tableInsert(outTokens, stringTok)
 			tableRemove(tokenStack) -- the string
@@ -3290,7 +3292,7 @@ local function astNodeToMetaprogram(buffer, ast, ln, lnMeta, asMacroArgExpr)
 			tableInsert(buffer, ")")
 		end
 
-		-- Use trailing comma if the user does.
+		-- Use trailing semicolon if the user does.
 		for i = #ast.valueTokens, 1, -1 do
 			if isToken(ast.valueTokens[i], "punctuation", ";") then
 				if    current_parsingAndMeta_isDebug
@@ -3378,7 +3380,13 @@ local function _processFileOrString(params, isFile)
 
 	local specialFirstLine, rest = luaUnprocessed:match"^(#[^\r\n]*\r?\n?)(.*)$"
 	if specialFirstLine then
-		luaUnprocessed = rest
+		specialFirstLine = specialFirstLine:gsub("\r", "") -- Normalize line breaks. (Assume the input is either "\n" or "\r\n".)
+		luaUnprocessed   = rest
+	end
+
+	-- Ensure there's a newline at the end of the code, otherwise there will be problems down the line.
+	if not (luaUnprocessed == "" or luaUnprocessed:find"\n%s*$") then
+		luaUnprocessed = luaUnprocessed .. "\n"
 	end
 
 	local tokens = _tokenize(luaUnprocessed, virtualPathIn, true, params.backtickStrings, params.jitSyntax)
@@ -3509,7 +3517,7 @@ local function _processFileOrString(params, isFile)
 
 		if not chunk then
 			local ln, _err = err:match"^.-:(%d+): (.*)"
-			errorOnLine(pathOut, (tonumber(ln) or 0), nil, "%s", (_err or err))
+			errorOnLine(pathOut, (tonumber(ln) or 0), nil, "Output is invalid Lua. (%s)", (_err or err))
 		end
 	end
 

tests/quickTestHandler.lua tests/quickTestHandlerFunction.luatests/quickTestHandler.lua renamed to tests/quickTestHandlerFunction.lua

@@ -3,6 +3,8 @@
 --=  Test message handler
 --=  for use with preprocess-cl.lua
 --=
+--=  Alternative #2: A single catch-all message handler.
+--=
 --============================================================]]
 
 --
@@ -26,28 +28,6 @@ end
 -- The module is expected to return one or multiple message handlers.
 --
 
--- [[ Alternative #1: Multiple specific message handlers.
-return {
-	beforemeta = function(path)
-		print("... Now processing "..path)
-	end,
-
-	aftermeta = function(path, luaString)
-		-- Remove comments (quick and dirty).
-		luaString = luaString
-			:gsub("%-%-%[%[.-%]%]", "") -- Multi-line.
-			:gsub("%-%-[^\n]*",     "") -- Single line.
-
-		return luaString
-	end,
-
-	filedone = function(path, outputPath)
-		print("... Done with "..path.." (writing to "..outputPath..")")
-	end,
-}
---]]
-
---[[ Alternative #2: A single catch-all message handler.
 return function(message, ...)
 	if message == "beforemeta" then
 		local path = ...
@@ -68,4 +48,3 @@ return function(message, ...)
 		print("... Done with "..path.." (writing to "..outputPath..")")
 	end
 end
---]]

tests/quickTestHandlerTable.lua

@@ -0,0 +1,48 @@
+--[[============================================================
+--=
+--=  Test message handler
+--=  for use with preprocess-cl.lua
+--=
+--=  Alternative #1: Multiple specific message handlers
+--=
+--============================================================]]
+
+--
+-- This module shares environment with the processed files,
+-- making it a good place to put things all files use/share.
+--
+
+_G.IS_DEVELOPER = true
+
+math.tau = 2*math.pi
+
+function _G.flipTable(t)
+	local flipped = {}
+	for k, v in pairs(t) do
+		flipped[v] = k
+	end
+	return flipped
+end
+
+--
+-- The module is expected to return one or multiple message handlers.
+--
+
+return {
+	beforemeta = function(path)
+		print("... Now processing "..path)
+	end,
+
+	aftermeta = function(path, luaString)
+		-- Remove comments (quick and dirty).
+		luaString = luaString
+			:gsub("%-%-%[%[.-%]%]", "") -- Multi-line.
+			:gsub("%-%-[^\n]*",     "") -- Single line.
+
+		return luaString
+	end,
+
+	filedone = function(path, outputPath)
+		print("... Done with "..path.." (writing to "..outputPath..")")
+	end,
+}

tests/runQuickTestUsingHandler.cmd

@@ -19,7 +19,11 @@ IF NOT EXIST temp  MD temp
 
 
 
-%_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandler.lua tests/quickTest.lua2p || EXIT /B 1
-REM %_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandler.lua --outputpaths tests/quickTest.lua2p temp/quickTest.output.lua || EXIT /B 1
+%_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandlerTable.lua tests/quickTest.lua2p || EXIT /B 1
+REM %_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandlerTable.lua --outputpaths tests/quickTest.lua2p temp/quickTest.output.lua || EXIT /B 1
 
+%_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandlerFunction.lua tests/quickTest.lua2p || EXIT /B 1
+REM %_lua% ./preprocess-cl.lua --debug --saveinfo=temp/info.lua --handler=tests/quickTestHandlerFunction.lua --outputpaths tests/quickTest.lua2p temp/quickTest.output.lua || EXIT /B 1
+
+ECHO. & ECHO Running quickTest.lua...
 %_lua% -e"io.stdout:setvbuf'no' io.stderr:setvbuf'no'" tests/quickTest.lua || EXIT /B 1