fbc: token pasting operator '##' allows pasting of single '_' characters

- previously fbc unable to handle single '_' token pasting single since in such cases the '_' is always seen as line continutation.
Example:
#define join1(a,b) a##_ b
#define join2(a,b) a _##b
#define join3(a,b) a##_##b

- because '_' line continuation and '##' token pasting are handled nearly entirely in the lexer, no token checks can be made
- adds lexGetLookAheadChar2( ) so that the sequence '_##' can be checked before tokenizing
- plus previous commits (Skyfish) allow handling the sequence '##_' to escape the '_' character and paste it in on expansion

Author: jayrm

changelog.txt

@@ -138,6 +138,7 @@ Version 1.08.0
 - sf.net #917: optimize 'm += s' string concatenations to fix the long compile times in the gcc backend (which makes heavy use of string building).
 - github #217: C backend, fix gcc array out of bounds warning when compiled with -O2 or higher optimizations and accessing non-zero lower bound fixed length string arrays
 - C backend: inline asm - don't add rsp/esp to the clobber list, it's deprecated in newer gcc versions and silently ignored in older versions 
+- github #309: token pasting operator '##' allows pasting of single '_' characters
 
 
 Version 1.07.0

src/compiler/lex.bas

@@ -98,7 +98,8 @@ sub lexInit _
 	next
 
 	lex.ctx->currchar = UINVALID
-	lex.ctx->lahdchar = UINVALID
+	lex.ctx->lahdchar1 = UINVALID
+	lex.ctx->lahdchar2 = UINVALID
 
 	if( is_fb_eval ) then
 		lex.ctx->linenum = (lex.ctx-1)->linenum
@@ -279,15 +280,21 @@ private function hReadChar _
 end function
 
 sub lexEatChar( )
-	if( lex.ctx->lahdchar = UINVALID ) then
+	if( lex.ctx->lahdchar1 = UINVALID ) then
 		'' No look-ahead char, read next char and force the next
 		'' lexCurrentChar() to update the current char.
 		hSkipChar( )
 		lex.ctx->currchar = UINVALID
+	elseif( lex.ctx->lahdchar2 = UINVALID ) then
+		'' Look ahead char is the next current char
+		lex.ctx->currchar = lex.ctx->lahdchar1
+		lex.ctx->lahdchar1 = UINVALID
 	else
 		'' Look ahead char is the next current char
-		lex.ctx->currchar = lex.ctx->lahdchar
-		lex.ctx->lahdchar = UINVALID
+		'' and second look ahead char becomes look ahead char
+		lex.ctx->currchar = lex.ctx->lahdchar1
+		lex.ctx->lahdchar1 = lex.ctx->lahdchar2
+		lex.ctx->lahdchar2 = UINVALID
 	end if
 end sub
 
@@ -351,20 +358,38 @@ function lexGetLookAheadChar _
 		byval skipwhitespc as integer = FALSE _
 	) as uinteger
 
-	if( lex.ctx->lahdchar = UINVALID ) then
+	if( lex.ctx->lahdchar1 = UINVALID ) then
 		hSkipChar( )
-		lex.ctx->lahdchar = hReadChar( )
+		lex.ctx->lahdchar1 = hReadChar( )
 	end if
 
 	if( skipwhitespc ) then
-		do while( (lex.ctx->lahdchar = CHAR_TAB) or (lex.ctx->lahdchar = CHAR_SPACE) )
+		do while( (lex.ctx->lahdchar1 = CHAR_TAB) or (lex.ctx->lahdchar1 = CHAR_SPACE) )
 			lex.ctx->after_space = TRUE
 			hSkipChar( )
-			lex.ctx->lahdchar = hReadChar( )
+			lex.ctx->lahdchar1 = hReadChar( )
 		loop
 	end if
 
-	function = lex.ctx->lahdchar
+	function = lex.ctx->lahdchar1
+
+end function
+
+'':::::
+function lexGetLookAheadChar2 _
+	( _
+	) as uinteger
+
+	'' internally, should never use this function unless there
+	'' is already a character in the look aead
+	assert( lex.ctx->lahdchar1 <> UINVALID ) 
+
+	if( lex.ctx->lahdchar2 = UINVALID ) then
+		hSkipChar( )
+		lex.ctx->lahdchar2 = hReadChar( )
+	end if
+
+	function = lex.ctx->lahdchar2
 
 end function
 
@@ -489,7 +514,7 @@ private sub hReadIdentifier _
 			lexEatChar( )
 
 		'' '#'?
-		case FB_TK_DBLTYPECHAR
+		case FB_TK_DBLTYPECHAR '' alias for CHAR_SHARP
 			'' isn't it a '##'?
 			if( lexGetLookAheadChar( ) <> FB_TK_DBLTYPECHAR ) then
 				dtype = FB_DATATYPE_DOUBLE
@@ -897,7 +922,7 @@ private sub hReadFloatNumber _
 		end if
 
 	'' '#'?
-	case FB_TK_DBLTYPECHAR
+	case FB_TK_DBLTYPECHAR '' alias for CHAR_SHARP
 		t.dtype = FB_DATATYPE_DOUBLE
 
 		if( (flags and LEXCHECK_NOSUFFIX) = 0 ) then
@@ -1201,7 +1226,7 @@ private sub hReadNumber( byref t as FBTOKEN, byval flags as LEXCHECK )
 				end if
 
 			'' '#'
-			case FB_TK_DBLTYPECHAR
+			case FB_TK_DBLTYPECHAR '' alias for CHAR_SHARP
 				if( have_u_suffix = FALSE ) then
 					'' isn't it a '##'?
 					if( lexGetLookAheadChar( ) <> FB_TK_DBLTYPECHAR ) then
@@ -1632,13 +1657,19 @@ re_read:
 					 CHAR_0 to CHAR_9, CHAR_UNDER
 					exit do
 
-				'' otherwise, skip until new-line is found
-				case else
-					lexEatChar( )
-					islinecont = TRUE
-					continue do
+				'' could it be '_##'?
+				case CHAR_SHARP
+					if( lexGetLookAheadChar2( ) = CHAR_SHARP ) then
+						exit do
+					end if
+
 				end select
 
+				'' otherwise, skip until new-line is found
+				lexEatChar( )
+				islinecont = TRUE
+				continue do
+
 			'' else, take it as-is
 			else
 				exit do

src/compiler/lex.bi

@@ -84,7 +84,8 @@ type LEX_TKCTX
 	tail			as FBTOKEN ptr
 
 	currchar		as uinteger					'' current char
-	lahdchar		as uinteger					'' look ahead char
+	lahdchar1		as uinteger					'' look ahead first char
+	lahdchar2		as uinteger					'' look ahead second char
 
 	linenum 		as integer
 	lasttk_id 		as integer
@@ -234,6 +235,10 @@ declare function lexGetLookAheadChar _
 		byval skipwhitespc as integer = FALSE _
 	) as uinteger
 
+declare function lexGetLookAheadChar2 _
+	( _
+	) as uinteger
+
 declare sub lexEatChar( )
 
 declare function lexPeekCurrentLine _

src/compiler/pp.bas

@@ -550,7 +550,11 @@ function ppReadLiteral _
 			case CHAR_SHARP
 				lexSkipToken( LEX_FLAGS )
 				lexSkipToken( LEX_FLAGS or LEXCHECK_NOLINECONT)
-				if *lexGetText( ) <> "_" then     '' Is only '##_'?
+
+				'' we can't check lexGetToken( ) here because a single '_' will
+				'' return as FB_TK_ID, so we need to do a lexGetText( ) check
+				'' Is only '##_'?
+				if *lexGetText( ) <> "_" then
 					DZstrConcatAssign( text, "##" )
 				end if
 
@@ -701,7 +705,11 @@ function ppReadLiteralW _
 			case CHAR_SHARP
 				lexSkipToken( LEX_FLAGS )
 				lexSkipToken( LEX_FLAGS or LEXCHECK_NOLINECONT)
-				if *lexGetText( ) <> "_" then     '' Is only '##_'?
+
+				'' we can't check lexGetToken( ) here because a single '_' will
+				'' return as FB_TK_ID, so we need to do a lexGetText( ) check
+				'' Is only '##_'?
+				if *lexGetText( ) <> "_" then
 					DWstrConcatAssignA( text, "##" )
 				end if
 

tests/pp/token-pasting.bas

@@ -0,0 +1,170 @@
+#include "fbcunit.bi"
+
+#define tostr(a)   #a
+
+SUITE( fbc_tests.pp.token_pasting )
+
+	TEST( plain )
+
+		#define join1(a,b) a##b
+		#define join2(a,b) a ##b
+		#define join3(a,b) a## b
+		#define join4(a,b) a ## b
+		#define join5(a,b) a  ##  b
+
+		CU_ASSERT_EQUAL( tostr(join1(X,Y)), tostr(XY) )
+		CU_ASSERT_EQUAL( tostr(join2(X,Y)), tostr(X Y) )
+		CU_ASSERT_EQUAL( tostr(join3(X,Y)), tostr(X Y) )
+		CU_ASSERT_EQUAL( tostr(join4(X,Y)), tostr(X  Y) )
+		CU_ASSERT_EQUAL( tostr(join5(X,Y)), tostr(X  Y) )
+
+	END_TEST
+
+	TEST( under1 )
+
+		#define join_01(a) a##_
+		#define join_02(a) _##a
+		#define join_03(a) a##__
+		#define join_04(a) __##a
+
+		CU_ASSERT_EQUAL( tostr(join_01(X)), tostr(X_) )
+		CU_ASSERT_EQUAL( tostr(join_02(X)), tostr(_X) )
+		CU_ASSERT_EQUAL( tostr(join_03(X)), tostr(X__) )
+		CU_ASSERT_EQUAL( tostr(join_04(X)), tostr(__X) )
+
+	END_TEST
+
+	TEST( under_1 )
+
+		#define join_01(_a) _a##_
+		#define join_02(_a) _##_a
+		#define join_03(_a) _a##__
+		#define join_04(_a) __##_a
+
+		CU_ASSERT_EQUAL( tostr(join_01(X)), tostr(X_) )
+		CU_ASSERT_EQUAL( tostr(join_02(X)), tostr(_X) )
+		CU_ASSERT_EQUAL( tostr(join_03(X)), tostr(X__) )
+		CU_ASSERT_EQUAL( tostr(join_04(X)), tostr(__X) )
+
+	END_TEST
+
+	TEST( under1_ )
+
+		#define join_01(a_) a_##_
+		#define join_02(a_) _##a_
+		#define join_03(a_) a_##__
+		#define join_04(a_) __##a_
+
+		CU_ASSERT_EQUAL( tostr(join_01(X)), tostr(X_) )
+		CU_ASSERT_EQUAL( tostr(join_02(X)), tostr(_X) )
+		CU_ASSERT_EQUAL( tostr(join_03(X)), tostr(X__) )
+		CU_ASSERT_EQUAL( tostr(join_04(X)), tostr(__X) )
+
+	END_TEST
+
+	TEST( under_1_ )
+
+		#define join_01(_a_) _a_##_
+		#define join_02(_a_) _##_a_
+		#define join_03(_a_) _a_##__
+		#define join_04(_a_) __##_a_
+
+		CU_ASSERT_EQUAL( tostr(join_01(X)), tostr(X_) )
+		CU_ASSERT_EQUAL( tostr(join_02(X)), tostr(_X) )
+		CU_ASSERT_EQUAL( tostr(join_03(X)), tostr(X__) )
+		CU_ASSERT_EQUAL( tostr(join_04(X)), tostr(__X) )
+
+	END_TEST
+
+	TEST( under2 )
+
+		#define join_01(a,b) a##_##b
+		#define join_02(a,b) a ##_##b
+		#define join_03(a,b) a## _##b
+		#define join_04(a,b) a ## _##b
+		#define join_05(a,b) a##_ ##b
+		#define join_06(a,b) a##_##b
+		#define join_07(a,b) a ##_##b
+		#define join_08(a,b) a## _##b
+		#define join_09(a,b) a ## _##b
+		#define join_10(a,b) a##_ ##b
+
+		CU_ASSERT_EQUAL( tostr(join_01(X,Y)), tostr(X_Y) )
+		CU_ASSERT_EQUAL( tostr(join_02(X,Y)), tostr(X _Y) )
+		CU_ASSERT_EQUAL( tostr(join_03(X,Y)), tostr(X _Y) )
+		CU_ASSERT_EQUAL( tostr(join_04(X,Y)), tostr(X  _Y) )
+		CU_ASSERT_EQUAL( tostr(join_05(X,Y)), tostr(X_ Y) )
+		CU_ASSERT_EQUAL( tostr(join_06(X,Y)), tostr(X_Y) )
+		CU_ASSERT_EQUAL( tostr(join_07(X,Y)), tostr(X _Y) )
+		CU_ASSERT_EQUAL( tostr(join_08(X,Y)), tostr(X _Y) )
+		CU_ASSERT_EQUAL( tostr(join_09(X,Y)), tostr(X  _Y) )
+		CU_ASSERT_EQUAL( tostr(join_10(X,Y)), tostr(X_ Y) )
+
+		'' Parser behaviour:
+		'' - everything after a line continuation token '_' is ignored
+		'' - the following expressions are poorly formed because the line
+		''   continuation either continues to the following define
+		''   or the following statment
+
+		/'
+		#define join_11(a,b) a ##_ ##b
+		#define join_12(a,b) a## _ ##b
+		#define join_13(a,b) a ## _ ##b
+		#define join_14(a,b) a ##_ ##b
+		#define join_15(a,b) a## _ ##b
+		#define join_16(a,b) a ## _ ##b
+
+		CU_ASSERT_EQUAL( tostr(join_11(X,Y)), tostr(X _ Y) )
+		CU_ASSERT_EQUAL( tostr(join_12(X,Y)), tostr(X _ Y) )
+		CU_ASSERT_EQUAL( tostr(join_13(X,Y)), tostr(X  _ Y) )
+		CU_ASSERT_EQUAL( tostr(join_14(X,Y)), tostr(X _ Y) )
+		CU_ASSERT_EQUAL( tostr(join_15(X,Y)), tostr(X _ Y) )
+		CU_ASSERT_EQUAL( tostr(join_16(X,Y)), tostr(X  _ Y) )
+		'/
+
+	END_TEST
+
+	''
+	'' test case from commit: 91e58fe7da978e464f976dc3f94378fbf3d1fbb1
+	''
+
+	Type _MAP_ENTRY
+		id As integer
+		pA As integer
+	End Type
+	#macro BEGIN_ENTRIESMAP()
+	Function _GetMapEntries() As _MAP_ENTRY Ptr
+			Static As _MAP_ENTRY _entries(0 To ...) = { ##_ 
+	#endmacro
+	#macro END_ENTRIESMAP()
+			(0, 0)}
+			Return @_entries(0)
+		End Function
+	#endmacro
+	#define _INTERFACE_ENTRY(x, y) (x, y), ##_
+
+	#macro GENMAP()
+
+	BEGIN_ENTRIESMAP()
+	_INTERFACE_ENTRY(1, 2)
+	_INTERFACE_ENTRY(3, 4)
+	_INTERFACE_ENTRY(5, 6)
+	END_ENTRIESMAP()
+
+	#endmacro
+
+	GENMAP()
+
+	TEST( line_pasting )
+
+		var p = _GetMapEntries()
+		CU_ASSERT_EQUAL( p[0].id, 1 )
+		CU_ASSERT_EQUAL( p[0].pA, 2 )
+		CU_ASSERT_EQUAL( p[1].id, 3 )
+		CU_ASSERT_EQUAL( p[1].pA, 4 )
+		CU_ASSERT_EQUAL( p[2].id, 5 )
+		CU_ASSERT_EQUAL( p[2].pA, 6 )
+
+	END_TEST	
+
+END_SUITE