Fix inability to handle gaps in the token assignments

Author: sharwell

tool/src/main/java/org/antlr/codegen/CodeGenerator.java

@@ -861,10 +861,11 @@ protected void genTokenTypeConstants(ST code) {
 	protected void genTokenTypeNames(ST code) {
 		for (int t=Label.MIN_TOKEN_TYPE; t<=grammar.getMaxTokenType(); t++) {
 			String tokenName = grammar.getTokenDisplayName(t);
-			if ( tokenName!=null ) {
+			if (tokenName != null) {
 				tokenName=target.getTargetStringLiteralFromString(tokenName, true);
-				code.add("tokenNames", tokenName);
 			}
+
+			code.add("tokenNames", tokenName);
 		}
 	}
 

tool/src/main/resources/org/antlr/codegen/templates/ActionScript/ActionScript.stg

@@ -173,7 +173,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 public class <grammar.recognizerName> extends <if(actions.(actionScope).superClass)><actions.(actionScope).superClass><else><@superClassName><superClass><@end><endif> {
 <if(grammar.grammarIsRoot)>
     public static const tokenNames:Array = [
-        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+        "\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
     ];<\n>
 <endif>
     <tokens:{it |public static const <it.name>:int=<it.type>;}; separator="\n">

tool/src/main/resources/org/antlr/codegen/templates/C/C.stg

@@ -1195,7 +1195,7 @@ pANTLR3_UINT8   <name>TokenNames[<length(tokenNames)>+4]
         (pANTLR3_UINT8) "\<EOR>",
         (pANTLR3_UINT8) "\<DOWN>",
         (pANTLR3_UINT8) "\<UP>",
-        <tokenNames:{it |(pANTLR3_UINT8) <it>}; separator=",\n">
+        <tokenNames:{it |(pANTLR3_UINT8) <it>}; separator=",\n", null="(pANTLR3_UINT8) \"<invalid>\"">
        };
 <endif>
 

tool/src/main/resources/org/antlr/codegen/templates/CSharp2/CSharp2.stg

@@ -287,7 +287,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 {
 <if(grammar.grammarIsRoot)>
 	internal static readonly string[] tokenNames = new string[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
 	};
 <endif>
 	<tokens:{it|public const int <it.name; format="id">=<it.type>;}; separator="\n">

tool/src/main/resources/org/antlr/codegen/templates/CSharp3/CSharp3.stg

@@ -282,7 +282,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 {
 <if(grammar.grammarIsRoot)>
 	internal static readonly string[] tokenNames = new string[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">
 	};
 <endif>
 	<tokens:{it|public const int <it.name; format="id">=<it.type>;}; separator="\n">

tool/src/main/resources/org/antlr/codegen/templates/Cpp/Cpp.stg

@@ -821,7 +821,7 @@ ANTLR_UINT8* <name>TokenNames[<length(tokenNames)>+4]
         (ANTLR_UINT8*) "\<EOR>",
         (ANTLR_UINT8*) "\<DOWN>",
         (ANTLR_UINT8*) "\<UP>",
-        <tokenNames:{it |(ANTLR_UINT8*) <it>}; separator=",\n">
+        <tokenNames:{it |(ANTLR_UINT8*) <it>}; separator=",\n", null="(ANTLR_UINT8*) \"<invalid>\"">
        };
 <endif>
 

tool/src/main/resources/org/antlr/codegen/templates/Delphi/Delphi.stg

@@ -321,7 +321,7 @@ type
         '\<EOR>',
         '\<DOWN>',
         '\<UP>',
-        <tokenNames; separator=",\n">);<\n>
+        <tokenNames; separator=",\n", null="'<invalid>'">);<\n>
 <endif>
   public
     const

tool/src/main/resources/org/antlr/codegen/templates/Java/Java.stg

@@ -208,7 +208,7 @@ genericParser(grammar, name, scopes, tokens, tokenNames, rules, numRules,
 public class <grammar.recognizerName> extends <@superClassName><superClass><@end> {
 <if(grammar.grammarIsRoot)>
 	public static final String[] tokenNames = new String[] {
-		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", wrap="\n\t\t">
+		"\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; null="\"<invalid>\"", separator=", ", wrap="\n\t\t">
 	};
 <endif>
 	<tokens:{it |public static final int <it.name>=<it.type>;}; separator="\n">

tool/src/main/resources/org/antlr/codegen/templates/JavaScript/JavaScript.stg

@@ -217,7 +217,7 @@ org.antlr.lang.augmentObject(<grammar.recognizerName>.prototype, {
 // public class variables
 org.antlr.lang.augmentObject(<grammar.recognizerName>, {
 <if(grammar.grammarIsRoot)>
-    tokenNames: ["\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ">],<\n>
+    tokenNames: ["\<invalid>", "\<EOR>", "\<DOWN>", "\<UP>", <tokenNames; separator=", ", null="\"<invalid>\"">],<\n>
 <endif>
     <bitsets:bitset(name={FOLLOW_<it.name>_in_<it.inName><it.tokenIndex>},
                     words64=it.bits); separator=",\n">

tool/src/main/resources/org/antlr/codegen/templates/ObjC/ObjC.stg

@@ -712,7 +712,7 @@ genericParser(  grammar,
 {
     #pragma mark Bitsets
     <bitsets:{it | <bitsetInit(name={FOLLOW_<it.name>_in_<it.inName><it.tokenIndex>}, words64=it.bits)>}>
-    [BaseRecognizer setTokenNames:[[AMutableArray arrayWithObjects:@"\<invalid>", @"\<EOR>", @"\<DOWN>", @"\<UP>", <tokenNames:{it | @<it>}; separator=", ", wrap="\n ">, nil] retain]];
+    [BaseRecognizer setTokenNames:[[AMutableArray arrayWithObjects:@"\<invalid>", @"\<EOR>", @"\<DOWN>", @"\<UP>", <tokenNames:{it | @<it>}; separator=", ", null="@\"<invalid>\"", wrap="\n ">, nil] retain]];
     [BaseRecognizer setGrammarFileName:@"<fileName>"];
     <synpreds:{pred | <synpred(pred)>}>
 }