[parser] Clean up behaviour regarding dictionaries and #ifdef.

c-lipka · c-lipka · commit c661589cc471 · 2018-09-17T17:45:18.000+02:00
diff --git a/changes.txt b/changes.txt
@@ -58,6 +58,9 @@ Changed Behaviour
     for now, due to their orientation being poorly defined.
   - An age-old bug in the inbuilt `f_enneper` isosurface function has been
     fixed; the function now results in the originally intended shape.
+  - The `defined()` pseudo-function now returns `true` (while printing a
+    warning) if applied to reserved words. The `#ifdef` and `#ifndef` directives
+    also behave accordingly.
 
 New Features
 ------------
diff --git a/source/parser/parser.h b/source/parser/parser.h
@@ -104,8 +104,7 @@ typedef unsigned short SymTableEntryRefCount;
 
 // Special symbol tables
 enum {
-    SYM_TABLE_RESERVED = 0,        // reserved words
-    SYM_TABLE_GLOBAL,              // identifiers declared using #declare (or #local in top-level file), #function, #macro, etc.
+    SYM_TABLE_GLOBAL = 0,       ///< Identifiers declared using #declare (or #local in top-level file), #function, #macro, etc.
 };
 
 /// Structure holding information about a symbol
diff --git a/source/parser/parser_tokenizer.cpp b/source/parser/parser_tokenizer.cpp
@@ -357,7 +357,14 @@ void Parser::Get_Token ()
                 break;
 
             default:
-                Write_Token(mToken.raw);
+                if (parseRawIdentifiers || (mToken.raw.isPseudoIdentifier && (!Parsing_Directive || Inside_Ifdef)))
+                    Read_Symbol(mToken.raw);
+                else
+                {
+                    if (mToken.raw.isReservedWord && Inside_Ifdef)
+                        Warning("Trying to test whether a reserved keyword is defined. Test result may not be what you expect.");
+                    Write_Token(mToken.raw);
+                }
                 break;
         }
     }
@@ -448,28 +455,26 @@ void Parser::Read_Symbol(const RawToken& rawToken)
     RawToken nextRawToken;
     bool haveNextRawToken;
 
-    /* If its a reserved keyword, write it and return */
-    Temp_Entry = Find_Symbol(SYM_TABLE_RESERVED, rawToken.lexeme.text.c_str());
-    if (Temp_Entry != nullptr)
+    if (rawToken.isReservedWord && !parseRawIdentifiers)
     {
-        POV_PARSER_ASSERT(false);
+        // Normally, this function shouldn't be called with reserved words.
+        // Exceptions are a few keywords that behave like identifiers in certain contexts,
+        // such as `global` and `local` which may behave like dictionaries.
+        POV_PARSER_ASSERT(rawToken.isPseudoIdentifier);
 
-        if (!Parsing_Directive && (Temp_Entry->Token_Number == LOCAL_TOKEN))
+        if (rawToken.id == LOCAL_TOKEN)
         {
+            POV_PARSER_ASSERT(!Parsing_Directive || Inside_Ifdef);
             pseudoDictionary = Table_Index;
         }
-        else if (!Parsing_Directive && (Temp_Entry->Token_Number == GLOBAL_TOKEN))
+        else if (rawToken.id == GLOBAL_TOKEN)
         {
+            POV_PARSER_ASSERT(!Parsing_Directive || Inside_Ifdef);
             pseudoDictionary = SYM_TABLE_GLOBAL;
         }
-        else if (Inside_Ifdef)
-        {
-            Warning("Tried to test whether a reserved keyword is defined. Test result may not be what you expect.");
-        }
         else
         {
-            Write_Token (Temp_Entry->Token_Number, rawToken);
-            return;
+            POV_PARSER_ASSERT(false);
         }
     }
 
@@ -490,7 +495,7 @@ void Parser::Read_Symbol(const RawToken& rawToken)
         {
             /* Search tables from newest to oldest */
             int firstIndex = Table_Index;
-            int lastIndex  = SYM_TABLE_RESERVED+1; // index SYM_TABLE_RESERVED is reserved for reserved words, not identifiers
+            int lastIndex  = SYM_TABLE_GLOBAL;
             for (Local_Index = firstIndex; Local_Index >= lastIndex; Local_Index--)
             {
                 /* See if it's a previously declared identifier. */
@@ -628,11 +633,19 @@ void Parser::Read_Symbol(const RawToken& rawToken)
                                 table = Tables [pseudoDictionary];
                                 pseudoDictionary = -1;
 
-                                if (!haveNextRawToken || (nextRawToken.lexeme.category != Lexeme::kOther) ||
+                                if (!haveNextRawToken ||
+                                    (nextRawToken.lexeme.category != Lexeme::kOther) ||
                                     ((nextRawToken.lexeme.text != "[") && (nextRawToken.lexeme.text != ".")))
                                 {
-                                    Get_Token(); // ensures the error is reported at the right token
-                                    Expectation_Error("'[' or '.'");
+                                    if (Inside_Ifdef)
+                                    {
+                                        Warning("Trying to test whether a reserved keyword is defined. Test result may not be what you expect.");
+                                    }
+                                    else
+                                    {
+                                        Get_Token(); // ensures the error is reported at the right token
+                                        Expectation_Error("'[' or '.'");
+                                    }
                                 }
                             }
                             else
@@ -2079,14 +2092,7 @@ void Parser::init_sym_tables()
 {
     int i;
 
-    Add_Sym_Table();
-
-    for (i = 0; Reserved_Words[i].Token_Name != nullptr; i++)
-    {
-        Add_Symbol(SYM_TABLE_RESERVED,Reserved_Words[i].Token_Name,Reserved_Words[i].Token_Number);
-    }
-
-    Add_Sym_Table();
+    Add_Sym_Table(); // global symbols
 }
 
 Parser::SYM_TABLE *Parser::Create_Sym_Table (bool copyNames)
@@ -2289,7 +2295,7 @@ SYM_ENTRY *Parser::Add_Symbol (int Index,const char *Name,TokenId Number)
 {
     SYM_ENTRY *New;
 
-    New = Create_Entry (Name, Number, (Index != SYM_TABLE_RESERVED));
+    New = Create_Entry(Name, Number, true);
     Add_Entry(Index,New);
 
     return(New);
diff --git a/source/parser/rawtokenizer.cpp b/source/parser/rawtokenizer.cpp
@@ -79,7 +79,10 @@ TokenId RawToken::GetTokenId() const
 //******************************************************************************
 
 RawTokenizer::KnownWordInfo::KnownWordInfo() :
-    id(int(NOT_A_TOKEN))
+    id(int(NOT_A_TOKEN)),
+    expressionId(NOT_A_TOKEN),
+    isReservedWord(false),
+    isPseudoIdentifier(false)
 {}
 
 //******************************************************************************
@@ -93,8 +96,11 @@ RawTokenizer::RawTokenizer() :
             continue;
         if (strchr(i->Token_Name, ' ') != nullptr)
             continue;
-        mKnownWords[i->Token_Name].id = i->Token_Number;
-        mKnownWords[i->Token_Name].expressionId = GetExpressionId(i->Token_Number);
+        KnownWordInfo& knownWord = mKnownWords[i->Token_Name];
+        knownWord.id = i->Token_Number;
+        knownWord.expressionId = GetExpressionId(i->Token_Number);
+        knownWord.isReservedWord = true;
+        knownWord.isPseudoIdentifier = ((knownWord.id == GLOBAL_TOKEN) || (knownWord.id == LOCAL_TOKEN));
     }
 }
 
@@ -142,6 +148,8 @@ bool RawTokenizer::GetNextDirective(RawToken& token)
     token.id = TokenId::HASH_TOKEN;
     token.expressionId = TokenId::HASH_TOKEN;
     token.value = nullptr;
+    token.isReservedWord = false;
+    token.isPseudoIdentifier = false;
 
     return true;
 }
@@ -160,6 +168,8 @@ bool RawTokenizer::ProcessWordLexeme(RawToken& token)
     token.id = i.id;
     token.expressionId = i.expressionId;
     token.value = nullptr;
+    token.isReservedWord = i.isReservedWord;
+    token.isPseudoIdentifier = i.isPseudoIdentifier;
 
     return true;
 }
@@ -174,6 +184,9 @@ bool RawTokenizer::ProcessFloatLiteralLexeme(RawToken& token)
     if (sscanf(token.lexeme.text.c_str(), POV_DBL_FORMAT_STRING, &token.floatValue) == 0)
         return false;
 
+    token.isReservedWord = false;
+    token.isPseudoIdentifier = false;
+
     return true;
 }
 
@@ -292,6 +305,8 @@ bool RawTokenizer::ProcessStringLiteralLexeme(RawToken& token)
     }
 
     token.value = pValue;
+    token.isReservedWord = false;
+    token.isPseudoIdentifier = false;
 
     return true;
 }
@@ -386,6 +401,8 @@ bool RawTokenizer::ProcessOtherLexeme(RawToken& token)
     token.id = int(tokenId);
     token.expressionId = GetExpressionId(tokenId);
     token.value = nullptr;
+    token.isReservedWord = false;
+    token.isPseudoIdentifier = false;
 
     return true;
 }
@@ -405,6 +422,8 @@ bool RawTokenizer::ProcessSignatureLexeme(RawToken& token)
     token.id = int(tokenId);
     token.expressionId = GetExpressionId(tokenId);
     token.value = nullptr;
+    token.isReservedWord = false;
+    token.isPseudoIdentifier = false;
 
     return true;
 }
diff --git a/source/parser/rawtokenizer.h b/source/parser/rawtokenizer.h
@@ -146,6 +146,14 @@ struct RawToken
     /// _cooked tokenizer_ to carry the value of the identifier.
     ConstValuePtr value;
 
+    /// Whether the token is a reserved word.
+    /// @note
+    ///     This flag is _not_ set for operators.
+    bool isReservedWord : 1;
+
+    /// Whether the token is a keyword with identifier-like properties.
+    bool isPseudoIdentifier : 1;
+
     /// Helper function to get the token ID.
     /// For identifiers, this function returns @ref IDENTIFIER_TOKEN. For all
     /// other tokens, this function returns @ref id as a @ref TokenId value.
@@ -230,6 +238,8 @@ class RawTokenizer
     {
         int     id;
         TokenId expressionId;
+        bool    isReservedWord     : 1;
+        bool    isPseudoIdentifier : 1;
         KnownWordInfo();
     };
 
