Merge branch 'refactor/tokenizer' into autobuild/tokenizer

Author: c-lipka

source/base/version.h

@@ -100,7 +100,7 @@
 /// where `N` is a serial number starting at 1 in each phase, `TIME` is the number of minutes
 /// since 2000-01-01 00:00, and `FEATURE` is an arbitrary alphanumeric moniker for a particular
 /// experimental feature.
-#define POV_RAY_PRERELEASE          "x.tokenizer.9675110"
+#define POV_RAY_PRERELEASE          "x.tokenizer.9684878"
 
 #if defined(DOXYGEN) && !defined(POV_RAY_PRERELEASE)
     // Work around doxygen being unable to document undefined macros.

source/core/coretypes.h

@@ -610,18 +610,27 @@ inline void intrusive_ptr_release(GenericFunctionContextFactory* f) { if (!(--f-
 typedef intrusive_ptr<GenericFunctionContextFactory>    GenericFunctionContextFactoryIPtr;
 typedef GenericFunctionContextFactory*                  GenericFunctionContextFactoryTPtr;
 
-struct SourceInfo : MessageContext
+struct SourcePosition
 {
-    UCS2String  fileName;
     POV_LONG    line;
     POV_LONG    column;
     POV_OFF_T   offset;
+    SourcePosition() = default;
+    SourcePosition(const SourcePosition&) = default;
+    SourcePosition(POV_LONG l, POV_LONG c, POV_OFF_T o) : line(l), column(c), offset(o) {}
+};
+
+struct SourceInfo : MessageContext
+{
+    UCS2String      fileName;
+    SourcePosition  position;
     SourceInfo() = default;
-    SourceInfo(const MessageContext& o) : fileName(o.GetFileName()), line(o.GetLine()), column(o.GetColumn()), offset(o.GetOffset()) {}
+    SourceInfo(const MessageContext& o) : fileName(o.GetFileName()), position(o.GetLine(), o.GetColumn(), o.GetOffset()) {}
+    SourceInfo(const UCS2String& fn, SourcePosition& p) : fileName(fn), position(p) {}
     virtual UCS2String GetFileName() const override { return fileName; }
-    virtual POV_LONG GetLine() const override { return line; }
-    virtual POV_LONG GetColumn() const override { return column; }
-    virtual POV_OFF_T GetOffset() const override { return offset; }
+    virtual POV_LONG GetLine() const override { return position.line; }
+    virtual POV_LONG GetColumn() const override { return position.column; }
+    virtual POV_OFF_T GetOffset() const override { return position.offset; }
 };
 
 struct CustomFunctionSourceInfo : SourceInfo

source/parser/fncode.cpp

@@ -124,9 +124,7 @@ FNCode::FNCode(Parser *pa, FunctionCode *f, bool is_local, const char *n)
     else
         function->sourceInfo.name = POV_STRDUP("");
     function->sourceInfo.fileName = parser->UCS2_strdup(parser->mToken.sourceFile->Name());
-    function->sourceInfo.line = parser->mToken.raw.lexeme.position.line;
-    function->sourceInfo.column = parser->mToken.raw.lexeme.position.column;
-    function->sourceInfo.offset = parser->mToken.raw.lexeme.position.offset;
+    function->sourceInfo.position = parser->mToken.raw.lexeme.position;
     function->flags = 0;
     function->private_copy_method = nullptr;
     function->private_destroy_method = nullptr;

source/parser/parser.cpp

@@ -9014,7 +9014,7 @@ bool Parser::Parse_RValue (TokenId Previous, TokenId *NumberPtr, void **DataPtr,
     void *Temp_Data;
     POV_PARAM *New_Par;
     bool Found=true;
-    int Temp_Count=3000000;
+    int Temp_Count=3000000; // TODO FIXME - magic value!
     bool Old_Ok=Ok_To_Declare;
     int Terms;
     bool function_identifier;
@@ -9142,7 +9142,7 @@ bool Parser::Parse_RValue (TokenId Previous, TokenId *NumberPtr, void **DataPtr,
             // get the number of tokens found
             Temp_Count -= token_count;
 
-            // no tokens have been found or a fucntion call had no parameters in parenthesis
+            // no tokens have been found or a function call had no parameters in parenthesis
             if (!((Temp_Count==-1) || (Temp_Count==TOKEN_OVERFLOW_RESET_COUNT)) && had_callable_identifier)
                 Error("Identifier expected, incomplete function call or spline call found instead.");
 
@@ -9457,7 +9457,6 @@ void Parser::Destroy_Ident_Data(void *Data, int Type)
 {
     int i;
     POV_ARRAY *a;
-    DATA_FILE *Temp_File;
 
     if(Data == nullptr)
         return;
@@ -9563,10 +9562,7 @@ void Parser::Destroy_Ident_Data(void *Data, int Type)
             POV_FREE(Data);
             break;
         case FILE_ID_TOKEN:
-            Temp_File = reinterpret_cast<DATA_FILE *>(Data);
-            Temp_File->In_File = nullptr;
-            Temp_File->Out_File = nullptr;
-            POV_FREE(Data);
+            delete reinterpret_cast<DATA_FILE *>(Data);
             break;
         case FUNCT_ID_TOKEN:
         case VECTFUNCT_ID_TOKEN:

source/parser/parser.h

@@ -305,10 +305,31 @@ class Parser : public SceneTask
 
         struct DATA_FILE
         {
-            shared_ptr<pov_base::ITextStream> In_File;
+            shared_ptr<RawTokenizer>    inTokenizer;
+            RawToken                    inToken;
             shared_ptr<pov_base::OTextStream> Out_File;
+            bool inUngetToken   : 1;
             bool busyParsing    : 1; ///< `true` if parsing a statement related to the file, `false` otherwise.
-            bool R_Flag         : 1;
+
+            bool ReadNextToken()
+            {
+                POV_PARSER_ASSERT(inTokenizer != nullptr);
+                if (!inUngetToken && !inTokenizer->GetNextToken(inToken))
+                {
+                    inToken.id = END_OF_FILE_TOKEN;
+                    return false;
+                }
+                inUngetToken = false;
+                return true;
+            }
+            void UnReadToken()
+            {
+                POV_PARSER_ASSERT(!inUngetToken);
+                if (inToken.id != END_OF_FILE_TOKEN)
+                    inUngetToken = true;
+            }
+
+            DATA_FILE() : inUngetToken(false), busyParsing(false) {}
         };
 
         // constructor
@@ -409,7 +430,7 @@ class Parser : public SceneTask
         void Release_Entry_Reference (SYM_TABLE *table, SYM_ENTRY *Entry);
         SYM_ENTRY *Destroy_Entry (SYM_ENTRY *Entry, bool destroyName);
         bool Parse_Ifdef_Param ();
-        int Parse_For_Param (char**, DBL*, DBL*);
+        int Parse_For_Param (UTF8String&, DBL*, DBL*);
 
         // parstxtr.h/parstxtr.cpp
         TEXTURE *Parse_Texture (void);
@@ -471,7 +492,6 @@ class Parser : public SceneTask
         void Parse_Float_Param2 (DBL *Val1, DBL *Val2);
         void Init_Random_Generators (void);
         void Destroy_Random_Generators (void);
-        DBL Parse_Signed_Float(void);
 
         // function.h/function.cpp
         FUNCTION_PTR Parse_Function(void);
@@ -586,8 +606,6 @@ class Parser : public SceneTask
 
         int line_count;
 
-        bool readingExternalFile;
-
         vector<RawTokenizer::HotBookmark> maIncludeStack;
 
         struct CS_ENTRY
@@ -598,11 +616,11 @@ class Parser : public SceneTask
             bool Macro_Same_Flag;
             bool Switch_Case_Ok_Flag;
             Macro *PMac;
-            char* Loop_Identifier;
+            UTF8String Loop_Identifier;
             DBL For_Loop_End;
             DBL For_Loop_Step;
-            CS_ENTRY() : Cond_Type(BUSY_COND), PMac(nullptr), Loop_Identifier(nullptr) {}
-            ~CS_ENTRY() { POV_PARSER_ASSERT(Loop_Identifier == nullptr); }
+            CS_ENTRY() : Cond_Type(BUSY_COND), PMac(nullptr) {}
+            ~CS_ENTRY() {}
         };
 
         vector<CS_ENTRY> Cond_Stack;
@@ -738,6 +756,7 @@ class Parser : public SceneTask
         void Parse_Read(void);
         void Parse_Write(void);
         int Parse_Read_Value(DATA_FILE *User_File, TokenId Previous, TokenId *NumberPtr, void **DataPtr);
+        bool Parse_Read_Float_Value(DBL& val, DATA_FILE *User_File);
         void Check_Macro_Vers(void);
         DBL Parse_Cond_Param(void);
         void Parse_Cond_Param2(DBL *V1,DBL *V2);

source/parser/parser_expressions.cpp

@@ -3904,41 +3904,4 @@ void Parser::Destroy_Random_Generators()
     Number_Of_Random_Generators = 0;
 }
 
-DBL Parser::Parse_Signed_Float(void)
-{
-    DBL Sign=1.0;
-    DBL Val=0.0;
-    bool old_allow_id = Allow_Identifier_In_Call;
-    Allow_Identifier_In_Call = false;
-
-    EXPECT
-        CASE (PLUS_TOKEN)
-        END_CASE
-
-        CASE (DASH_TOKEN)
-            Sign=-1.0;
-            Get_Token();
-            // FALLTHROUGH
-        CASE (FLOAT_FUNCT_TOKEN)
-            if (mToken.Function_Id==FLOAT_TOKEN)
-            {
-                Val = Sign * mToken.Token_Float;
-                EXIT
-            }
-            else
-            {
-                Parse_Error(FLOAT_TOKEN);
-            }
-        END_CASE
-
-        OTHERWISE
-            Parse_Error(FLOAT_TOKEN);
-        END_CASE
-    END_EXPECT
-
-    Allow_Identifier_In_Call = old_allow_id;
-
-    return(Val);
-}
-
 }