Merge branch 'refactor/tokenizer' into autobuild/tokenizer

Author: c-lipka

source/base/version.h

@@ -100,7 +100,7 @@
 /// where `N` is a serial number starting at 1 in each phase, `TIME` is the number of minutes
 /// since 2000-01-01 00:00, and `FEATURE` is an arbitrary alphanumeric moniker for a particular
 /// experimental feature.
-#define POV_RAY_PRERELEASE          "x.tokenizer.9841908"
+#define POV_RAY_PRERELEASE          "x.tokenizer.9844488"
 
 #if defined(DOXYGEN) && !defined(POV_RAY_PRERELEASE)
     // Work around doxygen being unable to document undefined macros.

source/core/material/pattern.cpp

@@ -9287,8 +9287,6 @@ void Read_Density_File(IStream *file, DENSITY_FILE *df)
         }
         else
             throw POV_EXCEPTION_STRING("Invalid density file size");
-
-        delete file;
     }
 }
 

source/core/math/randomsequence.cpp

@@ -226,7 +226,7 @@ Vector2d Uniform2dOnSquare(SequentialDoubleGeneratorPtr source)
 {
     double x = (*source)();
     double y = (*source)();
-    return Vector2d((*source)(), (*source)());
+    return Vector2d(x, y);
 }
 
 Vector2d Uniform2dOnDisc(SequentialDoubleGeneratorPtr source)

source/core/support/simplevector.h

@@ -627,6 +627,23 @@ class VectorPool
     {
         if (initialPoolSize != 0)
             mPool.reserve(initialPoolSize);
+#ifdef POV_CORE_DEBUG
+        mAllocCount = 0;
+#endif
+    }
+
+    ~VectorPool()
+    {
+        for (auto&& p : mPool)
+        {
+            delete p;
+#ifdef POV_CORE_DEBUG
+            --mAllocCount;
+#endif
+        }
+#ifdef POV_CORE_DEBUG
+        POV_CORE_ASSERT(mAllocCount == 0);
+#endif
     }
 
     VectorPool(const VectorPool&) = delete;
@@ -638,6 +655,9 @@ class VectorPool
         if (mPool.empty())
         {
             p = new VECTOR_T();
+#ifdef POV_CORE_DEBUG
+            ++mAllocCount;
+#endif
         }
         else
         {
@@ -660,6 +680,9 @@ class VectorPool
 
     vector<VECTOR_T*> mPool;
     size_t mSizeHint;
+#ifdef POV_CORE_DEBUG
+    size_t mAllocCount;
+#endif
 };
 
 //******************************************************************************

source/parser/scanner.cpp

@@ -338,11 +338,14 @@ void Scanner::SetInputStream(SourcePtr stream)
 
 bool Scanner::SetInputStream(SourcePtr stream, const Bookmark& bookmark)
 {
+    bool seekOk;
+
     if ((mpSource == stream) && (bookmark.offset >= mBase) &&
         ((bookmark.offset - mBase) < (mpBufferEnd - maBuffer)))
     {
         // Bookmark is already loaded in the buffer.
         // Just advance/rewind the next character accordingly.
+        seekOk = true;
         mpNextChar = maBuffer + (bookmark.offset - mBase);
     }
     else
@@ -353,11 +356,12 @@ bool Scanner::SetInputStream(SourcePtr stream, const Bookmark& bookmark)
         mpSource = stream;
 
         // Refill the buffer.
-        mEndOfStream = !mpSource->seekg(bookmark.offset);
+        seekOk = mpSource->seekg(bookmark.offset);
         mBase = mpSource->tellg();
         mpBufferEnd = maBuffer;
         mpNextChar = maBuffer;
-        if (!mEndOfStream)
+        mEndOfStream = !seekOk;
+        if (seekOk)
             RefillBuffer();
     }
 
@@ -370,7 +374,7 @@ bool Scanner::SetInputStream(SourcePtr stream, const Bookmark& bookmark)
     mNominalEndOfLine = bookmark.nominalEndOfLine;
     mAllowNestedBlockComments = bookmark.allowNestedBlockComments;
 
-    return !mEndOfStream;
+    return seekOk;
 }
 
 void pov_parser::Scanner::SetStringEncoding(StringEncoding encoding)