Merge pull request Tencent#1424 from ylavic/file_input_streams

Optimize FileReadStream and BasicIStreamWrapper.

Author: miloyip

include/rapidjson/istreamwrapper.h

@@ -48,57 +48,71 @@ template <typename StreamType>
 class BasicIStreamWrapper {
 public:
     typedef typename StreamType::char_type Ch;
-    BasicIStreamWrapper(StreamType& stream) : stream_(stream), count_(), peekBuffer_() {}
 
-    Ch Peek() const { 
-        typename StreamType::int_type c = stream_.peek();
-        return RAPIDJSON_LIKELY(c != StreamType::traits_type::eof()) ? static_cast<Ch>(c) : static_cast<Ch>('\0');
+    //! Constructor.
+    /*!
+        \param stream stream opened for read.
+    */
+    BasicIStreamWrapper(StreamType &stream) : stream_(stream), buffer_(peekBuffer_), bufferSize_(4), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 
+        Read();
     }
 
-    Ch Take() { 
-        typename StreamType::int_type c = stream_.get();
-        if (RAPIDJSON_LIKELY(c != StreamType::traits_type::eof())) {
-            count_++;
-            return static_cast<Ch>(c);
-        }
-        else
-            return '\0';
+    //! Constructor.
+    /*!
+        \param stream stream opened for read.
+        \param buffer user-supplied buffer.
+        \param bufferSize size of buffer in bytes. Must >=4 bytes.
+    */
+    BasicIStreamWrapper(StreamType &stream, char* buffer, size_t bufferSize) : stream_(stream), buffer_(buffer), bufferSize_(bufferSize), bufferLast_(0), current_(buffer_), readCount_(0), count_(0), eof_(false) { 
+        RAPIDJSON_ASSERT(bufferSize >= 4);
+        Read();
     }
 
-    // tellg() may return -1 when failed. So we count by ourself.
-    size_t Tell() const { return count_; }
+    Ch Peek() const { return *current_; }
+    Ch Take() { Ch c = *current_; Read(); return c; }
+    size_t Tell() const { return count_ + static_cast<size_t>(current_ - buffer_); }
 
-    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
+    // Not implemented
     void Put(Ch) { RAPIDJSON_ASSERT(false); }
-    void Flush() { RAPIDJSON_ASSERT(false); }
+    void Flush() { RAPIDJSON_ASSERT(false); } 
+    Ch* PutBegin() { RAPIDJSON_ASSERT(false); return 0; }
     size_t PutEnd(Ch*) { RAPIDJSON_ASSERT(false); return 0; }
 
     // For encoding detection only.
     const Ch* Peek4() const {
-        RAPIDJSON_ASSERT(sizeof(Ch) == 1); // Only usable for byte stream.
-        int i;
-        bool hasError = false;
-        for (i = 0; i < 4; ++i) {
-            typename StreamType::int_type c = stream_.get();
-            if (c == StreamType::traits_type::eof()) {
-                hasError = true;
-                stream_.clear();
-                break;
-            }
-            peekBuffer_[i] = static_cast<Ch>(c);
-        }
-        for (--i; i >= 0; --i)
-            stream_.putback(peekBuffer_[i]);
-        return !hasError ? peekBuffer_ : 0;
+        return (current_ + 4 - !eof_ <= bufferLast_) ? current_ : 0;
     }
 
 private:
+    BasicIStreamWrapper();
     BasicIStreamWrapper(const BasicIStreamWrapper&);
     BasicIStreamWrapper& operator=(const BasicIStreamWrapper&);
 
-    StreamType& stream_;
-    size_t count_;  //!< Number of characters read. Note:
-    mutable Ch peekBuffer_[4];
+    void Read() {
+        if (current_ < bufferLast_)
+            ++current_;
+        else if (!eof_) {
+            count_ += readCount_;
+            readCount_ = bufferSize_;
+            bufferLast_ = buffer_ + readCount_ - 1;
+            current_ = buffer_;
+
+            if (!stream_.read(buffer_, static_cast<std::streamsize>(bufferSize_))) {
+                readCount_ = static_cast<size_t>(stream_.gcount());
+                *(bufferLast_ = buffer_ + readCount_) = '\0';
+                eof_ = true;
+            }
+        }
+    }
+
+    StreamType &stream_;
+    Ch peekBuffer_[4], *buffer_;
+    size_t bufferSize_;
+    Ch *bufferLast_;
+    Ch *current_;
+    size_t readCount_;
+    size_t count_;  //!< Number of characters read
+    bool eof_;
 };
 
 typedef BasicIStreamWrapper<std::istream> IStreamWrapper;

test/perftest/rapidjsontest.cpp

@@ -21,9 +21,12 @@
 #include "rapidjson/prettywriter.h"
 #include "rapidjson/stringbuffer.h"
 #include "rapidjson/filereadstream.h"
+#include "rapidjson/istreamwrapper.h"
 #include "rapidjson/encodedstream.h"
 #include "rapidjson/memorystream.h"
 
+#include <fstream>
+
 #ifdef RAPIDJSON_SSE2
 #define SIMD_SUFFIX(name) name##_SSE2
 #elif defined(RAPIDJSON_SSE42)
@@ -463,6 +466,77 @@ TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_FileReadStream)) {
     }
 }
 
+TEST_F(RapidJson, IStreamWrapper) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is(filename_, std::ios::in | std::ios::binary);
+        char buffer[65536];
+        IStreamWrapper isw(is, buffer, sizeof(buffer));
+        while (isw.Take() != '\0')
+            ;
+        is.close();
+    }
+}
+
+TEST_F(RapidJson, IStreamWrapper_Unbuffered) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is(filename_, std::ios::in | std::ios::binary);
+        IStreamWrapper isw(is);
+        while (isw.Take() != '\0')
+            ;
+        is.close();
+    }
+}
+
+TEST_F(RapidJson, IStreamWrapper_Setbuffered) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is;
+        char buffer[65536];
+        is.rdbuf()->pubsetbuf(buffer, sizeof(buffer));
+        is.open(filename_, std::ios::in | std::ios::binary);
+        IStreamWrapper isw(is);
+        while (isw.Take() != '\0')
+            ;
+        is.close();
+    }
+}
+
+TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper)) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is(filename_, std::ios::in | std::ios::binary);
+        char buffer[65536];
+        IStreamWrapper isw(is, buffer, sizeof(buffer));
+        BaseReaderHandler<> h;
+        Reader reader;
+        reader.Parse(isw, h);
+        is.close();
+    }
+}
+
+TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper_Unbuffered)) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is(filename_, std::ios::in | std::ios::binary);
+        IStreamWrapper isw(is);
+        BaseReaderHandler<> h;
+        Reader reader;
+        reader.Parse(isw, h);
+        is.close();
+    }
+}
+
+TEST_F(RapidJson, SIMD_SUFFIX(ReaderParse_DummyHandler_IStreamWrapper_Setbuffered)) {
+    for (size_t i = 0; i < kTrialCount; i++) {
+        std::ifstream is;
+        char buffer[65536];
+        is.rdbuf()->pubsetbuf(buffer, sizeof(buffer));
+        is.open(filename_, std::ios::in | std::ios::binary);
+        IStreamWrapper isw(is);
+        BaseReaderHandler<> h;
+        Reader reader;
+        reader.Parse(isw, h);
+        is.close();
+    }
+}
+
 TEST_F(RapidJson, StringBuffer) {
     StringBuffer sb;
     for (int i = 0; i < 32 * 1024 * 1024; i++)