- Switched to mapped files for serving static files.

Tomenz · Tomenz · commit 4eec10ba5c56 · 2025-11-02T19:52:55.000+01:00
- Added a test script for checking.
diff --git a/HttpServ.cpp b/HttpServ.cpp
@@ -31,6 +31,7 @@
 #include "CommonLib/UrlCode.h"
 #include "SpawnProcess.h"
 #include "FastCgi/FastCgi.h"
+#include "MappedFile.h"
 
 using namespace std;
 using namespace std::placeholders;
@@ -2489,33 +2490,57 @@ void CHttpServ::DoAction(const MetaSocketData soMetaDa, const uint8_t httpVers,
     }
 
     // Load file
-    fstream fin(FN_CA(strItemPath), ios_base::in | ios_base::binary);
-    if (fin.is_open() == true)
+    MappedFile map;
+    if (map.open(strItemPath) == true)
     {
-        uint64_t nFSize = stFileInfo.st_size;
+        //std::deque<std::tuple<uint8_t, uint8_t*, uint64_t>> queMem2Send;
+        //queMem2Send.emplace_back(0, map.data(), map.size()); // Dummy packet to indicate the start of the data stream
+
+        uint64_t nFSize = map.size();
         if (vecRanges.size() == 1)  // Momentan nur 1 Range
         {
-            nFSize = vecRanges[0].second - vecRanges[0].first;
-            fin.seekg(vecRanges[0].first, ios_base::beg);
-            umPhpHeaders.emplace_back(make_pair("Content-Range", "bytes " + to_string(vecRanges[0].first) + "-" + to_string(vecRanges[0].second) + "/" + to_string(stFileInfo.st_size)));
-            iStatus = 206;
+            //OutputDebugStringA(std::string("Range requested, nFSize=" + std::to_string(nFSize) + ", Range from: " + std::to_string(vecRanges[0].first) + " bis " + std::to_string(vecRanges[0].second) + "\r\n").c_str());
+            if (vecRanges[0].first < nFSize && vecRanges[0].second <= nFSize && vecRanges[0].second >= vecRanges[0].first)
+            {
+                nFSize = vecRanges[0].second - vecRanges[0].first;
+                map.setOffset(vecRanges[0].first);
+                umPhpHeaders.emplace_back(make_pair("Content-Range", "bytes " + to_string(vecRanges[0].first) + "-" + to_string(vecRanges[0].second) + "/" + to_string(stFileInfo.st_size)));
+                iStatus = 206;
+            }
+            else
+                OutputDebugStringA(std::string("Invalid Range requested, nFSize=" + std::to_string(nFSize) + ", Range from: " + std::to_string(vecRanges[0].first) + " bis " + std::to_string(vecRanges[0].second) + "\r\n").c_str());
         }
         else
             umPhpHeaders.emplace_back(make_pair("Accept-Ranges", "bytes"));
 
         const auto acceptencoding = lstHeaderFields.find("accept-encoding");
         if (acceptencoding != end(lstHeaderFields))
         {
+            //OutputDebugStringA(string("accept-encoding: " + lstHeaderFields.find("accept-encoding")->second + "\r\n").c_str());
             // http://www.filesignatures.net/index.php?page=all
             if (find_if(begin(m_vHostParam[szHost].m_vDeflateTyps), end(m_vHostParam[szHost].m_vDeflateTyps), [&](const string& strType) noexcept { return strType == strMineType ? true : false; }) != end(m_vHostParam[szHost].m_vDeflateTyps))
             {
                 if (acceptencoding->second.find("br") != string::npos) iHeaderFlag |= BROTLICODING;
                 else if (acceptencoding->second.find("gzip") != string::npos) iHeaderFlag |= GZIPENCODING;
                 else if (acceptencoding->second.find("deflate") != string::npos) iHeaderFlag |= DEFLATEENCODING;
+
+                const auto xPrefCompAlgo = lstHeaderFields.find("x-prefcompalgo");
+                if (xPrefCompAlgo != end(lstHeaderFields))
+                {
+                    //OutputDebugStringA(string("x-prefcompalgo: " + xPrefCompAlgo->second + "\r\n").c_str());
+                    iHeaderFlag &= ~BROTLICODING;
+                    iHeaderFlag &= ~DEFLATEENCODING;
+                    iHeaderFlag &= ~GZIPENCODING;
+                    if (xPrefCompAlgo->second == "gzip")
+                        iHeaderFlag |= GZIPENCODING;
+                    else if (xPrefCompAlgo->second == "deflate")
+                        iHeaderFlag |= DEFLATEENCODING;
+                    else if (xPrefCompAlgo->second == "br")
+                        iHeaderFlag |= BROTLICODING;
+                }
             }
         }
 
-        //iHeaderFlag &= ~(GZIPENCODING | DEFLATEENCODING);
         if (iHeaderFlag & GZIPENCODING || iHeaderFlag & DEFLATEENCODING)
         {
             if (httpVers < 2)
@@ -2534,68 +2559,55 @@ void CHttpServ::DoAction(const MetaSocketData soMetaDa, const uint8_t httpVers,
             GZipPack gzipEncoder;
             if (gzipEncoder.Init((iHeaderFlag & DEFLATEENCODING) ? true : false) == Z_OK)
             {
-                unique_ptr<unsigned char[]> srcBuf(new unsigned char[nSizeSendBuf]);
-                unique_ptr<unsigned char[]> dstBuf(new unsigned char[nSizeSendBuf]);
-
-                uint64_t nBytesTransferred = 0;
                 int iResult = 0;
-                do
-                {
-                    const streamsize nBytesRead = fin.read(reinterpret_cast<char*>(srcBuf.get()), nSizeSendBuf).gcount();
-                    if (nBytesRead == 0)
-                        break;
-                    nBytesTransferred += nBytesRead;
+                gzipEncoder.InitBuffer(map.data(), static_cast<uint32_t>(nFSize));
 
-                    gzipEncoder.InitBuffer(srcBuf.get(), static_cast<uint32_t>(nBytesRead));
-                    const int nFlush = nBytesTransferred == nFSize ? Z_FINISH : Z_NO_FLUSH;
+                unique_ptr<unsigned char[]> dstBuf(new unsigned char[nSizeSendBuf]);
+                size_t nBytesConverted;
+                do
+                {   // Get next compressed chunk
+                    nBytesConverted = nSizeSendBuf - nHttp2Offset;
+                    iResult = gzipEncoder.Inflate(dstBuf.get() + nHttp2Offset, &nBytesConverted, Z_FINISH);
 
-                    size_t nBytesConverted;
-                    do
+                    // Send compressed chunk
+                    size_t nOffset = 0;
+                    while ((iResult == Z_OK || iResult == Z_STREAM_END) && ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset) != 0 && patStop.load() == false && fnIsStreamReset(nStreamId) == false)
                     {
-                        nBytesConverted = nSizeSendBuf - nHttp2Offset;
-                        iResult = gzipEncoder.Inflate(dstBuf.get() + nHttp2Offset, &nBytesConverted, nFlush);
-
-                        size_t nOffset = 0;
-                        while ((iResult == Z_OK || iResult == Z_STREAM_END) && ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset) != 0 && patStop.load() == false && fnIsStreamReset(nStreamId) == false)
-                        {
-                            int64_t nStreamWndSize = INT32_MAX;
-                            if (fnGetStreamWindowSize(nStreamWndSize) == false)
-                                break;  // Stream Item was removed, probably the stream was reset
-
-                            size_t nSendBufLen;
-                            if (fnSendQueueReady(nStreamWndSize, nSendBufLen, static_cast<uint64_t>(nSizeSendBuf - nHttp2Offset), ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset)) == false)
-                                continue;
-
-                            if (httpVers == 2)
-                            {
-                                bool bLastPaket = false;
-                                if (iResult == Z_STREAM_END && ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset) == nSendBufLen) // Letztes Paket
-                                    bLastPaket = true;
-                                BuildHttp2Frame(dstBuf.get() + nOffset, nSendBufLen, 0x0, bLastPaket == true ? 0x1 : 0x0, nStreamId);
-                            }
-                            else
-                            {
-                                stringstream ss;
-                                ss << hex << ::uppercase << nSendBufLen << "\r\n";
-                                soMetaDa.fSocketWrite(ss.str().c_str(), ss.str().size());
-                            }
-                            soMetaDa.fSocketWrite(dstBuf.get() + nOffset, nSendBufLen + nHttp2Offset);
-                            if (httpVers < 2)
-                                soMetaDa.fSocketWrite("\r\n", 2);
-                            soMetaDa.fResetTimer();
+                        int64_t nStreamWndSize = INT32_MAX;
+                        if (fnGetStreamWindowSize(nStreamWndSize) == false)
+                            break;  // Stream Item was removed, probably the stream was reset
 
-                            if (fnUpdateStreamParam(nSendBufLen) == -1)
-                                break;  // Stream Item was removed, probably the stream was reset
+                        size_t nSendBufLen;
+                        if (fnSendQueueReady(nStreamWndSize, nSendBufLen, static_cast<uint64_t>(nSizeSendBuf - nHttp2Offset), ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset)) == false)
+                            continue;
 
-                            //nBytesConverted += nSendBufLen;
-                            nOffset += nSendBufLen;
+                        if (httpVers == 2)
+                        {
+                            bool bLastPaket = false;
+                            if (iResult == Z_STREAM_END && ((nSizeSendBuf - nHttp2Offset) - nBytesConverted - nOffset) == nSendBufLen) // Letztes Paket
+                                bLastPaket = true;
+                            BuildHttp2Frame(dstBuf.get() + nOffset, nSendBufLen, 0x0, bLastPaket == true ? 0x1 : 0x0, nStreamId);
+                        }
+                        else
+                        {
+                            stringstream ss;
+                            ss << hex << ::uppercase << nSendBufLen << "\r\n";
+                            soMetaDa.fSocketWrite(ss.str().c_str(), ss.str().size());
                         }
+                        soMetaDa.fSocketWrite(dstBuf.get() + nOffset, nSendBufLen + nHttp2Offset);
+                        if (httpVers < 2)
+                            soMetaDa.fSocketWrite("\r\n", 2);
+                        soMetaDa.fResetTimer();
+
+                        if (fnUpdateStreamParam(nSendBufLen) == -1)
+                            break;  // Stream Item was removed, probably the stream was reset
 
-                        fnResetReservierteWindowSize();
+                        nOffset += nSendBufLen;
+                    }
 
-                    } while (iResult == Z_OK && nBytesConverted == 0 && patStop.load() == false && fnIsStreamReset(nStreamId) == false);
+                    fnResetReservierteWindowSize();
 
-                } while (iResult == Z_OK && patStop.load() == false && fnIsStreamReset(nStreamId) == false);
+                } while (iResult == Z_OK && nBytesConverted == 0 && patStop.load() == false && fnIsStreamReset(nStreamId) == false);
 
                 if (httpVers < 2 && patStop.load() == false)
                     soMetaDa.fSocketWrite("0\r\n\r\n", 5);
@@ -2619,35 +2631,21 @@ void CHttpServ::DoAction(const MetaSocketData soMetaDa, const uint8_t httpVers,
             BrotliEncoderState* s = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr);
             BrotliEncoderSetParameter(s, BROTLI_PARAM_QUALITY, (uint32_t)9);
             BrotliEncoderSetParameter(s, BROTLI_PARAM_LGWIN, (uint32_t)0);
-/*                if (dictionary_path != NULL) {
-                size_t dictionary_size = 0;
-                uint8_t* dictionary = ReadDictionary(dictionary_path, &dictionary_size);
-                BrotliEncoderSetCustomDictionary(s, dictionary_size, dictionary);
-                free(dictionary);
-            }
-*/
-            unique_ptr<unsigned char[]> srcBuf(new unsigned char[nSizeSendBuf]);
+
             unique_ptr<unsigned char[]> dstBuf(new unsigned char[nSizeSendBuf]);
 
-            size_t nBytIn = 0;
-            const uint8_t* input = nullptr;
+            size_t nBytIn = nFSize;
+            const uint8_t* input = map.data();
             size_t nBytOut = nSizeSendBuf - nHttp2Offset;
             uint8_t* output = dstBuf.get() + nHttp2Offset;
 
-            uint64_t nBytesTransferred = 0;
             while (patStop.load() == false && fnIsStreamReset(nStreamId) == false)
             {
-                if (nBytIn == 0)
-                {
-                    const streamsize nBytesRead = fin.read(reinterpret_cast<char*>(srcBuf.get()), nSizeSendBuf).gcount();
-                    nBytIn = static_cast<size_t>(nBytesRead);
-                    input = srcBuf.get();
-                    nBytesTransferred += nBytIn;
-                }
-
+                // Nächsten Chunk komprimieren
                 if (!BrotliEncoderCompressStream(s, nBytIn == 0 ? BROTLI_OPERATION_FINISH : BROTLI_OPERATION_PROCESS, &nBytIn, &input, &nBytOut, &output, nullptr))
                     break;
 
+                // Senden des komprimierten Chunks
                 size_t nOffset = 0;
                 while (nBytOut != nSizeSendBuf - nHttp2Offset && patStop.load() == false && fnIsStreamReset(nStreamId) == false)
                 {
@@ -2704,9 +2702,6 @@ void CHttpServ::DoAction(const MetaSocketData soMetaDa, const uint8_t httpVers,
                     soMetaDa.fSocketWrite(caBuffer, nHeaderLen + nHttp2Offset);
             }
             soMetaDa.fResetTimer();
-
-            auto apBuf = make_unique<uint8_t[]>(nSizeSendBuf + nHttp2Offset + 2);
-
             uint64_t nBytesTransferred = 0;
             while (nBytesTransferred < nFSize && patStop.load() == false && fnIsStreamReset(nStreamId) == false)
             {
@@ -2718,20 +2713,22 @@ void CHttpServ::DoAction(const MetaSocketData soMetaDa, const uint8_t httpVers,
                 if (fnSendQueueReady(nStreamWndSize, nSendBufLen, static_cast<uint64_t>(nSizeSendBuf - nHttp2Offset), nFSize - nBytesTransferred) == false)
                     continue;
 
-                nBytesTransferred += nSendBufLen;
-                fin.read(reinterpret_cast<char*>(apBuf.get()) + nHttp2Offset, nSendBufLen);
-
                 if (httpVers == 2)
-                    BuildHttp2Frame(apBuf.get(), nSendBufLen, 0x0, (nFSize - nBytesTransferred == 0 ? 0x1 : 0x0), nStreamId);
-                soMetaDa.fSocketWrite(apBuf.get(), nSendBufLen + nHttp2Offset);
+                {
+                    BuildHttp2Frame(caBuffer, nSendBufLen, 0x0, (nFSize - (nBytesTransferred + nSendBufLen) == 0 ? 0x1 : 0x0), nStreamId);
+                    soMetaDa.fSocketWrite(caBuffer, nHttp2Offset);
+                }
+                soMetaDa.fSocketWrite(map.data(), nSendBufLen);
                 soMetaDa.fResetTimer();
 
                 if (fnUpdateStreamParam(nSendBufLen) == -1)
                     break;  // Stream Item was removed, probably the stream was reset
+
+                nBytesTransferred += nSendBufLen;
+                map.addOffset(nSendBufLen);
             }
             fnResetReservierteWindowSize();
         }
-        fin.close();
 
         CLogFile::GetInstance(m_vHostParam[szHost].m_strLogFile) << soMetaDa.strIpClient << " - - [" << CLogFile::LOGTYPES::PUTTIME << "] \""
             << itMethode->second << " " << lstHeaderFields.find(":path")->second
diff --git a/MappedFile.cpp b/MappedFile.cpp
@@ -110,3 +110,9 @@ void MappedFile::setOffset(const uint64_t off)
  {
     offset = off;
  }
+
+void MappedFile::addOffset(const uint64_t off)
+{
+    offset += off;
+}
+
diff --git a/MappedFile.h b/MappedFile.h
@@ -25,6 +25,8 @@ class MappedFile
     const uint8_t* data() const;
     uint64_t size() const;
     void setOffset(const uint64_t offset);
+    void addOffset(const uint64_t offset);
+    uint64_t getOffset() const { return offset; }
 
 private:
     std::wstring filePath;
diff --git a/test/test.sh b/test/test.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+# Generating reference files
+echo "Generating reference files..."
+if [ ! -f test/ref_499.bin ]; then
+    dd if=test/output.bin of=test/ref_499.bin bs=1 skip=500 count=499 status=none
+fi
+if [ ! -f test/ref_73456.bin ]; then
+    dd if=test/output.bin of=test/ref_73456.bin bs=1 skip=50000 status=none
+fi
+if [ ! -f test/ref_10000.txt ]; then
+    dd if=test/output.txt of=test/ref_10000.txt bs=1 skip=50000 count=10000 status=none
+fi
+if [ ! -f test/ref_73456.txt ]; then
+    dd if=test/output.txt of=test/ref_73456.txt bs=1 skip=50000 status=none
+fi
+
+# Performing tests with curl
+echo "Performing tests with curl..."
+
+# http1.1 tests
+# Full file
+curl --ipv4 --http1.1 --dump-header test/ref_header1.out http://hauck-thomas.de/output.bin -o test/ref_file1.out
+diff -q test/output.bin test/ref_file1.out && echo "Http 1.1 Full file test passed." || echo "Http 1.1 Full file test failed!"
+
+curl --ipv4 --http1.1 --compressed --dump-header test/ref_header2.out http://hauck-thomas.de/output.txt -o test/ref_file2.txt
+diff -q test/output.txt test/ref_file2.txt && echo "Http 1.1 Full compressed(br) file test passed." || echo "Http 1.1 Full compressed(br) file test failed!"
+
+curl --ipv4 --http1.1 --compressed --dump-header test/ref_header3.out -H "X-PrefCompAlgo: deflate" http://hauck-thomas.de/output.txt -o test/ref_file3.txt
+diff -q test/output.txt test/ref_file3.txt && echo "Http 1.1 Full compressed(deflate) file test passed." || echo "Http 1.1 Full compressed(deflate) file test failed!"
+
+
+# Partial file
+curl --ipv4 --http1.1 --range 500-999 --dump-header test/ref_header4.out https://hauck-thomas.de/output.bin -o test/ref_file4.out
+diff -q test/ref_499.bin test/ref_file4.out && echo "Http 1.1 partial file test passed." || echo "Http 1.1 partial file test failed!"
+
+curl --ipv4 --http1.1 --range 50000-60000 --compressed --dump-header test/ref_header5.out https://hauck-thomas.de/output.txt -o test/ref_file5.txt
+diff -q test/ref_10000.txt test/ref_file5.txt && echo "Http 1.1 partial compressed(br) file test passed." || echo "Http 1.1 partial compressed(br) file test failed!"
+
+curl --ipv4 --http1.1 --range 50000-60000 --compressed --dump-header test/ref_header6.out -H "X-PrefCompAlgo: deflate" https://hauck-thomas.de/output.txt -o test/ref_file6.txt
+diff -q test/ref_10000.txt test/ref_file6.txt && echo "Http 1.1 partial compressed(deflate) file test passed." || echo "Http 1.1 partial compressed(deflate) file test failed!"
+
+# http2 tests
+# Full file
+curl --ipv4 --http2 --dump-header test/ref_header7.out https://hauck-thomas.de/output.bin -o test/ref_file7.out
+diff -q test/output.bin test/ref_file7.out && echo "Http2 Full file test passed." || echo "Http2 Full file test failed!"
+
+curl --ipv4 --http2 --compressed --dump-header test/ref_header8.out https://hauck-thomas.de/output.txt -o test/ref_file8.txt
+diff -q test/output.txt test/ref_file8.txt && echo "Http2 Full compressed(br) file test passed." || echo "Http2 Full compressed(br) file test failed!"
+
+curl --ipv4 --http2 --compressed --dump-header test/ref_header9.out -H "X-PrefCompAlgo: deflate" https://hauck-thomas.de/output.txt -o test/ref_file9.txt
+diff -q test/output.txt test/ref_file9.txt && echo "Http2 Full compressed(deflate) file test passed." || echo "Http2 Full compressed(deflate) file test failed!"
+
+# Partial file
+curl --ipv4 --http2 --range 500-999 --dump-header test/ref_header10.out https://hauck-thomas.de/output.bin -o test/ref_file10.out
+diff -q test/ref_499.bin test/ref_file10.out && echo "Http2 partial file test passed." || echo "Http2 partial file test failed!"
+
+curl --ipv4 --http2 --range 50000- --compressed --dump-header test/ref_header11.out https://hauck-thomas.de/output.txt -o test/ref_file11.txt
+diff -q test/ref_73456.txt test/ref_file11.txt && echo "Http2 partial compressed(br) file test passed." || echo "Http2 partial compressed(br) file test failed!"
+
+curl --ipv4 --http2 --range 50000- --compressed --dump-header test/ref_header12.out -H "X-PrefCompAlgo: deflate" https://hauck-thomas.de/output.txt -o test/ref_file12.txt
+diff -q test/ref_73456.txt test/ref_file12.txt && echo "Http2 partial compressed(deflate) file test passed." || echo "Http2 partial compressed(deflate) file test failed!"
diff --git a/test/write_bytes.py b/test/write_bytes.py

Original file line number	Diff line number	Diff line change
`@@ -110,3 +110,9 @@ void MappedFile::setOffset(const uint64_t off)`
`110`	`110`	`{`
`111`	`111`	`offset = off;`
`112`	`112`	`}`
	`113`	`+`
	`114`	`+void MappedFile::addOffset(const uint64_t off)`
	`115`	`+{`
	`116`	`+ offset += off;`
	`117`	`+}`
	`118`	`+`