Optimize base64 writing to XML and reuse Enc buffer (#72)

metsma · web-flow · commit 86b154a58be7 · 2025-08-28T14:53:45.000+03:00
Signed-off-by: Raul Metsma &lt;raul@metsma.ee&gt;
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -56,13 +56,24 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Expose Android NDK env
+      shell: bash
+      run: |
+        echo "ANDROID_NDK_HOME=$ANDROID_NDK_LATEST_HOME" >> "$GITHUB_ENV"
+        echo "ANDROID_NDK_ROOT=$ANDROID_NDK_LATEST_HOME" >> "$GITHUB_ENV"
+    - name: Cache vcpkg
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/vcpkg_cache
+        key: vcpkg-${{ matrix.target }}-${{ hashFiles('.github/workflows/*', 'vcpkg.json', 'CMakeLists.txt', '**/CMakeLists.txt', 'CMakePresets.json') }}
     - name: Prepare vcpkg
       uses: lukka/run-vcpkg@v11
       with:
-        vcpkgGitCommitId: 031ad89ce6c575df35a8e58707ad2c898446c63e
+        vcpkgGitCommitId: 085820b35f4ef5ad54967c8a46fb822e53c4be33
         vcpkgJsonGlob: ./vcpkg.json
         runVcpkgInstall: true
       env:
+        VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg_cache,readwrite
         VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }}
     - name: Build
       run: |
@@ -95,14 +106,21 @@ jobs:
       run: |
         brew update
         brew install --formula flatbuffers swig doxygen boost
+    - name: Cache vcpkg
+      if: matrix.target != 'macos'
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/vcpkg_cache
+        key: vcpkg-${{ matrix.target }}-${{ hashFiles('.github/workflows/*', 'vcpkg.json', 'CMakeLists.txt', '**/CMakeLists.txt', 'CMakePresets.json') }}
     - name: Prepare vcpkg
       if: matrix.target != 'macos'
       uses: lukka/run-vcpkg@v11
       with:
-        vcpkgGitCommitId: 031ad89ce6c575df35a8e58707ad2c898446c63e
+        vcpkgGitCommitId: 085820b35f4ef5ad54967c8a46fb822e53c4be33
         vcpkgJsonGlob: ./vcpkg.json
         runVcpkgInstall: true
       env:
+        VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg_cache,readwrite
         VCPKG_DEFAULT_TRIPLET: ${{ matrix.triplet }}
     - name: Build
       run: |
@@ -132,14 +150,20 @@ jobs:
     steps:
     - name: Checkout
       uses: actions/checkout@v4
+    - name: Cache vcpkg
+      uses: actions/cache@v4
+      with:
+        path: ${{ github.workspace }}/vcpkg_cache
+        key: vcpkg-${{ matrix.image }}-${{ matrix.platform }}-${{ hashFiles('.github/workflows/*', 'vcpkg.json', 'CMakeLists.txt', '**/CMakeLists.txt', 'CMakePresets.json') }}
     - name: Prepare vcpkg
       uses: lukka/run-vcpkg@v11
       with:
-        vcpkgGitCommitId: 031ad89ce6c575df35a8e58707ad2c898446c63e
+        vcpkgGitCommitId: 085820b35f4ef5ad54967c8a46fb822e53c4be33
         vcpkgJsonGlob: ./vcpkg.json
         runVcpkgInstall: true
         runVcpkgFormatString: "[`install`, `--recurse`, `--clean-after-build`, `--x-install-root`, `$[env.VCPKG_INSTALLED_DIR]`, `--triplet`, `$[env.VCPKG_DEFAULT_TRIPLET]`, `--x-feature`, `tests`]"
       env:
+        VCPKG_BINARY_SOURCES: clear;files,${{ github.workspace }}/vcpkg_cache,readwrite
         VCPKG_INSTALLED_DIR: ${{ github.workspace }}/build/vcpkg_installed
     - name: Install dependencies
       run: winget install --silent --accept-source-agreements --accept-package-agreements swig doxygen
diff --git a/cdoc/CDoc1Writer.cpp b/cdoc/CDoc1Writer.cpp
@@ -59,7 +59,7 @@ struct CDoc1Writer::Private final: public XMLWriter
     std::vector<Recipient> rcpts;
 
     int64_t writeEncryptionProperties(bool use_ddoc);
-    int64_t writeKeyInfo(bool use_ddoc, const Crypto::Key& transportKey);
+    int64_t writeKeyInfo(bool use_ddoc, const std::vector<Recipient> &rcpts, const Crypto::Key& transportKey);
     int64_t writeRecipient(const std::vector<uint8_t> &recipient, const Crypto::Key& transportKey);
 };
 
@@ -90,7 +90,7 @@ int64_t CDoc1Writer::Private::writeEncryptionProperties(bool use_ddoc)
     return writeEndElement(Private::DENC); // EncryptedData
 }
 
-int64_t CDoc1Writer::Private::writeKeyInfo(bool use_ddoc, const Crypto::Key& transportKey)
+int64_t CDoc1Writer::Private::writeKeyInfo(bool use_ddoc, const std::vector<Recipient> &rcpts, const Crypto::Key& transportKey)
 {
     RET_ERROR(writeStartElement(Private::DENC, "EncryptedData",
         {{"MimeType", use_ddoc ? "http://www.sk.ee/DigiDoc/v1.3.0/digidoc.xsd" : "application/octet-stream"}}));
@@ -230,43 +230,38 @@ int64_t CDoc1Writer::Private::writeRecipient(const std::vector<uint8_t> &recipie
 libcdoc::result_t
 CDoc1Writer::encrypt(libcdoc::MultiDataSource& src, const std::vector<libcdoc::Recipient>& keys)
 {
+    if(keys.empty())
+        return WORKFLOW_ERROR;
     RET_ERROR(beginEncryption());
     d->rcpts = keys;
     Crypto::Key transportKey = Crypto::generateKey(d->method);
 	int n_components = src.getNumComponents();
 	bool use_ddoc = (n_components > 1) || (n_components == libcdoc::NOT_IMPLEMENTED);
 
-    RET_ERROR(d->writeKeyInfo(use_ddoc, transportKey));
-    RET_ERROR(d->writeElement(Private::DENC, "CipherData", [&]() -> int64_t {
-        std::vector<uint8_t> data;
-        data.reserve(16384);
-        VectorConsumer vcons(data);
-        EncryptionConsumer enc(vcons, d->method, transportKey);
-        std::string name;
-        int64_t size;
-        if(use_ddoc) {
-            DDOCWriter ddoc(enc);
-            result_t result;
-            for (result = src.next(name, size); result == OK; result = src.next(name, size)) {
-                std::vector<uint8_t> contents;
-                VectorConsumer vcons(contents);
-                RET_ERROR(src.readAll(vcons));
-                RET_ERROR(vcons.close());
-                RET_ERROR(ddoc.addFile(name, "application/octet-stream", contents));
-                d->files.push_back({name, contents.size()});
+    RET_ERROR(d->writeKeyInfo(use_ddoc, keys, transportKey));
+    RET_ERROR(d->writeElement(Private::DENC, "CipherData", [&] {
+        return d->writeBase64Element(Private::DENC, "CipherValue", [&](DataConsumer &dst) -> int64_t {
+            EncryptionConsumer enc(dst, d->method, transportKey);
+            std::string name;
+            int64_t size;
+            if(use_ddoc) {
+                DDOCWriter ddoc(enc);
+                result_t result;
+                for (result = src.next(name, size); result == OK; result = src.next(name, size)) {
+                    RET_ERROR(ddoc.addFile(name, "application/octet-stream", size, src));
+                    d->files.push_back({name, size_t(size)});
+                }
+                if(result != END_OF_STREAM)
+                    return result;
+            } else {
+                RET_ERROR(src.next(name, size));
+                if(auto rv = src.readAll(enc); rv >= 0)
+                    d->files.push_back({std::move(name), size_t(rv)});
+                else
+                    return rv;
             }
-            if(result != END_OF_STREAM)
-                return result;
-        } else {
-            RET_ERROR(src.next(name, size));
-            if(auto rv = src.readAll(enc); rv >= 0)
-                d->files.push_back({std::move(name), size_t(rv)});
-            else
-                return rv;
-        }
-        RET_ERROR(enc.close());
-        RET_ERROR(vcons.close());
-        return d->writeBase64Element(Private::DENC, "CipherValue", data);
+            return enc.close();
+        });
     }));
     RET_ERROR(d->writeEncryptionProperties(use_ddoc));
     d.reset();
@@ -318,22 +313,19 @@ CDoc1Writer::finishEncryption()
 	bool use_ddoc = d->files.size() > 1;
 	libcdoc::Crypto::Key transportKey = libcdoc::Crypto::generateKey(d->method);
 
-    RET_ERROR(d->writeKeyInfo(use_ddoc, transportKey));
+    RET_ERROR(d->writeKeyInfo(use_ddoc, d->rcpts, transportKey));
     RET_ERROR(d->writeElement(Private::DENC, "CipherData", [&] {
-        std::vector<uint8_t> data;
-        data.reserve(16384);
-        VectorConsumer vcons(data);
-        EncryptionConsumer enc(vcons, d->method, transportKey);
-        if(use_ddoc)
-        {
-            for(DDOCWriter ddoc(enc); const FileEntry& file : d->files)
-                RET_ERROR(ddoc.addFile(file.name, "application/octet-stream", file.data));
-        }
-        else
-            RET_ERROR(VectorSource(d->files.back().data).readAll(enc));
-        RET_ERROR(enc.close());
-        RET_ERROR(vcons.close());
-        return d->writeBase64Element(Private::DENC, "CipherValue", data);
+        return d->writeBase64Element(Private::DENC, "CipherValue", [&](DataConsumer &dst) -> int64_t {
+            EncryptionConsumer enc(dst, d->method, transportKey);
+            if(use_ddoc)
+            {
+                for(DDOCWriter ddoc(enc); const FileEntry& file : d->files)
+                    RET_ERROR(ddoc.addFile(file.name, "application/octet-stream", file.data));
+            }
+            else
+                RET_ERROR(VectorSource(d->files.back().data).readAll(enc));
+            return enc.close();
+        });
     }));
     RET_ERROR(d->writeEncryptionProperties(use_ddoc));
     d.reset();
diff --git a/cdoc/Crypto.cpp b/cdoc/Crypto.cpp
@@ -573,11 +573,11 @@ EncryptionConsumer::write(const uint8_t *src, size_t size)
         return OK;
     if(error != OK)
         return error;
-    std::vector<uint8_t> data(size + EVP_CIPHER_CTX_block_size(ctx.get()) - 1);
-    int len = int(data.size());
-    if(SSL_FAILED(EVP_CipherUpdate(ctx.get(), data.data(), &len, src, int(size)), "EVP_CipherUpdate"))
+    buf.resize(std::max(buf.size(), size + EVP_CIPHER_CTX_block_size(ctx.get()) - 1));
+    int len = int(buf.size());
+    if(SSL_FAILED(EVP_CipherUpdate(ctx.get(), buf.data(), &len, src, int(size)), "EVP_CipherUpdate"))
         return CRYPTO_ERROR;
-    return dst.write(data.data(), size_t(len));
+    return dst.write(buf.data(), size_t(len));
 }
 
 result_t
@@ -592,11 +592,11 @@ EncryptionConsumer::writeAAD(const std::vector<uint8_t> &data)
 result_t
 EncryptionConsumer::close()
 {
-    std::vector<uint8_t> data(EVP_CIPHER_CTX_block_size(ctx.get()));
-    int len = int(data.size());
-    if(SSL_FAILED(EVP_CipherFinal(ctx.get(), data.data(), &len), "EVP_CipherFinal"))
+    buf.resize(std::max(buf.size(), size_t(EVP_CIPHER_CTX_block_size(ctx.get()))));
+    int len = int(buf.size());
+    if(SSL_FAILED(EVP_CipherFinal(ctx.get(), buf.data(), &len), "EVP_CipherFinal"))
         return CRYPTO_ERROR;
-    if(auto rv = dst.write(data.data(), size_t(len)); rv < 0)
+    if(auto rv = dst.write(buf.data(), size_t(len)); rv < 0)
         return rv;
     std::array<uint8_t, 16> tag {};
     if(EVP_CIPHER_CTX_mode(ctx.get()) == EVP_CIPH_GCM_MODE)
diff --git a/cdoc/Crypto.h b/cdoc/Crypto.h
@@ -124,7 +124,7 @@ class Crypto
 
 struct EncryptionConsumer final : public DataConsumer {
     EncryptionConsumer(DataConsumer &dst, const std::string &method, const Crypto::Key &key);
-	EncryptionConsumer(DataConsumer &dst, const EVP_CIPHER *cipher, const Crypto::Key &key);
+    EncryptionConsumer(DataConsumer &dst, const EVP_CIPHER *cipher, const Crypto::Key &key);
     CDOC_DISABLE_MOVE_COPY(EncryptionConsumer)
     result_t write(const uint8_t *src, size_t size) final;
     result_t writeAAD(const std::vector<uint8_t> &data);
@@ -135,6 +135,7 @@ struct EncryptionConsumer final : public DataConsumer {
     unique_free_t<EVP_CIPHER_CTX> ctx;
     DataConsumer &dst;
     result_t error = OK;
+    std::vector<uint8_t> buf;
 };
 
 }; // namespace libcdoc
diff --git a/cdoc/DDocWriter.cpp b/cdoc/DDocWriter.cpp
@@ -18,6 +18,8 @@
 
 #include "DDocWriter.h"
 
+#include "Io.h"
+
 using namespace libcdoc;
 
 /**
@@ -38,6 +40,24 @@ DDOCWriter::~DDOCWriter() noexcept
     writeEndElement(DDOC); // SignedDoc
 }
 
+/**
+ * Add File to container
+ * @param file Filename
+ * @param mime File mime type
+ * @param size File size
+ * @param data File content
+ */
+int64_t DDOCWriter::addFile(const std::string &file, const std::string &mime, size_t size, libcdoc::DataSource& src)
+{
+    return writeBase64Element(DDOC, "DataFile", [&src](DataConsumer &dst){ return src.readAll(dst); }, {
+        {"ContentType", "EMBEDDED_BASE64"},
+        {"Filename", file},
+        {"Id", "D" + std::to_string(fileCount++)},
+        {"MimeType", mime},
+        {"Size", std::to_string(size)}
+    });
+}
+
 /**
  * Add File to container
  * @param file Filename
diff --git a/cdoc/DDocWriter.h b/cdoc/DDocWriter.h
@@ -21,13 +21,15 @@
 #include "XmlWriter.h"
 
 namespace libcdoc {
+struct DataSource;
 
 class DDOCWriter final: public XMLWriter
 {
 public:
     DDOCWriter(DataConsumer &dst);
     ~DDOCWriter() noexcept final;
 
+    int64_t addFile(const std::string &name, const std::string &mime, size_t size, libcdoc::DataSource &src);
     int64_t addFile(const std::string &name, const std::string &mime, const std::vector<unsigned char> &data);
 
 private:
diff --git a/cdoc/XmlWriter.cpp b/cdoc/XmlWriter.cpp
@@ -24,6 +24,8 @@
 
 #include <libxml/xmlwriter.h>
 
+#include <array>
+
 using namespace libcdoc;
 
 using pcxmlChar = xmlChar *;
@@ -111,6 +113,66 @@ int64_t XMLWriter::writeElement(const NS &ns, const std::string &name, const std
     return writeEndElement(ns);
 }
 
+int64_t XMLWriter::writeBase64Element(const NS &ns, const std::string &name, const std::function<int64_t(DataConsumer&)> &f, const std::map<std::string, std::string> &attr)
+{
+    if(auto rv = writeStartElement(ns, name, attr); rv != OK)
+        return rv;
+
+    struct Base64Consumer : public DataConsumer {
+        xmlTextWriterPtr w;
+        std::array<uint8_t, 3> buf {}; // buffer up to 2 leftover bytes
+        size_t bufSize = 0;
+        Base64Consumer(xmlTextWriterPtr _w) : w(_w) {}
+        result_t write(const uint8_t *src, size_t size) final {
+            if(!src || size == 0)
+                return OK;
+
+            size_t pos = 0;
+            if(bufSize > 0) {
+                pos = std::min(buf.size() - bufSize, size);
+                std::copy(src, src + pos, buf.begin() + bufSize);
+                bufSize += pos;
+                if (bufSize < 3) {
+                    return result_t(size);
+                }
+                if (xmlTextWriterWriteBase64(w, reinterpret_cast<const char*>(buf.data()), 0, buf.size()) == -1)
+                    return IO_ERROR;
+                bufSize = 0;
+            }
+
+            // Write largest contiguous chunk with length multiple of 3
+            size_t remaining = size - pos;
+            if(size_t fullTriples = remaining - (remaining % 3); fullTriples > 0) {
+                if (xmlTextWriterWriteBase64(w, reinterpret_cast<const char*>(src), pos, fullTriples) == -1)
+                    return IO_ERROR;
+                pos += fullTriples;
+            }
+
+            // Buffer leftover (0..2) bytes for next write/close
+            if(bufSize = size - pos; bufSize > 0) {
+                std::copy(src + pos, src + size, buf.begin());
+            }
+
+            return result_t(size);
+        }
+        result_t close() final {
+            if (bufSize > 0) {
+                // write remaining 1..2 bytes so base64 padding is applied only at the end
+                if(xmlTextWriterWriteBase64(w, reinterpret_cast<const char*>(buf.data()), 0, bufSize) == -1)
+                    return IO_ERROR;
+            }
+            bufSize = 0;
+            return OK;
+        }
+        bool isError() final { return false; }
+    } base64Consumer {d->w.get()};
+    if(auto rv = f(base64Consumer); rv < 0)
+        return rv;
+    if(auto rv = base64Consumer.close(); rv < 0)
+        return rv;
+    return writeEndElement(ns);
+}
+
 int64_t XMLWriter::writeBase64Element(const NS &ns, const std::string &name, const std::vector<xmlChar> &data, const std::map<std::string, std::string> &attr)
 {
     if(auto rv = writeStartElement(ns, name, attr); rv != OK)
diff --git a/cdoc/XmlWriter.h b/cdoc/XmlWriter.h
@@ -41,6 +41,7 @@ class XMLWriter
     int64_t writeEndElement(const NS &ns);
     int64_t writeElement(const NS &ns, const std::string &name, const std::function<int64_t()> &f = nullptr);
     int64_t writeElement(const NS &ns, const std::string &name, const std::map<std::string, std::string> &attr, const std::function<int64_t()> &f = nullptr);
+    int64_t writeBase64Element(const NS &ns, const std::string &name, const std::function<int64_t(DataConsumer &)> &f, const std::map<std::string, std::string> &attr = {});
     int64_t writeBase64Element(const NS &ns, const std::string &name, const std::vector<unsigned char> &data, const std::map<std::string, std::string> &attr = {});
     int64_t writeTextElement(const NS &ns, const std::string &name, const std::map<std::string, std::string> &attr, const std::string &data);
 
diff --git a/vcpkg.json b/vcpkg.json
@@ -21,7 +21,7 @@
   "features": {
     "tests": { "description": "Build tests", "dependencies": ["boost-test"] }
   },
-  "builtin-baseline": "031ad89ce6c575df35a8e58707ad2c898446c63e",
+  "builtin-baseline": "085820b35f4ef5ad54967c8a46fb822e53c4be33",
   "vcpkg-configuration": {
     "overlay-triplets": ["./vcpkg-triplets"]
   }

Original file line number	Diff line number	Diff line change
`@@ -21,7 +21,7 @@`
`21`	`21`	`"features": {`
`22`	`22`	`"tests": { "description": "Build tests", "dependencies": ["boost-test"] }`
`23`	`23`	`},`
`24`		`- "builtin-baseline": "031ad89ce6c575df35a8e58707ad2c898446c63e",`
	`24`	`+ "builtin-baseline": "085820b35f4ef5ad54967c8a46fb822e53c4be33",`
`25`	`25`	`"vcpkg-configuration": {`
`26`	`26`	`"overlay-triplets": ["./vcpkg-triplets"]`
`27`	`27`	`}`