Add Zip Read object to avoid create temporary stream objects

metsma · metsma · commit c5d8ed5a7c8f · 2025-12-04T17:03:28.000+02:00
IB-8746

Signed-off-by: Raul Metsma &lt;raul@metsma.ee&gt;
diff --git a/src/ASiC_E.cpp b/src/ASiC_E.cpp
@@ -129,7 +129,7 @@ void ASiC_E::addAdESSignature(istream &data)
         THROW("'%s' format is not supported", mediaType().c_str());
     try
     {
-        loadSignatures(data, d->unique_name());
+        loadSignatures(XMLDocument::openStream(data), d->unique_name());
     }
     catch(const Exception &e)
     {
@@ -149,9 +149,9 @@ unique_ptr<Container> ASiC_E::openInternal(const string &path)
     return unique_ptr<Container>(new ASiC_E(path));
 }
 
-void ASiC_E::loadSignatures(istream &data, const string &file)
+void ASiC_E::loadSignatures(XMLDocument &&doc, const string &file)
 {
-    auto signatures = make_shared<Signatures>(data, mediaType());
+    auto signatures = make_shared<Signatures>(std::move(doc), mediaType());
     d->signatures.emplace(file, signatures.get());
     for(auto s = signatures->signature(); s; s++)
         addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));
@@ -170,8 +170,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
 
     try
     {
-        auto manifestdata = z.extract<stringstream>("META-INF/manifest.xml");
-        auto doc = XMLDocument::openStream(manifestdata, {"manifest", MANIFEST_NS});
+        auto doc = XMLDocument::open(z.read("META-INF/manifest.xml"), {"manifest", MANIFEST_NS});
         doc.validateSchema(File::path(Conf::instance()->xsdPath(), "OpenDocument_manifest_v1_2.xsd"));
 
         set<string_view> manifestFiles;
@@ -219,8 +218,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
             {
                 try
                 {
-                    auto data = z.extract<stringstream>(file);
-                    loadSignatures(data, file);
+                    loadSignatures(XMLDocument::open(z.read(file)), file);
                 }
                 catch(const Exception &e)
                 {
diff --git a/src/ASiC_E.h b/src/ASiC_E.h
@@ -54,7 +54,7 @@ namespace digidoc
           ASiC_E(const std::string &path);
           DISABLE_COPY(ASiC_E);
           void canSave() final;
-          void loadSignatures(std::istream &data, const std::string &file);
+          void loadSignatures(XMLDocument &&doc, const std::string &file);
           void parseManifestAndLoadFiles(const ZipSerialize &z);
           void save(const ZipSerialize &s) final;
 
diff --git a/src/ASiC_S.cpp b/src/ASiC_S.cpp
@@ -66,8 +66,7 @@ ASiC_S::ASiC_S(const string &path)
         {
             if(!signatures().empty())
                 THROW("Can not add signature to ASiC-S container which already contains a signature.");
-            auto data = z.extract<stringstream>(file);
-            auto signatures = make_shared<Signatures>(data, mediaType());
+            auto signatures = make_shared<Signatures>(XMLDocument::open(z.read(file)), mediaType());
             for(auto s = signatures->signature(); s; s++)
                 addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));
         }
diff --git a/src/ASiContainer.cpp b/src/ASiContainer.cpp
@@ -90,7 +90,7 @@ ZipSerialize ASiContainer::load(const string &path, bool mimetypeRequired, const
 
     if(list.front() == "mimetype")
     {
-        d->mimetype = readMimetype(z);
+        d->mimetype = z.mimetype();
         if(!contains(supported, d->mimetype))
             THROW("Incorrect mimetype '%s'", d->mimetype.c_str());
     }
@@ -285,26 +285,3 @@ const ZipSerialize::Properties& ASiContainer::zproperty(string_view file) const
         return i->second;
     return d->properties[string(file)] = { appInfo(), time(nullptr), 0 };
 }
-
-/**
- * Reads and parses container mimetype. Checks that the mimetype is supported.
- *
- * @param path path to container directory.
- * @throws IOException exception is thrown if there was error reading mimetype file from disk.
- * @throws ContainerException exception is thrown if the parsed mimetype is incorrect.
- */
-string ASiContainer::readMimetype(const ZipSerialize &z)
-{
-    DEBUG("ASiContainer::readMimetype()");
-    string text = z.extract<stringstream>("mimetype").str();
-    text.erase(text.find_last_not_of(" \n\r\f\t\v") + 1);
-    if(text.empty())
-        THROW("Failed to read mimetype.");
-    // Contains UTF-16 BOM
-    if(text.find("\xFF\xEF") == 0 || text.find("\xEF\xFF") == 0)
-        THROW("Mimetype file must be UTF-8 format.");
-    // contains UTF-8 BOM, remove
-    if(text.find("\xEF\xBB\xBF") == 0)
-        text.erase(text.cbegin(), text.cbegin() + 3);
-    return text;
-}
diff --git a/src/ASiContainer.h b/src/ASiContainer.h
@@ -58,8 +58,6 @@ namespace digidoc
           void save(const std::string &path) override;
           std::vector<Signature*> signatures() const override;
 
-          static std::string readMimetype(const ZipSerialize &z);
-
       protected:
           ASiContainer(std::string_view mimetype);
 
diff --git a/src/DataFile.cpp b/src/DataFile.cpp
@@ -24,6 +24,7 @@
 #include "util/log.h"
 #include "util/ZipSerialize.h"
 
+#include <array>
 #include <fstream>
 #include <sstream>
 
@@ -102,12 +103,24 @@ DataFilePrivate::DataFilePrivate(const ZipSerialize &z, string filename, string
     , m_filename(std::move(filename))
     , m_mediatype(std::move(mediatype))
 {
-    auto prop = z.properties(m_filename);
-    d->size.emplace(prop.size);
-    if(d->size.value() > MAX_MEM_FILE)
-        m_is = make_unique<fstream>(z.extract<fstream>(m_filename));
+    auto r = z.read(m_filename);
+    d->size.emplace(r.size);
+    if(r.size > MAX_MEM_FILE)
+    {
+        auto fs = make_unique<fstream>(util::File::tempFileName(), fstream::in|fstream::out|fstream::binary|fstream::trunc);
+        if(!fs->is_open())
+            THROW("Failed to open destination file");
+        array<char,10240> buf{};
+        for(size_t size = 0, currentStreamSize = 0;
+             (size = r(buf.data(), buf.size())) > 0; currentStreamSize += size)
+        {
+            if(!fs->write(buf.data(), size))
+                THROW("Failed to write '%s' data to stream. Stream size: %d", m_filename.c_str(), currentStreamSize);
+        }
+        m_is = std::move(fs);
+    }
     else
-        m_is = make_unique<stringstream>(z.extract<stringstream>(m_filename));
+        m_is = make_unique<stringstream>(r.operator string());
 }
 
 void DataFilePrivate::digest(const Digest &digest) const
diff --git a/src/SiVaContainer.cpp b/src/SiVaContainer.cpp
@@ -164,12 +164,10 @@ SiVaContainer::SiVaContainer(const string &path, ContainerOpenCB *cb, bool useHa
     else if(File::fileExtension(path, {"asice", "sce", "asics", "scs"}))
     {
         ZipSerialize z(path, false);
-        vector<string> list = z.list();
-        if(list.front() != "mimetype")
-            THROW("Missing mimetype");
-        if(d->mediaType = ASiContainer::readMimetype(z);
+        if(d->mediaType = z.mimetype();
             d->mediaType != ASiContainer::MIMETYPE_ASIC_E && d->mediaType != ASiContainer::MIMETYPE_ASIC_S)
             THROW("Unknown file");
+        vector<string> list = z.list();
         if(none_of(list.cbegin(), list.cend(), [](const string &file) {
                 return file.starts_with("META-INF/") && util::File::fileExtension(file, {"p7s"});
             }))
diff --git a/src/SignatureTST.cpp b/src/SignatureTST.cpp
@@ -68,22 +68,21 @@ struct SignatureTST::Data {
 SignatureTST::SignatureTST(bool manifest, const ZipSerialize &z, ASiC_S *asicSDoc)
     : asicSDoc(asicSDoc)
 {
-    auto data = z.extract<stringstream>("META-INF/timestamp.tst").str();
+    string data = z.read("META-INF/timestamp.tst");
     timestampToken = make_unique<TS>((const unsigned char*)data.data(), data.size());
     if(manifest)
     {
         XMLSchema schema(util::File::path(Conf::instance()->xsdPath(), "en_31916201v010101.xsd"));
         string file = "META-INF/ASiCArchiveManifest.xml";
         string mime = "text/xml";
         while(!file.empty()) {
-            auto xml = z.extract<stringstream>(file);
+            stringstream xml(z.read(file).operator string());
             XMLDocument doc = XMLDocument::openStream(xml, {"ASiCManifest", ASiContainer::ASIC_NS});
             schema.validate(doc);
             auto ref = doc/"SigReference";
             string uri = util::File::fromUriPath(ref["URI"]);
-            string tst = z.extract<stringstream>(uri).str();
             metadata.emplace_back(std::move(file), std::move(mime), xml.str());
-            metadata.emplace_back(std::move(uri), string(ref["MimeType"]), std::move(tst));
+            metadata.emplace_back(std::move(uri), string(ref["MimeType"]), z.read(uri));
             file.clear();
 
             for(auto ref = doc/"DataObjectReference"; ref; ref++)
diff --git a/src/SignatureXAdES_B.cpp b/src/SignatureXAdES_B.cpp
@@ -197,8 +197,8 @@ Signatures::Signatures()
     addNS(XADES_NS, "xades");
 }
 
-Signatures::Signatures(istream &data, string_view mediaType)
-    : XMLDocument(openStream(data))
+Signatures::Signatures(XMLDocument &&doc, string_view mediaType)
+    : XMLDocument(std::move(doc))
 {
     /* http://www.etsi.org/deliver/etsi_ts/102900_102999/102918/01.03.01_60/ts_102918v010301p.pdf
      * 6.2.2
diff --git a/src/SignatureXAdES_B.h b/src/SignatureXAdES_B.h
@@ -40,7 +40,7 @@ namespace digidoc
     {
     public:
         explicit Signatures();
-        Signatures(std::istream &data, std::string_view mediaType);
+        Signatures(XMLDocument &&doc, std::string_view mediaType);
 
         constexpr XMLNode signature() const noexcept
         {
diff --git a/src/XMLDocument.h b/src/XMLDocument.h
@@ -307,13 +307,19 @@ struct XMLDocument: public unique_free_d<xmlFreeDoc>, public XMLNode
         : XMLDocument(path.empty() ? nullptr : xmlParseFile(path.c_str()), n)
     {}
 
-    static XMLDocument openStream(std::istream &is, const XMLName &name = {}, bool hugeFile = false)
-    {
-        auto ctxt = make_unique_ptr<xmlFreeParserCtxt>(xmlCreateIOParserCtxt(nullptr, nullptr, [](void *context, char *buffer, int len) -> int {
-            auto *is = static_cast<std::istream *>(context);
-            is->read(buffer, len);
-            return is->good() || is->eof() ? int(is->gcount()) : -1;
-        }, nullptr, &is, XML_CHAR_ENCODING_NONE));
+    template <typename F>
+    static XMLDocument open(F &&f, const XMLName &name = {}, bool hugeFile = false)
+    requires (std::is_invocable_r_v<size_t, F, char*, size_t>)
+    {
+        auto ctxt = make_unique_ptr<xmlFreeParserCtxt>(xmlCreateIOParserCtxt(nullptr, nullptr, [](void *context, char *buffer, int len) noexcept -> int {
+            try {
+                auto *f = static_cast<F*>(context);
+                return (*f)(buffer, size_t(len));
+            } catch(...) {
+                ERR("Failed to read from stream");
+                return -1;
+            }
+        }, nullptr, &f, XML_CHAR_ENCODING_NONE));
         ctxt->options |= XML_PARSE_NOENT|XML_PARSE_DTDLOAD|XML_PARSE_DTDATTR|XML_PARSE_NONET|XML_PARSE_NODICT;
 #if LIBXML_VERSION >= 21300
         ctxt->options |= XML_PARSE_NO_XXE;
@@ -338,6 +344,14 @@ struct XMLDocument: public unique_free_d<xmlFreeDoc>, public XMLNode
         return {ctxt->myDoc, name};
     }
 
+    static XMLDocument openStream(std::istream &is, const XMLName &name = {}, bool hugeFile = false)
+    {
+        return open([&is](char *data, size_t size) {
+            is.read(data, size);
+            return is.good() || is.eof() ? int(is.gcount()) : -1;
+        }, name, hugeFile);
+    }
+
     static XMLDocument create(std::string_view name = {}, std::string_view href = {}, std::string_view prefix = {}) noexcept
     {
         XMLDocument doc(xmlNewDoc(nullptr));
@@ -400,13 +414,18 @@ struct XMLDocument: public unique_free_d<xmlFreeDoc>, public XMLNode
     }
 
     template <typename F>
-    std::enable_if_t<std::is_invocable_v<F, const char*, size_t>, bool>
-    save(F &&f, bool format = false) const noexcept
-    {
-        auto *buf = xmlOutputBufferCreateIO([](void *context, const char *buffer, int len) {
-            auto *f = static_cast<F*>(context);
-            (*f)(buffer, size_t(len));
-            return len;
+    bool save(F &&f, bool format = false) const noexcept
+    requires (std::is_invocable_v<F, const char*, size_t>)
+    {
+        auto *buf = xmlOutputBufferCreateIO([](void *context, const char *buffer, int len) noexcept {
+            try {
+                auto *f = static_cast<F*>(context);
+                (*f)(buffer, size_t(len));
+                return len;
+            } catch(...) {
+                ERR("Failed to write XML file to stream");
+                return -1;
+            }
         }, nullptr, &f, nullptr);
         return xmlSaveFormatFileTo(buf, get(), "UTF-8", format) > 0;
     }
diff --git a/src/util/ZipSerialize.cpp b/src/util/ZipSerialize.cpp
@@ -31,11 +31,6 @@
 #include <minizip/iowin32.h>
 #endif
 
-#include <algorithm>
-#include <array>
-#include <fstream>
-#include <sstream>
-
 using namespace digidoc;
 using namespace std;
 
@@ -107,19 +102,19 @@ vector<string> ZipSerialize::list() const
 }
 
 /**
- * Extracts current file from ZIP file to stream.
+ * Reads current file from ZIP file.
  *
  * @param file path to opened ZIP in file.
- * @throws Exception throws exception if the extraction of the current file fails from ZIP
- *         file or creating new file to disk failed.
+ * @throws Exception throws exception if the extraction of the current file fails from ZIP file
  */
-template<class T>
-T ZipSerialize::extract(string_view file) const
+ZipSerialize::Read ZipSerialize::read(string_view file) const
 {
-    DEBUG("ZipSerializePrivate::extract(%.*s)", int(file.size()), file.data());
-    T os;
+    if(!d)
+        THROW("Zip file is not open");
+
+    DEBUG("ZipSerialize::read(%.*s)", int(file.size()), file.data());
     if(file.empty() || file.back() == '/')
-        return os;
+        return {{nullptr, unzCloseCurrentFile}, 0};
 
     int unzResult = unzLocateFile(d.get(), file.data(), 1);
     if(unzResult != UNZ_OK)
@@ -129,37 +124,12 @@ T ZipSerialize::extract(string_view file) const
     if(unzResult != UNZ_OK)
         THROW("Failed to open '%.*s' inside ZIP container. ZLib error: %d", int(file.size()), file.data(), unzResult);
 
-    if constexpr(std::is_same_v<T, fstream>)
-    {
-        os.open(util::File::tempFileName(), fstream::in|fstream::out|fstream::binary|fstream::trunc);
-        if(!os.is_open())
-            THROW("Failed to open destination file");
-    }
-    array<char,10240> buf{};
-    for(int currentStreamSize = 0;
-         (unzResult = unzReadCurrentFile(d.get(), buf.data(), buf.size())) > UNZ_EOF; currentStreamSize += unzResult)
-    {
-        if(!os.write(buf.data(), unzResult))
-        {
-            unzCloseCurrentFile(d.get());
-            THROW("Failed to write '%.*s' data to stream. Stream size: %d", int(file.size()), file.data(), currentStreamSize);
-        }
-    }
-    if(unzResult < UNZ_EOF)
-    {
-        unzCloseCurrentFile(d.get());
-        THROW("Failed to read bytes from '%.*s' inside ZIP container. ZLib error: %d", int(file.size()), file.data(), unzResult);
-    }
+    unz_file_info info {};
+    unzGetCurrentFileInfo(d.get(), &info, nullptr, 0, nullptr, 0, nullptr, 0);
 
-    unzResult = unzCloseCurrentFile(d.get());
-    if(unzResult != UNZ_OK)
-        THROW("Failed to close '%.*s' inside ZIP container. ZLib error: %d", int(file.size()), file.data(), unzResult);
-    return os;
+    return {{d.get(), unzCloseCurrentFile}, size_t(info.uncompressed_size)};
 }
 
-template fstream ZipSerialize::extract<fstream>(string_view file) const;
-template stringstream ZipSerialize::extract<stringstream>(string_view file) const;
-
 /**
  * Add new file to ZIP container.
  *
@@ -194,6 +164,27 @@ ZipSerialize::Write ZipSerialize::addFile(string_view containerPath, const Prope
     return {{d.get(), zipCloseFileInZip}};
 }
 
+/**
+ * Reads mimetype.
+ *
+ * @throws IOException exception is thrown if there was error reading mimetype file.
+ */
+string ZipSerialize::mimetype() const
+{
+    DEBUG("ZipSerialize::mimetype()");
+    string text = read("mimetype");
+    text.erase(text.find_last_not_of(" \n\r\f\t\v") + 1);
+    if(text.empty())
+        THROW("Failed to read mimetype.");
+    // Contains UTF-16 BOM
+    if(text.find("\xFF\xEF") == 0 || text.find("\xEF\xFF") == 0)
+        THROW("Mimetype file must be UTF-8 format.");
+    // contains UTF-8 BOM, remove
+    if(text.find("\xEF\xBB\xBF") == 0)
+        text.erase(text.cbegin(), text.cbegin() + 3);
+    return text;
+}
+
 ZipSerialize::Properties ZipSerialize::properties(const string &file) const
 {
     int unzResult = unzLocateFile(d.get(), file.c_str(), 1);
@@ -222,6 +213,14 @@ ZipSerialize::Properties ZipSerialize::properties(const string &file) const
     return prop;
 }
 
+size_t ZipSerialize::Read::operator()(void *data, size_t size) const
+{
+    auto result = unzReadCurrentFile(d.get(), data, size);
+    if(result >= UNZ_EOF)
+        return size_t(result);
+    THROW("Failed to read bytes from ZIP container. ZLib error: %d", result);
+}
+
 void ZipSerialize::Write::operator()(const void *data, size_t size) const
 {
     if(auto result = zipWriteInFileInZip(d.get(), data, unsigned(size)); result != ZIP_OK)
diff --git a/src/util/ZipSerialize.h b/src/util/ZipSerialize.h

Original file line number	Diff line number	Diff line change
`@@ -129,7 +129,7 @@ void ASiC_E::addAdESSignature(istream &data)`
`129`	`129`	`THROW("'%s' format is not supported", mediaType().c_str());`
`130`	`130`	`try`
`131`	`131`	`{`
`132`		`- loadSignatures(data, d->unique_name());`
	`132`	`+ loadSignatures(XMLDocument::openStream(data), d->unique_name());`
`133`	`133`	`}`
`134`	`134`	`catch(const Exception &e)`
`135`	`135`	`{`
`@@ -149,9 +149,9 @@ unique_ptr<Container> ASiC_E::openInternal(const string &path)`
`149`	`149`	`return unique_ptr<Container>(new ASiC_E(path));`
`150`	`150`	`}`
`151`	`151`
`152`		`-void ASiC_E::loadSignatures(istream &data, const string &file)`
	`152`	`+void ASiC_E::loadSignatures(XMLDocument &&doc, const string &file)`
`153`	`153`	`{`
`154`		`- auto signatures = make_shared<Signatures>(data, mediaType());`
	`154`	`+ auto signatures = make_shared<Signatures>(std::move(doc), mediaType());`
`155`	`155`	`d->signatures.emplace(file, signatures.get());`
`156`	`156`	`for(auto s = signatures->signature(); s; s++)`
`157`	`157`	`addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));`
`@@ -170,8 +170,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)`
`170`	`170`
`171`	`171`	`try`
`172`	`172`	`{`
`173`		`- auto manifestdata = z.extract<stringstream>("META-INF/manifest.xml");`
`174`		`- auto doc = XMLDocument::openStream(manifestdata, {"manifest", MANIFEST_NS});`
	`173`	`+ auto doc = XMLDocument::open(z.read("META-INF/manifest.xml"), {"manifest", MANIFEST_NS});`
`175`	`174`	`doc.validateSchema(File::path(Conf::instance()->xsdPath(), "OpenDocument_manifest_v1_2.xsd"));`
`176`	`175`
`177`	`176`	`set<string_view> manifestFiles;`
`@@ -219,8 +218,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)`
`219`	`218`	`{`
`220`	`219`	`try`
`221`	`220`	`{`
`222`		`- auto data = z.extract<stringstream>(file);`
`223`		`- loadSignatures(data, file);`
	`221`	`+ loadSignatures(XMLDocument::open(z.read(file)), file);`
`224`	`222`	`}`
`225`	`223`	`catch(const Exception &e)`
`226`	`224`	`{`
Original file line number	Diff line number	Diff line change
`@@ -66,8 +66,7 @@ ASiC_S::ASiC_S(const string &path)`
`66`	`66`	`{`
`67`	`67`	`if(!signatures().empty())`
`68`	`68`	`THROW("Can not add signature to ASiC-S container which already contains a signature.");`
`69`		`- auto data = z.extract<stringstream>(file);`
`70`		`- auto signatures = make_shared<Signatures>(data, mediaType());`
	`69`	`+ auto signatures = make_shared<Signatures>(XMLDocument::open(z.read(file)), mediaType());`
`71`	`70`	`for(auto s = signatures->signature(); s; s++)`
`72`	`71`	`addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));`
`73`	`72`	`}`
Original file line number	Diff line number	Diff line change
`@@ -197,8 +197,8 @@ Signatures::Signatures()`
`197`	`197`	`addNS(XADES_NS, "xades");`
`198`	`198`	`}`
`199`	`199`
`200`		`-Signatures::Signatures(istream &data, string_view mediaType)`
`201`		`- : XMLDocument(openStream(data))`
	`200`	`+Signatures::Signatures(XMLDocument &&doc, string_view mediaType)`
	`201`	`+ : XMLDocument(std::move(doc))`
`202`	`202`	`{`
`203`	`203`	`/* http://www.etsi.org/deliver/etsi_ts/102900_102999/102918/01.03.01_60/ts_102918v010301p.pdf`
`204`	`204`	`* 6.2.2`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ namespace digidoc`
`40`	`40`	`{`
`41`	`41`	`public:`
`42`	`42`	`explicit Signatures();`
`43`		`- Signatures(std::istream &data, std::string_view mediaType);`
	`43`	`+ Signatures(XMLDocument &&doc, std::string_view mediaType);`
`44`	`44`
`45`	`45`	`constexpr XMLNode signature() const noexcept`
`46`	`46`	`{`