Check duplicate files in Zip all code paths (#626)

IB-8230

Signed-off-by: Raul Metsma <[email protected]>

Author: metsma

src/ASiC_E.cpp

@@ -26,7 +26,6 @@
 #include "crypto/Digest.h"
 #include "crypto/Signer.h"
 #include "util/File.h"
-#include "util/ZipSerialize.h"
 
 #include <algorithm>
 #include <set>
@@ -65,7 +64,7 @@ ASiC_E::ASiC_E(const string &path)
     , d(make_unique<Private>())
 {
     auto zip = load(path, true, {MIMETYPE_ASIC_E, MIMETYPE_ADOC});
-    parseManifestAndLoadFiles(*zip);
+    parseManifestAndLoadFiles(zip);
 }
 
 ASiC_E::~ASiC_E()
@@ -99,7 +98,7 @@ void ASiC_E::save(const string &path)
 
     stringstream mimetype;
     mimetype << mediaType();
-    s.addFile("mimetype", mimetype, zproperty("mimetype"), ZipSerialize::DontCompress);
+    s.addFile("mimetype", mimetype, zproperty("mimetype"), false);
 
     stringstream manifest;
     createManifest(manifest);
@@ -193,32 +192,21 @@ void ASiC_E::createManifest(ostream &os)
  * Parses manifest file and checks that files described in manifest exist, also
  * checks that no extra file do exist that are not described in manifest.xml.
  *
- * Note: If non-ascii characters are present in XML data, we depend on the LANG variable to be set properly
- * (see iconv --list for the list of supported encoding values for libiconv).
- *
  * @param path directory on disk of the BDOC container.
  * @throws Exception exception is thrown if the manifest.xml file parsing failed.
  */
 void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
 {
     DEBUG("ASiC_E::readManifest()");
 
-    const vector<string> &list = z.list();
-    auto mcount = size_t(count(list.cbegin(), list.cend(), "META-INF/manifest.xml"));
-    if(mcount < 1)
-        THROW("Manifest file is missing");
-    if(mcount > 1)
-        THROW("Found multiple manifest files");
-
     try
     {
-        stringstream manifestdata;
-        z.extract("META-INF/manifest.xml", manifestdata);
+        auto manifestdata = z.extract<stringstream>("META-INF/manifest.xml");
+        auto doc = XMLDocument::openStream(manifestdata, {"manifest", MANIFEST_NS});
+        doc.validateSchema(File::path(Conf::instance()->xsdPath(), "OpenDocument_manifest_v1_2.xsd"));
 
         set<string_view> manifestFiles;
         bool mimeFound = false;
-        auto doc = XMLDocument::openStream(manifestdata, {"manifest", MANIFEST_NS});
-        doc.validateSchema(File::path(Conf::instance()->xsdPath(), "OpenDocument_manifest_v1_2.xsd"));
         for(auto file = doc/"file-entry"; file; file++)
         {
             auto full_path = file[{"full-path", MANIFEST_NS}];
@@ -239,24 +227,18 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
             if(full_path.back() == '/') // Skip Directory entries
                 continue;
 
-            auto fcount = size_t(count(list.cbegin(), list.cend(), full_path));
-            if(fcount < 1)
-                THROW("File described in manifest '%s' does not exist in container.", full_path.data());
-            if(fcount > 1)
-                THROW("Found multiple references of file '%s' in zip container.", full_path.data());
-
             manifestFiles.insert(full_path);
             if(mediaType() == MIMETYPE_ADOC &&
                (full_path.compare(0, 9, "META-INF/") == 0 ||
                 full_path.compare(0, 9, "metadata/") == 0))
-                d->metadata.push_back(new DataFilePrivate(dataStream(string(full_path), z), string(full_path), string(media_type)));
+                d->metadata.push_back(new DataFilePrivate(dataStream(full_path, z), string(full_path), string(media_type)));
             else
-                addDataFilePrivate(dataStream(string(full_path), z), string(full_path), string(media_type));
+                addDataFilePrivate(dataStream(full_path, z), string(full_path), string(media_type));
         }
         if(!mimeFound)
             THROW("Manifest is missing mediatype file entry.");
 
-        for(const string &file: list)
+        for(const string &file: z.list())
         {
             /**
              * http://www.etsi.org/deliver/etsi_ts/102900_102999/102918/01.03.01_60/ts_102918v010301p.pdf
@@ -266,12 +248,9 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
             if(file.compare(0, 9, "META-INF/") == 0 &&
                file.find("signatures") != string::npos)
             {
-                if(count(list.begin(), list.end(), file) > 1)
-                    THROW("Multiple signature files with same name found '%s'", file.c_str());
                 try
                 {
-                    stringstream data;
-                    z.extract(file, data);
+                    auto data = z.extract<stringstream>(file);
                     auto signatures = make_shared<Signatures>(data, this);
                     for(auto s = signatures->signature(); s; s++)
                         addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));
@@ -285,7 +264,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
 
             if(file == "mimetype" || file.compare(0, 8,"META-INF") == 0)
                 continue;
-            if(manifestFiles.find(file) == manifestFiles.end())
+            if(manifestFiles.count(file) == 0)
                 THROW("File '%s' found in container is not described in manifest.", file.c_str());
         }
     }

src/ASiC_E.h

@@ -23,8 +23,6 @@
 
 namespace digidoc
 {
-    class ZipSerialize;
-
     /**
      * Implements the BDOC specification of the signed digital document container.
      * Container can contain several files and all these files can be signed using

src/ASiC_S.cpp

@@ -23,7 +23,6 @@
 #include "SignatureXAdES_LTA.h"
 #include "util/File.h"
 #include "util/log.h"
-#include "util/ZipSerialize.h"
 
 #include <algorithm>
 #include <sstream>
@@ -46,7 +45,7 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
     auto z = load(path, false, {mediaType()});
     static const string_view metaInf = "META-INF/";
 
-    for(const string &file: z->list())
+    for(const string &file: z.list())
     {
         if(file == "mimetype" ||
             (metaInf.size() < file.size() && file.compare(0, metaInf.size(), metaInf) == 0))
@@ -55,16 +54,13 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
             {
                 if(!signatures().empty())
                     THROW("Can not add signature to ASiC-S container which already contains a signature.");
-                stringstream data;
-                z->extract(file, data);
-                addSignature(make_unique<SignatureTST>(data, this));
+                addSignature(make_unique<SignatureTST>(z.extract<stringstream>(file).str(), this));
             }
             if(file == "META-INF/signatures.xml")
             {
                 if(!signatures().empty())
                     THROW("Can not add signature to ASiC-S container which already contains a signature.");
-                stringstream data;
-                z->extract(file, data);
+                auto data = z.extract<stringstream>(file);
                 auto signatures = make_shared<Signatures>(data, this);
                 for(auto s = signatures->signature(); s; s++)
                     addSignature(make_unique<SignatureXAdES_LTA>(signatures, s, this));
@@ -77,7 +73,7 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
         {
             if(!dataFiles().empty())
                 THROW("Can not add document to ASiC-S container which already contains a document.");
-            addDataFile(dataStream(file, *z), file, "application/octet-stream");
+            addDataFile(dataStream(file, z), file, "application/octet-stream");
         }
     }
 
@@ -139,7 +135,7 @@ bool ASiC_S::isContainerSimpleFormat(const string &path)
     {
         ZipSerialize z(path, false);
         vector<string> list = z.list();
-        return !list.empty() && list.front() == "mimetype" && readMimetype(z) == MIMETYPE_ASIC_S;
+        return list.front() == "mimetype" && readMimetype(z) == MIMETYPE_ASIC_S;
     }
     catch(const Exception &)
     {

src/ASiC_S.h

@@ -23,8 +23,6 @@
 
 namespace digidoc
 {
-    class ZipSerialize;
-
     /**
     * Implements the ASiC-S specification of the timestamped digital document container.
     * Container contains a single datafile object and one time assertion file.

src/ASiContainer.cpp

@@ -42,7 +42,7 @@ class ASiContainer::Private
     string mimetype, path;
     vector<DataFile*> documents;
     vector<Signature*> signatures;
-    map<string, ZipSerialize::Properties> properties;
+    map<string, ZipSerialize::Properties, std::less<>> properties;
 };
 
 /**
@@ -62,29 +62,26 @@ ASiContainer::ASiContainer(string_view mimetype)
  * @param supported supported mimetypes.
  * @return returns zip serializer for the container.
  */
-unique_ptr<ZipSerialize> ASiContainer::load(const string &path, bool mimetypeRequired, const set<string_view> &supported)
+ZipSerialize ASiContainer::load(const string &path, bool mimetypeRequired, const set<string_view> &supported)
 {
     DEBUG("ASiContainer::ASiContainer(path = '%s')", path.c_str());
-    auto z = make_unique<ZipSerialize>(d->path = path, false);
-
-    vector<string> list = z->list();
-    if(list.empty())
-        THROW("Failed to parse container");
+    ZipSerialize z(d->path = path, false);
+    vector<string> list = z.list();
 
     // ETSI TS 102 918: mimetype has to be the first in the archive
     if(mimetypeRequired && list.front() != "mimetype")
         THROW("required mimetype not found");
 
     if(list.front() == "mimetype")
     {
-        d->mimetype = readMimetype(*z);
+        d->mimetype = readMimetype(z);
         if(supported.find(d->mimetype) == supported.cend())
             THROW("Incorrect mimetype '%s'", d->mimetype.c_str());
     }
     DEBUG("mimetype = '%s'", d->mimetype.c_str());
 
     for(const string &file: list)
-        d->properties[file] = z->properties(file);
+        d->properties[file] = z.properties(file);
 
     return z;
 }
@@ -134,15 +131,11 @@ vector<Signature *> ASiContainer::signatures() const
  * @param z Zip container.
  * @return returns data as a stream.
  */
-unique_ptr<iostream> ASiContainer::dataStream(const string &path, const ZipSerialize &z) const
+unique_ptr<iostream> ASiContainer::dataStream(string_view path, const ZipSerialize &z) const
 {
-    unique_ptr<iostream> data;
-    if(d->properties[path].size > MAX_MEM_FILE)
-        data = make_unique<fstream>(File::tempFileName(), fstream::in|fstream::out|fstream::binary|fstream::trunc);
-    else
-        data = make_unique<stringstream>();
-    z.extract(path, *data);
-    return data;
+    if(auto i = d->properties.find(path); i != d->properties.cend() && i->second.size > MAX_MEM_FILE)
+        return make_unique<fstream>(z.extract<fstream>(path));
+    return make_unique<stringstream>(z.extract<stringstream>(path));
 }
 
 /**
@@ -275,9 +268,7 @@ const ZipSerialize::Properties& ASiContainer::zproperty(const string &file) cons
 string ASiContainer::readMimetype(const ZipSerialize &z)
 {
     DEBUG("ASiContainer::readMimetype()");
-    stringstream is;
-    z.extract("mimetype", is);
-    string text = is.str();
+    string text = z.extract<stringstream>("mimetype").str();
     text.erase(text.find_last_not_of(" \n\r\f\t\v") + 1);
     if(text.empty())
         THROW("Failed to read mimetype.");

src/ASiContainer.h

@@ -28,11 +28,11 @@ namespace digidoc
 {
     /**
      * Base class for the ASiC (Associated Signature Container) documents.
-     * Implements the operations and data structures common for more specific ASiC 
+     * Implements the operations and data structures common for more specific ASiC
      * signature containers like ASiC-S and ASiC-E (e.g. Estonian BDoc).
      * See standards ETSI TS 102 918, ETSI TS 103 171, ETSI TS 103 174 for details.
      *
-     * Contains methods for detecting the container type and manipulating the container's 
+     * Contains methods for detecting the container type and manipulating the container's
      * zip archive.
      */
     class ASiContainer: public Container
@@ -60,8 +60,8 @@ namespace digidoc
 
           void addDataFilePrivate(std::unique_ptr<std::istream> is, std::string fileName, std::string mediaType);
           Signature* addSignature(std::unique_ptr<Signature> &&signature);
-          std::unique_ptr<std::iostream> dataStream(const std::string &path, const ZipSerialize &z) const;
-          std::unique_ptr<ZipSerialize> load(const std::string &path, bool requireMimetype, const std::set<std::string_view> &supported);
+          std::unique_ptr<std::iostream> dataStream(std::string_view path, const ZipSerialize &z) const;
+          ZipSerialize load(const std::string &path, bool requireMimetype, const std::set<std::string_view> &supported);
           void deleteSignature(Signature* s);
 
           void zpath(const std::string &file);

src/SiVaContainer.cpp

@@ -159,7 +159,7 @@ SiVaContainer::SiVaContainer(const string &path, ContainerOpenCB *cb, bool useHa
         static const string_view metaInf = "META-INF/";
         ZipSerialize z(path, false);
         vector<string> list = z.list();
-        if(list.empty() || list.front() != "mimetype")
+        if(list.front() != "mimetype")
             THROW("Missing mimetype");
         if(d->mediaType = ASiContainer::readMimetype(z);
             d->mediaType != ASiContainer::MIMETYPE_ASIC_E && d->mediaType != ASiContainer::MIMETYPE_ASIC_S)
@@ -173,13 +173,9 @@ SiVaContainer::SiVaContainer(const string &path, ContainerOpenCB *cb, bool useHa
         {
             if(file == "mimetype" || file.rfind(metaInf, 0) == 0)
                 continue;
-            const auto directory = File::directory(file);
-            if(directory.empty() || directory == "/" || directory == "./")
-            {
-                auto data = make_unique<stringstream>();
-                z.extract(file, *data);
-                d->dataFiles.push_back(new DataFilePrivate(std::move(data), file, "application/octet-stream"));
-            }
+            if(const auto directory = File::directory(file);
+                directory.empty() || directory == "/" || directory == "./")
+                d->dataFiles.push_back(new DataFilePrivate(make_unique<stringstream>(z.extract<stringstream>(file)), file, "application/octet-stream"));
         }
     }
     else

src/SignatureTST.cpp

@@ -29,19 +29,10 @@
 using namespace digidoc;
 using namespace std;
 
-SignatureTST::SignatureTST(istream &is, ASiC_S *asicSDoc): asicSDoc(asicSDoc)
-{
-    is.seekg(0, istream::end);
-    istream::pos_type pos = is.tellg();
-    const auto size = pos < 0 ? 0 : (unsigned long)pos;
-    is.clear();
-    is.seekg(0, istream::beg);
-
-    vector<unsigned char> buf(size, 0);
-    is.read((char*)buf.data(), streamsize(buf.size()));
-
-    timestampToken = make_unique<TS>(buf.data(), buf.size());
-}
+SignatureTST::SignatureTST(const string &data, ASiC_S *asicSDoc)
+    : asicSDoc(asicSDoc)
+    , timestampToken(make_unique<TS>((const unsigned char*)data.data(), data.size()))
+{}
 
 SignatureTST::~SignatureTST() = default;
 
@@ -93,8 +84,7 @@ void SignatureTST::validate() const
     try
     {
         const string digestMethod = timestampToken->digestMethod();
-        const auto *dataFile = static_cast<const DataFilePrivate*>(asicSDoc->dataFiles().front());
-        timestampToken->verify(dataFile->calcDigest(digestMethod));
+        timestampToken->verify(asicSDoc->dataFiles().front()->calcDigest(digestMethod));
 
         if(!Exception::hasWarningIgnore(Exception::ReferenceDigestWeak) &&
             Digest::isWeakDigest(digestMethod))

src/SignatureTST.h

@@ -31,7 +31,7 @@ class TS;
 class SignatureTST final: public Signature
 {
 public:
-    SignatureTST(std::istream &sigdata, ASiC_S *asicSDoc);
+    SignatureTST(const std::string &data, ASiC_S *asicSDoc);
     ~SignatureTST();
 
     std::string trustedSigningTime() const final;