Skip to content

Commit ad9523e

Browse files
metsmametsma
committed
ZIP Streaming
IB-7625 Signed-off-by: Raul Metsma <[email protected]>
1 parent 14e2f1f commit ad9523e

File tree

11 files changed

+142
-82
lines changed

11 files changed

+142
-82
lines changed

‎src/ASiC_E.cpp

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ ASiC_E::ASiC_E(const string &path)
6767
: ASiContainer(MIMETYPE_ASIC_E)
6868
, d(make_unique<Private>())
6969
{
70-
auto zip = load(path, true, {MIMETYPE_ASIC_E, MIMETYPE_ADOC});
70+
auto *zip = load(path, true, {MIMETYPE_ASIC_E, MIMETYPE_ADOC});
7171
parseManifestAndLoadFiles(*zip);
7272
}
7373

@@ -229,12 +229,11 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
229229

230230
try
231231
{
232-
stringstream manifestdata;
233-
z.extract("META-INF/manifest.xml", manifestdata);
232+
unique_ptr<istream> manifestdata = z.stream("META-INF/manifest.xml");
234233
xml_schema::Properties p;
235234
p.schema_location(ASiC_E::MANIFEST_NAMESPACE,
236235
File::fullPathUrl(Conf::instance()->xsdPath() + "/OpenDocument_manifest.xsd"));
237-
unique_ptr<xercesc::DOMDocument> doc = SecureDOMParser(p.schema_location(), true).parseIStream(manifestdata);
236+
unique_ptr<xercesc::DOMDocument> doc = SecureDOMParser(p.schema_location(), true).parseIStream(*manifestdata);
238237
unique_ptr<manifest::Manifest> manifest = manifest::manifest(*doc, {}, p);
239238

240239
set<string> manifestFiles;
@@ -267,9 +266,9 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
267266
if(mediaType() == MIMETYPE_ADOC &&
268267
(file.full_path().compare(0, 9, "META-INF/") == 0 ||
269268
file.full_path().compare(0, 9, "metadata/") == 0))
270-
d->metadata.push_back(new DataFilePrivate(dataStream(file.full_path(), z), file.full_path(), file.media_type()));
269+
d->metadata.push_back(dataFile(file.full_path(), file.media_type()));
271270
else
272-
addDataFilePrivate(dataStream(file.full_path(), z), file.full_path(), file.media_type());
271+
addDataFilePrivate(file.full_path(), file.media_type());
273272
}
274273
if(!mimeFound)
275274
THROW("Manifest is missing mediatype file entry.");
@@ -288,9 +287,7 @@ void ASiC_E::parseManifestAndLoadFiles(const ZipSerialize &z)
288287
THROW("Multiple signature files with same name found '%s'", file.c_str());
289288
try
290289
{
291-
stringstream data;
292-
z.extract(file, data);
293-
auto signatures = make_shared<Signatures>(data, this);
290+
auto signatures = make_shared<Signatures>(*z.stream(file), this);
294291
for(size_t i = 0, count = signatures->count(); i < count; ++i)
295292
addSignature(make_unique<SignatureXAdES_LTA>(signatures, i, this));
296293
}

‎src/ASiC_S.cpp

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ ASiC_S::ASiC_S(): ASiContainer(MIMETYPE_ASIC_S)
4343
*/
4444
ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
4545
{
46-
auto z = load(path, false, {mediaType()});
46+
auto *z = load(path, false, {mediaType()});
4747
static const string_view metaInf = "META-INF/";
4848

4949
for(const string &file: z->list())
@@ -55,17 +55,13 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
5555
{
5656
if(!signatures().empty())
5757
THROW("Can not add signature to ASiC-S container which already contains a signature.");
58-
stringstream data;
59-
z->extract(file, data);
60-
addSignature(make_unique<SignatureTST>(data, this));
58+
addSignature(make_unique<SignatureTST>(*z->stream(file), this));
6159
}
6260
if(file == "META-INF/signatures.xml")
6361
{
6462
if(!signatures().empty())
6563
THROW("Can not add signature to ASiC-S container which already contains a signature.");
66-
stringstream data;
67-
z->extract(file, data);
68-
auto signatures = make_shared<Signatures>(data, this);
64+
auto signatures = make_shared<Signatures>(*z->stream(file), this);
6965
for(size_t i = 0, count = signatures->count(); i < count; ++i)
7066
addSignature(make_unique<SignatureXAdES_LTA>(signatures, i, this));
7167
}
@@ -77,7 +73,7 @@ ASiC_S::ASiC_S(const string &path): ASiContainer(MIMETYPE_ASIC_S)
7773
{
7874
if(!dataFiles().empty())
7975
THROW("Can not add document to ASiC-S container which already contains a document.");
80-
addDataFile(dataStream(file, *z), file, "application/octet-stream");
76+
addDataFilePrivate(file, "application/octet-stream");
8177
}
8278
}
8379

‎src/ASiContainer.cpp

Lines changed: 18 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ class ASiContainer::Private
4646
vector<DataFile*> documents;
4747
vector<Signature*> signatures;
4848
map<string, ZipSerialize::Properties> properties;
49+
unique_ptr<ZipSerialize> z;
4950
};
5051

5152
const string_view ASiContainer::ASICE_EXTENSION = "asice";
@@ -76,12 +77,12 @@ ASiContainer::ASiContainer(const string &mimetype)
7677
* @param supported supported mimetypes.
7778
* @return returns zip serializer for the container.
7879
*/
79-
unique_ptr<ZipSerialize> ASiContainer::load(const string &path, bool mimetypeRequired, const set<string> &supported)
80+
ZipSerialize* ASiContainer::load(const string &path, bool mimetypeRequired, const set<string> &supported)
8081
{
8182
DEBUG("ASiContainer::ASiContainer(path = '%s')", path.c_str());
82-
auto z = make_unique<ZipSerialize>(d->path = path, false);
83+
d->z = make_unique<ZipSerialize>(d->path = path, false);
8384

84-
vector<string> list = z->list();
85+
vector<string> list = d->z->list();
8586
if(list.empty())
8687
THROW("Failed to parse container");
8788

@@ -91,16 +92,16 @@ unique_ptr<ZipSerialize> ASiContainer::load(const string &path, bool mimetypeReq
9192
// ETSI TS 102 918: mimetype has to be the first in the archive
9293
if(list.front() == "mimetype")
9394
{
94-
d->mimetype = readMimetype(*z);
95+
d->mimetype = readMimetype(*d->z);
9596
DEBUG("mimetype = '%s'", d->mimetype.c_str());
9697
if(supported.find(d->mimetype) == supported.cend())
9798
THROW("Incorrect mimetype '%s'", d->mimetype.c_str());
9899
}
99100

100101
for(const string &file: list)
101-
d->properties[file] = z->properties(file);
102+
d->properties[file] = d->z->properties(file);
102103

103-
return z;
104+
return d->z.get();
104105
}
105106

106107
string ASiContainer::mediaType() const
@@ -138,25 +139,9 @@ vector<Signature *> ASiContainer::signatures() const
138139
return d->signatures;
139140
}
140141

141-
/**
142-
* <p>
143-
* Read a datafile from container.
144-
* </p>
145-
* If expected size of the data is too big, then stream is written to temp file.
146-
*
147-
* @param path name of the file in zip container stream is used to read from.
148-
* @param z Zip container.
149-
* @return returns data as a stream.
150-
*/
151-
unique_ptr<iostream> ASiContainer::dataStream(const string &path, const ZipSerialize &z) const
142+
DataFilePrivate* ASiContainer::dataFile(const std::string &path, const std::string &mediaType) const
152143
{
153-
unique_ptr<iostream> data;
154-
if(d->properties[path].size > MAX_MEM_FILE)
155-
data = make_unique<fstream>(File::tempFileName(), fstream::in|fstream::out|fstream::binary|fstream::trunc);
156-
else
157-
data = make_unique<stringstream>();
158-
z.extract(path, *data);
159-
return data;
144+
return new DataFilePrivate(d->z->stream(path), path, mediaType, d->properties[path].size);
160145
}
161146

162147
/**
@@ -191,15 +176,16 @@ void ASiContainer::addDataFile(const string &path, const string &mediaType)
191176
*data << file.rdbuf();
192177
is = std::move(data);
193178
}
194-
addDataFilePrivate(std::move(is), fileName, mediaType);
179+
d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType, prop.size));
195180
}
196181

197182
void ASiContainer::addDataFile(unique_ptr<istream> is, const string &fileName, const string &mediaType)
198183
{
199184
addDataFileChecks(fileName, mediaType);
200185
if(fileName.find_last_of("/\\") != string::npos)
201186
THROW("Document file '%s' cannot contain directory path.", fileName.c_str());
202-
addDataFilePrivate(std::move(is), fileName, mediaType);
187+
istream::pos_type pos = is->tellg();
188+
d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType, pos < 0 ? 0 : (unsigned long)pos));
203189
}
204190

205191
void ASiContainer::addDataFileChecks(const string &fileName, const string &mediaType)
@@ -214,9 +200,9 @@ void ASiContainer::addDataFileChecks(const string &fileName, const string &media
214200
THROW("MediaType does not meet format requirements (RFC2045, section 5.1) '%s'.", mediaType.c_str());
215201
}
216202

217-
void ASiContainer::addDataFilePrivate(unique_ptr<istream> is, const string &fileName, const string &mediaType)
203+
void ASiContainer::addDataFilePrivate(const string &fileName, const string &mediaType)
218204
{
219-
d->documents.push_back(new DataFilePrivate(std::move(is), fileName, mediaType));
205+
d->documents.push_back(dataFile(fileName, mediaType));
220206
}
221207

222208
/**
@@ -298,17 +284,16 @@ void ASiContainer::zproperty(const string &file, ZipSerialize::Properties &&prop
298284
string ASiContainer::readMimetype(const ZipSerialize &z)
299285
{
300286
DEBUG("ASiContainer::readMimetype()");
301-
stringstream is;
302-
z.extract("mimetype", is);
287+
auto is = z.stream("mimetype");
303288
string text;
304-
is >> text;
289+
*is >> text;
305290
if(!is)
306291
THROW("Failed to read mimetype.");
307292
// Contains UTF-16 BOM
308-
if(text.find("\xFF\xEF") == 0 || text.find("\xEF\xFF") == 0)
293+
if(text.rfind("\xFF\xEF", 0) == 0 || text.rfind("\xEF\xFF", 0) == 0)
309294
THROW("Mimetype file must be UTF-8 format.");
310295
// contains UTF-8 BOM, remove
311-
if(text.find("\xEF\xBB\xBF") == 0)
296+
if(text.rfind("\xEF\xBB\xBF", 0) == 0)
312297
text.erase(text.cbegin(), text.cbegin() + 3);
313298
return text;
314299
}

‎src/ASiContainer.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
namespace digidoc
2828
{
29+
class DataFilePrivate;
2930
/**
3031
* Base class for the ASiC (Associated Signature Container) documents.
3132
* Implements the operations and data structures common for more specific ASiC
@@ -63,10 +64,10 @@ namespace digidoc
6364
protected:
6465
ASiContainer(const std::string &mimetype);
6566

66-
void addDataFilePrivate(std::unique_ptr<std::istream> is, const std::string &fileName, const std::string &mediaType);
67+
void addDataFilePrivate(const std::string &fileName, const std::string &mediaType);
6768
Signature* addSignature(std::unique_ptr<Signature> &&signature);
68-
std::unique_ptr<std::iostream> dataStream(const std::string &path, const ZipSerialize &z) const;
69-
std::unique_ptr<ZipSerialize> load(const std::string &path, bool requireMimetype, const std::set<std::string> &supported);
69+
DataFilePrivate *dataFile(const std::string &path, const std::string &mediaType) const;
70+
ZipSerialize* load(const std::string &path, bool requireMimetype, const std::set<std::string> &supported);
7071
void deleteSignature(Signature* s);
7172

7273
void zpath(const std::string &file);

‎src/DataFile.cpp

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -87,16 +87,13 @@ DataFile::DataFile() = default;
8787
DataFile::~DataFile() = default;
8888

8989

90-
DataFilePrivate::DataFilePrivate(unique_ptr<istream> &&is, string filename, string mediatype, string id)
90+
DataFilePrivate::DataFilePrivate(unique_ptr<istream> &&is, string filename, string mediatype, unsigned long size, string id)
9191
: m_is(std::move(is))
9292
, m_id(std::move(id))
9393
, m_filename(std::move(filename))
9494
, m_mediatype(std::move(mediatype))
95-
{
96-
m_is->seekg(0, istream::end);
97-
istream::pos_type pos = m_is->tellg();
98-
m_size = pos < 0 ? 0 : (unsigned long)pos;
99-
}
95+
, m_size(size)
96+
{}
10097

10198
vector<unsigned char> DataFilePrivate::calcDigest(const string &method) const
10299
{

‎src/DataFile_p.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ namespace digidoc
3030
class DataFilePrivate final: public DataFile
3131
{
3232
public:
33-
DataFilePrivate(std::unique_ptr<std::istream> &&is, std::string filename, std::string mediatype, std::string id = {});
33+
DataFilePrivate(std::unique_ptr<std::istream> &&is, std::string filename, std::string mediatype, unsigned long size, std::string id = {});
3434

3535
std::string id() const final { return m_id.empty() ? m_filename : m_id; }
3636
std::string fileName() const final { return m_filename; }

‎src/SiVaContainer.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ SiVaContainer::SiVaContainer(const string &path, bool useHashCode)
163163
else if(ext == "pdf")
164164
{
165165
d->mediaType = "application/pdf";
166-
d->dataFiles.push_back(new DataFilePrivate(std::move(ifs), fileName, "application/pdf"));
166+
d->dataFiles.push_back(new DataFilePrivate(std::move(ifs), fileName, "application/pdf", File::fileSize(d->path)));
167167
}
168168
else if(find(asic.cbegin(), asic.cend(), ext) != asic.cend())
169169
{
@@ -185,9 +185,8 @@ SiVaContainer::SiVaContainer(const string &path, bool useHashCode)
185185
const auto directory = File::directory(file);
186186
if(directory.empty() || directory == "/" || directory == "./")
187187
{
188-
auto data = make_unique<stringstream>();
189-
z.extract(file, *data);
190-
d->dataFiles.push_back(new DataFilePrivate(std::move(data), file, "application/octet-stream"));
188+
auto properties = z.properties(file);
189+
d->dataFiles.push_back(new DataFilePrivate(z.stream(file), file, "application/octet-stream", properties.size));
191190
}
192191
}
193192
}
@@ -381,6 +380,7 @@ unique_ptr<istream> SiVaContainer::parseDDoc(bool useHashCode)
381380
d->dataFiles.push_back(new DataFilePrivate(make_unique<stringstream>(base64_decode(b64)),
382381
xml::transcode<char>(item->getAttribute(cpXMLCh(u"Filename"))),
383382
xml::transcode<char>(item->getAttribute(cpXMLCh(u"MimeType"))),
383+
0,
384384
xml::transcode<char>(item->getAttribute(cpXMLCh(u"Id")))));
385385
}
386386

‎src/SignatureTST.cpp

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,21 +26,16 @@
2626
#include "util/DateTime.h"
2727
#include "util/log.h"
2828

29+
#include <sstream>
30+
2931
using namespace digidoc;
3032
using namespace std;
3133

3234
SignatureTST::SignatureTST(istream &is, ASiC_S *asicSDoc): asicSDoc(asicSDoc)
3335
{
34-
is.seekg(0, istream::end);
35-
istream::pos_type pos = is.tellg();
36-
const auto size = pos < 0 ? 0 : (unsigned long)pos;
37-
is.clear();
38-
is.seekg(0, istream::beg);
39-
40-
vector<unsigned char> buf(size, 0);
41-
is.read((char*)buf.data(), streamsize(buf.size()));
42-
43-
timestampToken = make_unique<TS>(buf.data(), buf.size());
36+
stringstream data;
37+
data << is.rdbuf();
38+
timestampToken = make_unique<TS>(data.str());
4439
}
4540

4641
SignatureTST::~SignatureTST() = default;

‎src/crypto/TS.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ class TS
3131
{
3232
public:
3333
TS(const std::string &url, const Digest &digest);
34+
template <class Container>
35+
inline TS(const Container &data): TS((const unsigned char*)data.data(), data.size()) {}
3436
TS(const unsigned char *data = nullptr, size_t size = 0);
3537

3638
X509Cert cert() const;

0 commit comments

Comments
 (0)