-
Notifications
You must be signed in to change notification settings - Fork 721
Use file content heuristics to decide file reader. #1962
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: dev
Are you sure you want to change the base?
Changes from 10 commits
02de760
d2b6339
685dd9f
40dee69
f1e3e18
8da1790
3ad51e2
15c2000
17af8d4
46418ec
3d713ab
a2391ec
ea328d7
db86c3e
4aed9bd
a83ae2b
916e872
022529f
169fcd2
db8c848
3e74912
f1613c4
bc2bacd
58ac45d
3b4b5ad
18379b4
8a4f6f8
7776e0e
4f52f59
88ebfff
41fe188
c8ae4f8
c7cab2b
6d55077
682eeac
07804da
9c4fc08
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
#define LOG_MODULE PcapLogModuleFileDevice | ||
|
||
#include <cerrno> | ||
#include <array> | ||
#include <algorithm> | ||
#include "PcapFileDevice.h" | ||
#include "light_pcapng_ext.h" | ||
#include "Logger.h" | ||
|
@@ -28,32 +30,186 @@ namespace pcpp | |
{ | ||
return reinterpret_cast<light_pcapng_t*>(pcapngHandle); | ||
} | ||
|
||
struct pcap_file_header | ||
{ | ||
uint32_t magic; | ||
uint16_t version_major; | ||
uint16_t version_minor; | ||
int32_t thiszone; | ||
uint32_t sigfigs; | ||
uint32_t snaplen; | ||
uint32_t linktype; | ||
}; | ||
|
||
struct packet_header | ||
{ | ||
uint32_t tv_sec; | ||
uint32_t tv_usec; | ||
uint32_t caplen; | ||
uint32_t len; | ||
}; | ||
|
||
/// @brief Check if a stream is seekable. | ||
/// @param stream The stream to check. | ||
/// @return True if the stream supports seek operations, false otherwise. | ||
bool isStreamSeekable(std::istream& stream) | ||
{ | ||
auto pos = stream.tellg(); | ||
if (stream.fail()) | ||
{ | ||
stream.clear(); | ||
return false; | ||
} | ||
|
||
if (stream.seekg(pos).fail()) | ||
{ | ||
stream.clear(); | ||
return false; | ||
} | ||
|
||
return true; | ||
} | ||
|
||
class StreamPositionCheckpoint | ||
{ | ||
public: | ||
explicit StreamPositionCheckpoint(std::istream& stream) : m_Stream(stream), m_Pos(stream.tellg()) | ||
{} | ||
|
||
~StreamPositionCheckpoint() | ||
{ | ||
m_Stream.seekg(m_Pos); | ||
} | ||
|
||
private: | ||
std::istream& m_Stream; | ||
std::streampos m_Pos; | ||
}; | ||
|
||
enum class CaptureFileFormat | ||
{ | ||
Unknown, | ||
Pcap, | ||
PcapNG, | ||
Snoop, | ||
}; | ||
Comment on lines
+95
to
+101
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. ditto: this can be an enum inside of There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I suppose. It has internal linkage so it doesn't really matter. But then we would end up with really long case names: There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I guess it's fine? It's all internal anyway... |
||
|
||
/// @brief Heuristic file format detector that scans the magic number of the file format header. | ||
class CaptureFileFormatDetector | ||
{ | ||
public: | ||
/// @brief Checks a content stream for the magic number and determines the type. | ||
/// @param content A content stream that contains the file content. | ||
/// @return A CaptureFileFormat value with the detected content type. | ||
CaptureFileFormat detectFormat(std::istream& content) | ||
{ | ||
// Check if the stream supports seeking. | ||
if (!isStreamSeekable(content)) | ||
{ | ||
throw std::runtime_error("Heuristic file format detection requires seekable stream"); | ||
} | ||
|
||
if (isPcapFile(content)) | ||
return CaptureFileFormat::Pcap; | ||
|
||
// PcapNG backend can support ZstdCompressed Pcap files, so we assume an archive is compressed PcapNG. | ||
if (isPcapNgFile(content) || isZstdArchive(content)) | ||
return CaptureFileFormat::PcapNG; | ||
|
||
if (isSnoopFile(content)) | ||
return CaptureFileFormat::Snoop; | ||
|
||
return CaptureFileFormat::Unknown; | ||
} | ||
|
||
private: | ||
bool isPcapFile(std::istream& content) | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
constexpr std::array<uint32_t, 4> pcapMagicNumbers = { | ||
0xa1'b2'c3'd4, // regular pcap, microsecond-precision | ||
0xd4'c3'b2'a1, // regular pcap, microsecond-precision (byte-swapped) | ||
0xa1'b2'3c'4d, // regular pcap, nanosecond-precision | ||
0x4d'3c'b2'a1 // regular pcap, nanosecond-precision (byte-swapped) | ||
}; | ||
Dimi1010 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
||
StreamPositionCheckpoint checkpoint(content); | ||
|
||
pcap_file_header header; | ||
content.read(reinterpret_cast<char*>(&header), sizeof(header)); | ||
if (content.gcount() != sizeof(header)) | ||
{ | ||
return false; | ||
} | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
return std::find(pcapMagicNumbers.begin(), pcapMagicNumbers.end(), header.magic) != | ||
pcapMagicNumbers.end(); | ||
} | ||
|
||
bool isPcapNgFile(std::istream& content) | ||
{ | ||
constexpr std::array<uint32_t, 1> pcapMagicNumbers = { | ||
0x0A'0D'0D'0A, // pcapng magic number (palindrome) | ||
}; | ||
|
||
StreamPositionCheckpoint checkpoint(content); | ||
|
||
uint32_t magic = 0; | ||
content.read(reinterpret_cast<char*>(&magic), sizeof(magic)); | ||
if (content.gcount() != sizeof(magic)) | ||
{ | ||
return false; | ||
} | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
return std::find(pcapMagicNumbers.begin(), pcapMagicNumbers.end(), magic) != pcapMagicNumbers.end(); | ||
} | ||
|
||
bool isSnoopFile(std::istream& content) | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
{ | ||
constexpr std::array<uint64_t, 2> snoopMagicNumbers = { | ||
0x73'6E'6F'6F'70'00'00'00, // snoop magic number, "snoop" in ASCII | ||
0x00'00'00'70'6F'6F'6E'73 // snoop magic number, "snoop" in ASCII (byte-swapped) | ||
}; | ||
|
||
StreamPositionCheckpoint checkpoint(content); | ||
|
||
uint64_t magic = 0; | ||
content.read(reinterpret_cast<char*>(&magic), sizeof(magic)); | ||
if (content.gcount() != sizeof(magic)) | ||
{ | ||
return false; | ||
} | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
return std::find(snoopMagicNumbers.begin(), snoopMagicNumbers.end(), magic) != snoopMagicNumbers.end(); | ||
} | ||
|
||
bool isZstdArchive(std::istream& content) | ||
{ | ||
constexpr std::array<uint32_t, 2> zstdMagicNumbers = { | ||
0x28'B5'2F'FD, // zstd archive magic number | ||
0xFD'2F'B5'28, // zstd archive magic number (byte-swapped) | ||
}; | ||
|
||
StreamPositionCheckpoint checkpoint(content); | ||
|
||
uint32_t magic = 0; | ||
content.read(reinterpret_cast<char*>(&magic), sizeof(magic)); | ||
if (content.gcount() != sizeof(magic)) | ||
{ | ||
return false; | ||
} | ||
seladb marked this conversation as resolved.
Show resolved
Hide resolved
|
||
|
||
return std::find(zstdMagicNumbers.begin(), zstdMagicNumbers.end(), magic) != zstdMagicNumbers.end(); | ||
} | ||
}; | ||
|
||
} // namespace | ||
|
||
template <typename T, size_t N> constexpr size_t ARRAY_SIZE(T (&)[N]) | ||
{ | ||
return N; | ||
} | ||
|
||
struct pcap_file_header | ||
{ | ||
uint32_t magic; | ||
uint16_t version_major; | ||
uint16_t version_minor; | ||
int32_t thiszone; | ||
uint32_t sigfigs; | ||
uint32_t snaplen; | ||
uint32_t linktype; | ||
}; | ||
|
||
struct packet_header | ||
{ | ||
uint32_t tv_sec; | ||
uint32_t tv_usec; | ||
uint32_t caplen; | ||
uint32_t len; | ||
}; | ||
|
||
static bool checkNanoSupport() | ||
{ | ||
#if defined(PCAP_TSTAMP_PRECISION_NANO) | ||
|
@@ -130,6 +286,27 @@ namespace pcpp | |
return new PcapFileReaderDevice(fileName); | ||
} | ||
|
||
std::unique_ptr<IFileReaderDevice> IFileReaderDevice::createReader(const std::string& fileName) | ||
{ | ||
std::ifstream fileContent(fileName, std::ios_base::binary); | ||
if (fileContent.fail()) | ||
{ | ||
throw std::runtime_error("Could not open: " + fileName); | ||
} | ||
|
||
switch (CaptureFileFormatDetector().detectFormat(fileContent)) | ||
{ | ||
case CaptureFileFormat::Pcap: | ||
return std::make_unique<PcapFileReaderDevice>(fileName); | ||
case CaptureFileFormat::PcapNG: | ||
return std::make_unique<PcapNgFileReaderDevice>(fileName); | ||
case CaptureFileFormat::Snoop: | ||
return std::make_unique<SnoopFileReaderDevice>(fileName); | ||
default: | ||
return nullptr; | ||
} | ||
} | ||
|
||
uint64_t IFileReaderDevice::getFileSize() const | ||
{ | ||
std::ifstream fileStream(m_FileName.c_str(), std::ifstream::ate | std::ifstream::binary); | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -754,7 +754,7 @@ PTF_TEST_CASE(TestPcapNgFileReadWriteAdv) | |
|
||
PTF_ASSERT_EQUAL(packetCount, 161); | ||
|
||
// ------- | ||
// ------- IFileReaderDevice::getReader() Factory | ||
|
||
// copy the .zstd file to a similar file with .zst extension | ||
std::ifstream zstdFile(EXAMPLE2_PCAPNG_ZSTD_WRITE_PATH, std::ios::binary); | ||
|
@@ -763,26 +763,57 @@ PTF_TEST_CASE(TestPcapNgFileReadWriteAdv) | |
zstdFile.close(); | ||
zstFile.close(); | ||
|
||
pcpp::IFileReaderDevice* genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAP_PATH); | ||
FileReaderTeardown genericReaderTeardown1(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAPNG_PATH); | ||
FileReaderTeardown genericReaderTeardown2(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE_PCAPNG_ZSTD_WRITE_PATH); | ||
FileReaderTeardown genericReaderTeardown3(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
{ | ||
pcpp::IFileReaderDevice* genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAP_PATH); | ||
FileReaderTeardown genericReaderTeardown1(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAPNG_PATH); | ||
FileReaderTeardown genericReaderTeardown2(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE_PCAPNG_ZSTD_WRITE_PATH); | ||
FileReaderTeardown genericReaderTeardown3(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAPNG_ZST_WRITE_PATH); | ||
FileReaderTeardown genericReaderTeardown4(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
|
||
genericReader->close(); | ||
} | ||
|
||
genericReader = pcpp::IFileReaderDevice::getReader(EXAMPLE2_PCAPNG_ZST_WRITE_PATH); | ||
FileReaderTeardown genericReaderTeardown4(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader)); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
// ------- IFileReaderDevice::createReader() Factory | ||
// TODO: Move to a separate unit test. | ||
Dimi1010 marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
||
genericReader->close(); | ||
{ | ||
PTF_ASSERT_RAISES(pcpp::IFileReaderDevice::createReader("BogusFile"), std::runtime_error, | ||
"Could not open: BogusFile"); | ||
|
||
auto genericReader = pcpp::IFileReaderDevice::createReader(EXAMPLE2_PCAP_PATH); | ||
PTF_ASSERT_NOT_NULL(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapFileReaderDevice*>(genericReader.get())); | ||
genericReader->close(); | ||
|
||
genericReader = pcpp::IFileReaderDevice::createReader(EXAMPLE2_PCAPNG_PATH); | ||
PTF_ASSERT_NOT_NULL(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader.get())); | ||
|
||
genericReader = pcpp::IFileReaderDevice::createReader(EXAMPLE_PCAPNG_ZSTD_WRITE_PATH); | ||
PTF_ASSERT_NOT_NULL(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader.get())); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
genericReader->close(); | ||
|
||
genericReader = pcpp::IFileReaderDevice::createReader(EXAMPLE2_PCAPNG_ZST_WRITE_PATH); | ||
PTF_ASSERT_NOT_NULL(genericReader); | ||
PTF_ASSERT_NOT_NULL(dynamic_cast<pcpp::PcapNgFileReaderDevice*>(genericReader.get())); | ||
PTF_ASSERT_TRUE(genericReader->open()); | ||
genericReader->close(); | ||
} | ||
|
||
// ------- | ||
|
||
|
Uh oh!
There was an error while loading. Please reload this page.