Skip to content

Commit b8acf0c

Browse files
SirLynixSirLynix
committed
[Tools] Add --skip-unchanged option to archiver and compiler
Introduces a --skip-unchanged option to nzsla and nzslc, preventing overwriting output files if the content has not changed.
1 parent ea6e9a6 commit b8acf0c

File tree

7 files changed

+101
-34
lines changed

7 files changed

+101
-34
lines changed

‎src/NZSL/Parser.cpp‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2060,14 +2060,9 @@ namespace nzsl
20602060
if (!inputFile)
20612061
throw std::runtime_error("failed to open " + Nz::PathToString(sourcePath));
20622062

2063-
inputFile.seekg(0, std::ios::end);
2064-
2065-
std::streamsize length = inputFile.tellg();
2066-
2067-
inputFile.seekg(0, std::ios::beg);
2068-
2069-
std::vector<char> content(Nz::SafeCast<std::size_t>(length));
2070-
if (length > 0 && !inputFile.read(&content[0], length))
2063+
std::size_t fileSize = Nz::SafeCaster(std::filesystem::file_size(sourcePath));
2064+
std::vector<char> content(fileSize);
2065+
if (fileSize > 0 && !inputFile.read(&content[0], fileSize))
20712066
throw std::runtime_error("failed to read " + Nz::PathToString(sourcePath));
20722067

20732068
return Parse(std::string_view(content.data(), content.size()), Nz::PathToString(sourcePath));

‎src/ShaderArchiver/Archiver.cpp‎

Lines changed: 42 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,8 @@ namespace nzsla
1717
m_isArchiving(false),
1818
m_isShowing(false),
1919
m_isVerbose(false),
20-
m_outputToStdout(false)
20+
m_outputToStdout(false),
21+
m_skipUnchangedOutput(false)
2122
{
2223
}
2324

@@ -26,6 +27,7 @@ namespace nzsla
2627
m_isArchiving = m_options.count("archive") > 0;
2728
m_isShowing = m_options.count("show") > 0;
2829
m_isVerbose = m_options.count("verbose") > 0;
30+
m_skipUnchangedOutput = m_options.count("skip-unchanged") > 0;
2931

3032
if (m_options.count("input") == 0)
3133
throw cxxopts::exceptions::specification("no input file");
@@ -84,7 +86,8 @@ namespace nzsla
8486

8587
options.add_options("archive")
8688
("a,archive", "Archives the input shaders to an archive.")
87-
("header", "Generates an includable header file.");
89+
("header", "Generates an includable header file.")
90+
("skip-unchanged", "After compilation, compare the output with the current output file and skip writing if the content is the same", cxxopts::value<bool>()->default_value("false"));
8891

8992
options.add_options("compression")
9093
("c,compress", "Compression algorithm", cxxopts::value<std::string>()->implicit_value("lz4hc"), "[none|lz4hc]");
@@ -156,13 +159,23 @@ namespace nzsla
156159
assert(outputHeader);
157160
fmt::print("{}", ToHeader(archiveData.data(), archiveData.size()));
158161
}
159-
else if (outputHeader)
162+
else
160163
{
161-
std::string headerFile = ToHeader(archiveData.data(), archiveData.size());
162-
WriteFileContent(outputFilePath, headerFile.data(), headerFile.size());
164+
const void* data = archiveData.data();
165+
std::size_t size = archiveData.size();
166+
167+
std::string headerFile;
168+
if (outputHeader)
169+
{
170+
headerFile = ToHeader(archiveData.data(), archiveData.size());
171+
data = headerFile.data();
172+
size = headerFile.size();
173+
}
174+
175+
bool written = WriteFileContent(outputFilePath, data, size);
176+
if (m_isVerbose)
177+
fmt::print("{} file {}\n", (written) ? "Generated" : "Skipped", Nz::PathToString(std::filesystem::absolute(outputFilePath)));
163178
}
164-
else
165-
WriteFileContent(outputFilePath, archiveData.data(), archiveData.size());
166179
}
167180

168181
void Archiver::DoShow()
@@ -238,13 +251,34 @@ namespace nzsla
238251
return std::move(ss).str();
239252
}
240253

241-
void Archiver::WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size)
254+
bool Archiver::WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size)
242255
{
256+
if (m_skipUnchangedOutput)
257+
{
258+
std::ifstream inputFile(filePath, std::ios::in | std::ios::binary);
259+
if (inputFile)
260+
{
261+
std::size_t fileSize = Nz::SafeCaster(std::filesystem::file_size(filePath));
262+
if (fileSize == size)
263+
{
264+
if (size == 0)
265+
return false;
266+
267+
// Compare file content
268+
std::vector<char> content(fileSize);
269+
if (inputFile.read(&content[0], fileSize) && std::memcmp(&content[0], data, size) == 0)
270+
return false;
271+
}
272+
}
273+
}
274+
243275
std::ofstream outputFile(filePath, std::ios::out | std::ios::binary | std::ios::trunc);
244276
if (!outputFile)
245277
throw std::runtime_error(fmt::format("failed to open {}, reason: {}", Nz::PathToString(filePath), std::strerror(errno)));
246278

247279
if (!outputFile.write(static_cast<const char*>(data), size))
248280
throw std::runtime_error(fmt::format("failed to write {}, reason: {}", Nz::PathToString(filePath), std::strerror(errno)));
281+
282+
return true;
249283
}
250284
}

‎src/ShaderArchiver/Archiver.hpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ namespace nzsla
4040
void DoShow();
4141
std::vector<std::uint8_t> ReadFileContent(const std::filesystem::path& filePath);
4242
std::string ToHeader(const void* data, std::size_t size);
43-
void WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size);
43+
bool WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size);
4444

4545
std::vector<std::filesystem::path> m_inputFiles;
4646
std::filesystem::path m_outputPath;
@@ -49,6 +49,7 @@ namespace nzsla
4949
bool m_isShowing;
5050
bool m_isVerbose;
5151
bool m_outputToStdout;
52+
bool m_skipUnchangedOutput;
5253
};
5354
}
5455

‎src/ShaderCompiler/Compiler.cpp‎

Lines changed: 38 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ namespace nzslc
5757
m_profiling(false),
5858
m_outputToStdout(false),
5959
m_skipOutput(false),
60+
m_skipUnchangedOutput(false),
6061
m_verbose(false),
6162
m_iterationCount(1)
6263
{
@@ -108,6 +109,7 @@ namespace nzslc
108109
if (m_options.count("measure") > 0)
109110
m_profiling = m_options["measure"].as<bool>();
110111

112+
m_skipUnchangedOutput = m_options.count("skip-unchanged") > 0;
111113
m_verbose = m_options.count("verbose") > 0;
112114
}
113115

@@ -231,7 +233,8 @@ You can also specify -header as a suffix (ex: --compile=glsl-header) to generate
231233
("d,debug-level", "Debug level to generate", cxxopts::value<std::string>(), "[none|minimal|regular|full]")
232234
("m,module", "Module file or directory", cxxopts::value<std::vector<std::string>>())
233235
("optimize", "Optimize shader code")
234-
("p,partial", "Allow partial compilation");
236+
("p,partial", "Allow partial compilation")
237+
("skip-unchanged", "After compilation, compare the output with the current output file and skip writing if the content is the same", cxxopts::value<bool>()->default_value("false"));
235238

236239
options.add_options("glsl output")
237240
("gl-es", "Generate GLSL ES instead of GLSL", cxxopts::value<bool>()->default_value("false"))
@@ -568,6 +571,12 @@ You can also specify -header as a suffix (ex: --compile=glsl-header) to generate
568571
}
569572
}
570573

574+
nzsl::Ast::ModulePtr Compiler::Deserialize(const std::uint8_t* data, std::size_t size)
575+
{
576+
nzsl::Deserializer deserializer(data, size);
577+
return nzsl::Ast::DeserializeShader(deserializer);
578+
}
579+
571580
void Compiler::PrintTime()
572581
{
573582
long long fullTime = std::max(m_steps[0].time, 1LL); //< prevent a divison by zero
@@ -620,18 +629,18 @@ You can also specify -header as a suffix (ex: --compile=glsl-header) to generate
620629

621630
void Compiler::OutputFile(std::filesystem::path filePath, const void* data, std::size_t size, bool disallowHeader)
622631
{
632+
std::string headerFile;
623633
if (m_outputHeader && !disallowHeader)
624634
{
625-
std::string headerFile = ToHeader(data, size);
626-
635+
headerFile = ToHeader(data, size);
627636
filePath.replace_extension(Nz::PathToString(filePath.extension()) + ".h");
628-
WriteFileContent(filePath, headerFile.data(), headerFile.size());
637+
data = headerFile.data();
638+
size = headerFile.size();
629639
}
630-
else
631-
WriteFileContent(filePath, data, size);
632640

641+
bool written = WriteFileContent(filePath, data, size);
633642
if (m_verbose)
634-
fmt::print("Generated file {}\n", Nz::PathToString(std::filesystem::absolute(filePath)));
643+
fmt::print("{} file {}\n", (written) ? "Generated" : "Skipped", Nz::PathToString(std::filesystem::absolute(filePath)));
635644
}
636645

637646
void Compiler::OutputToStdout(std::string_view str)
@@ -752,12 +761,6 @@ You can also specify -header as a suffix (ex: --compile=glsl-header) to generate
752761
return func();
753762
}
754763

755-
nzsl::Ast::ModulePtr Compiler::Deserialize(const std::uint8_t* data, std::size_t size)
756-
{
757-
nzsl::Deserializer deserializer(data, size);
758-
return nzsl::Ast::DeserializeShader(deserializer);
759-
}
760-
761764
nzsl::Ast::ModulePtr Compiler::Parse(std::string_view sourceContent, const std::string& filePath)
762765
{
763766
std::vector<nzsl::Token> tokens = nzsl::Tokenize(sourceContent, filePath);
@@ -811,13 +814,34 @@ You can also specify -header as a suffix (ex: --compile=glsl-header) to generate
811814
return std::move(ss).str();
812815
}
813816

814-
void Compiler::WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size)
817+
bool Compiler::WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size)
815818
{
819+
if (m_skipUnchangedOutput)
820+
{
821+
std::ifstream inputFile(filePath, std::ios::in | std::ios::binary);
822+
if (inputFile)
823+
{
824+
std::size_t fileSize = Nz::SafeCaster(std::filesystem::file_size(filePath));
825+
if (fileSize == size)
826+
{
827+
if (size == 0)
828+
return false;
829+
830+
// Compare file content
831+
std::vector<char> content(fileSize);
832+
if (inputFile.read(&content[0], fileSize) && std::memcmp(&content[0], data, size) == 0)
833+
return false;
834+
}
835+
}
836+
}
837+
816838
std::ofstream outputFile(filePath, std::ios::out | std::ios::binary | std::ios::trunc);
817839
if (!outputFile)
818840
throw std::runtime_error(fmt::format("failed to open {}, reason: {}", Nz::PathToString(filePath), std::strerror(errno)));
819841

820842
if (!outputFile.write(static_cast<const char*>(data), size))
821843
throw std::runtime_error(fmt::format("failed to write {}, reason: {}", Nz::PathToString(filePath), std::strerror(errno)));
844+
845+
return true;
822846
}
823847
}

‎src/ShaderCompiler/Compiler.hpp‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ namespace nzslc
5858
void CompileToNZSL(std::filesystem::path outputPath, const nzsl::Ast::Module& module);
5959
void CompileToNZSLB(std::filesystem::path outputPath, const nzsl::Ast::Module& module);
6060
void CompileToSPV(std::filesystem::path outputPath, nzsl::Ast::Module& module, bool textual);
61+
nzsl::Ast::ModulePtr Deserialize(const std::uint8_t* data, std::size_t size);
6162
void PrintTime();
6263
void OutputFile(std::filesystem::path filePath, const void* data, std::size_t size, bool disallowHeader = false);
6364
void OutputToStdout(std::string_view str);
@@ -66,13 +67,12 @@ namespace nzslc
6667
template<typename F, typename... Args> auto Step(std::enable_if_t<!std::is_member_function_pointer_v<F>, std::string_view> stepName, F&& func, Args&&... args) -> decltype(std::invoke(func, std::forward<Args>(args)...));
6768
template<typename F, typename... Args> auto Step(std::enable_if_t<std::is_member_function_pointer_v<F>, std::string_view> stepName, F&& func, Args&&... args) -> decltype(std::invoke(func, this, std::forward<Args>(args)...));
6869
template<typename F> auto StepInternal(std::string_view stepName, F&& func) -> decltype(func());
69-
nzsl::Ast::ModulePtr Deserialize(const std::uint8_t* data, std::size_t size);
70+
bool WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size);
7071

7172
static nzsl::Ast::ModulePtr Parse(std::string_view sourceContent, const std::string& filePath);
7273
static std::vector<std::uint8_t> ReadFileContent(const std::filesystem::path& filePath);
7374
static std::string ReadSourceFileContent(const std::filesystem::path& filePath);
7475
static std::string ToHeader(const void* data, std::size_t size);
75-
static void WriteFileContent(const std::filesystem::path& filePath, const void* data, std::size_t size);
7676

7777
struct StepTime
7878
{
@@ -91,6 +91,7 @@ namespace nzslc
9191
bool m_outputHeader;
9292
bool m_outputToStdout;
9393
bool m_skipOutput;
94+
bool m_skipUnchangedOutput;
9495
bool m_verbose;
9596
unsigned int m_iterationCount;
9697
};

‎tests/src/Tests/NzslaTests.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,12 @@ TEST_CASE("Standalone archiver", "[NZSLA]")
3535

3636
// Archive each modules
3737
ExecuteCommand("./nzsla --archive -o test_files/test_archive.nzsla ../resources/modules/Archive/InstanceData.nzslb ../resources/modules/Archive/LightData.nzslb ../resources/modules/Archive/SkeletalData.nzslb ../resources/modules/Archive/SkinningData.nzslb ../resources/modules/Archive/ViewerData.nzslb");
38+
39+
// Archive again with --skip-unchanged and ensure file wasn't modified
40+
std::filesystem::file_time_type archiveModifiedTime = std::filesystem::last_write_time(Nz::Utf8Path("test_files/test_archive.nzsla"));
41+
ExecuteCommand("./nzsla --skip-unchanged --verbose --archive -o test_files/test_archive.nzsla ../resources/modules/Archive/InstanceData.nzslb ../resources/modules/Archive/LightData.nzslb ../resources/modules/Archive/SkeletalData.nzslb ../resources/modules/Archive/SkinningData.nzslb ../resources/modules/Archive/ViewerData.nzslb", "Skipped file .+test_archive.nzsla");
42+
CHECK(archiveModifiedTime == std::filesystem::last_write_time(Nz::Utf8Path("test_files/test_archive.nzsla")));
43+
3844
ExecuteCommand("./nzsla test_files/test_archive.nzsla", {}, R"(archive info for test_files/test_archive.nzsla
3945
4046
5 module(s) are stored in this archive:

‎tests/src/Tests/NzslcTests.cpp‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
#include <Tests/ToolUtils.hpp>
22
#include <NazaraUtils/CallOnExit.hpp>
3+
#include <NazaraUtils/PathUtils.hpp>
34
#include <NZSL/Config.hpp>
45
#include <catch2/catch_test_macros.hpp>
56
#include <fmt/format.h>
@@ -41,7 +42,7 @@ TEST_CASE("Standalone compiler", "[NZSLC]")
4142
ExecuteCommand("./nzslc ../resources/modules/Data/DataStruct.nzslb"); //< validation
4243

4344
// Try to generate a full shader based on partial compilation result
44-
ExecuteCommand("./nzslc --compile=glsl,glsl-header,nzsl,nzsl-header,nzslb,nzslb-header,spv,spv-header,spv-txt --gl-bindingmap --debug-level=regular -o test_files -m ../resources/modules/Color.nzslb -m ../resources/modules/Data/OutputStruct.nzslb -m ../resources/modules/Data/DataStruct.nzslb ../resources/Shader.nzslb");
45+
ExecuteCommand("./nzslc --skip-unchanged --verbose --compile=glsl,glsl-header,nzsl,nzsl-header,nzslb,nzslb-header,spv,spv-header,spv-dis --gl-bindingmap --debug-level=regular -o test_files -m ../resources/modules/Color.nzslb -m ../resources/modules/Data/OutputStruct.nzslb -m ../resources/modules/Data/DataStruct.nzslb ../resources/Shader.nzslb");
4546

4647
// Validate generated files
4748
ExecuteCommand("./nzslc test_files/Shader.nzsl");
@@ -55,5 +56,10 @@ TEST_CASE("Standalone compiler", "[NZSLC]")
5556
CheckHeaderMatch("test_files/Shader.nzsl");
5657
CheckHeaderMatch("test_files/Shader.nzslb");
5758
CheckHeaderMatch("test_files/Shader.spv");
59+
60+
// Generate the same shader a second time with --skip-unchanged and ensure file wasn't modified
61+
std::filesystem::file_time_type shaderModifiedTime = std::filesystem::last_write_time(Nz::Utf8Path("test_files/Shader.nzsl"));
62+
ExecuteCommand("./nzslc --skip-unchanged --verbose --compile=spv --debug-level=regular -o test_files -m ../resources/modules/Color.nzslb -m ../resources/modules/Data/OutputStruct.nzslb -m ../resources/modules/Data/DataStruct.nzslb ../resources/Shader.nzslb", "Skipped file .+Shader.spv");
63+
CHECK(shaderModifiedTime == std::filesystem::last_write_time(Nz::Utf8Path("test_files/Shader.nzsl")));
5864
}
5965
}

0 commit comments

Comments
 (0)