Skip to content

Commit d503322

Browse files
jviottijviotti
committed
Avoid copying dependencies in the vast majority of dispatches
Signed-off-by: Juan Cruz Viotti <jv@jviotti.com>
1 parent d4b4243 commit d503322

File tree

9 files changed

+465
-470
lines changed

9 files changed

+465
-470
lines changed

‎src/build/build.cc‎

Lines changed: 109 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,39 @@
22

33
#include <sourcemeta/core/jsonschema.h>
44

5-
#include <cassert> // assert
6-
#include <chrono> // std::chrono::nanoseconds, std::chrono::duration_cast
7-
#include <cstdint> // std::int64_t
8-
#include <fstream> // std::ofstream, std::ifstream
5+
#include <algorithm> // std::ranges::none_of
6+
#include <cassert> // assert
7+
#include <chrono> // std::chrono::nanoseconds, std::chrono::duration_cast
8+
#include <cstdint> // std::int64_t
9+
#include <fstream> // std::ofstream, std::ifstream
910

1011
#include <mutex> // std::unique_lock
1112
#include <string> // std::string
1213
#include <string_view> // std::string_view
1314

1415
static constexpr std::string_view DEPENDENCIES_FILE{"deps.txt"};
1516

16-
namespace sourcemeta::one {
17+
using mark_type = sourcemeta::one::Build::mark_type;
18+
using Entry = sourcemeta::one::Build::Entry;
1719

18-
auto Build::dependency(std::filesystem::path node)
19-
-> std::pair<DependencyKind, std::filesystem::path> {
20-
return {DependencyKind::Static, std::move(node)};
20+
static auto mark_locked(const std::unordered_map<std::string, Entry> &entries,
21+
const std::filesystem::path &path)
22+
-> std::optional<mark_type> {
23+
assert(path.is_absolute());
24+
const auto match{entries.find(path.native())};
25+
if (match != entries.end() && match->second.file_mark.has_value()) {
26+
return match->second.file_mark;
27+
}
28+
29+
try {
30+
return std::filesystem::last_write_time(path);
31+
} catch (const std::filesystem::filesystem_error &) {
32+
return std::nullopt;
33+
}
2134
}
2235

36+
namespace sourcemeta::one {
37+
2338
Build::Build(const std::filesystem::path &output_root)
2439
: root{(static_cast<void>(std::filesystem::create_directories(output_root)),
2540
std::filesystem::canonical(output_root))},
@@ -31,13 +46,13 @@ Build::Build(const std::filesystem::path &output_root)
3146
map_entry.is_directory = entry.is_directory();
3247
}
3348

34-
const auto deps_path{this->root / DEPENDENCIES_FILE};
35-
if (!std::filesystem::exists(deps_path)) {
49+
const auto dependencies_path{this->root / DEPENDENCIES_FILE};
50+
if (!std::filesystem::exists(dependencies_path)) {
3651
return;
3752
}
3853

3954
try {
40-
std::ifstream stream{deps_path};
55+
std::ifstream stream{dependencies_path};
4156
if (!stream.is_open()) {
4257
return;
4358
}
@@ -46,7 +61,8 @@ Build::Build(const std::filesystem::path &output_root)
4661
std::istreambuf_iterator<char>()};
4762

4863
std::string current_key;
49-
Build::Dependencies current_deps;
64+
std::vector<std::filesystem::path> current_static_dependencies;
65+
std::vector<std::filesystem::path> current_dynamic_dependencies;
5066
std::size_t position{0};
5167

5268
while (position < contents.size()) {
@@ -67,33 +83,34 @@ Build::Build(const std::filesystem::path &output_root)
6783
switch (tag) {
6884
case 't':
6985
if (!current_key.empty()) {
70-
this->entries_[current_key].dependencies = std::move(current_deps);
71-
current_deps = {};
86+
auto &previous_entry{this->entries_[current_key]};
87+
previous_entry.static_dependencies =
88+
std::move(current_static_dependencies);
89+
previous_entry.dynamic_dependencies =
90+
std::move(current_dynamic_dependencies);
91+
current_static_dependencies = {};
92+
current_dynamic_dependencies = {};
7293
}
7394

7495
current_key = this->root_string;
7596
current_key += '/';
7697
current_key += value;
7798
break;
7899
case 's':
79-
current_deps.emplace_back(
80-
Build::DependencyKind::Static,
100+
current_static_dependencies.emplace_back(
81101
(this->root / std::string{value}).lexically_normal());
82-
83102
break;
84103
case 'd':
85-
current_deps.emplace_back(
86-
Build::DependencyKind::Dynamic,
104+
current_dynamic_dependencies.emplace_back(
87105
(this->root / std::string{value}).lexically_normal());
88-
89106
break;
90107
case 'm': {
91108
const auto space{value.find(' ')};
92109
if (space != std::string_view::npos) {
93110
const auto path_part{value.substr(0, space)};
94-
const auto ns_part{value.substr(space + 1)};
111+
const auto nanoseconds_part{value.substr(space + 1)};
95112
const std::chrono::nanoseconds nanoseconds{
96-
std::stoll(std::string{ns_part})};
113+
std::stoll(std::string{nanoseconds_part})};
97114
const auto mark_value{mark_type{
98115
std::chrono::duration_cast<mark_type::duration>(nanoseconds)}};
99116
std::string absolute_key{this->root_string};
@@ -112,7 +129,9 @@ Build::Build(const std::filesystem::path &output_root)
112129
}
113130

114131
if (!current_key.empty()) {
115-
this->entries_[current_key].dependencies = std::move(current_deps);
132+
auto &last_entry{this->entries_[current_key]};
133+
last_entry.static_dependencies = std::move(current_static_dependencies);
134+
last_entry.dynamic_dependencies = std::move(current_dynamic_dependencies);
116135
}
117136
this->has_previous_run = true;
118137
} catch (...) {
@@ -124,12 +143,14 @@ auto Build::has_dependencies(const std::filesystem::path &path) const -> bool {
124143
assert(path.is_absolute());
125144
std::shared_lock lock{this->mutex};
126145
const auto match{this->entries_.find(path.native())};
127-
return match != this->entries_.end() && !match->second.dependencies.empty();
146+
return match != this->entries_.end() &&
147+
(!match->second.static_dependencies.empty() ||
148+
!match->second.dynamic_dependencies.empty());
128149
}
129150

130151
auto Build::finish() -> void {
131-
const auto deps_path{this->root / DEPENDENCIES_FILE};
132-
std::ofstream stream{deps_path};
152+
const auto dependencies_path{this->root / DEPENDENCIES_FILE};
153+
std::ofstream stream{dependencies_path};
133154
assert(!stream.fail());
134155

135156
const auto root_prefix_size{this->root_string.size() + 1};
@@ -139,7 +160,8 @@ auto Build::finish() -> void {
139160
continue;
140161
}
141162

142-
if (!entry.second.dependencies.empty()) {
163+
if (!entry.second.static_dependencies.empty() ||
164+
!entry.second.dynamic_dependencies.empty()) {
143165
if (entry.first.size() > root_prefix_size &&
144166
entry.first.starts_with(this->root_string)) {
145167
stream << "t " << std::string_view{entry.first}.substr(root_prefix_size)
@@ -148,17 +170,27 @@ auto Build::finish() -> void {
148170
stream << "t " << entry.first << '\n';
149171
}
150172

151-
for (const auto &dependency : entry.second.dependencies) {
152-
const char kind_char{
153-
dependency.first == Build::DependencyKind::Dynamic ? 'd' : 's'};
154-
const auto &dep_string{dependency.second.native()};
155-
if (dep_string.size() > root_prefix_size &&
156-
dep_string.starts_with(this->root_string)) {
157-
stream << kind_char << ' '
158-
<< std::string_view{dep_string}.substr(root_prefix_size)
173+
for (const auto &dependency : entry.second.static_dependencies) {
174+
const auto &dependency_string{dependency.native()};
175+
if (dependency_string.size() > root_prefix_size &&
176+
dependency_string.starts_with(this->root_string)) {
177+
stream << "s "
178+
<< std::string_view{dependency_string}.substr(root_prefix_size)
159179
<< '\n';
160180
} else {
161-
stream << kind_char << ' ' << dep_string << '\n';
181+
stream << "s " << dependency_string << '\n';
182+
}
183+
}
184+
185+
for (const auto &dependency : entry.second.dynamic_dependencies) {
186+
const auto &dependency_string{dependency.native()};
187+
if (dependency_string.size() > root_prefix_size &&
188+
dependency_string.starts_with(this->root_string)) {
189+
stream << "d "
190+
<< std::string_view{dependency_string}.substr(root_prefix_size)
191+
<< '\n';
192+
} else {
193+
stream << "d " << dependency_string << '\n';
162194
}
163195
}
164196
}
@@ -179,7 +211,7 @@ auto Build::finish() -> void {
179211
stream.flush();
180212
stream.close();
181213
lock.unlock();
182-
this->track(deps_path);
214+
this->track(dependencies_path);
183215

184216
// Remove untracked files inside the output directory
185217
std::shared_lock read_lock{this->mutex};
@@ -192,55 +224,42 @@ auto Build::finish() -> void {
192224
}
193225
}
194226

195-
auto Build::refresh(const std::filesystem::path &path) -> void {
196-
const auto value{std::filesystem::file_time_type::clock::now()};
197-
std::unique_lock lock{this->mutex};
198-
this->entries_[path.native()].file_mark = value;
199-
}
200-
201-
auto Build::mark(const std::filesystem::path &path)
202-
-> std::optional<mark_type> {
203-
assert(path.is_absolute());
204-
205-
{
206-
std::shared_lock lock{this->mutex};
207-
const auto match{this->entries_.find(path.native())};
208-
if (match != this->entries_.end() && match->second.file_mark.has_value()) {
209-
return match->second.file_mark;
210-
}
227+
auto Build::dispatch_is_cached(const Entry &entry,
228+
const bool static_dependencies_match) const
229+
-> bool {
230+
if (!static_dependencies_match) {
231+
return false;
211232
}
212233

213-
// Output files should always have their marks cached
214-
// Only input files or new output files are not
215-
assert(!this->has_previous_run ||
216-
path.native().starts_with(this->root_string) == false ||
217-
!std::filesystem::exists(path));
234+
const auto check_mtime{[this, &entry](
235+
const std::filesystem::path &dependency_path) {
236+
const auto dependency_mark{mark_locked(this->entries_, dependency_path)};
237+
return !dependency_mark.has_value() ||
238+
dependency_mark.value() > entry.file_mark.value();
239+
}};
218240

219-
try {
220-
const auto value{std::filesystem::last_write_time(path)};
221-
std::unique_lock lock{this->mutex};
222-
this->entries_[path.native()].file_mark = value;
223-
return value;
224-
} catch (const std::filesystem::filesystem_error &) {
225-
return std::nullopt;
226-
}
241+
return std::ranges::none_of(entry.static_dependencies, check_mtime) &&
242+
std::ranges::none_of(entry.dynamic_dependencies, check_mtime);
227243
}
228244

229-
auto Build::mark_locked(const std::filesystem::path &path) const
230-
-> std::optional<mark_type> {
231-
assert(path.is_absolute());
232-
const auto match{this->entries_.find(path.native())};
233-
if (match != this->entries_.end() && match->second.file_mark.has_value()) {
234-
return match->second.file_mark;
235-
}
245+
auto Build::dispatch_commit(
246+
const std::filesystem::path &destination,
247+
std::vector<std::filesystem::path> &&static_dependencies,
248+
std::vector<std::filesystem::path> &&dynamic_dependencies) -> void {
249+
assert(destination.is_absolute());
250+
assert(std::filesystem::exists(destination));
251+
this->refresh(destination);
252+
this->track(destination);
253+
std::unique_lock lock{this->mutex};
254+
auto &entry{this->entries_[destination.native()]};
255+
entry.static_dependencies = std::move(static_dependencies);
256+
entry.dynamic_dependencies = std::move(dynamic_dependencies);
257+
}
236258

237-
// For the locked variant used in dispatch cache-hit checks,
238-
// if the mark isn't cached, fall back to stat
239-
try {
240-
return std::filesystem::last_write_time(path);
241-
} catch (const std::filesystem::filesystem_error &) {
242-
return std::nullopt;
243-
}
259+
auto Build::refresh(const std::filesystem::path &path) -> void {
260+
const auto value{std::filesystem::file_time_type::clock::now()};
261+
std::unique_lock lock{this->mutex};
262+
this->entries_[path.native()].file_mark = value;
244263
}
245264

246265
auto Build::track(const std::filesystem::path &path) -> void {
@@ -272,31 +291,24 @@ auto Build::is_untracked_file(const std::filesystem::path &path) const -> bool {
272291
return match == this->entries_.cend() || !match->second.tracked;
273292
}
274293

275-
auto Build::output_write_json(const std::filesystem::path &path,
276-
const sourcemeta::core::JSON &document) -> void {
277-
assert(path.is_absolute());
278-
std::filesystem::create_directories(path.parent_path());
279-
std::ofstream stream{path};
280-
assert(!stream.fail());
281-
sourcemeta::core::stringify(document, stream);
282-
this->track(path);
283-
}
284-
285294
auto Build::write_json_if_different(const std::filesystem::path &path,
286295
const sourcemeta::core::JSON &document)
287296
-> void {
288297
if (std::filesystem::exists(path)) {
289298
const auto current{sourcemeta::core::read_json(path)};
290-
if (current != document) {
291-
this->output_write_json(path, document);
292-
this->refresh(path);
293-
} else {
299+
if (current == document) {
294300
this->track(path);
301+
return;
295302
}
296-
} else {
297-
this->output_write_json(path, document);
298-
this->refresh(path);
299303
}
304+
305+
assert(path.is_absolute());
306+
std::filesystem::create_directories(path.parent_path());
307+
std::ofstream stream{path};
308+
assert(!stream.fail());
309+
sourcemeta::core::stringify(document, stream);
310+
this->track(path);
311+
this->refresh(path);
300312
}
301313

302314
} // namespace sourcemeta::one

0 commit comments

Comments
 (0)