Skip to content

Commit a737e1f

Browse files
gitstergitster
committed
Merge branch 'mt/parallel-checkout-part-3'
The final part of "parallel checkout". * mt/parallel-checkout-part-3: ci: run test round with parallel-checkout enabled parallel-checkout: add tests related to .gitattributes t0028: extract encoding helpers to lib-encoding.sh parallel-checkout: add tests related to path collisions parallel-checkout: add tests for basic operations checkout-index: add parallel checkout support builtin/checkout.c: complete parallel checkout support make_transient_cache_entry(): optionally alloc from mem_pool
2 parents 644f4a2 + 87094fc commit a737e1f

16 files changed

+734
-49
lines changed

‎builtin/checkout--worker.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ static void packet_to_pc_item(const char *buffer, int len,
3939
}
4040

4141
memset(pc_item, 0, sizeof(*pc_item));
42-
pc_item->ce = make_empty_transient_cache_entry(fixed_portion->name_len);
42+
pc_item->ce = make_empty_transient_cache_entry(fixed_portion->name_len, NULL);
4343
pc_item->ce->ce_namelen = fixed_portion->name_len;
4444
pc_item->ce->ce_mode = fixed_portion->ce_mode;
4545
memcpy(pc_item->ce->name, variant, pc_item->ce->ce_namelen);

‎builtin/checkout-index.c

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "cache-tree.h"
1313
#include "parse-options.h"
1414
#include "entry.h"
15+
#include "parallel-checkout.h"
1516

1617
#define CHECKOUT_ALL 4
1718
static int nul_term_line;
@@ -115,7 +116,7 @@ static int checkout_file(const char *name, const char *prefix)
115116
return -1;
116117
}
117118

118-
static void checkout_all(const char *prefix, int prefix_length)
119+
static int checkout_all(const char *prefix, int prefix_length)
119120
{
120121
int i, errs = 0;
121122
struct cache_entry *last_ce = NULL;
@@ -144,11 +145,7 @@ static void checkout_all(const char *prefix, int prefix_length)
144145
}
145146
if (last_ce && to_tempfile)
146147
write_tempfile_record(last_ce->name, prefix);
147-
if (errs)
148-
/* we have already done our error reporting.
149-
* exit with the same code as die().
150-
*/
151-
exit(128);
148+
return !!errs;
152149
}
153150

154151
static const char * const builtin_checkout_index_usage[] = {
@@ -184,6 +181,7 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix)
184181
int force = 0, quiet = 0, not_new = 0;
185182
int index_opt = 0;
186183
int err = 0;
184+
int pc_workers, pc_threshold;
187185
struct option builtin_checkout_index_options[] = {
188186
OPT_BOOL('a', "all", &all,
189187
N_("check out all files in the index")),
@@ -238,6 +236,10 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix)
238236
hold_locked_index(&lock_file, LOCK_DIE_ON_ERROR);
239237
}
240238

239+
get_parallel_checkout_configs(&pc_workers, &pc_threshold);
240+
if (pc_workers > 1)
241+
init_parallel_checkout();
242+
241243
/* Check out named files first */
242244
for (i = 0; i < argc; i++) {
243245
const char *arg = argv[i];
@@ -277,12 +279,16 @@ int cmd_checkout_index(int argc, const char **argv, const char *prefix)
277279
strbuf_release(&buf);
278280
}
279281

282+
if (all)
283+
err |= checkout_all(prefix, prefix_length);
284+
285+
if (pc_workers > 1)
286+
err |= run_parallel_checkout(&state, pc_workers, pc_threshold,
287+
NULL, NULL);
288+
280289
if (err)
281290
return 1;
282291

283-
if (all)
284-
checkout_all(prefix, prefix_length);
285-
286292
if (is_lock_file_locked(&lock_file) &&
287293
write_locked_index(&the_index, &lock_file, COMMIT_LOCK))
288294
die("Unable to write new index file");

‎builtin/checkout.c

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
#include "wt-status.h"
2828
#include "xdiff-interface.h"
2929
#include "entry.h"
30+
#include "parallel-checkout.h"
3031

3132
static const char * const checkout_usage[] = {
3233
N_("git checkout [<options>] <branch>"),
@@ -230,7 +231,8 @@ static int checkout_stage(int stage, const struct cache_entry *ce, int pos,
230231
return error(_("path '%s' does not have their version"), ce->name);
231232
}
232233

233-
static int checkout_merged(int pos, const struct checkout *state, int *nr_checkouts)
234+
static int checkout_merged(int pos, const struct checkout *state,
235+
int *nr_checkouts, struct mem_pool *ce_mem_pool)
234236
{
235237
struct cache_entry *ce = active_cache[pos];
236238
const char *path = ce->name;
@@ -291,11 +293,10 @@ static int checkout_merged(int pos, const struct checkout *state, int *nr_checko
291293
if (write_object_file(result_buf.ptr, result_buf.size, blob_type, &oid))
292294
die(_("Unable to add merge result for '%s'"), path);
293295
free(result_buf.ptr);
294-
ce = make_transient_cache_entry(mode, &oid, path, 2);
296+
ce = make_transient_cache_entry(mode, &oid, path, 2, ce_mem_pool);
295297
if (!ce)
296298
die(_("make_cache_entry failed for path '%s'"), path);
297299
status = checkout_entry(ce, state, NULL, nr_checkouts);
298-
discard_cache_entry(ce);
299300
return status;
300301
}
301302

@@ -359,19 +360,27 @@ static int checkout_worktree(const struct checkout_opts *opts,
359360
int nr_checkouts = 0, nr_unmerged = 0;
360361
int errs = 0;
361362
int pos;
363+
int pc_workers, pc_threshold;
364+
struct mem_pool ce_mem_pool;
362365

363366
state.force = 1;
364367
state.refresh_cache = 1;
365368
state.istate = &the_index;
366369

370+
mem_pool_init(&ce_mem_pool, 0);
371+
get_parallel_checkout_configs(&pc_workers, &pc_threshold);
367372
init_checkout_metadata(&state.meta, info->refname,
368373
info->commit ? &info->commit->object.oid : &info->oid,
369374
NULL);
370375

371376
enable_delayed_checkout(&state);
372377

378+
if (pc_workers > 1)
379+
init_parallel_checkout();
380+
373381
/* TODO: audit for interaction with sparse-index. */
374382
ensure_full_index(&the_index);
383+
375384
for (pos = 0; pos < active_nr; pos++) {
376385
struct cache_entry *ce = active_cache[pos];
377386
if (ce->ce_flags & CE_MATCHED) {
@@ -387,10 +396,15 @@ static int checkout_worktree(const struct checkout_opts *opts,
387396
&nr_checkouts, opts->overlay_mode);
388397
else if (opts->merge)
389398
errs |= checkout_merged(pos, &state,
390-
&nr_unmerged);
399+
&nr_unmerged,
400+
&ce_mem_pool);
391401
pos = skip_same_name(ce, pos) - 1;
392402
}
393403
}
404+
if (pc_workers > 1)
405+
errs |= run_parallel_checkout(&state, pc_workers, pc_threshold,
406+
NULL, NULL);
407+
mem_pool_discard(&ce_mem_pool, should_validate_cache_entries());
394408
remove_marked_cache_entries(&the_index, 1);
395409
remove_scheduled_dirs();
396410
errs |= finish_delayed_checkout(&state, &nr_checkouts);

‎builtin/difftool.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ static int checkout_path(unsigned mode, struct object_id *oid,
323323
struct cache_entry *ce;
324324
int ret;
325325

326-
ce = make_transient_cache_entry(mode, oid, path, 0);
326+
ce = make_transient_cache_entry(mode, oid, path, 0, NULL);
327327
ret = checkout_entry(ce, state, NULL, NULL);
328328

329329
discard_cache_entry(ce);

‎cache.h

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -370,16 +370,20 @@ struct cache_entry *make_empty_cache_entry(struct index_state *istate,
370370
size_t name_len);
371371

372372
/*
373-
* Create a cache_entry that is not intended to be added to an index.
374-
* Caller is responsible for discarding the cache_entry
375-
* with `discard_cache_entry`.
373+
* Create a cache_entry that is not intended to be added to an index. If
374+
* `ce_mem_pool` is not NULL, the entry is allocated within the given memory
375+
* pool. Caller is responsible for discarding "loose" entries with
376+
* `discard_cache_entry()` and the memory pool with
377+
* `mem_pool_discard(ce_mem_pool, should_validate_cache_entries())`.
376378
*/
377379
struct cache_entry *make_transient_cache_entry(unsigned int mode,
378380
const struct object_id *oid,
379381
const char *path,
380-
int stage);
382+
int stage,
383+
struct mem_pool *ce_mem_pool);
381384

382-
struct cache_entry *make_empty_transient_cache_entry(size_t name_len);
385+
struct cache_entry *make_empty_transient_cache_entry(size_t len,
386+
struct mem_pool *ce_mem_pool);
383387

384388
/*
385389
* Discard cache entry.

‎ci/run-build-and-tests.sh

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ linux-gcc)
2626
export GIT_TEST_ADD_I_USE_BUILTIN=1
2727
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
2828
export GIT_TEST_WRITE_REV_INDEX=1
29+
export GIT_TEST_CHECKOUT_WORKERS=2
2930
make test
3031
;;
3132
linux-clang)

‎parallel-checkout.c

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#include "sigchain.h"
99
#include "streaming.h"
1010
#include "thread-utils.h"
11+
#include "trace2.h"
1112

1213
struct pc_worker {
1314
struct child_process cp;
@@ -34,6 +35,20 @@ static const int DEFAULT_NUM_WORKERS = 1;
3435

3536
void get_parallel_checkout_configs(int *num_workers, int *threshold)
3637
{
38+
char *env_workers = getenv("GIT_TEST_CHECKOUT_WORKERS");
39+
40+
if (env_workers && *env_workers) {
41+
if (strtol_i(env_workers, 10, num_workers)) {
42+
die("invalid value for GIT_TEST_CHECKOUT_WORKERS: '%s'",
43+
env_workers);
44+
}
45+
if (*num_workers < 1)
46+
*num_workers = online_cpus();
47+
48+
*threshold = 0;
49+
return;
50+
}
51+
3752
if (git_config_get_int("checkout.workers", num_workers))
3853
*num_workers = DEFAULT_NUM_WORKERS;
3954
else if (*num_workers < 1)
@@ -326,6 +341,7 @@ void write_pc_item(struct parallel_checkout_item *pc_item,
326341
if (dir_sep && !has_dirs_only_path(path.buf, dir_sep - path.buf,
327342
state->base_dir_len)) {
328343
pc_item->status = PC_ITEM_COLLIDED;
344+
trace2_data_string("pcheckout", NULL, "collision/dirname", path.buf);
329345
goto out;
330346
}
331347

@@ -341,6 +357,8 @@ void write_pc_item(struct parallel_checkout_item *pc_item,
341357
* call should have already caught these cases.
342358
*/
343359
pc_item->status = PC_ITEM_COLLIDED;
360+
trace2_data_string("pcheckout", NULL,
361+
"collision/basename", path.buf);
344362
} else {
345363
error_errno("failed to open file '%s'", path.buf);
346364
pc_item->status = PC_ITEM_FAILED;

‎read-cache.c

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -839,8 +839,11 @@ struct cache_entry *make_empty_cache_entry(struct index_state *istate, size_t le
839839
return mem_pool__ce_calloc(find_mem_pool(istate), len);
840840
}
841841

842-
struct cache_entry *make_empty_transient_cache_entry(size_t len)
842+
struct cache_entry *make_empty_transient_cache_entry(size_t len,
843+
struct mem_pool *ce_mem_pool)
843844
{
845+
if (ce_mem_pool)
846+
return mem_pool__ce_calloc(ce_mem_pool, len);
844847
return xcalloc(1, cache_entry_size(len));
845848
}
846849

@@ -874,8 +877,11 @@ struct cache_entry *make_cache_entry(struct index_state *istate,
874877
return ret;
875878
}
876879

877-
struct cache_entry *make_transient_cache_entry(unsigned int mode, const struct object_id *oid,
878-
const char *path, int stage)
880+
struct cache_entry *make_transient_cache_entry(unsigned int mode,
881+
const struct object_id *oid,
882+
const char *path,
883+
int stage,
884+
struct mem_pool *ce_mem_pool)
879885
{
880886
struct cache_entry *ce;
881887
int len;
@@ -886,7 +892,7 @@ struct cache_entry *make_transient_cache_entry(unsigned int mode, const struct o
886892
}
887893

888894
len = strlen(path);
889-
ce = make_empty_transient_cache_entry(len);
895+
ce = make_empty_transient_cache_entry(len, ce_mem_pool);
890896

891897
oidcpy(&ce->oid, oid);
892898
memcpy(ce->name, path, len);

‎t/README

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -439,6 +439,10 @@ GIT_TEST_WRITE_REV_INDEX=<boolean>, when true enables the
439439
GIT_TEST_SPARSE_INDEX=<boolean>, when true enables index writes to use the
440440
sparse-index format by default.
441441

442+
GIT_TEST_CHECKOUT_WORKERS=<n> overrides the 'checkout.workers' setting
443+
to <n> and 'checkout.thresholdForParallelism' to 0, forcing the
444+
execution of the parallel-checkout code.
445+
442446
Naming Tests
443447
------------
444448

‎t/lib-encoding.sh

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
# Encoding helpers
2+
3+
test_lazy_prereq NO_UTF16_BOM '
4+
test $(printf abc | iconv -f UTF-8 -t UTF-16 | wc -c) = 6
5+
'
6+
7+
test_lazy_prereq NO_UTF32_BOM '
8+
test $(printf abc | iconv -f UTF-8 -t UTF-32 | wc -c) = 12
9+
'
10+
11+
write_utf16 () {
12+
if test_have_prereq NO_UTF16_BOM
13+
then
14+
printf '\376\377'
15+
fi &&
16+
iconv -f UTF-8 -t UTF-16
17+
}
18+
19+
write_utf32 () {
20+
if test_have_prereq NO_UTF32_BOM
21+
then
22+
printf '\0\0\376\377'
23+
fi &&
24+
iconv -f UTF-8 -t UTF-32
25+
}

0 commit comments

Comments
 (0)