Skip to content

Commit 7ee055b

Browse files
gitstergitster
committed
Merge branch 'ps/ref-backend-migration-optim'
The migration procedure between two ref backends has been optimized. * ps/ref-backend-migration-optim: reftable: rename scratch buffer refs: adapt `initial_transaction` flag to be unsigned reftable/block: optimize allocations by using scratch buffer reftable/block: rename `block_writer::buf` variable reftable/writer: optimize allocations by using a scratch buffer refs: don't normalize log messages with `REF_SKIP_CREATE_REFLOG` refs: skip collision checks in initial transactions refs: use "initial" transaction semantics to migrate refs refs/files: support symbolic and root refs in initial transaction refs: introduce "initial" transaction flag refs/files: move logic to commit initial transaction refs: allow passing flags when setting up a transaction
2 parents a5dd262 + ef46ad0 commit 7ee055b

23 files changed

+249
-253
lines changed

‎branch.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -627,7 +627,7 @@ void create_branch(struct repository *r,
627627
else
628628
msg = xstrfmt("branch: Created from %s", start_name);
629629
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
630-
&err);
630+
0, &err);
631631
if (!transaction ||
632632
ref_transaction_update(transaction, ref.buf,
633633
&oid, forcing ? NULL : null_oid(),

‎builtin/clone.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,7 +574,7 @@ static void write_remote_refs(const struct ref *local_refs)
574574
struct strbuf err = STRBUF_INIT;
575575

576576
t = ref_store_transaction_begin(get_main_ref_store(the_repository),
577-
&err);
577+
REF_TRANSACTION_FLAG_INITIAL, &err);
578578
if (!t)
579579
die("%s", err.buf);
580580

@@ -586,7 +586,7 @@ static void write_remote_refs(const struct ref *local_refs)
586586
die("%s", err.buf);
587587
}
588588

589-
if (initial_ref_transaction_commit(t, &err))
589+
if (ref_transaction_commit(t, &err))
590590
die("%s", err.buf);
591591

592592
strbuf_release(&err);

‎builtin/fast-import.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1647,7 +1647,7 @@ static int update_branch(struct branch *b)
16471647
}
16481648
}
16491649
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1650-
&err);
1650+
0, &err);
16511651
if (!transaction ||
16521652
ref_transaction_update(transaction, b->name, &b->oid, &old_oid,
16531653
NULL, NULL, 0, msg, &err) ||
@@ -1682,7 +1682,7 @@ static void dump_tags(void)
16821682
struct ref_transaction *transaction;
16831683

16841684
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1685-
&err);
1685+
0, &err);
16861686
if (!transaction) {
16871687
failure |= error("%s", err.buf);
16881688
goto cleanup;

‎builtin/fetch.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -665,7 +665,7 @@ static int s_update_ref(const char *action,
665665
*/
666666
if (!transaction) {
667667
transaction = our_transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
668-
&err);
668+
0, &err);
669669
if (!transaction) {
670670
ret = STORE_REF_ERROR_OTHER;
671671
goto out;
@@ -1667,7 +1667,7 @@ static int do_fetch(struct transport *transport,
16671667

16681668
if (atomic_fetch) {
16691669
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1670-
&err);
1670+
0, &err);
16711671
if (!transaction) {
16721672
retcode = -1;
16731673
goto cleanup;

‎builtin/receive-pack.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1849,7 +1849,7 @@ static void execute_commands_non_atomic(struct command *commands,
18491849
continue;
18501850

18511851
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1852-
&err);
1852+
0, &err);
18531853
if (!transaction) {
18541854
rp_error("%s", err.buf);
18551855
strbuf_reset(&err);
@@ -1878,7 +1878,7 @@ static void execute_commands_atomic(struct command *commands,
18781878
const char *reported_error = "atomic push failure";
18791879

18801880
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
1881-
&err);
1881+
0, &err);
18821882
if (!transaction) {
18831883
rp_error("%s", err.buf);
18841884
strbuf_reset(&err);

‎builtin/replace.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -201,7 +201,7 @@ static int replace_object_oid(const char *object_ref,
201201
}
202202

203203
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
204-
&err);
204+
0, &err);
205205
if (!transaction ||
206206
ref_transaction_update(transaction, ref.buf, repl, &prev,
207207
NULL, NULL, 0, NULL, &err) ||

‎builtin/tag.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -681,7 +681,7 @@ int cmd_tag(int argc,
681681
}
682682

683683
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
684-
&err);
684+
0, &err);
685685
if (!transaction ||
686686
ref_transaction_update(transaction, ref.buf, &object, &prev,
687687
NULL, NULL,

‎builtin/update-ref.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -612,7 +612,7 @@ static void update_refs_stdin(void)
612612
int i, j;
613613

614614
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
615-
&err);
615+
0, &err);
616616
if (!transaction)
617617
die("%s", err.buf);
618618

@@ -680,7 +680,7 @@ static void update_refs_stdin(void)
680680
*/
681681
state = cmd->state;
682682
transaction = ref_store_transaction_begin(get_main_ref_store(the_repository),
683-
&err);
683+
0, &err);
684684
if (!transaction)
685685
die("%s", err.buf);
686686

‎refs.c

Lines changed: 32 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -918,7 +918,7 @@ int refs_delete_ref(struct ref_store *refs, const char *msg,
918918
struct ref_transaction *transaction;
919919
struct strbuf err = STRBUF_INIT;
920920

921-
transaction = ref_store_transaction_begin(refs, &err);
921+
transaction = ref_store_transaction_begin(refs, 0, &err);
922922
if (!transaction ||
923923
ref_transaction_delete(transaction, refname, old_oid,
924924
NULL, flags, msg, &err) ||
@@ -1116,13 +1116,15 @@ int read_ref_at(struct ref_store *refs, const char *refname,
11161116
}
11171117

11181118
struct ref_transaction *ref_store_transaction_begin(struct ref_store *refs,
1119+
unsigned int flags,
11191120
struct strbuf *err)
11201121
{
11211122
struct ref_transaction *tr;
11221123
assert(err);
11231124

11241125
CALLOC_ARRAY(tr, 1);
11251126
tr->ref_store = refs;
1127+
tr->flags = flags;
11261128
return tr;
11271129
}
11281130

@@ -1186,8 +1188,9 @@ struct ref_update *ref_transaction_add_update(
11861188
oidcpy(&update->new_oid, new_oid);
11871189
if ((flags & REF_HAVE_OLD) && old_oid)
11881190
oidcpy(&update->old_oid, old_oid);
1191+
if (!(flags & REF_SKIP_CREATE_REFLOG))
1192+
update->msg = normalize_reflog_message(msg);
11891193

1190-
update->msg = normalize_reflog_message(msg);
11911194
return update;
11921195
}
11931196

@@ -1309,7 +1312,7 @@ int refs_update_ref(struct ref_store *refs, const char *msg,
13091312
struct strbuf err = STRBUF_INIT;
13101313
int ret = 0;
13111314

1312-
t = ref_store_transaction_begin(refs, &err);
1315+
t = ref_store_transaction_begin(refs, 0, &err);
13131316
if (!t ||
13141317
ref_transaction_update(t, refname, new_oid, old_oid, NULL, NULL,
13151318
flags, msg, &err) ||
@@ -2120,7 +2123,7 @@ int refs_update_symref(struct ref_store *refs, const char *ref,
21202123
struct strbuf err = STRBUF_INIT;
21212124
int ret = 0;
21222125

2123-
transaction = ref_store_transaction_begin(refs, &err);
2126+
transaction = ref_store_transaction_begin(refs, 0, &err);
21242127
if (!transaction ||
21252128
ref_transaction_update(transaction, ref, NULL, NULL,
21262129
target, NULL, REF_NO_DEREF,
@@ -2316,7 +2319,7 @@ int ref_transaction_commit(struct ref_transaction *transaction,
23162319
}
23172320

23182321
ret = refs->be->transaction_finish(refs, transaction, err);
2319-
if (!ret)
2322+
if (!ret && !(transaction->flags & REF_TRANSACTION_FLAG_INITIAL))
23202323
run_transaction_hook(transaction, "committed");
23212324
return ret;
23222325
}
@@ -2325,6 +2328,7 @@ int refs_verify_refname_available(struct ref_store *refs,
23252328
const char *refname,
23262329
const struct string_list *extras,
23272330
const struct string_list *skip,
2331+
unsigned int initial_transaction,
23282332
struct strbuf *err)
23292333
{
23302334
const char *slash;
@@ -2333,8 +2337,6 @@ int refs_verify_refname_available(struct ref_store *refs,
23332337
struct strbuf referent = STRBUF_INIT;
23342338
struct object_id oid;
23352339
unsigned int type;
2336-
struct ref_iterator *iter;
2337-
int ok;
23382340
int ret = -1;
23392341

23402342
/*
@@ -2364,7 +2366,8 @@ int refs_verify_refname_available(struct ref_store *refs,
23642366
if (skip && string_list_has_string(skip, dirname.buf))
23652367
continue;
23662368

2367-
if (!refs_read_raw_ref(refs, dirname.buf, &oid, &referent,
2369+
if (!initial_transaction &&
2370+
!refs_read_raw_ref(refs, dirname.buf, &oid, &referent,
23682371
&type, &ignore_errno)) {
23692372
strbuf_addf(err, _("'%s' exists; cannot create '%s'"),
23702373
dirname.buf, refname);
@@ -2389,21 +2392,26 @@ int refs_verify_refname_available(struct ref_store *refs,
23892392
strbuf_addstr(&dirname, refname + dirname.len);
23902393
strbuf_addch(&dirname, '/');
23912394

2392-
iter = refs_ref_iterator_begin(refs, dirname.buf, NULL, 0,
2393-
DO_FOR_EACH_INCLUDE_BROKEN);
2394-
while ((ok = ref_iterator_advance(iter)) == ITER_OK) {
2395-
if (skip &&
2396-
string_list_has_string(skip, iter->refname))
2397-
continue;
2395+
if (!initial_transaction) {
2396+
struct ref_iterator *iter;
2397+
int ok;
23982398

2399-
strbuf_addf(err, _("'%s' exists; cannot create '%s'"),
2400-
iter->refname, refname);
2401-
ref_iterator_abort(iter);
2402-
goto cleanup;
2403-
}
2399+
iter = refs_ref_iterator_begin(refs, dirname.buf, NULL, 0,
2400+
DO_FOR_EACH_INCLUDE_BROKEN);
2401+
while ((ok = ref_iterator_advance(iter)) == ITER_OK) {
2402+
if (skip &&
2403+
string_list_has_string(skip, iter->refname))
2404+
continue;
2405+
2406+
strbuf_addf(err, _("'%s' exists; cannot create '%s'"),
2407+
iter->refname, refname);
2408+
ref_iterator_abort(iter);
2409+
goto cleanup;
2410+
}
24042411

2405-
if (ok != ITER_DONE)
2406-
BUG("error while iterating over references");
2412+
if (ok != ITER_DONE)
2413+
BUG("error while iterating over references");
2414+
}
24072415

24082416
extra_refname = find_descendant_ref(dirname.buf, extras, skip);
24092417
if (extra_refname)
@@ -2487,14 +2495,6 @@ int refs_reflog_expire(struct ref_store *refs,
24872495
cleanup_fn, policy_cb_data);
24882496
}
24892497

2490-
int initial_ref_transaction_commit(struct ref_transaction *transaction,
2491-
struct strbuf *err)
2492-
{
2493-
struct ref_store *refs = transaction->ref_store;
2494-
2495-
return refs->be->initial_transaction_commit(refs, transaction, err);
2496-
}
2497-
24982498
void ref_transaction_for_each_queued_update(struct ref_transaction *transaction,
24992499
ref_transaction_for_each_queued_update_fn cb,
25002500
void *cb_data)
@@ -2530,7 +2530,7 @@ int refs_delete_refs(struct ref_store *refs, const char *logmsg,
25302530
* individual updates can't fail, so we can pack all of the
25312531
* updates into a single transaction.
25322532
*/
2533-
transaction = ref_store_transaction_begin(refs, &err);
2533+
transaction = ref_store_transaction_begin(refs, 0, &err);
25342534
if (!transaction) {
25352535
ret = error("%s", err.buf);
25362536
goto out;
@@ -2836,7 +2836,8 @@ int repo_migrate_ref_storage_format(struct repository *repo,
28362836
if (ret < 0)
28372837
goto done;
28382838

2839-
transaction = ref_store_transaction_begin(new_refs, errbuf);
2839+
transaction = ref_store_transaction_begin(new_refs, REF_TRANSACTION_FLAG_INITIAL,
2840+
errbuf);
28402841
if (!transaction)
28412842
goto done;
28422843

@@ -2861,13 +2862,6 @@ int repo_migrate_ref_storage_format(struct repository *repo,
28612862
if (ret < 0)
28622863
goto done;
28632864

2864-
/*
2865-
* TODO: we might want to migrate to `initial_ref_transaction_commit()`
2866-
* here, which is more efficient for the files backend because it would
2867-
* write new refs into the packed-refs file directly. At this point,
2868-
* the files backend doesn't handle pseudo-refs and symrefs correctly
2869-
* though, so this requires some more work.
2870-
*/
28712865
ret = ref_transaction_commit(transaction, errbuf);
28722866
if (ret < 0)
28732867
goto done;

‎refs.h

Lines changed: 24 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -101,13 +101,16 @@ int refs_read_symbolic_ref(struct ref_store *ref_store, const char *refname,
101101
* both "foo" and with "foo/bar/baz" but not with "foo/bar" or
102102
* "foo/barbados".
103103
*
104+
* If `initial_transaction` is truish, then all collision checks with
105+
* preexisting refs are skipped.
106+
*
104107
* extras and skip must be sorted.
105108
*/
106-
107109
int refs_verify_refname_available(struct ref_store *refs,
108110
const char *refname,
109111
const struct string_list *extras,
110112
const struct string_list *skip,
113+
unsigned int initial_transaction,
111114
struct strbuf *err);
112115

113116
int refs_ref_exists(struct ref_store *refs, const char *refname);
@@ -214,11 +217,9 @@ char *repo_default_branch_name(struct repository *r, int quiet);
214217
*
215218
* Or
216219
*
217-
* - Call `initial_ref_transaction_commit()` if the ref database is
218-
* known to be empty and have no other writers (e.g. during
219-
* clone). This is likely to be much faster than
220-
* `ref_transaction_commit()`. `ref_transaction_prepare()` should
221-
* *not* be called before `initial_ref_transaction_commit()`.
220+
* - Call `ref_transaction_begin()` with REF_TRANSACTION_FLAG_INITIAL if the
221+
* ref database is known to be empty and have no other writers (e.g. during
222+
* clone). This is likely to be much faster than without the flag.
222223
*
223224
* - Then finally, call `ref_transaction_free()` to free the
224225
* `ref_transaction` data structure.
@@ -234,7 +235,7 @@ char *repo_default_branch_name(struct repository *r, int quiet);
234235
* struct strbuf err = STRBUF_INIT;
235236
* int ret = 0;
236237
*
237-
* transaction = ref_store_transaction_begin(refs, &err);
238+
* transaction = ref_store_transaction_begin(refs, 0, &err);
238239
* if (!transaction ||
239240
* ref_transaction_update(...) ||
240241
* ref_transaction_create(...) ||
@@ -579,11 +580,27 @@ enum action_on_err {
579580
UPDATE_REFS_QUIET_ON_ERR
580581
};
581582

583+
enum ref_transaction_flag {
584+
/*
585+
* The ref transaction is part of the initial creation of the ref store
586+
* and can thus assume that the ref store is completely empty. This
587+
* allows the backend to perform the transaction more efficiently by
588+
* skipping certain checks.
589+
*
590+
* It is a bug to set this flag when there might be other processes
591+
* accessing the repository or if there are existing references that
592+
* might conflict with the ones being created. All old_oid values must
593+
* either be absent or null_oid.
594+
*/
595+
REF_TRANSACTION_FLAG_INITIAL = (1 << 0),
596+
};
597+
582598
/*
583599
* Begin a reference transaction. The reference transaction must
584600
* be freed by calling ref_transaction_free().
585601
*/
586602
struct ref_transaction *ref_store_transaction_begin(struct ref_store *refs,
603+
unsigned int flags,
587604
struct strbuf *err);
588605

589606
/*
@@ -797,20 +814,6 @@ int ref_transaction_commit(struct ref_transaction *transaction,
797814
int ref_transaction_abort(struct ref_transaction *transaction,
798815
struct strbuf *err);
799816

800-
/*
801-
* Like ref_transaction_commit(), but optimized for creating
802-
* references when originally initializing a repository (e.g., by "git
803-
* clone"). It writes the new references directly to packed-refs
804-
* without locking the individual references.
805-
*
806-
* It is a bug to call this function when there might be other
807-
* processes accessing the repository or if there are existing
808-
* references that might conflict with the ones being created. All
809-
* old_oid values must either be absent or null_oid.
810-
*/
811-
int initial_ref_transaction_commit(struct ref_transaction *transaction,
812-
struct strbuf *err);
813-
814817
/*
815818
* Execute the given callback function for each of the reference updates which
816819
* have been queued in the given transaction. `old_oid` and `new_oid` may be

0 commit comments

Comments
 (0)