Skip to content

Commit dabecb9

Browse files
KarthikNayakgitster
KarthikNayak
authored and
gitster
committed
for-each-ref: introduce a '--start-after' option
The `git-for-each-ref(1)` command is used to iterate over references present in a repository. In large repositories with millions of references, it would be optimal to paginate this output such that we can start iteration from a given reference. This would avoid having to iterate over all references from the beginning each time when paginating through results. The previous commit added 'seek' functionality to the reference backends. Utilize this and expose a '--start-after' option in 'git-for-each-ref(1)'. When used, the reference iteration seeks to the lexicographically next reference and iterates from there onward. This enables efficient pagination workflows, where the calling script can remember the last provided reference and use that as the starting point for the next set of references: git for-each-ref --count=100 git for-each-ref --count=100 --start-after=refs/heads/branch-100 git for-each-ref --count=100 --start-after=refs/heads/branch-200 Since the reference iterators only allow seeking to a specified marker via the `ref_iterator_seek()`, we introduce a helper function `start_ref_iterator_after()`, which seeks to next reference by simply adding (char) 1 to the marker. We must note that pagination always continues from the provided marker, as such any concurrent reference updates lexicographically behind the marker will not be output. Document the same. Signed-off-by: Karthik Nayak <[email protected]> Signed-off-by: Junio C Hamano <[email protected]>
1 parent 526530a commit dabecb9

File tree

5 files changed

+272
-19
lines changed

5 files changed

+272
-19
lines changed

‎Documentation/git-for-each-ref.adoc

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ SYNOPSIS
1414
[--points-at=<object>]
1515
[--merged[=<object>]] [--no-merged[=<object>]]
1616
[--contains[=<object>]] [--no-contains[=<object>]]
17-
[--exclude=<pattern> ...]
17+
[--exclude=<pattern> ...] [--start-after=<marker>]
1818

1919
DESCRIPTION
2020
-----------
@@ -108,6 +108,14 @@ TAB %(refname)`.
108108
--include-root-refs::
109109
List root refs (HEAD and pseudorefs) apart from regular refs.
110110

111+
--start-after=<marker>::
112+
Allows paginating the output by skipping references up to and including the
113+
specified marker. When paging, it should be noted that references may be
114+
deleted, modified or added between invocations. Output will only yield those
115+
references which follow the marker lexicographically. Output begins from the
116+
first reference that would come after the marker alphabetically. Cannot be
117+
used with general pattern matching or custom sort options.
118+
111119
FIELD NAMES
112120
-----------
113121

‎builtin/for-each-ref.c

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ static char const * const for_each_ref_usage[] = {
1313
N_("git for-each-ref [--points-at <object>]"),
1414
N_("git for-each-ref [--merged [<commit>]] [--no-merged [<commit>]]"),
1515
N_("git for-each-ref [--contains [<commit>]] [--no-contains [<commit>]]"),
16+
N_("git for-each-ref [--start-after <marker>]"),
1617
NULL
1718
};
1819

@@ -44,6 +45,7 @@ int cmd_for_each_ref(int argc,
4445
OPT_GROUP(""),
4546
OPT_INTEGER( 0 , "count", &format.array_opts.max_count, N_("show only <n> matched refs")),
4647
OPT_STRING( 0 , "format", &format.format, N_("format"), N_("format to use for the output")),
48+
OPT_STRING( 0 , "start-after", &filter.start_after, N_("start-start"), N_("start iteration after the provided marker")),
4749
OPT__COLOR(&format.use_color, N_("respect format colors")),
4850
OPT_REF_FILTER_EXCLUDE(&filter),
4951
OPT_REF_SORT(&sorting_options),
@@ -79,6 +81,9 @@ int cmd_for_each_ref(int argc,
7981
if (verify_ref_format(&format))
8082
usage_with_options(for_each_ref_usage, opts);
8183

84+
if (filter.start_after && sorting_options.nr > 1)
85+
die(_("cannot use --start-after with custom sort options"));
86+
8287
sorting = ref_sorting_options(&sorting_options);
8388
ref_sorting_set_sort_flags_all(sorting, REF_SORTING_ICASE, icase);
8489
filter.ignore_case = icase;
@@ -100,6 +105,9 @@ int cmd_for_each_ref(int argc,
100105
filter.name_patterns = argv;
101106
}
102107

108+
if (filter.start_after && filter.name_patterns && filter.name_patterns[0])
109+
die(_("cannot use --start-after with patterns"));
110+
103111
if (include_root_refs)
104112
flags |= FILTER_REFS_ROOT_REFS | FILTER_REFS_DETACHED_HEAD;
105113

‎ref-filter.c

Lines changed: 60 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -2683,6 +2683,41 @@ static int filter_exclude_match(struct ref_filter *filter, const char *refname)
26832683
return match_pattern(filter->exclude.v, refname, filter->ignore_case);
26842684
}
26852685

2686+
/*
2687+
* We need to seek to the reference right after a given marker but excluding any
2688+
* matching references. So we seek to the lexicographically next reference.
2689+
*/
2690+
static int start_ref_iterator_after(struct ref_iterator *iter, const char *marker)
2691+
{
2692+
struct strbuf sb = STRBUF_INIT;
2693+
int ret;
2694+
2695+
strbuf_addstr(&sb, marker);
2696+
strbuf_addch(&sb, 1);
2697+
2698+
ret = ref_iterator_seek(iter, sb.buf, 0);
2699+
2700+
strbuf_release(&sb);
2701+
return ret;
2702+
}
2703+
2704+
static int for_each_fullref_with_seek(struct ref_filter *filter, each_ref_fn cb,
2705+
void *cb_data, unsigned int flags)
2706+
{
2707+
struct ref_iterator *iter;
2708+
int ret = 0;
2709+
2710+
iter = refs_ref_iterator_begin(get_main_ref_store(the_repository), "",
2711+
NULL, 0, flags);
2712+
if (filter->start_after)
2713+
ret = start_ref_iterator_after(iter, filter->start_after);
2714+
2715+
if (ret)
2716+
return ret;
2717+
2718+
return do_for_each_ref_iterator(iter, cb, cb_data);
2719+
}
2720+
26862721
/*
26872722
* This is the same as for_each_fullref_in(), but it tries to iterate
26882723
* only over the patterns we'll care about. Note that it _doesn't_ do a full
@@ -2694,8 +2729,8 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
26942729
{
26952730
if (filter->kind & FILTER_REFS_ROOT_REFS) {
26962731
/* In this case, we want to print all refs including root refs. */
2697-
return refs_for_each_include_root_refs(get_main_ref_store(the_repository),
2698-
cb, cb_data);
2732+
return for_each_fullref_with_seek(filter, cb, cb_data,
2733+
DO_FOR_EACH_INCLUDE_ROOT_REFS);
26992734
}
27002735

27012736
if (!filter->match_as_path) {
@@ -2704,8 +2739,7 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
27042739
* prefixes like "refs/heads/" etc. are stripped off,
27052740
* so we have to look at everything:
27062741
*/
2707-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2708-
"", NULL, cb, cb_data);
2742+
return for_each_fullref_with_seek(filter, cb, cb_data, 0);
27092743
}
27102744

27112745
if (filter->ignore_case) {
@@ -2714,14 +2748,12 @@ static int for_each_fullref_in_pattern(struct ref_filter *filter,
27142748
* so just return everything and let the caller
27152749
* sort it out.
27162750
*/
2717-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2718-
"", NULL, cb, cb_data);
2751+
return for_each_fullref_with_seek(filter, cb, cb_data, 0);
27192752
}
27202753

27212754
if (!filter->name_patterns[0]) {
27222755
/* no patterns; we have to look at everything */
2723-
return refs_for_each_fullref_in(get_main_ref_store(the_repository),
2724-
"", filter->exclude.v, cb, cb_data);
2756+
return for_each_fullref_with_seek(filter, cb, cb_data, 0);
27252757
}
27262758

27272759
return refs_for_each_fullref_in_prefixes(get_main_ref_store(the_repository),
@@ -3189,6 +3221,7 @@ void filter_is_base(struct repository *r,
31893221

31903222
static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref_fn fn, void *cb_data)
31913223
{
3224+
const char *prefix = NULL;
31923225
int ret = 0;
31933226

31943227
filter->kind = type & FILTER_REFS_KIND_MASK;
@@ -3207,19 +3240,28 @@ static int do_filter_refs(struct ref_filter *filter, unsigned int type, each_ref
32073240
* of filter_ref_kind().
32083241
*/
32093242
if (filter->kind == FILTER_REFS_BRANCHES)
3210-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3211-
"refs/heads/", NULL,
3212-
fn, cb_data);
3243+
prefix = "refs/heads/";
32133244
else if (filter->kind == FILTER_REFS_REMOTES)
3214-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3215-
"refs/remotes/", NULL,
3216-
fn, cb_data);
3245+
prefix = "refs/remotes/";
32173246
else if (filter->kind == FILTER_REFS_TAGS)
3218-
ret = refs_for_each_fullref_in(get_main_ref_store(the_repository),
3219-
"refs/tags/", NULL, fn,
3220-
cb_data);
3221-
else if (filter->kind & FILTER_REFS_REGULAR)
3247+
prefix = "refs/tags/";
3248+
3249+
if (prefix) {
3250+
struct ref_iterator *iter;
3251+
3252+
iter = refs_ref_iterator_begin(get_main_ref_store(the_repository),
3253+
"", NULL, 0, 0);
3254+
3255+
if (filter->start_after)
3256+
ret = start_ref_iterator_after(iter, filter->start_after);
3257+
else if (prefix)
3258+
ret = ref_iterator_seek(iter, prefix, 1);
3259+
3260+
if (!ret)
3261+
ret = do_for_each_ref_iterator(iter, fn, cb_data);
3262+
} else if (filter->kind & FILTER_REFS_REGULAR) {
32223263
ret = for_each_fullref_in_pattern(filter, fn, cb_data);
3264+
}
32233265

32243266
/*
32253267
* When printing all ref types, HEAD is already included,

‎ref-filter.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ struct ref_array {
6464

6565
struct ref_filter {
6666
const char **name_patterns;
67+
const char *start_after;
6768
struct strvec exclude;
6869
struct oid_array points_at;
6970
struct commit_list *with_commit;

‎t/t6302-for-each-ref-filter.sh

Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -541,4 +541,198 @@ test_expect_success 'validate worktree atom' '
541541
test_cmp expect actual
542542
'
543543

544+
test_expect_success 'start after with empty value' '
545+
cat >expect <<-\EOF &&
546+
refs/heads/main
547+
refs/heads/main_worktree
548+
refs/heads/side
549+
refs/odd/spot
550+
refs/tags/annotated-tag
551+
refs/tags/doubly-annotated-tag
552+
refs/tags/doubly-signed-tag
553+
refs/tags/foo1.10
554+
refs/tags/foo1.3
555+
refs/tags/foo1.6
556+
refs/tags/four
557+
refs/tags/one
558+
refs/tags/signed-tag
559+
refs/tags/three
560+
refs/tags/two
561+
EOF
562+
git for-each-ref --format="%(refname)" --start-after="" >actual &&
563+
test_cmp expect actual
564+
'
565+
566+
test_expect_success 'start after a specific reference' '
567+
cat >expect <<-\EOF &&
568+
refs/tags/annotated-tag
569+
refs/tags/doubly-annotated-tag
570+
refs/tags/doubly-signed-tag
571+
refs/tags/foo1.10
572+
refs/tags/foo1.3
573+
refs/tags/foo1.6
574+
refs/tags/four
575+
refs/tags/one
576+
refs/tags/signed-tag
577+
refs/tags/three
578+
refs/tags/two
579+
EOF
580+
git for-each-ref --format="%(refname)" --start-after=refs/odd/spot >actual &&
581+
test_cmp expect actual
582+
'
583+
584+
test_expect_success 'start after a specific reference with partial match' '
585+
cat >expect <<-\EOF &&
586+
refs/odd/spot
587+
refs/tags/annotated-tag
588+
refs/tags/doubly-annotated-tag
589+
refs/tags/doubly-signed-tag
590+
refs/tags/foo1.10
591+
refs/tags/foo1.3
592+
refs/tags/foo1.6
593+
refs/tags/four
594+
refs/tags/one
595+
refs/tags/signed-tag
596+
refs/tags/three
597+
refs/tags/two
598+
EOF
599+
git for-each-ref --format="%(refname)" --start-after=refs/odd/sp >actual &&
600+
test_cmp expect actual
601+
'
602+
603+
test_expect_success 'start after, just behind a specific reference' '
604+
cat >expect <<-\EOF &&
605+
refs/odd/spot
606+
refs/tags/annotated-tag
607+
refs/tags/doubly-annotated-tag
608+
refs/tags/doubly-signed-tag
609+
refs/tags/foo1.10
610+
refs/tags/foo1.3
611+
refs/tags/foo1.6
612+
refs/tags/four
613+
refs/tags/one
614+
refs/tags/signed-tag
615+
refs/tags/three
616+
refs/tags/two
617+
EOF
618+
git for-each-ref --format="%(refname)" --start-after=refs/odd/parrot >actual &&
619+
test_cmp expect actual
620+
'
621+
622+
test_expect_success 'start after with specific directory match' '
623+
cat >expect <<-\EOF &&
624+
refs/odd/spot
625+
refs/tags/annotated-tag
626+
refs/tags/doubly-annotated-tag
627+
refs/tags/doubly-signed-tag
628+
refs/tags/foo1.10
629+
refs/tags/foo1.3
630+
refs/tags/foo1.6
631+
refs/tags/four
632+
refs/tags/one
633+
refs/tags/signed-tag
634+
refs/tags/three
635+
refs/tags/two
636+
EOF
637+
git for-each-ref --format="%(refname)" --start-after=refs/odd >actual &&
638+
test_cmp expect actual
639+
'
640+
641+
test_expect_success 'start after with specific directory and trailing slash' '
642+
cat >expect <<-\EOF &&
643+
refs/odd/spot
644+
refs/tags/annotated-tag
645+
refs/tags/doubly-annotated-tag
646+
refs/tags/doubly-signed-tag
647+
refs/tags/foo1.10
648+
refs/tags/foo1.3
649+
refs/tags/foo1.6
650+
refs/tags/four
651+
refs/tags/one
652+
refs/tags/signed-tag
653+
refs/tags/three
654+
refs/tags/two
655+
EOF
656+
git for-each-ref --format="%(refname)" --start-after=refs/odd/ >actual &&
657+
test_cmp expect actual
658+
'
659+
660+
test_expect_success 'start after, just behind a specific directory' '
661+
cat >expect <<-\EOF &&
662+
refs/odd/spot
663+
refs/tags/annotated-tag
664+
refs/tags/doubly-annotated-tag
665+
refs/tags/doubly-signed-tag
666+
refs/tags/foo1.10
667+
refs/tags/foo1.3
668+
refs/tags/foo1.6
669+
refs/tags/four
670+
refs/tags/one
671+
refs/tags/signed-tag
672+
refs/tags/three
673+
refs/tags/two
674+
EOF
675+
git for-each-ref --format="%(refname)" --start-after=refs/lost >actual &&
676+
test_cmp expect actual
677+
'
678+
679+
test_expect_success 'start after, overflow specific reference length' '
680+
cat >expect <<-\EOF &&
681+
refs/tags/annotated-tag
682+
refs/tags/doubly-annotated-tag
683+
refs/tags/doubly-signed-tag
684+
refs/tags/foo1.10
685+
refs/tags/foo1.3
686+
refs/tags/foo1.6
687+
refs/tags/four
688+
refs/tags/one
689+
refs/tags/signed-tag
690+
refs/tags/three
691+
refs/tags/two
692+
EOF
693+
git for-each-ref --format="%(refname)" --start-after=refs/odd/spotnew >actual &&
694+
test_cmp expect actual
695+
'
696+
697+
test_expect_success 'start after, overflow specific reference path' '
698+
cat >expect <<-\EOF &&
699+
refs/tags/annotated-tag
700+
refs/tags/doubly-annotated-tag
701+
refs/tags/doubly-signed-tag
702+
refs/tags/foo1.10
703+
refs/tags/foo1.3
704+
refs/tags/foo1.6
705+
refs/tags/four
706+
refs/tags/one
707+
refs/tags/signed-tag
708+
refs/tags/three
709+
refs/tags/two
710+
EOF
711+
git for-each-ref --format="%(refname)" --start-after=refs/odd/spot/new >actual &&
712+
test_cmp expect actual
713+
'
714+
715+
test_expect_success 'start after, last reference' '
716+
cat >expect <<-\EOF &&
717+
EOF
718+
git for-each-ref --format="%(refname)" --start-after=refs/tags/two >actual &&
719+
test_cmp expect actual
720+
'
721+
722+
test_expect_success 'start after used with a pattern' '
723+
cat >expect <<-\EOF &&
724+
fatal: cannot use --start-after with patterns
725+
EOF
726+
test_must_fail git for-each-ref --format="%(refname)" --start-after=refs/odd/spot refs/tags 2>actual &&
727+
test_cmp expect actual
728+
'
729+
730+
test_expect_success 'start after used with custom sort order' '
731+
cat >expect <<-\EOF &&
732+
fatal: cannot use --start-after with custom sort options
733+
EOF
734+
test_must_fail git for-each-ref --format="%(refname)" --start-after=refs/odd/spot --sort=author 2>actual &&
735+
test_cmp expect actual
736+
'
737+
544738
test_done

0 commit comments

Comments
 (0)