Skip to content

Commit 7626fc2

Browse files
authored
Add GitLab group match regex filter (#617)
1 parent 0e00d0b commit 7626fc2

File tree

9 files changed

+368
-8
lines changed

9 files changed

+368
-8
lines changed

CHANGELOG.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,15 @@ All notable changes to this project will be documented in this file.
33

44
The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
55

6+
## [1.11.9] - unreleased
7+
### Added
8+
- GHORG_GITLAB_GROUP_MATCH_REGEX; thanks @batagy
9+
### Changed
10+
### Deprecated
11+
### Removed
12+
### Fixed
13+
### Security
14+
615
## [1.11.8] - 2/1/26
716
### Added
817
- Bitbucket Cloud API token authentication support; thanks @dean-tate

cmd/clone.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,11 @@ func cloneFunc(cmd *cobra.Command, argz []string) {
187187
os.Setenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX", prefix)
188188
}
189189

190+
if cmd.Flags().Changed("gitlab-group-match-regex") {
191+
regex := cmd.Flag("gitlab-group-match-regex").Value.String()
192+
os.Setenv("GHORG_GITLAB_GROUP_MATCH_REGEX", regex)
193+
}
194+
190195
if cmd.Flags().Changed("match-regex") {
191196
regex := cmd.Flag("match-regex").Value.String()
192197
os.Setenv("GHORG_MATCH_REGEX", regex)
@@ -1264,6 +1269,12 @@ func PrintConfigs() {
12641269
if os.Getenv("GHORG_EXCLUDE_MATCH_PREFIX") != "" {
12651270
colorlog.PrintInfo("* Exclude Prefix: " + os.Getenv("GHORG_EXCLUDE_MATCH_PREFIX"))
12661271
}
1272+
if os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX") != "" {
1273+
colorlog.PrintInfo("* GL Grp Match : " + os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX"))
1274+
}
1275+
if os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX") != "" {
1276+
colorlog.PrintInfo("* GL Grp Exclude: " + os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX"))
1277+
}
12671278
if os.Getenv("GHORG_INCLUDE_SUBMODULES") == "true" {
12681279
colorlog.PrintInfo("* Submodules : " + os.Getenv("GHORG_INCLUDE_SUBMODULES"))
12691280
}

cmd/root.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ var (
4141
excludeMatchRegex string
4242
config string
4343
gitlabGroupExcludeMatchRegex string
44+
gitlabGroupMatchRegex string
4445
ghorgIgnorePath string
4546
ghorgOnlyPath string
4647
targetReposPath string
@@ -346,6 +347,7 @@ func InitConfig() {
346347
getOrSetDefaults("GHORG_MATCH_PREFIX")
347348
getOrSetDefaults("GHORG_EXCLUDE_MATCH_PREFIX")
348349
getOrSetDefaults("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX")
350+
getOrSetDefaults("GHORG_GITLAB_GROUP_MATCH_REGEX")
349351
getOrSetDefaults("GHORG_IGNORE_PATH")
350352
getOrSetDefaults("GHORG_RECLONE_PATH")
351353
getOrSetDefaults("GHORG_QUIET")
@@ -420,6 +422,7 @@ func init() {
420422
cloneCmd.Flags().StringVarP(&matchRegex, "match-regex", "", "", "GHORG_MATCH_REGEX - Only clone repos that match name to regex provided")
421423
cloneCmd.Flags().StringVarP(&excludeMatchRegex, "exclude-match-regex", "", "", "GHORG_EXCLUDE_MATCH_REGEX - Exclude cloning repos that match name to regex provided")
422424
cloneCmd.Flags().StringVarP(&gitlabGroupExcludeMatchRegex, "gitlab-group-exclude-match-regex", "", "", "GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX - Exclude cloning gitlab groups that match name to regex provided")
425+
cloneCmd.Flags().StringVarP(&gitlabGroupMatchRegex, "gitlab-group-match-regex", "", "", "GHORG_GITLAB_GROUP_MATCH_REGEX - Only clone gitlab groups that match name to regex provided")
423426
cloneCmd.Flags().StringVarP(&ghorgIgnorePath, "ghorgignore-path", "", "", "GHORG_IGNORE_PATH - If you want to set a path other than $HOME/.config/ghorg/ghorgignore for your ghorgignore")
424427
cloneCmd.Flags().StringVarP(&ghorgOnlyPath, "ghorgonly-path", "", "", "GHORG_ONLY_PATH - If you want to set a path other than $HOME/.config/ghorg/ghorgonly for your ghorgonly")
425428
cloneCmd.Flags().StringVarP(&exitCodeOnCloneInfos, "exit-code-on-clone-infos", "", "", "GHORG_EXIT_CODE_ON_CLONE_INFOS - Allows you to control the exit code when ghorg runs into a problem (info level message) cloning a repo from the remote. Info messages will appear after a clone is complete, similar to success messages. (default 0)")

cmd/version.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
"github.com/spf13/cobra"
77
)
88

9-
const ghorgVersion = "v1.11.8"
9+
const ghorgVersion = "v1.11.9"
1010

1111
var versionCmd = &cobra.Command{
1212
Use: "version",

sample-conf.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -261,6 +261,10 @@ GHORG_PRESERVE_DIRECTORY_STRUCTURE: false
261261
# flag (--insecure-gitlab-client)
262262
GHORG_INSECURE_GITLAB_CLIENT: false
263263

264+
# Only clone gitlab groups that match name to regex provided
265+
# flag (--gitlab-group-match-regex)
266+
GHORG_GITLAB_GROUP_MATCH_REGEX:
267+
264268
# Exclude gitlab groups by regex
265269
# flag (--gitlab-group-exclude-match-regex)
266270
GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX:

scm/gitlab.go

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -80,8 +80,17 @@ func (c Gitlab) GetOrgRepos(targetOrg string) ([]Repo, error) {
8080
allGroups = append(allGroups, targetOrg)
8181
}
8282

83-
if os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX") != "" {
84-
allGroups = filterGitlabGroupByExcludeMatchRegex(allGroups)
83+
// Group-level filters only apply to all-groups mode where there are multiple
84+
// top-level groups to select from. For single-group clones, the per-repo
85+
// filter in filter() handles PathWithNamespace-based matching instead.
86+
if gitLabAllGroups {
87+
if os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX") != "" {
88+
allGroups = filterGitlabGroupByMatchRegex(allGroups)
89+
}
90+
91+
if os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX") != "" {
92+
allGroups = filterGitlabGroupByExcludeMatchRegex(allGroups)
93+
}
8594
}
8695

8796
for i, group := range allGroups {
@@ -512,6 +521,15 @@ func (c Gitlab) filter(group string, ps []*gitlab.Project) []Repo {
512521
continue
513522
}
514523

524+
// Apply GitLab group match regex to repository path (include only matching)
525+
if os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX") != "" {
526+
regex := os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX")
527+
re := regexp.MustCompile(regex)
528+
if re.FindString(p.PathWithNamespace) == "" {
529+
continue // Skip this repository as it does not match the include pattern
530+
}
531+
}
532+
515533
// Apply GitLab group exclude regex to repository path
516534
if os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX") != "" {
517535
regex := os.Getenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX")
@@ -599,6 +617,20 @@ func filterGitlabGroupByExcludeMatchRegex(groups []string) []string {
599617
return filteredGroups
600618
}
601619

620+
func filterGitlabGroupByMatchRegex(groups []string) []string {
621+
filteredGroups := []string{}
622+
regex := fmt.Sprint(os.Getenv("GHORG_GITLAB_GROUP_MATCH_REGEX"))
623+
re := regexp.MustCompile(regex)
624+
625+
for i, grp := range groups {
626+
if re.FindString(grp) != "" {
627+
filteredGroups = append(filteredGroups, groups[i])
628+
}
629+
}
630+
631+
return filteredGroups
632+
}
633+
602634
// ToSlug converts a title into a URL-friendly slug.
603635
func ToSlug(title string) string {
604636
// Convert to lowercase

scm/gitlab_test.go

Lines changed: 204 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,204 @@
1+
package scm
2+
3+
import (
4+
"os"
5+
"testing"
6+
)
7+
8+
func TestFilterGitlabGroupByMatchRegex(t *testing.T) {
9+
testCases := []struct {
10+
name string
11+
regex string
12+
groups []string
13+
expectedGroups []string
14+
}{
15+
{
16+
name: "matches specific group names",
17+
regex: "^(subgroup-a|subgroup-b)$",
18+
groups: []string{"subgroup-a", "subgroup-b", "subgroup-c", "subgroup-d"},
19+
expectedGroups: []string{"subgroup-a", "subgroup-b"},
20+
},
21+
{
22+
name: "matches groups with partial regex",
23+
regex: "subgroup-a",
24+
groups: []string{"subgroup-a", "subgroup-ab", "subgroup-b"},
25+
expectedGroups: []string{"subgroup-a", "subgroup-ab"},
26+
},
27+
{
28+
name: "no matches returns empty",
29+
regex: "^nonexistent$",
30+
groups: []string{"subgroup-a", "subgroup-b"},
31+
expectedGroups: []string{},
32+
},
33+
{
34+
name: "matches all groups",
35+
regex: ".*",
36+
groups: []string{"subgroup-a", "subgroup-b", "subgroup-c"},
37+
expectedGroups: []string{"subgroup-a", "subgroup-b", "subgroup-c"},
38+
},
39+
{
40+
name: "case insensitive matching",
41+
regex: "(?i:^SUBGROUP-A$)",
42+
groups: []string{"subgroup-a", "subgroup-b", "Subgroup-A"},
43+
expectedGroups: []string{"subgroup-a", "Subgroup-A"},
44+
},
45+
{
46+
name: "matches numeric group IDs",
47+
regex: "^(123|456)$",
48+
groups: []string{"123", "456", "789", "101"},
49+
expectedGroups: []string{"123", "456"},
50+
},
51+
{
52+
name: "empty groups list",
53+
regex: "anything",
54+
groups: []string{},
55+
expectedGroups: []string{},
56+
},
57+
{
58+
name: "matches groups with path-like names",
59+
regex: "my-org/subgroup",
60+
groups: []string{"my-org/subgroup-a", "my-org/subgroup-b", "other-org/group-c"},
61+
expectedGroups: []string{"my-org/subgroup-a", "my-org/subgroup-b"},
62+
},
63+
}
64+
65+
for _, tc := range testCases {
66+
t.Run(tc.name, func(t *testing.T) {
67+
os.Setenv("GHORG_GITLAB_GROUP_MATCH_REGEX", tc.regex)
68+
defer os.Unsetenv("GHORG_GITLAB_GROUP_MATCH_REGEX")
69+
70+
result := filterGitlabGroupByMatchRegex(tc.groups)
71+
if len(result) == 0 && len(tc.expectedGroups) == 0 {
72+
return // Both empty, test passes
73+
}
74+
if len(result) != len(tc.expectedGroups) {
75+
t.Errorf("Expected %d groups, got %d. Expected: %v, Got: %v",
76+
len(tc.expectedGroups), len(result), tc.expectedGroups, result)
77+
return
78+
}
79+
for i, group := range result {
80+
if group != tc.expectedGroups[i] {
81+
t.Errorf("Expected group at index %d to be %s, got %s", i, tc.expectedGroups[i], group)
82+
}
83+
}
84+
})
85+
}
86+
}
87+
88+
func TestFilterGitlabGroupByExcludeMatchRegex(t *testing.T) {
89+
testCases := []struct {
90+
name string
91+
regex string
92+
groups []string
93+
expectedGroups []string
94+
}{
95+
{
96+
name: "excludes specific group names",
97+
regex: "^(subgroup-a|subgroup-b)$",
98+
groups: []string{"subgroup-a", "subgroup-b", "subgroup-c", "subgroup-d"},
99+
expectedGroups: []string{"subgroup-c", "subgroup-d"},
100+
},
101+
{
102+
name: "no exclusions when nothing matches",
103+
regex: "^nonexistent$",
104+
groups: []string{"subgroup-a", "subgroup-b"},
105+
expectedGroups: []string{"subgroup-a", "subgroup-b"},
106+
},
107+
{
108+
name: "excludes all groups",
109+
regex: "subgroup",
110+
groups: []string{"subgroup-a", "subgroup-b"},
111+
expectedGroups: []string{},
112+
},
113+
{
114+
name: "empty groups list",
115+
regex: "anything",
116+
groups: []string{},
117+
expectedGroups: []string{},
118+
},
119+
}
120+
121+
for _, tc := range testCases {
122+
t.Run(tc.name, func(t *testing.T) {
123+
os.Setenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX", tc.regex)
124+
defer os.Unsetenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX")
125+
126+
result := filterGitlabGroupByExcludeMatchRegex(tc.groups)
127+
if len(result) == 0 && len(tc.expectedGroups) == 0 {
128+
return // Both empty, test passes
129+
}
130+
if len(result) != len(tc.expectedGroups) {
131+
t.Errorf("Expected %d groups, got %d. Expected: %v, Got: %v",
132+
len(tc.expectedGroups), len(result), tc.expectedGroups, result)
133+
return
134+
}
135+
for i, group := range result {
136+
if group != tc.expectedGroups[i] {
137+
t.Errorf("Expected group at index %d to be %s, got %s", i, tc.expectedGroups[i], group)
138+
}
139+
}
140+
})
141+
}
142+
}
143+
144+
func TestFilterGitlabGroupMatchAndExcludeCombined(t *testing.T) {
145+
// Test that match and exclude work together:
146+
// First include only matching, then exclude from those
147+
t.Run("match then exclude narrows results", func(t *testing.T) {
148+
groups := []string{"subgroup-a", "subgroup-b", "subgroup-c", "other-group"}
149+
150+
// First apply match: include only subgroup-* groups
151+
os.Setenv("GHORG_GITLAB_GROUP_MATCH_REGEX", "^subgroup-")
152+
defer os.Unsetenv("GHORG_GITLAB_GROUP_MATCH_REGEX")
153+
154+
matched := filterGitlabGroupByMatchRegex(groups)
155+
156+
// Then apply exclude: remove subgroup-c from the result
157+
os.Setenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX", "^subgroup-c$")
158+
defer os.Unsetenv("GHORG_GITLAB_GROUP_EXCLUDE_MATCH_REGEX")
159+
160+
result := filterGitlabGroupByExcludeMatchRegex(matched)
161+
162+
expected := []string{"subgroup-a", "subgroup-b"}
163+
if len(result) != len(expected) {
164+
t.Errorf("Expected %d groups, got %d. Expected: %v, Got: %v",
165+
len(expected), len(result), expected, result)
166+
return
167+
}
168+
for i, group := range result {
169+
if group != expected[i] {
170+
t.Errorf("Expected group at index %d to be %s, got %s", i, expected[i], group)
171+
}
172+
}
173+
})
174+
}
175+
176+
func TestFilterGitlabGroupMatchRegexWithAlternation(t *testing.T) {
177+
// This is the primary use case from the feature request:
178+
// User wants to include only 3 subgroups out of 20
179+
t.Run("include only 3 out of many subgroups using alternation", func(t *testing.T) {
180+
// Simulate 10 subgroups
181+
groups := []string{
182+
"subgroup-1", "subgroup-2", "subgroup-3", "subgroup-4", "subgroup-5",
183+
"subgroup-6", "subgroup-7", "subgroup-8", "subgroup-9", "subgroup-10",
184+
}
185+
186+
// Include only 3 specific subgroups
187+
os.Setenv("GHORG_GITLAB_GROUP_MATCH_REGEX", "^(subgroup-2|subgroup-5|subgroup-8)$")
188+
defer os.Unsetenv("GHORG_GITLAB_GROUP_MATCH_REGEX")
189+
190+
result := filterGitlabGroupByMatchRegex(groups)
191+
192+
expected := []string{"subgroup-2", "subgroup-5", "subgroup-8"}
193+
if len(result) != len(expected) {
194+
t.Errorf("Expected %d groups, got %d. Expected: %v, Got: %v",
195+
len(expected), len(result), expected, result)
196+
return
197+
}
198+
for i, group := range result {
199+
if group != expected[i] {
200+
t.Errorf("Expected group at index %d to be %s, got %s", i, expected[i], group)
201+
}
202+
}
203+
})
204+
}

0 commit comments

Comments
 (0)