Skip to content

Commit 6011769

Browse files
tstrombergtstromberg
authored
Merge pull request tinkerbell#33 from mmlb/concurrencize
Allow linters to run concurrently
2 parents 432b78e + e4a2229 commit 6011769

File tree

5 files changed

+171
-76
lines changed

5 files changed

+171
-76
lines changed

‎.envrc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
has nix && use nix
2+
dotenv_if_exists
3+
PATH_add bin
4+
path_add GOBIN bin

‎Makefile‎

Lines changed: 60 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
# BEGIN: lint-install .
33
# http://github.com/tinkerbell/lint-install
44

5-
GOLINT_VERSION ?= v1.42.1
6-
HADOLINT_VERSION ?= v2.7.0
7-
SHELLCHECK_VERSION ?= v0.7.2
8-
YAMLLINT_VERSION ?= 1.26.3
9-
LINT_OS := $(shell uname)
5+
.PHONY: lint
6+
lint: _lint
7+
108
LINT_ARCH := $(shell uname -m)
9+
LINT_OS := $(shell uname)
10+
LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
1111
LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
1212

1313
# shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation
@@ -17,43 +17,72 @@ ifeq ($(LINT_OS),Darwin)
1717
endif
1818
endif
1919

20-
LINT_LOWER_OS = $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
21-
GOLINT_CONFIG = $(LINT_ROOT)/.golangci.yml
22-
YAMLLINT_ROOT = out/linters/yamllint-$(YAMLLINT_VERSION)
20+
LINTERS :=
21+
FIXERS :=
2322

24-
.PHONY: lint
25-
lint: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) $(YAMLLINT_ROOT)/dist/bin/yamllint
26-
out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run
27-
out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) $(shell find . -name "*Dockerfile")
28-
out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh")
29-
PYTHONPATH=$(YAMLLINT_ROOT)/dist $(YAMLLINT_ROOT)/dist/bin/yamllint .
30-
31-
.PHONY: fix
32-
fix: out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
33-
out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run --fix
34-
out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh") -f diff | git apply -p2 -
35-
36-
out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck:
23+
SHELLCHECK_VERSION ?= v0.8.0
24+
SHELLCHECK_BIN := out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
25+
$(SHELLCHECK_BIN):
3726
mkdir -p out/linters
3827
rm -rf out/linters/shellcheck-*
39-
curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_LOWER_OS).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
40-
mv out/linters/shellcheck-$(SHELLCHECK_VERSION) out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
28+
curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_OS_LOWER).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
29+
mv out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck $@
30+
rm -rf out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck
31+
32+
LINTERS += shellcheck-lint
33+
shellcheck-lint: $(SHELLCHECK_BIN)
34+
$(SHELLCHECK_BIN) $(shell find . -name "*.sh")
4135

42-
out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH):
36+
FIXERS += shellcheck-fix
37+
shellcheck-fix: $(SHELLCHECK_BIN)
38+
$(SHELLCHECK_BIN) $(shell find . -name "*.sh") -f diff | { read -t 1 line || exit 0; { echo "$$line" && cat; } | git apply -p2; }
39+
40+
HADOLINT_VERSION ?= v2.8.0
41+
HADOLINT_BIN := out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
42+
$(HADOLINT_BIN):
4343
mkdir -p out/linters
4444
rm -rf out/linters/hadolint-*
45-
curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
46-
chmod u+x out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
45+
curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > $@
46+
chmod u+x $@
47+
48+
LINTERS += hadolint-lint
49+
hadolint-lint: $(HADOLINT_BIN)
50+
$(HADOLINT_BIN) $(shell find . -name "*Dockerfile")
4751

48-
out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH):
52+
GOLANGCI_LINT_CONFIG := $(LINT_ROOT)/.golangci.yml
53+
GOLANGCI_LINT_VERSION ?= v1.43.0
54+
GOLANGCI_LINT_BIN := out/linters/golangci-lint-$(GOLANGCI_LINT_VERSION)-$(LINT_ARCH)
55+
$(GOLANGCI_LINT_BIN):
4956
mkdir -p out/linters
5057
rm -rf out/linters/golangci-lint-*
51-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLINT_VERSION)
52-
mv out/linters/golangci-lint out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
58+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLANGCI_LINT_VERSION)
59+
mv out/linters/golangci-lint $@
60+
61+
LINTERS += golangci-lint-lint
62+
golangci-lint-lint: $(GOLANGCI_LINT_BIN)
63+
$(GOLANGCI_LINT_BIN) run
5364

54-
$(YAMLLINT_ROOT)/dist/bin/yamllint:
65+
FIXERS += golangci-lint-fix
66+
golangci-lint-fix: $(GOLANGCI_LINT_BIN)
67+
$(GOLANGCI_LINT_BIN) run --fix
68+
69+
YAMLLINT_VERSION ?= 1.26.3
70+
YAMLLINT_ROOT := out/linters/yamllint-$(YAMLLINT_VERSION)
71+
YAMLLINT_BIN := $(YAMLLINT_ROOT)/dist/bin/yamllint
72+
$(YAMLLINT_BIN):
5573
mkdir -p out/linters
5674
rm -rf out/linters/yamllint-*
5775
curl -sSfL https://github.com/adrienverge/yamllint/archive/refs/tags/v$(YAMLLINT_VERSION).tar.gz | tar -C out/linters -zxf -
58-
cd $(YAMLLINT_ROOT) && pip3 install . -t dist
76+
cd $(YAMLLINT_ROOT) && pip3 install --target dist .
77+
78+
LINTERS += yamllint-lint
79+
yamllint-lint: $(YAMLLINT_BIN)
80+
PYTHONPATH=$(YAMLLINT_ROOT)/dist $(YAMLLINT_ROOT)/dist/bin/yamllint .
81+
82+
.PHONY: _lint $(LINTERS)
83+
_lint: $(LINTERS)
84+
85+
.PHONY: fix $(FIXERS)
86+
fix: $(FIXERS)
87+
5988
# END: lint-install .

‎Makefile.tmpl‎

Lines changed: 62 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,12 @@
22
# BEGIN: lint-install {{.Args}}
33
# http://github.com/tinkerbell/lint-install
44

5-
{{ if .Go }}GOLINT_VERSION ?= v1.42.1{{ end }}
6-
{{ if .Dockerfile}}HADOLINT_VERSION ?= v2.7.0{{ end }}
7-
{{ if .Shell}}SHELLCHECK_VERSION ?= v0.7.2{{ end }}
8-
{{ if .YAML}}YAMLLINT_VERSION ?= 1.26.3{{ end }}
9-
LINT_OS := $(shell uname)
5+
.PHONY: lint
6+
lint: _lint
7+
108
LINT_ARCH := $(shell uname -m)
9+
LINT_OS := $(shell uname)
10+
LINT_OS_LOWER := $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]')
1111
LINT_ROOT := $(shell dirname $(realpath $(firstword $(MAKEFILE_LIST))))
1212

1313
# shellcheck and hadolint lack arm64 native binaries: rely on x86-64 emulation
@@ -17,51 +17,84 @@ ifeq ($(LINT_OS),Darwin)
1717
endif
1818
endif
1919

20-
{{ if .Shell }}LINT_LOWER_OS = $(shell echo $(LINT_OS) | tr '[:upper:]' '[:lower:]'){{ end }}
21-
{{ if .Go }}GOLINT_CONFIG = $(LINT_ROOT)/.golangci.yml{{ end }}
22-
{{ if .YAML }}YAMLLINT_ROOT = out/linters/yamllint-$(YAMLLINT_VERSION){{ end }}
23-
24-
.PHONY: lint
25-
lint: {{ if .Shell }}out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck {{ end }}{{ if .Dockerfile }}out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH) {{ end }}{{ if .Go}}out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) {{ end }}{{ if .YAML}}$(YAMLLINT_ROOT)/dist/bin/yamllint{{ end }}
26-
{{- range .LintCommands }}
27-
{{ .}}{{ end}}
28-
29-
.PHONY: fix
30-
fix: {{ if .Shell }}out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck {{ end }}{{ if .Go}}out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH){{ end }}
31-
{{- range .FixCommands }}
32-
{{ .}}{{ end}}
20+
LINTERS :=
21+
FIXERS :=
3322

3423
{{ if .Shell -}}
35-
out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck:
24+
SHELLCHECK_VERSION ?= v0.8.0
25+
SHELLCHECK_BIN := out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
26+
$(SHELLCHECK_BIN):
3627
mkdir -p out/linters
3728
rm -rf out/linters/shellcheck-*
38-
curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_LOWER_OS).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
39-
mv out/linters/shellcheck-$(SHELLCHECK_VERSION) out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)
29+
curl -sSfL https://github.com/koalaman/shellcheck/releases/download/$(SHELLCHECK_VERSION)/shellcheck-$(SHELLCHECK_VERSION).$(LINT_OS_LOWER).$(LINT_ARCH).tar.xz | tar -C out/linters -xJf -
30+
mv out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck $@
31+
rm -rf out/linters/shellcheck-$(SHELLCHECK_VERSION)/shellcheck
32+
33+
LINTERS += shellcheck-lint
34+
shellcheck-lint: $(SHELLCHECK_BIN)
35+
{{ .LintCommands.shellcheck }}
36+
37+
FIXERS += shellcheck-fix
38+
shellcheck-fix: $(SHELLCHECK_BIN)
39+
{{ .FixCommands.shellcheck }}
4040

4141
{{ end -}}
42+
4243
{{ if .Dockerfile -}}
43-
out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH):
44+
HADOLINT_VERSION ?= v2.8.0
45+
HADOLINT_BIN := out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
46+
$(HADOLINT_BIN):
4447
mkdir -p out/linters
4548
rm -rf out/linters/hadolint-*
46-
curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
47-
chmod u+x out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)
49+
curl -sfL https://github.com/hadolint/hadolint/releases/download/v2.6.1/hadolint-$(LINT_OS)-$(LINT_ARCH) > $@
50+
chmod u+x $@
51+
52+
LINTERS += hadolint-lint
53+
hadolint-lint: $(HADOLINT_BIN)
54+
{{ .LintCommands.hadolint }}
4855

4956
{{ end -}}
57+
5058
{{ if .Go -}}
51-
out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH):
59+
GOLANGCI_LINT_CONFIG := $(LINT_ROOT)/.golangci.yml
60+
GOLANGCI_LINT_VERSION ?= v1.43.0
61+
GOLANGCI_LINT_BIN := out/linters/golangci-lint-$(GOLANGCI_LINT_VERSION)-$(LINT_ARCH)
62+
$(GOLANGCI_LINT_BIN):
5263
mkdir -p out/linters
5364
rm -rf out/linters/golangci-lint-*
54-
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLINT_VERSION)
55-
mv out/linters/golangci-lint out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)
65+
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b out/linters $(GOLANGCI_LINT_VERSION)
66+
mv out/linters/golangci-lint $@
67+
68+
LINTERS += golangci-lint-lint
69+
golangci-lint-lint: $(GOLANGCI_LINT_BIN)
70+
{{ index .LintCommands "golangci-lint" }}
71+
72+
FIXERS += golangci-lint-fix
73+
golangci-lint-fix: $(GOLANGCI_LINT_BIN)
74+
{{ index .FixCommands "golangci-lint" }}
5675

5776
{{ end -}}
5877

5978
{{ if .YAML -}}
60-
$(YAMLLINT_ROOT)/dist/bin/yamllint:
79+
YAMLLINT_VERSION ?= 1.26.3
80+
YAMLLINT_ROOT := out/linters/yamllint-$(YAMLLINT_VERSION)
81+
YAMLLINT_BIN := $(YAMLLINT_ROOT)/dist/bin/yamllint
82+
$(YAMLLINT_BIN):
6183
mkdir -p out/linters
6284
rm -rf out/linters/yamllint-*
6385
curl -sSfL https://github.com/adrienverge/yamllint/archive/refs/tags/v$(YAMLLINT_VERSION).tar.gz | tar -C out/linters -zxf -
64-
cd $(YAMLLINT_ROOT) && pip3 install . -t dist
86+
cd $(YAMLLINT_ROOT) && pip3 install --target dist .
87+
88+
LINTERS += yamllint-lint
89+
yamllint-lint: $(YAMLLINT_BIN)
90+
{{ .LintCommands.yamllint }}
91+
6592
{{ end -}}
6693

94+
.PHONY: _lint $(LINTERS)
95+
_lint: $(LINTERS)
96+
97+
.PHONY: fix $(FIXERS)
98+
fix: $(FIXERS)
99+
67100
# END: lint-install {{.Args}}

‎lint-install.go‎

Lines changed: 22 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,8 @@ type Config struct {
5151
Dockerfile string
5252
Shell string
5353
YAML string
54-
LintCommands []string
55-
FixCommands []string
54+
LintCommands map[string]string
55+
FixCommands map[string]string
5656
}
5757

5858
// applicableLinters returns a list of languages with known linters within a given directory.
@@ -255,24 +255,28 @@ func goLintCmd(root string, level string, fix bool) string {
255255

256256
klog.Infof("found %d modules within %s: %s", len(found), root, found)
257257
if len(found) == 0 || (len(found) == 1 && found[0] == strings.Trim(root, "/")) {
258-
return fmt.Sprintf("out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH) run%s", suffix)
258+
return fmt.Sprintf("$(GOLANGCI_LINT_BIN) run%s", suffix)
259259
}
260260

261-
return fmt.Sprintf(`find . -name go.mod -execdir "$(LINT_ROOT)/out/linters/golangci-lint-$(GOLINT_VERSION)-$(LINT_ARCH)" run -c "$(GOLINT_CONFIG)"%s \;`, suffix)
261+
return fmt.Sprintf(`find . -name go.mod -execdir "$(GOLANGCI_LINT_BIN)" run -c "$(GOLINT_CONFIG)"%s \;`, suffix)
262262
}
263263

264264
// shellLintCmd returns the appropriate shell lint command for a project.
265265
func shellLintCmd(_ string, level string, fix bool) string {
266266
suffix := ""
267267

268268
if fix {
269-
// patch(1) doesn't support patching from stdin on all platforms, so we use git apply instead
270-
suffix = " -f diff | git apply -p2 -"
269+
// Use git apply instead of patch(1) because it doesn't support patching from stdin on all platforms.
270+
// Everything after the first | is so we don't call git apply if there's nothing to apply, which would cause git apply to return an error.
271+
// This is the cross platform way to do what gnu xargs does with its --no-run-if-empty switch.
272+
// read -t 1 will read the first line from stdin but timeout after 1 second if empty
273+
// if its not empty it will run the rest of the line after && which echoes the line and uses cat to pipe the rest to git
274+
suffix = ` -f diff | { read -t 1 line || exit 0; { echo "$$line" && cat; } | git apply -p2; }`
271275
} else if level == "warn" {
272276
suffix = " || true"
273277
}
274278

275-
return fmt.Sprintf(`out/linters/shellcheck-$(SHELLCHECK_VERSION)-$(LINT_ARCH)/shellcheck $(shell find . -name "*.sh")%s`, suffix)
279+
return fmt.Sprintf(`$(SHELLCHECK_BIN) $(shell find . -name "*.sh")%s`, suffix)
276280
}
277281

278282
// dockerLintCmd returns the appropriate docker lint command for a project.
@@ -282,7 +286,7 @@ func dockerLintCmd(_ string, level string) string {
282286
f = " --no-fail"
283287
}
284288

285-
return fmt.Sprintf(`out/linters/hadolint-$(HADOLINT_VERSION)-$(LINT_ARCH)%s $(shell find . -name "*Dockerfile")`, f)
289+
return fmt.Sprintf(`$(HADOLINT_BIN)%s $(shell find . -name "*Dockerfile")`, f)
286290
}
287291

288292
// yamlLintCmd returns the appropriate yamllint command for a project.
@@ -313,14 +317,16 @@ func main() {
313317
}
314318

315319
cfg := Config{
316-
Args: strings.Join(os.Args[1:], " "),
317-
Makefile: *makeFileName,
320+
Args: strings.Join(os.Args[1:], " "),
321+
Makefile: *makeFileName,
322+
LintCommands: make(map[string]string),
323+
FixCommands: make(map[string]string),
318324
}
319325

320326
if needs[Go] {
321327
cfg.Go = *goFlag
322-
cfg.LintCommands = append(cfg.LintCommands, goLintCmd(root, cfg.Go, false))
323-
cfg.FixCommands = append(cfg.FixCommands, goLintCmd(root, cfg.Go, true))
328+
cfg.LintCommands["golangci-lint"] = goLintCmd(root, cfg.Go, false)
329+
cfg.FixCommands["golangci-lint"] = goLintCmd(root, cfg.Go, true)
324330

325331
diff, err := updateFile(root, ".golangci.yml", goLintConfig, *dryRunFlag)
326332
if err != nil {
@@ -344,16 +350,16 @@ func main() {
344350
}
345351
if needs[Dockerfile] {
346352
cfg.Dockerfile = *dockerfileFlag
347-
cfg.LintCommands = append(cfg.LintCommands, dockerLintCmd(root, cfg.Dockerfile))
353+
cfg.LintCommands["hadolint"] = dockerLintCmd(root, cfg.Dockerfile)
348354
}
349355
if needs[Shell] {
350356
cfg.Shell = *shellFlag
351-
cfg.LintCommands = append(cfg.LintCommands, shellLintCmd(root, cfg.Shell, false))
352-
cfg.FixCommands = append(cfg.FixCommands, shellLintCmd(root, cfg.Shell, true))
357+
cfg.LintCommands["shellcheck"] = shellLintCmd(root, cfg.Shell, false)
358+
cfg.FixCommands["shellcheck"] = shellLintCmd(root, cfg.Shell, true)
353359
}
354360
if needs[YAML] {
355361
cfg.YAML = *yamlFlag
356-
cfg.LintCommands = append(cfg.LintCommands, yamlLintCmd(root, cfg.Shell))
362+
cfg.LintCommands["yamllint"] = yamlLintCmd(root, cfg.Shell)
357363

358364
diff, err := updateFile(root, ".yamllint", yamlLintConfig, *dryRunFlag)
359365
if err != nil {

‎shell.nix‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
let _pkgs = import <nixpkgs> { };
2+
in { pkgs ? import (_pkgs.fetchFromGitHub {
3+
owner = "NixOS";
4+
repo = "nixpkgs";
5+
#branch@date: nixpkgs-unstable@2021-11-12
6+
rev = "2fbba4b4416446721ebfb2e0bfcc9e45d8ddb4d2";
7+
sha256 = "1yw2p38pdvx63n21g6pmn79xjpxa93p1qpcadrlmg0j0zjnxkwr8";
8+
}) { } }:
9+
10+
with pkgs;
11+
12+
mkShell {
13+
buildInputs = [
14+
git
15+
gnumake
16+
go
17+
nixfmt
18+
nodePackages.prettier
19+
python3Packages.pip
20+
python3Packages.setuptools
21+
python3Packages.wheel
22+
];
23+
}

0 commit comments

Comments
 (0)