Skip to content

Commit a9e3b70

Browse files
omusomus
authored
Create GHA extract shell scripts action (#1)
1 parent f11aac0 commit a9e3b70

14 files changed

+485
-2
lines changed

‎.editorconfig‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
# https://editorconfig.org/
2+
3+
# https://manpages.debian.org/testing/shfmt/shfmt.1.en.html#EXAMPLES
4+
[*.sh]
5+
indent_style = space
6+
indent_size = 4
7+
shell_variant = bash # --language-variant
8+
binary_next_line = false
9+
switch_case_indent = true # --case-indent
10+
space_redirects = false
11+
keep_padding = false
12+
function_next_line = false # --func-next-line

‎.github/workflows/integration-tests.yaml‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
---
2+
name: Integration Tests
3+
on:
4+
pull_request:
5+
paths:
6+
- "action.yaml"
7+
- "gha_extract_shell_scripts.py"
8+
- ".github/workflows/integration-tests.yaml"
9+
10+
jobs:
11+
test:
12+
name: Test
13+
runs-on: ubuntu-latest
14+
permissions:
15+
contents: read
16+
steps:
17+
- uses: actions/checkout@v4
18+
- name: Run action
19+
id: self
20+
uses: ./
21+
- name: Target step
22+
run: |
23+
echo "${{ env.greeting }}, $name"
24+
env:
25+
greeting: Hello
26+
name: Integration Tests
27+
- name: Test extracted
28+
run: |
29+
if [[ -f "$output_file" ]]; then
30+
echo "Output:"
31+
cat -n "$output_file"
32+
echo "Expected:"
33+
cat -n <<<"$expected"
34+
else
35+
find "${output_dir:?}"
36+
exit 1
37+
fi
38+
diff --color=always "${output_file:?}" <(echo "${expected:?}")
39+
env:
40+
output_dir: ${{ steps.self.outputs.output-dir }}
41+
output_file: ${{ steps.self.outputs.output-dir }}/integration-tests.yaml/job=Test/step=Target_step.sh
42+
expected: |-
43+
#!/usr/bin/env bash
44+
set -e
45+
# shellcheck disable=SC2016,SC2034
46+
greeting='Hello'
47+
# shellcheck disable=SC2016,SC2034
48+
name='Integration Tests'
49+
# ---
50+
echo ":env.greeting:, $name"

‎.github/workflows/shell.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
---
2+
name: Shell
3+
on:
4+
pull_request:
5+
paths:
6+
- "**.sh"
7+
- ".github/workflows/*"
8+
- "action.yaml"
9+
- "gha_extract_shell_scripts.py"
10+
11+
jobs:
12+
lint-format:
13+
name: Lint & Format
14+
needs: workflow-scripts
15+
# These permissions are needed to:
16+
# - Checkout the Git repo (`contents: read`)
17+
# - Post a comments on PRs: https://github.com/luizm/action-sh-checker#secrets
18+
permissions:
19+
contents: read
20+
pull-requests: write
21+
runs-on: ubuntu-latest
22+
steps:
23+
- uses: actions/checkout@v4
24+
- name: Extract workflow shell scripts
25+
id: extract
26+
uses: ./
27+
- uses: luizm/action-sh-checker@c6edb3de93e904488b413636d96c6a56e3ad671a # v0.8.0
28+
env:
29+
GITHUB_TOKEN: ${{ github.token }}
30+
with:
31+
sh_checker_comment: true
32+
# Support investigating linting/formatting errors
33+
- uses: actions/upload-artifact@v4
34+
if: ${{ !cancelled() }}
35+
with:
36+
name: workflow-scripts
37+
path: ${{ steps.extract.outputs.output-dir }}

‎.github/workflows/unit-tests.yaml‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
---
2+
name: Unit Tests
3+
on:
4+
pull_request:
5+
paths:
6+
- "**/*.py"
7+
- ".github/workflows/unit-tests.yaml"
8+
9+
jobs:
10+
test:
11+
name: Test
12+
runs-on: ubuntu-latest
13+
permissions:
14+
contents: read
15+
steps:
16+
- uses: actions/checkout@v4
17+
- name: Set up Python
18+
uses: actions/setup-python@v5
19+
with:
20+
python-version: "3.x"
21+
- name: Install dependencies
22+
run: |
23+
python -m pip install --upgrade pip
24+
pip install -r requirements.txt
25+
- name: Test with unittest
26+
run: |
27+
python test/test_reference.py

‎.github/workflows/yaml.yaml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
---
2+
# https://yamllint.readthedocs.io/en/stable/integration.html#integration-with-github-actions
3+
name: YAML
4+
on:
5+
pull_request:
6+
paths:
7+
- "**/*.yaml"
8+
- "**/*.yml"
9+
jobs:
10+
lint:
11+
name: Lint
12+
runs-on: ubuntu-latest
13+
steps:
14+
- uses: actions/checkout@v4
15+
- name: Install yamllint
16+
run: pip install yamllint
17+
- name: Lint YAML files
18+
run: yamllint . --format=github

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
__pycache__

‎.yamllint.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
---
2+
rules:
3+
indentation:
4+
spaces: 2
5+
indent-sequences: true
6+
document-start:
7+
present: true
8+
new-line-at-end-of-file: enable

‎README.md‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,65 @@
1-
# inline-workflow-shell-scripts
2-
Extracts inline shell scripts within GitHub Action workflows
1+
# GHA Extract Shell Scripts
2+
3+
Processes the GitHub Action workflows contained within `.github/workflows` and extracts all steps which contain an embedded shell script for the purpose of running linting and formatting. Each workflow step containing a shell script will be written out to a file to make it easy to use existing tooling such as `shellcheck` and `shfmt`.
4+
5+
## Example
6+
7+
```yaml
8+
---
9+
name: Shell
10+
on:
11+
pull_request:
12+
paths:
13+
- "**.sh"
14+
- ".github/workflows/*"
15+
16+
jobs:
17+
lint-format:
18+
name: Lint & Format
19+
# These permissions are needed to:
20+
# - Checkout the Git repo (`contents: read`)
21+
# - Post a comments on PRs: https://github.com/luizm/action-sh-checker#secrets
22+
permissions:
23+
contents: read
24+
pull-requests: write
25+
runs-on: ubuntu-latest
26+
steps:
27+
- uses: actions/checkout@v4
28+
- name: Extract workflow shell scripts
29+
id: extract
30+
uses: beacon-biosignals/gha-extract-shell-scripts@v1
31+
- uses: luizm/action-sh-checker@c6edb3de93e904488b413636d96c6a56e3ad671a # v0.8.0
32+
env:
33+
GITHUB_TOKEN: ${{ github.token }}
34+
with:
35+
sh_checker_comment: true
36+
# Support investigating linting/formatting errors
37+
- uses: actions/upload-artifact@v4
38+
if: ${{ failure() }}
39+
with:
40+
name: workflow-scripts
41+
path: ${{ steps.extract.outputs.output-dir }}
42+
```
43+
44+
## Inputs
45+
46+
The `gha-extract-shell-scripts` action supports the following inputs:
47+
48+
| Name | Description | Required | Example |
49+
|:---------------------|:------------|:---------|:--------|
50+
| `output-dir` | Allows the user to specify the name of the directory containing the extracted workflow shell script steps. Defaults to `workflow_scripts`. | No | `workflow_scripts` |
51+
| `shellcheck-disable` | Ignore all the specified errors within the extracted shell scripts. | No | `SC2016,SC2050` |
52+
53+
## Outputs
54+
55+
| Name | Description | Example |
56+
|:-------------|:------------|:--------|
57+
| `output-dir` | The name of the directory containing the various extracted workflow shell script steps. | `workflow_scripts` |
58+
59+
## Permissions
60+
61+
The following [job permissions](https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs) are required to run this action:
62+
63+
```yaml
64+
permissions: {}
65+
```

‎action.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
---
2+
inputs:
3+
output-dir:
4+
default: "workflow_scripts"
5+
shellcheck-disable:
6+
default: ""
7+
outputs:
8+
output-dir:
9+
value: ${{ inputs.output-dir }}
10+
runs:
11+
using: composite
12+
steps:
13+
- name: Install dependencies
14+
shell: bash
15+
run: |
16+
venv="$(mktemp -d venv.XXXXXX)"
17+
python -m venv "$venv"
18+
source "$venv/bin/activate"
19+
python -m pip install -r "${GITHUB_ACTION_PATH}/requirements.txt"
20+
- name: Extract shell scripts
21+
shell: bash
22+
run: |
23+
args=()
24+
if [[ -n "$disable" ]]; then
25+
args+=(--disable "$disable")
26+
fi
27+
args+=("$input_dir" "$output_dir")
28+
python "${GITHUB_ACTION_PATH}/gha_extract_shell_scripts.py" "${args[@]}"
29+
env:
30+
disable: ${{ inputs.shellcheck-disable }}
31+
input_dir: .github/workflows
32+
output_dir: ${{ inputs.output-dir }}

‎gha_extract_shell_scripts.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#!/usr/bin/env python3
2+
3+
# Reads shell scripts from `run` steps in GitHub Actions workflows and outputs
4+
# them as files so that tools like `shfmt` or ShellCheck can operate on them.
5+
#
6+
# Arguments:
7+
# - Path to output directory where shell scripts will be written.
8+
9+
import os
10+
import re
11+
import sys
12+
13+
import argparse
14+
from pathlib import Path
15+
16+
import yaml
17+
18+
19+
def list_str(values):
20+
return values.split(',')
21+
22+
23+
def sanitize(path):
24+
# Needed filename replacements to satisfy both GHA artifacts and shellcheck.
25+
replacements = {
26+
" ": "_",
27+
"/": "-",
28+
'"': "",
29+
"(": "",
30+
")": "",
31+
"&": "",
32+
"$": "",
33+
}
34+
return path.translate(str.maketrans(replacements))
35+
36+
37+
# Replace any GHA placeholders, e.g. ${{ matrix.version }}.
38+
def sanitize_gha_expression(string):
39+
return re.sub(r"\${{\s*(.*?)\s*}}", r":\1:", string)
40+
41+
42+
def process_workflow_file(workflow_path: Path, output_dir: Path, ignored_errors=[]):
43+
with workflow_path.open() as f:
44+
workflow = yaml.safe_load(f)
45+
workflow_file = workflow_path.name
46+
# GHA allows workflow names to be defined as empty (e.g. `name:`)
47+
workflow_name = sanitize(workflow.get("name") or workflow_path.stem)
48+
workflow_default_shell = workflow.get("defaults", {}).get("run", {}).get("shell")
49+
workflow_env = workflow.get("env", {})
50+
count = 0
51+
print(f"Processing {workflow_path} ({workflow_name})")
52+
for job_key, job in workflow.get("jobs", {}).items():
53+
# GHA allows job names to be defined as empty (e.g. `name:`)
54+
job_name = sanitize(job.get("name") or job_key)
55+
job_default_shell = (
56+
job.get("defaults", {}).get("run", {}).get("shell", workflow_default_shell)
57+
)
58+
job_env = workflow_env | job.get("env", {})
59+
for i, step in enumerate(job.get("steps", [])):
60+
run = step.get("run")
61+
if not run:
62+
continue
63+
run = sanitize_gha_expression(run)
64+
shell = step.get("shell", job_default_shell)
65+
if shell and shell not in ["bash", "sh"]:
66+
print(f"Skipping command with unknown shell '{shell}'")
67+
continue
68+
env = job_env | step.get("env", {})
69+
# GHA allows step names to be defined as empty (e.g. `name:`)
70+
step_name = sanitize(step.get("name") or str(i + 1))
71+
script_path = (
72+
output_dir / workflow_file / f"job={job_name}" / f"step={step_name}.sh"
73+
)
74+
script_path.parent.mkdir(parents=True, exist_ok=True)
75+
with script_path.open("w") as f:
76+
# Default shell is bash.
77+
f.write(f"#!/usr/bin/env {shell or 'bash'}\n")
78+
# Ignore failure with GitHub expression variables such as:
79+
# - SC2050: `[[ "${{ github.ref }}" == "refs/heads/main" ]]`
80+
if ignored_errors:
81+
f.write(f"# shellcheck disable={','.join(ignored_errors)}\n")
82+
# Add a no-op command to ensure that additional shellcheck
83+
# disable directives aren't applied globally
84+
# https://github.com/koalaman/shellcheck/issues/657#issuecomment-213038218
85+
f.write("true\n")
86+
# Whether or not it was explicitly set determines the arguments.
87+
# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
88+
if not shell or shell == "sh":
89+
f.write("set -e\n")
90+
elif shell == "bash":
91+
f.write("set -eo pipefail\n")
92+
for k, v in env.items():
93+
f.write("# shellcheck disable=SC2016,SC2034\n")
94+
v = sanitize_gha_expression(str(v)).replace("'", "'\\''")
95+
f.write(f"{k}='{v}'\n")
96+
f.write("# ---\n")
97+
f.write(run)
98+
if not run.endswith("\n"):
99+
f.write("\n")
100+
count += 1
101+
print(f"Produced {count} files")
102+
103+
104+
if __name__ == "__main__":
105+
parser = argparse.ArgumentParser()
106+
parser.add_argument("input_dir", type=Path)
107+
parser.add_argument("output_dir", type=Path)
108+
parser.add_argument("--disable", type=list_str)
109+
args = parser.parse_args()
110+
111+
print(f"Outputting scripts to {args.output_dir}")
112+
args.output_dir.mkdir(parents=True, exist_ok=True)
113+
for file in os.listdir(args.input_dir):
114+
if file.endswith(".yaml") or file.endswith(".yml"):
115+
process_workflow_file(
116+
args.input_dir / file, args.output_dir, args.disable
117+
)

0 commit comments

Comments
 (0)