Skip to content

Commit 1675473

Browse files
pinin4fjordspinin4fjords
authored
Merge pull request #1559 from nf-core/nf-core-template-merge-3.3.1
Important! Template update for nf-core/tools v3.3.1
2 parents a60f6cf + 6afadbb commit 1675473

30 files changed

+544
-285
lines changed

‎.editorconfig‎

Lines changed: 0 additions & 37 deletions
This file was deleted.

‎.github/CONTRIBUTING.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ If you wish to contribute a new step, please use the following coding standards:
7878
5. Add any new parameters to `nextflow_schema.json` with help text (via the `nf-core pipelines schema build` tool).
7979
6. Add sanity checks and validation for all relevant parameters.
8080
7. Perform local tests to validate that the new code works as expected.
81-
8. If applicable, add a new test command in `.github/workflow/ci.yml`.
81+
8. If applicable, add a new test in the `tests` directory.
8282
9. Update MultiQC config `assets/multiqc_config.yml` so relevant suffixes, file name clean up and module plots are in the appropriate order. If applicable, add a [MultiQC](https://https://multiqc.info/) module.
8383
10. Add a description of the output files and if relevant any appropriate images from the MultiQC report to `docs/output.md`.
8484

‎.github/actions/get-shards/action.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: "Get number of shards"
2+
description: "Get the number of nf-test shards for the current CI job"
3+
inputs:
4+
max_shards:
5+
description: "Maximum number of shards allowed"
6+
required: true
7+
paths:
8+
description: "Component paths to test"
9+
required: false
10+
tags:
11+
description: "Tags to pass as argument for nf-test --tag parameter"
12+
required: false
13+
outputs:
14+
shard:
15+
description: "Array of shard numbers"
16+
value: ${{ steps.shards.outputs.shard }}
17+
total_shards:
18+
description: "Total number of shards"
19+
value: ${{ steps.shards.outputs.total_shards }}
20+
runs:
21+
using: "composite"
22+
steps:
23+
- name: Install nf-test
24+
uses: nf-core/setup-nf-test@v1
25+
with:
26+
version: ${{ env.NFT_VER }}
27+
- name: Get number of shards
28+
id: shards
29+
shell: bash
30+
run: |
31+
# Run nf-test with dynamic parameter
32+
nftest_output=$(nf-test test \
33+
--profile +docker \
34+
$(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
35+
--dry-run \
36+
--ci \
37+
--changed-since HEAD^) || {
38+
echo "nf-test command failed with exit code $?"
39+
echo "Full output: $nftest_output"
40+
exit 1
41+
}
42+
echo "nf-test dry-run output: $nftest_output"
43+
44+
# Default values for shard and total_shards
45+
shard="[]"
46+
total_shards=0
47+
48+
# Check if there are related tests
49+
if echo "$nftest_output" | grep -q 'No tests to execute'; then
50+
echo "No related tests found."
51+
else
52+
# Extract the number of related tests
53+
number_of_shards=$(echo "$nftest_output" | sed -n 's|.*Executed \([0-9]*\) tests.*|\1|p')
54+
if [[ -n "$number_of_shards" && "$number_of_shards" -gt 0 ]]; then
55+
shards_to_run=$(( $number_of_shards < ${{ inputs.max_shards }} ? $number_of_shards : ${{ inputs.max_shards }} ))
56+
shard=$(seq 1 "$shards_to_run" | jq -R . | jq -c -s .)
57+
total_shards="$shards_to_run"
58+
else
59+
echo "Unexpected output format. Falling back to default values."
60+
fi
61+
fi
62+
63+
# Write to GitHub Actions outputs
64+
echo "shard=$shard" >> $GITHUB_OUTPUT
65+
echo "total_shards=$total_shards" >> $GITHUB_OUTPUT
66+
67+
# Debugging output
68+
echo "Final shard array: $shard"
69+
echo "Total number of shards: $total_shards"

‎.github/actions/nf-test/action.yml‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
name: "nf-test Action"
2+
description: "Runs nf-test with common setup steps"
3+
inputs:
4+
profile:
5+
description: "Profile to use"
6+
required: true
7+
shard:
8+
description: "Shard number for this CI job"
9+
required: true
10+
total_shards:
11+
description: "Total number of test shards(NOT the total number of matrix jobs)"
12+
required: true
13+
paths:
14+
description: "Test paths"
15+
required: true
16+
tags:
17+
description: "Tags to pass as argument for nf-test --tag parameter"
18+
required: false
19+
runs:
20+
using: "composite"
21+
steps:
22+
- name: Setup Nextflow
23+
uses: nf-core/setup-nextflow@v2
24+
with:
25+
version: "${{ env.NXF_VERSION }}"
26+
27+
- name: Set up Python
28+
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
29+
with:
30+
python-version: "3.13"
31+
32+
- name: Install nf-test
33+
uses: nf-core/setup-nf-test@v1
34+
with:
35+
version: "${{ env.NFT_VER }}"
36+
install-pdiff: true
37+
38+
- name: Setup apptainer
39+
if: contains(inputs.profile, 'singularity')
40+
uses: eWaterCycle/setup-apptainer@main
41+
42+
- name: Set up Singularity
43+
if: contains(inputs.profile, 'singularity')
44+
shell: bash
45+
run: |
46+
mkdir -p $NXF_SINGULARITY_CACHEDIR
47+
mkdir -p $NXF_SINGULARITY_LIBRARYDIR
48+
49+
- name: Conda setup
50+
if: contains(inputs.profile, 'conda')
51+
uses: conda-incubator/setup-miniconda@505e6394dae86d6a5c7fbb6e3fb8938e3e863830 # v3
52+
with:
53+
auto-update-conda: true
54+
conda-solver: libmamba
55+
conda-remove-defaults: true
56+
57+
# TODO Skip failing conda tests and document their failures
58+
# https://github.com/nf-core/modules/issues/7017
59+
- name: Run nf-test
60+
shell: bash
61+
env:
62+
NFT_DIFF: ${{ env.NFT_DIFF }}
63+
NFT_DIFF_ARGS: ${{ env.NFT_DIFF_ARGS }}
64+
NFT_WORKDIR: ${{ env.NFT_WORKDIR }}
65+
run: |
66+
nf-test test \
67+
--profile=+${{ inputs.profile }} \
68+
$(if [ -n "${{ inputs.tags }}" ]; then echo "--tag ${{ inputs.tags }}"; fi) \
69+
--ci \
70+
--changed-since HEAD^ \
71+
--verbose \
72+
--tap=test.tap \
73+
--shard ${{ inputs.shard }}/${{ inputs.total_shards }}
74+
75+
# Save the absolute path of the test.tap file to the output
76+
echo "tap_file_path=$(realpath test.tap)" >> $GITHUB_OUTPUT
77+
78+
- name: Generate test summary
79+
if: always()
80+
shell: bash
81+
run: |
82+
# Add header if it doesn't exist (using a token file to track this)
83+
if [ ! -f ".summary_header" ]; then
84+
echo "# 🚀 nf-test results" >> $GITHUB_STEP_SUMMARY
85+
echo "" >> $GITHUB_STEP_SUMMARY
86+
echo "| Status | Test Name | Profile | Shard |" >> $GITHUB_STEP_SUMMARY
87+
echo "|:------:|-----------|---------|-------|" >> $GITHUB_STEP_SUMMARY
88+
touch .summary_header
89+
fi
90+
91+
if [ -f test.tap ]; then
92+
while IFS= read -r line; do
93+
if [[ $line =~ ^ok ]]; then
94+
test_name="${line#ok }"
95+
# Remove the test number from the beginning
96+
test_name="${test_name#* }"
97+
echo "| ✅ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
98+
elif [[ $line =~ ^not\ ok ]]; then
99+
test_name="${line#not ok }"
100+
# Remove the test number from the beginning
101+
test_name="${test_name#* }"
102+
echo "| ❌ | ${test_name} | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
103+
fi
104+
done < test.tap
105+
else
106+
echo "| ⚠️ | No test results found | ${{ inputs.profile }} | ${{ inputs.shard }}/${{ inputs.total_shards }} |" >> $GITHUB_STEP_SUMMARY
107+
fi
108+
109+
- name: Clean up
110+
if: always()
111+
shell: bash
112+
run: |
113+
sudo rm -rf /home/ubuntu/tests/

‎.github/workflows/awsfulltest.yml‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
name: nf-core AWS full size tests
2+
# This workflow is triggered on PRs opened against the main/master branch.
3+
# It can be additionally triggered manually with GitHub actions workflow dispatch button.
4+
# It runs the -profile 'test_full' on AWS batch
5+
6+
on:
7+
workflow_dispatch:
8+
pull_request_review:
9+
types: [submitted]
10+
release:
11+
types: [published]
12+
13+
jobs:
14+
run-platform:
15+
name: Run AWS full tests
16+
# run only if the PR is approved by at least 2 reviewers and against the master/main branch or manually triggered
17+
if: github.repository == 'nf-core/rnaseq' && github.event.review.state == 'approved' && (github.event.pull_request.base.ref == 'master' || github.event.pull_request.base.ref == 'main') || github.event_name == 'workflow_dispatch' || github.event_name == 'release'
18+
runs-on: ubuntu-latest
19+
strategy:
20+
matrix:
21+
aligner: ["star_salmon", "star_rsem"]
22+
steps:
23+
- name: Set revision variable
24+
id: revision
25+
run: |
26+
echo "revision=${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'release') && github.sha || 'dev' }}" >> "$GITHUB_OUTPUT"
27+
28+
- name: Launch workflow via Seqera Platform
29+
uses: seqeralabs/action-tower-launch@v2
30+
# TODO nf-core: You can customise AWS full pipeline tests as required
31+
# Add full size test data (but still relatively small datasets for few samples)
32+
# on the `test_full.config` test runs with only one set of parameters
33+
with:
34+
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
35+
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
36+
compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
37+
revision: ${{ steps.revision.outputs.revision }}
38+
workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnaseq/work-${{ steps.revision.outputs.revision }}
39+
parameters: |
40+
{
41+
"hook_url": "${{ secrets.MEGATESTS_ALERTS_SLACK_HOOK_URL }}",
42+
"aligner": "${{ matrix.aligner }}",
43+
"outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnaseq/results-${{ steps.revision.outputs.revision }}"
44+
}
45+
profiles: test_full
46+
47+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
48+
with:
49+
name: Seqera Platform debug log file
50+
path: |
51+
seqera_platform_action_*.log
52+
seqera_platform_action_*.json

‎.github/workflows/awstest.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: nf-core AWS test
2+
# This workflow can be triggered manually with the GitHub actions workflow dispatch button.
3+
# It runs the -profile 'test' on AWS batch
4+
5+
on:
6+
workflow_dispatch:
7+
jobs:
8+
run-platform:
9+
name: Run AWS tests
10+
if: github.repository == 'nf-core/rnaseq'
11+
runs-on: ubuntu-latest
12+
steps:
13+
# Launch workflow using Seqera Platform CLI tool action
14+
- name: Launch workflow via Seqera Platform
15+
uses: seqeralabs/action-tower-launch@v2
16+
with:
17+
workspace_id: ${{ secrets.TOWER_WORKSPACE_ID }}
18+
access_token: ${{ secrets.TOWER_ACCESS_TOKEN }}
19+
compute_env: ${{ secrets.TOWER_COMPUTE_ENV }}
20+
revision: ${{ github.sha }}
21+
workdir: s3://${{ secrets.AWS_S3_BUCKET }}/work/rnaseq/work-${{ github.sha }}
22+
parameters: |
23+
{
24+
"outdir": "s3://${{ secrets.AWS_S3_BUCKET }}/rnaseq/results-test-${{ github.sha }}"
25+
}
26+
profiles: test
27+
28+
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
29+
with:
30+
name: Seqera Platform debug log file
31+
path: |
32+
seqera_platform_action_*.log
33+
seqera_platform_action_*.json

0 commit comments

Comments
 (0)