Skip to content

Commit 0f3acb2

Browse files
ssjunnebossjunnebo
authored
Merge pull request #10 from ssjunnebo/more_tests
More tests and merging of transfer methods
2 parents 8b27259 + aebfad1 commit 0f3acb2

File tree

16 files changed

+338
-82
lines changed

16 files changed

+338
-82
lines changed

‎.github/workflows/check-pr-label.yml‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
name: Check Label on PR
2+
on:
3+
pull_request:
4+
types: [opened, synchronize, labeled, unlabeled]
5+
6+
jobs:
7+
check_pr_label:
8+
runs-on: ubuntu-latest
9+
steps:
10+
- name: Checkout PR
11+
uses: actions/checkout@v6
12+
with:
13+
fetch-depth: 0 # Fetch all history for all branches and tags
14+
15+
- name: Check if the PR contains the label validation or no validation
16+
id: check_pr_label
17+
if: |
18+
! contains( github.event.pull_request.labels.*.name, 'validation') && ! contains( github.event.pull_request.labels.*.name, 'no validation')
19+
run: |
20+
echo "Neither 'validation' nor 'no validation' labels are present."
21+
exit 1 # Exit with a failure

‎.github/workflows/check-version.yml‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
name: Check version update
2+
on:
3+
pull_request:
4+
branches: [ "master" ]
5+
permissions:
6+
contents: read
7+
jobs:
8+
check-version:
9+
runs-on: ubuntu-latest
10+
steps:
11+
- name: Checkout PR
12+
uses: actions/checkout@v6
13+
with:
14+
fetch-depth: 0 # Fetch all history for all branches and tags
15+
- name: Check version update
16+
run: |
17+
PR_NUMBER=${{ github.event.pull_request.number }}
18+
FILE_CHANGED=$(git diff --name-only ${{ github.event.pull_request.base.sha }} HEAD | grep 'pyproject.toml' || true)
19+
if [ -z "$FILE_CHANGED" ]; then
20+
echo "pyproject.toml was not changed in this PR. Please update the version."
21+
exit 1
22+
else
23+
echo "pyproject.toml was changed in this PR."
24+
fi
25+
VERSION_CHANGED=$(git diff ${{ github.event.pull_request.base.sha }} HEAD | grep 'version =' || true)
26+
if [ -z "$VERSION_CHANGED" ]; then
27+
echo "Version in pyproject.toml was not updated. Please update the version."
28+
exit 1
29+
else
30+
echo "Version in pyproject.toml was updated."
31+
fi

‎.github/workflows/lint-code.yml‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
name: Lint code
2+
on: [push, pull_request]
3+
4+
jobs:
5+
# Use ruff to check for code style violations
6+
ruff-check:
7+
runs-on: ubuntu-latest
8+
steps:
9+
- name: Checkout repo
10+
uses: actions/checkout@v6
11+
- name: Set up Python
12+
uses: actions/setup-python@v6
13+
with:
14+
python-version: "3.14"
15+
- name: Install dependencies
16+
run: |
17+
python -m pip install --upgrade pip
18+
pip install ruff
19+
- name: ruff --> Check for style violations
20+
# Configured in pyproject.toml
21+
run: ruff check .
22+
23+
# Use ruff to check code formatting
24+
ruff-format:
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: Checkout repo
28+
uses: actions/checkout@v6
29+
- name: Set up Python
30+
uses: actions/setup-python@v6
31+
with:
32+
python-version: "3.14"
33+
- name: Install dependencies
34+
run: |
35+
python -m pip install --upgrade pip
36+
pip install ruff
37+
- name: ruff --> Check code formatting
38+
run: ruff format --check .
39+
40+
# Use pip-check-reqs/pip-missing-reqs to check for missing dependencies
41+
requirements-check:
42+
runs-on: ubuntu-latest
43+
steps:
44+
- name: Checkout repository
45+
uses: actions/checkout@v6
46+
- name: Set up Python
47+
uses: actions/setup-python@v6
48+
with:
49+
python-version: "3.14"
50+
51+
- name: Install dependencies
52+
run: |
53+
python -m pip install --upgrade pip
54+
pip install pip-check-reqs
55+
56+
- name: Run pip-check-reqs/pip-missing-reqs
57+
run: |
58+
pip-missing-reqs .

.github/workflows/python-app.yml renamed to .github/workflows/test-code.yml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,22 +18,22 @@ jobs:
1818
runs-on: ubuntu-latest
1919

2020
steps:
21-
- uses: actions/checkout@v4
21+
- uses: actions/checkout@v6
2222
- name: Set up Python 3.14
23-
uses: actions/setup-python@v3
23+
uses: actions/setup-python@v6
2424
with:
2525
python-version: "3.14"
2626
- name: Install dependencies
2727
run: |
2828
python -m pip install --upgrade pip
29-
pip install flake8 pytest
29+
pip install pytest pytest-cov
3030
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
31-
- name: Lint with flake8
32-
run: |
33-
# stop the build if there are Python syntax errors or undefined names
34-
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
35-
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
36-
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
31+
- name: Install dataflow_transfer
32+
run: pip install -e .
3733
- name: Test with pytest
3834
run: |
39-
pytest
35+
pytest --cov --cov-branch --cov-report=xml
36+
- name: Upload coverage reports to Codecov
37+
uses: codecov/codecov-action@v5
38+
with:
39+
token: ${{ secrets.CODECOV_TOKEN }}

‎dataflow_transfer/cli.py‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
1-
import click
2-
import os
31
import logging
2+
import os
3+
4+
import click
45
import yaml
56

6-
from dataflow_transfer.dataflow_transfer import transfer_runs
77
from dataflow_transfer import log
8+
from dataflow_transfer.dataflow_transfer import transfer_runs
89
from dataflow_transfer.run_classes.registry import RUN_CLASS_REGISTRY
910

1011
logger = logging.getLogger(__name__)
1112

1213

1314
def load_config(config_file_path):
14-
with open(config_file_path, "r") as file:
15+
with open(config_file_path) as file:
1516
config = yaml.safe_load(file)
1617
return config
1718

‎dataflow_transfer/dataflow_transfer.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
import time
33

44
from dataflow_transfer.run_classes.registry import RUN_CLASS_REGISTRY
5-
from dataflow_transfer.utils.filesystem import get_run_dir, find_runs
5+
from dataflow_transfer.utils.filesystem import find_runs, get_run_dir
66

77
logger = logging.getLogger(__name__)
88

@@ -30,7 +30,7 @@ def process_run(run_dir, sequencer, config):
3030
## Sequencing ongoing. Start background transfer if not already running.
3131
if run.sequencing_ongoing:
3232
run.update_statusdb(status="sequencing_started")
33-
run.initiate_background_transfer()
33+
run.start_transfer(final=False)
3434
return
3535

3636
## Sequencing finished but transfer not complete. Start final transfer.
@@ -41,7 +41,7 @@ def process_run(run_dir, sequencer, config):
4141
"Will attempt final transfer again."
4242
)
4343
run.update_statusdb(status="sequencing_finished")
44-
run.do_final_transfer()
44+
run.start_transfer(final=True)
4545
return
4646

4747
## Final transfer completed successfully. Update statusdb.

‎dataflow_transfer/run_classes/__init__.py‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,11 @@
11
# This adds the run classes to the registry. Do not remove.
22

3-
from .registry import RUN_CLASS_REGISTRY
4-
5-
3+
from dataflow_transfer.run_classes.element_runs import AVITIRun # noqa: F401, I001
64
from dataflow_transfer.run_classes.illumina_runs import (
7-
NovaSeqXPlusRun,
8-
NextSeqRun,
9-
MiSeqRun,
5+
MiSeqRun, # noqa: F401
6+
NextSeqRun, # noqa: F401
7+
NovaSeqXPlusRun, # noqa: F401
108
)
11-
from dataflow_transfer.run_classes.ont_runs import PromethIONRun, MinIONRun
12-
from dataflow_transfer.run_classes.element_runs import AVITIRun
9+
from dataflow_transfer.run_classes.ont_runs import MinIONRun, PromethIONRun # noqa: F401
10+
11+
from .registry import RUN_CLASS_REGISTRY # noqa: F401

‎dataflow_transfer/run_classes/element_runs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from dataflow_transfer.run_classes.generic_runs import Run
2+
23
from .registry import register_run_class
34

45

‎dataflow_transfer/run_classes/generic_runs.py‎

Lines changed: 15 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,10 @@
1-
import os
21
import logging
2+
import os
33
import re
44
from datetime import datetime
5-
from dataflow_transfer.utils.statusdb import StatusdbSession
5+
66
import dataflow_transfer.utils.filesystem as fs
7+
from dataflow_transfer.utils.statusdb import StatusdbSession
78

89
logger = logging.getLogger(__name__)
910

@@ -65,53 +66,33 @@ def generate_rsync_command(self, is_final_sync=False):
6566
command_str += f"; echo $? > {self.final_rsync_exitcode_file}"
6667
return command_str
6768

68-
def initiate_background_transfer(self):
69+
def start_transfer(self, final=False):
6970
"""Start background rsync transfer to storage."""
70-
background_transfer_command = self.generate_rsync_command(is_final_sync=False)
71+
transfer_command = self.generate_rsync_command(is_final_sync=final)
7172
if fs.rsync_is_running(src=self.run_dir):
7273
logger.info(
7374
f"Rsync is already running for {self.run_dir}. Skipping background transfer initiation."
7475
)
7576
return
7677
try:
77-
fs.submit_background_process(background_transfer_command)
78+
fs.submit_background_process(transfer_command)
7879
logger.info(
79-
f"{self.run_id}: Started background rsync to {self.miarka_destination}"
80-
+ f" with the following command: '{background_transfer_command}'"
80+
f"{self.run_id}: Started rsync to {self.miarka_destination}"
81+
+ f" with the following command: '{transfer_command}'"
8182
)
8283
except Exception as e:
83-
logger.error(f"Failed to start background transfer for {self.run_id}: {e}")
84+
logger.error(f"Failed to start rsync for {self.run_id}: {e}")
8485
raise e
8586
rsync_info = {
86-
"command": background_transfer_command,
87+
"command": transfer_command,
8788
"destination_path": self.miarka_destination,
8889
}
89-
self.update_statusdb(status="transfer_started", additional_info=rsync_info)
90-
91-
def do_final_transfer(self):
92-
"""Start final rsync transfer to storage."""
93-
final_transfer_command = self.generate_rsync_command(is_final_sync=True)
94-
if fs.rsync_is_running(src=self.run_dir):
95-
logger.info(
96-
f"Rsync is already running for {self.run_dir}. Skipping final transfer initiation."
97-
)
98-
return
99-
try:
100-
fs.submit_background_process(final_transfer_command)
101-
logger.info(
102-
f"{self.run_id}: Started FINAL rsync to {self.miarka_destination}"
103-
+ f" with the following command: '{final_transfer_command}'"
90+
if final:
91+
self.update_statusdb(
92+
status="final_transfer_started", additional_info=rsync_info
10493
)
105-
except Exception as e:
106-
logger.error(f"Failed to start final transfer for {self.run_id}: {e}")
107-
raise e
108-
rsync_info = {
109-
"command": final_transfer_command,
110-
"destination_path": self.miarka_destination,
111-
}
112-
self.update_statusdb(
113-
status="final_transfer_started", additional_info=rsync_info
114-
)
94+
else:
95+
self.update_statusdb(status="transfer_started", additional_info=rsync_info)
11596

11697
@property
11798
def final_sync_successful(self):

‎dataflow_transfer/run_classes/illumina_runs.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
from dataflow_transfer.run_classes.generic_runs import Run
2+
23
from .registry import register_run_class
34

45

0 commit comments

Comments
 (0)