Skip to content

Commit 34fac89

Browse files
mvdbeekmvdbeek
authored
Merge pull request #20936 from mvdbeek/resource_requirements_tpv
Wire up and test resource requirement via tpv
2 parents 69387d5 + 992d46f commit 34fac89

File tree

9 files changed

+119
-6
lines changed

9 files changed

+119
-6
lines changed

‎client/src/components/Tool/ToolSourceSchema.json‎

Lines changed: 1 addition & 1 deletion
Large diffs are not rendered by default.

‎lib/galaxy/dependencies/conditional-requirements.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ ldap3==2.9.1
1313
python-pam
1414
galaxycloudrunner
1515
pkce
16-
total-perspective-vortex>=2.3.2,<3
16+
total-perspective-vortex>=3.1.0,<4
1717
openai
1818
# upper version constraint because of https://github.com/galaxyproject/galaxy/issues/20600
1919
redis>=5.3.0,<6

‎lib/galaxy/dependencies/dev-requirements.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ boto3==1.40.18
1818
botocore==1.40.18
1919
build==1.3.0
2020
cachecontrol==0.14.3
21+
cachetools==5.5.2
2122
cattrs==25.2.0
2223
certifi==2025.8.3
2324
cffi==2.0.0 ; (implementation_name != 'pypy' and os_name == 'nt') or platform_python_implementation != 'PyPy'
@@ -90,6 +91,7 @@ mistune==3.0.2
9091
more-itertools==10.8.0 ; platform_machine != 'ppc64le' and platform_machine != 's390x'
9192
msgpack==1.1.1
9293
multidict==6.6.4
94+
mypy==1.18.2
9395
mypy-extensions==1.1.0
9496
myst-parser==3.0.1 ; python_full_version < '3.10'
9597
myst-parser==4.0.1 ; python_full_version >= '3.10'
@@ -197,11 +199,16 @@ text-unidecode==1.3
197199
tinydb==4.8.2
198200
toml==0.10.2
199201
tomli==2.2.1 ; python_full_version <= '3.11'
202+
total-perspective-vortex==3.1.0
200203
trio==0.30.0
201204
trio-websocket==0.12.2
202205
tuspy==1.1.0
203206
twill==3.3.1
204207
twine==6.2.0
208+
types-cachetools==6.2.0.20250827
209+
types-requests==2.31.0.6 ; python_full_version < '3.10'
210+
types-requests==2.32.4.20250913 ; python_full_version >= '3.10'
211+
types-urllib3==1.26.25.14 ; python_full_version < '3.10'
205212
typing-extensions==4.14.1
206213
tzdata==2025.2 ; sys_platform == 'win32'
207214
url-normalize==2.2.1

‎lib/galaxy/dependencies/pinned-requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,8 @@ mypy-extensions==1.1.0
124124
networkx==3.2.1 ; python_full_version < '3.10'
125125
networkx==3.4.2 ; python_full_version == '3.10.*'
126126
networkx==3.5 ; python_full_version >= '3.11'
127-
nodejs-wheel==22.13.0
127+
nodejs-wheel==22.20.0
128+
nodejs-wheel-binaries==22.20.0
128129
numpy==2.0.2 ; python_full_version < '3.10'
129130
numpy==2.2.6 ; python_full_version == '3.10.*'
130131
numpy==2.3.3 ; python_full_version >= '3.11'

‎lib/galaxy/dependencies/pinned-test-requirements.txt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ backports-asyncio-runner==1.2.0 ; python_full_version < '3.11'
1212
boto3==1.40.18
1313
botocore==1.40.18
1414
cachecontrol==0.14.3
15+
cachetools==5.5.2
1516
cattrs==25.2.0
1617
certifi==2025.8.3
1718
cffi==2.0.0 ; implementation_name != 'pypy' and os_name == 'nt'
@@ -66,6 +67,7 @@ mirakuru==2.6.1
6667
mistune==3.0.2
6768
msgpack==1.1.1
6869
multidict==6.6.4
70+
mypy==1.18.2
6971
mypy-extensions==1.1.0
7072
numpy==2.0.2 ; python_full_version < '3.10'
7173
numpy==2.2.6 ; python_full_version == '3.10.*'
@@ -76,6 +78,7 @@ owlrl==6.0.2 ; python_full_version < '3.10'
7678
owlrl==7.1.4 ; python_full_version >= '3.10'
7779
packaging==24.2 ; python_full_version < '3.10'
7880
packaging==25.0 ; python_full_version >= '3.10'
81+
pathspec==0.12.1
7982
pfzy==0.3.4
8083
pillow==11.3.0
8184
pkce==1.0.3
@@ -145,10 +148,15 @@ text-unidecode==1.3
145148
tinydb==4.8.2
146149
toml==0.10.2
147150
tomli==2.2.1 ; python_full_version <= '3.11'
151+
total-perspective-vortex==3.1.0
148152
trio==0.30.0
149153
trio-websocket==0.12.2
150154
tuspy==1.1.0
151155
twill==3.3.1
156+
types-cachetools==6.2.0.20250827
157+
types-requests==2.31.0.6 ; python_full_version < '3.10'
158+
types-requests==2.32.4.20250913 ; python_full_version >= '3.10'
159+
types-urllib3==1.26.25.14 ; python_full_version < '3.10'
152160
typing-extensions==4.14.1
153161
tzdata==2025.2 ; sys_platform == 'win32'
154162
url-normalize==2.2.1

‎lib/galaxy/tool_util/deps/requirements.py‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,9 @@ def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> L
302302
for key in valid_key_set.intersection(set(r.keys())):
303303
value = r[key]
304304
key = cast(ResourceType, cwl_to_galaxy.get(key, key))
305-
rr.append(ResourceRequirement(value_or_expression=value, resource_type=key))
305+
if value is not None:
306+
# all resoure requirement fields are optional
307+
rr.append(ResourceRequirement(value_or_expression=value, resource_type=key))
306308
return rr
307309

308310

‎pyproject.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ dependencies = [
6060
"mercurial>=6.8.2", # Python 3.13 support
6161
"mrcfile",
6262
"msal",
63-
"nodejs-wheel==22",
63+
"nodejs-wheel>=22,<23",
6464
"numpy>=1.26.0", # Python 3.12 support
6565
"numpy>=2.1.0 ; python_version>='3.10'", # Python 3.13 support
6666
"openpyxl>=3.1.5", # Minimum version ever tested with
@@ -146,6 +146,7 @@ test = [
146146
"seletools",
147147
"statsd",
148148
"testfixtures",
149+
"total-perspective-vortex>=3.1.0,<4", # https://github.com/galaxyproject/total-perspective-vortex/pull/166
149150
"tuspy",
150151
"twill>=3.2.5", # Python 3.13 support
151152
"watchdog",

‎test/integration/embedded_pulsar_tpv_job_conf.yml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
runners:
2+
local:
3+
load: galaxy.jobs.runners.local:LocalJobRunner
4+
pulsar_embed:
5+
load: galaxy.jobs.runners.pulsar:PulsarEmbeddedJobRunner
6+
pulsar_app_config: # this doesn't start without disabling - filelock problem
7+
tool_dependency_dir: none
8+
conda_auto_init: false
9+
conda_auto_install: false
10+
11+
execution:
12+
default: tpv
13+
environments:
14+
tpv:
15+
runner: dynamic_tpv
16+
tpv_configs:
17+
- destinations:
18+
local:
19+
runner: local
20+
env:
21+
GALAXY_SLOTS: '{cores}'
22+
user_defined:
23+
runner: pulsar_embed
24+
env:
25+
GALAXY_SLOTS: '{cores}'
26+
scheduling:
27+
accept:
28+
- support_user_defined
29+
params:
30+
remote_metadata: true
31+
docker_enabled: true
32+
require_container: true
33+
docker_net: "none"
34+
global:
35+
default_inherits: default
36+
37+
tools:
38+
default:
39+
abstract: true
40+
cores: 1
41+
user_defined-*:
42+
scheduling:
43+
require:
44+
- support_user_defined

‎test/integration/test_user_defined_tool_job_conf.py‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
import os
44

5+
import yaml
6+
57
from galaxy.tool_util_models import UserToolSource
68
from galaxy_test.api.test_tools import TestsTools
79
from galaxy_test.base.populators import (
@@ -12,18 +14,39 @@
1214

1315
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
1416
EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_job_conf.yml")
17+
EMBEDDED_PULSAR_TPV_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_tpv_job_conf.yml")
18+
TOOL_WITH_RESOURCE_SPECIFICATION = yaml.safe_load(
19+
"""class: GalaxyUserTool
20+
id: resource_requirement
21+
version: "0.1"
22+
name: resource_requirement
23+
description: test resource requirement
24+
container: busybox
25+
requirements:
26+
- type: resource
27+
cores_min: 2
28+
shell_command: |
29+
echo $GALAXY_SLOTS > galaxy_cores.txt
30+
outputs:
31+
- name: output2
32+
type: data
33+
format: txt
34+
from_work_dir: galaxy_cores.txt
35+
"""
36+
)
1537

1638

1739
class TestUserDefinedToolRecommendedJobSetup(integration_util.IntegrationTestCase, TestsTools):
1840
"""Exercies how user defined tools could be run in production."""
1941

2042
framework_tool_and_types = True
43+
job_config_file = EMBEDDED_PULSAR_JOB_CONFIG_FILE
2144
dataset_populator: DatasetPopulator
2245

2346
@classmethod
2447
def handle_galaxy_config_kwds(cls, config):
2548
super().handle_galaxy_config_kwds(config)
26-
config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
49+
config["job_config_file"] = cls.job_config_file
2750
config["enable_celery_tasks"] = False
2851
config["metadata_strategy"] = "directory"
2952
config["admin_users"] = "[email protected]"
@@ -57,3 +80,30 @@ def test_user_defined_runs_in_correct_destination(self):
5780

5881
assert destination_params["Runner"] == "pulsar_embed"
5982
assert destination_params["require_container"]
83+
84+
85+
class TestUserDefinedToolRecommendedJobSetupTPV(TestUserDefinedToolRecommendedJobSetup):
86+
87+
job_config_file = EMBEDDED_PULSAR_TPV_JOB_CONFIG_FILE
88+
89+
def test_user_defined_applies_resource_requirements(self):
90+
with (
91+
self.dataset_populator.test_history() as history_id,
92+
self.dataset_populator.user_tool_execute_permissions(),
93+
):
94+
# Create a new dynamic tool.
95+
# This is a shell command tool that will echo the input dataset.
96+
dynamic_tool = self.dataset_populator.create_unprivileged_tool(
97+
UserToolSource(**TOOL_WITH_RESOURCE_SPECIFICATION)
98+
)
99+
# Run tool.
100+
response = self._run(
101+
history_id=history_id,
102+
tool_uuid=dynamic_tool["uuid"],
103+
inputs={},
104+
wait_for_job=True,
105+
assert_ok=True,
106+
)
107+
cores = response["outputs"][0]
108+
cores_content = self.dataset_populator.get_history_dataset_content(history_id, content_id=cores["id"])
109+
assert cores_content == "2.0\n"

0 commit comments

Comments
 (0)