Skip to content

Commit ef4c784

Browse files
authored
Merge branch 'main' into fix/autologin-singleuser
2 parents 23ec1a0 + a1fe7a6 commit ef4c784

File tree

24 files changed

+989
-356
lines changed

24 files changed

+989
-356
lines changed

api/pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,4 +109,4 @@ cpu = [
109109
"tensorboard==2.18.0",
110110
"tiktoken==0.8.0",
111111
"watchfiles==1.0.4",
112-
]
112+
]

api/transformerlab/galleries/interactive-gallery.json

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"description": "Remote VS Code development environment via tunnel",
77
"setup": "export DEBIAN_FRONTEND=noninteractive; sudo apt update && sudo apt install -y gnupg software-properties-common apt-transport-https wget && wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg && sudo install -o root -g root -m 644 packages.microsoft.gpg /usr/share/keyrings/ && echo \"deb [arch=amd64 signed-by=/usr/share/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main\" | sudo tee /etc/apt/sources.list.d/vscode.list && sudo apt update && sudo apt install -y code;",
88
"command": "code tunnel --accept-server-license-terms --disable-telemetry",
9-
"config_fields": []
9+
"env_parameters": []
1010
},
1111
{
1212
"id": "jupyter",
@@ -15,7 +15,7 @@
1515
"description": "Jupyter Lab notebook environment via ngrok tunnel",
1616
"setup": "export DEBIAN_FRONTEND=noninteractive; curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null ; echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list ; sudo apt update ; sudo apt install -y ngrok && pip install jupyter;",
1717
"command": "ngrok config add-authtoken $NGROK_AUTH_TOKEN; jupyter lab --ip=0.0.0.0 --port=8888 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password='' --notebook-dir=~ > /tmp/jupyter.log 2>&1 & sleep 3 && ngrok http 8888 --log=stdout 2>&1 | tee /tmp/ngrok.log; tail -f /tmp/jupyter.log /tmp/ngrok.log",
18-
"config_fields": [
18+
"env_parameters": [
1919
{
2020
"field_name": "ngrok Auth Token",
2121
"env_var": "NGROK_AUTH_TOKEN",
@@ -32,9 +32,9 @@
3232
"interactive_type": "vllm",
3333
"name": "vLLM Server",
3434
"description": "vLLM OpenAI-compatible API server via ngrok tunnel",
35-
"setup": "export DEBIAN_FRONTEND=noninteractive; curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null ; echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list ; sudo apt update ; sudo apt install -y ngrok; wget curl python3-pip && curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH=\"$HOME/.cargo/bin:$PATH\" && uv venv ~/vllm-venv && source ~/vllm-venv/bin/activate && uv pip install \"vllm>=0.11.0\" && uv pip install \"transformers>=4.57.1\" && uv pip install qwen-vl-utils==0.0.14 && uv pip install flashinfer-python flashinfer-cubin;",
35+
"setup": "export DEBIAN_FRONTEND=noninteractive && curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null && echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list && sudo apt update && sudo apt install -y ngrok curl python3-pip && curl -LsSf https://astral.sh/uv/install.sh | sh && export PATH=\"$HOME/.cargo/bin:$PATH\" && uv venv ~/vllm-venv && . ~/vllm-venv/bin/activate && uv pip install \"vllm>=0.11.0\" \"transformers>=4.57.1\" qwen-vl-utils==0.0.14 flashinfer-python flashinfer-cubin",
3636
"command": "ngrok config add-authtoken $NGROK_AUTH_TOKEN; source ~/vllm-venv/bin/activate && python -u -m vllm.entrypoints.openai.api_server --model $MODEL_NAME --tensor-parallel-size $TP_SIZE --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.9 > /tmp/vllm.log 2>&1 & sleep 10 && ngrok http 8000 --log=stdout 2>&1 | tee /tmp/ngrok.log; tail -f /tmp/vllm.log /tmp/ngrok.log",
37-
"config_fields": [
37+
"env_parameters": [
3838
{
3939
"field_name": "Model Name",
4040
"env_var": "MODEL_NAME",
@@ -76,9 +76,9 @@
7676
"interactive_type": "ollama",
7777
"name": "Ollama Server",
7878
"description": "Ollama model server via ngrok tunnel",
79-
"setup": "export DEBIAN_FRONTEND=noninteractive; curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null ; echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list ; sudo apt update ; sudo apt install -y ngrok; wget curl && curl -fsSL https://ollama.com/install.sh | sh && sudo apt update && sudo apt install -y pciutils lshw;",
79+
"setup": "export DEBIAN_FRONTEND=noninteractive; curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null ; echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list ; sudo apt update ; sudo apt install -y ngrok curl && curl -fsSL https://ollama.com/install.sh | sh && sudo apt update && sudo apt install -y pciutils lshw;",
8080
"command": "ngrok config add-authtoken $NGROK_AUTH_TOKEN; export OLLAMA_HOST=0.0.0.0:11434 && ollama serve > /tmp/ollama.log 2>&1 & sleep 3 && ollama pull $MODEL_NAME > /tmp/ollama-pull.log 2>&1 & sleep 5 && ngrok http 11434 --log=stdout 2>&1 | tee /tmp/ngrok.log; tail -f /tmp/ollama.log /tmp/ollama-pull.log /tmp/ngrok.log",
81-
"config_fields": [
81+
"env_parameters": [
8282
{
8383
"field_name": "Model Name",
8484
"env_var": "MODEL_NAME",
@@ -101,11 +101,11 @@
101101
{
102102
"id": "ssh",
103103
"interactive_type": "ssh",
104-
"name": "SSH via ngrok",
104+
"name": "SSH",
105105
"description": "SSH access via ngrok TCP tunnel",
106106
"setup": "export DEBIAN_FRONTEND=noninteractive; curl -sSL https://ngrok-agent.s3.amazonaws.com/ngrok.asc | sudo tee /etc/apt/trusted.gpg.d/ngrok.asc >/dev/null ; echo \"deb https://ngrok-agent.s3.amazonaws.com bookworm main\" | sudo tee /etc/apt/sources.list.d/ngrok.list ; sudo apt update ; sudo apt install ngrok;",
107107
"command": "ngrok config add-authtoken $NGROK_AUTH_TOKEN; echo USER_ID=$(whoami 2>/dev/null || basename $HOME 2>/dev/null || echo ''); ngrok tcp 22 --log=stdout 2>&1 | tee /tmp/ngrok.log; tail -f /tmp/ngrok.log",
108-
"config_fields": [
108+
"env_parameters": [
109109
{
110110
"field_name": "ngrok Auth Token",
111111
"env_var": "NGROK_AUTH_TOKEN",

api/transformerlab/routers/experiment/task.py

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,56 @@
2929
router = APIRouter(prefix="/task", tags=["task"])
3030

3131

32+
def process_env_parameters_to_env_vars(config: dict) -> dict:
33+
"""
34+
Process env_parameters from config/task.json and convert them to env_vars.
35+
36+
For each env_parameter:
37+
- If it has env_var and value, add to env_vars with that value
38+
- If it has only env_var (no value), add to env_vars with blank value
39+
40+
Args:
41+
config: Dictionary that may contain env_parameters
42+
43+
Returns:
44+
Updated config with env_vars populated from env_parameters
45+
"""
46+
if not isinstance(config, dict):
47+
return config
48+
49+
env_parameters = config.get("env_parameters", [])
50+
if not isinstance(env_parameters, list):
51+
return config
52+
53+
# Initialize env_vars if not present
54+
if "env_vars" not in config:
55+
config["env_vars"] = {}
56+
elif not isinstance(config["env_vars"], dict):
57+
# If env_vars exists but is not a dict, try to convert it
58+
try:
59+
if isinstance(config["env_vars"], str):
60+
config["env_vars"] = json.loads(config["env_vars"])
61+
else:
62+
config["env_vars"] = {}
63+
except (json.JSONDecodeError, TypeError):
64+
config["env_vars"] = {}
65+
66+
# Process each env_parameter
67+
for param in env_parameters:
68+
if not isinstance(param, dict):
69+
continue
70+
71+
env_var = param.get("env_var")
72+
if not env_var:
73+
continue
74+
75+
# If value is provided, use it; otherwise use blank string
76+
value = param.get("value", "")
77+
config["env_vars"][env_var] = value
78+
79+
return config
80+
81+
3282
@router.get("/list", summary="Returns all the tasks")
3383
async def task_get_all():
3484
tasks = task_service.task_get_all()
@@ -559,6 +609,9 @@ async def import_task_from_gallery(
559609
if github_repo_dir:
560610
task_config["github_directory"] = github_repo_dir
561611

612+
# Process env_parameters into env_vars if present
613+
task_config = process_env_parameters_to_env_vars(task_config)
614+
562615
# Get task name from config or use title
563616
task_name = task_config.get("name") or task_config.get("cluster_name") or title
564617

@@ -653,6 +706,9 @@ async def import_task_from_team_gallery(
653706
if github_repo_dir:
654707
task_config["github_directory"] = github_repo_dir
655708

709+
# Process env_parameters into env_vars if present
710+
task_config = process_env_parameters_to_env_vars(task_config)
711+
656712
# Get task name from config or use title
657713
task_name = task_config.get("name") or task_config.get("cluster_name") or title
658714

api/transformerlab/routers/tasks.py

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,17 @@
11
import json
2-
3-
from fastapi import APIRouter, Body, Depends, HTTPException, Query
42
from typing import Optional
5-
from werkzeug.utils import secure_filename
6-
from pydantic import BaseModel
73

4+
from fastapi import APIRouter, Body, Depends, HTTPException, Query
85
from lab import Dataset
9-
from transformerlab.services.job_service import job_create
6+
from pydantic import BaseModel
107
from transformerlab.models import model_helper
8+
from transformerlab.routers.auth import get_user_and_team
9+
from transformerlab.services.job_service import job_create
1110
from transformerlab.services.tasks_service import tasks_service
1211
from transformerlab.shared import galleries
13-
from transformerlab.shared.github_utils import (
14-
fetch_task_json_from_github_helper,
15-
fetch_task_json_from_github,
16-
)
17-
from transformerlab.routers.auth import get_user_and_team
12+
from transformerlab.shared.github_utils import fetch_task_json_from_github, fetch_task_json_from_github_helper
13+
from transformerlab.shared.task_utils import process_env_parameters_to_env_vars
14+
from werkzeug.utils import secure_filename
1815

1916
router = APIRouter(prefix="/tasks", tags=["tasks"])
2017

@@ -374,6 +371,9 @@ async def import_task_from_gallery(
374371
if github_repo_dir:
375372
task_config["github_directory"] = github_repo_dir
376373

374+
# Process env_parameters into env_vars if present
375+
task_config = process_env_parameters_to_env_vars(task_config)
376+
377377
# Get task name from config or use title
378378
task_name = task_config.get("name") or task_config.get("cluster_name") or title
379379

@@ -479,6 +479,9 @@ async def import_task_from_team_gallery(
479479
if github_repo_dir:
480480
task_config["github_directory"] = github_repo_dir
481481

482+
# Process env_parameters into env_vars if present
483+
task_config = process_env_parameters_to_env_vars(task_config)
484+
482485
# Get task name from config or use title
483486
task_name = task_config.get("name") or task_config.get("cluster_name") or title
484487

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import json
2+
3+
4+
def process_env_parameters_to_env_vars(config: dict) -> dict:
5+
"""
6+
Process env_parameters from config/task.json and convert them to env_vars.
7+
8+
For each env_parameter:
9+
- If it has env_var and value, add to env_vars with that value
10+
- If it has only env_var (no value), add to env_vars with blank value
11+
12+
Args:
13+
config: Dictionary that may contain env_parameters
14+
15+
Returns:
16+
Updated config with env_vars populated from env_parameters
17+
"""
18+
if not isinstance(config, dict):
19+
return config
20+
21+
env_parameters = config.get("env_parameters", [])
22+
if not isinstance(env_parameters, list):
23+
return config
24+
25+
# Initialize env_vars if not present
26+
if "env_vars" not in config:
27+
config["env_vars"] = {}
28+
elif not isinstance(config["env_vars"], dict):
29+
# If env_vars exists but is not a dict, try to convert it
30+
try:
31+
if isinstance(config["env_vars"], str):
32+
config["env_vars"] = json.loads(config["env_vars"])
33+
else:
34+
config["env_vars"] = {}
35+
except (json.JSONDecodeError, TypeError):
36+
config["env_vars"] = {}
37+
38+
# Process each env_parameter
39+
for param in env_parameters:
40+
if not isinstance(param, dict):
41+
continue
42+
43+
env_var = param.get("env_var")
44+
if not env_var:
45+
continue
46+
47+
# If value is provided, use it; otherwise use blank string
48+
value = param.get("value", "")
49+
config["env_vars"][env_var] = value
50+
51+
return config

cli/pyproject.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,5 +27,7 @@ where = ["src"]
2727

2828
[dependency-groups]
2929
dev = [
30+
"pytest",
3031
"textual>=6.11.0",
32+
"typer[testing]",
3133
]

cli/pytest.ini

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
[pytest]
2+
pythonpath = src

cli/src/transformerlab_cli/commands/job.py

Lines changed: 84 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,10 @@
11
import typer
2+
import os
3+
from pathlib import Path
24
from rich.console import Console
35
from rich.table import Table
46
from rich import print
5-
from rich.progress import Progress
7+
from rich.progress import Progress, SpinnerColumn, TextColumn, BarColumn, DownloadColumn, TransferSpeedColumn
68
from rich.panel import Panel
79
from rich.text import Text
810
from urllib.parse import urlparse
@@ -161,6 +163,75 @@ def list_artifacts(job_id: str, output_format: str = "pretty") -> None:
161163
console.print(f"[red]Error:[/red] Failed to fetch artifacts. Status code: {response.status_code}")
162164

163165

166+
def download_artifacts(job_id: str, output_dir: str = None) -> None:
167+
"""Download all artifacts for a job as a zip file."""
168+
if output_dir is None:
169+
output_dir = os.getcwd()
170+
else:
171+
output_dir = os.path.abspath(output_dir)
172+
Path(output_dir).mkdir(parents=True, exist_ok=True)
173+
174+
# Determine output filename
175+
filename = f"artifacts_{job_id}.zip"
176+
output_path = os.path.join(output_dir, filename)
177+
178+
# Check if file already exists
179+
if os.path.exists(output_path):
180+
console.print(f"[yellow]Warning:[/yellow] File {output_path} already exists. It will be overwritten.")
181+
182+
try:
183+
with console.status(f"[bold green]Downloading artifacts for job {job_id}...[/bold green]", spinner="dots"):
184+
response = api.get(f"/jobs/{job_id}/artifacts/download_all", timeout=300.0)
185+
186+
if response.status_code == 200:
187+
# Get filename from Content-Disposition header if available
188+
content_disposition = response.headers.get("Content-Disposition", "")
189+
if "filename=" in content_disposition:
190+
# Extract filename from Content-Disposition header
191+
filename_part = content_disposition.split("filename=")[1].strip('"')
192+
if filename_part:
193+
filename = filename_part
194+
output_path = os.path.join(output_dir, filename)
195+
196+
# Write the file with progress tracking
197+
content_length = response.headers.get("Content-Length")
198+
total_size = int(content_length) if content_length else None
199+
200+
with Progress(
201+
SpinnerColumn(),
202+
TextColumn("[progress.description]{task.description}"),
203+
BarColumn(),
204+
DownloadColumn(),
205+
TransferSpeedColumn(),
206+
) as progress:
207+
task = progress.add_task(f"[cyan]Downloading {filename}...", total=total_size)
208+
209+
with open(output_path, "wb") as f:
210+
if total_size:
211+
# Show progress if we know the total size
212+
downloaded = 0
213+
for chunk in response.iter_bytes(chunk_size=8192):
214+
if chunk:
215+
f.write(chunk)
216+
downloaded += len(chunk)
217+
progress.update(task, completed=downloaded)
218+
else:
219+
# Just write without progress if we don't know the size
220+
f.write(response.content)
221+
progress.update(task, completed=1, total=1)
222+
223+
console.print(f"[green]✓[/green] Successfully downloaded artifacts to: {output_path}")
224+
elif response.status_code == 404:
225+
console.print(f"[red]Error:[/red] No artifacts found for job {job_id}.")
226+
else:
227+
console.print(f"[red]Error:[/red] Failed to download artifacts. Status code: {response.status_code}")
228+
if response.text:
229+
console.print(f"[red]Response:[/red] {response.text[:200]}")
230+
231+
except Exception as e:
232+
console.print(f"[red]Error:[/red] Failed to download artifacts: {e}")
233+
234+
164235
@app.command("artifacts")
165236
def command_job_artifacts(
166237
job_id: str = typer.Argument(..., help="Job ID to list artifacts for"),
@@ -170,6 +241,18 @@ def command_job_artifacts(
170241
list_artifacts(job_id)
171242

172243

244+
@app.command("download")
245+
def command_job_download(
246+
job_id: str = typer.Argument(..., help="Job ID to download artifacts for"),
247+
output_dir: str = typer.Option(
248+
None, "--output", "-o", help="Output directory for the zip file (default: current directory)"
249+
),
250+
):
251+
"""Download all artifacts for a job as a zip file."""
252+
check_configs()
253+
download_artifacts(job_id, output_dir)
254+
255+
173256
@app.command("list")
174257
def command_job_list():
175258
"""List all jobs."""

0 commit comments

Comments
 (0)