diff --git a/docs/assets/stylesheets/extra.css b/docs/assets/stylesheets/extra.css index 4d97c82325..4952a41c58 100644 --- a/docs/assets/stylesheets/extra.css +++ b/docs/assets/stylesheets/extra.css @@ -790,6 +790,7 @@ body { display: flex; } + /* Comment to siwtch to sections in sidebar */ .md-sidebar__inner > .md-nav--primary > .md-nav__list:not(.md-post__meta) > .md-nav__item > .md-nav > .md-nav__list > .md-nav__item:not(.md-nav__item--section) { display: none; } @@ -861,7 +862,7 @@ body { } .md-sidebar--primary .md-nav__link, .md-sidebar--post .md-nav__link { - padding: 5px 15px 5px; + padding: 5px 15px 4px; margin-top: 0; } diff --git a/docs/blog/posts/amd-on-runpod.md b/docs/blog/posts/amd-on-runpod.md index aab962c9da..eb4668e506 100644 --- a/docs/blog/posts/amd-on-runpod.md +++ b/docs/blog/posts/amd-on-runpod.md @@ -42,7 +42,7 @@ you can now specify an AMD GPU under `resources`. Below are a few examples. Here's an example of a [service](../../docs/concepts/services.md) that deploys Llama 3.1 70B in FP16 using [TGI :material-arrow-top-right-thin:{ .external }](https://huggingface.co/docs/text-generation-inference/en/installation_amd){:target="_blank"}. -
+
```yaml type: service diff --git a/docs/blog/posts/beyond-kubernetes-2024-recap-and-whats-ahead.md b/docs/blog/posts/beyond-kubernetes-2024-recap-and-whats-ahead.md index be7868e285..79e026a0b4 100644 --- a/docs/blog/posts/beyond-kubernetes-2024-recap-and-whats-ahead.md +++ b/docs/blog/posts/beyond-kubernetes-2024-recap-and-whats-ahead.md @@ -110,7 +110,7 @@ efficient manner. ### NVIDIA -NVIDIA remains the top accelerator supported by `dstack`. Recently, we introduced a [NIM example](../../examples/deployment/nim/index.md) +NVIDIA remains the top accelerator supported by `dstack`. Recently, we introduced a [NIM example](../../examples/inference/nim/index.md) for model deployment, and we continue to enhance support for the rest of NVIDIA's ecosystem. ### AMD diff --git a/docs/blog/posts/dstack-metrics.md b/docs/blog/posts/dstack-metrics.md index 459ef6d3e5..07d80ab0ab 100644 --- a/docs/blog/posts/dstack-metrics.md +++ b/docs/blog/posts/dstack-metrics.md @@ -7,7 +7,7 @@ image: https://dstack.ai/static-assets/static-assets/images/dstack-stats-v2.png categories: - AMD - NVIDIA - - Monitoring + - Metrics --- # Monitoring essential GPU metrics via CLI diff --git a/docs/blog/posts/metrics-ui.md b/docs/blog/posts/metrics-ui.md index ba0cfde6b6..032115e5a6 100644 --- a/docs/blog/posts/metrics-ui.md +++ b/docs/blog/posts/metrics-ui.md @@ -5,7 +5,7 @@ description: "TBA" slug: metrics-ui image: https://dstack.ai/static-assets/static-assets/images/dstack-metrics-ui-v3-min.png categories: - - Monitoring + - Metrics - AMD - NVIDIA --- @@ -55,6 +55,6 @@ For persistent storage and long-term access to metrics, we still recommend setti metrics from `dstack`. !!! info "What's next?" - 1. See [Monitoring](../../docs/guides/monitoring.md) + 1. See [Metrics](../../docs/guides/metrics.md) 2. Check [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md) 3. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"} diff --git a/docs/blog/posts/mpi.md b/docs/blog/posts/mpi.md index 4516c4297e..70b3ed1650 100644 --- a/docs/blog/posts/mpi.md +++ b/docs/blog/posts/mpi.md @@ -101,5 +101,5 @@ as well as use MPI for other tasks. !!! info "What's next?" 1. Learn more about [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md) - 2. Check the [NCCL tests](../../examples/distributed-training/nccl-tests/index.md) example + 2. Check the [NCCL tests](../../examples/clusters/nccl-tests/index.md) example 2. Join [Discord :material-arrow-top-right-thin:{ .external }](https://discord.gg/u8SmfwPpMd){:target="_blank"} diff --git a/docs/blog/posts/prometheus.md b/docs/blog/posts/prometheus.md index 23e644bcfd..fbaee63c1e 100644 --- a/docs/blog/posts/prometheus.md +++ b/docs/blog/posts/prometheus.md @@ -5,7 +5,7 @@ description: "TBA" slug: prometheus image: https://dstack.ai/static-assets/static-assets/images/dstack-prometheus-v3.png categories: - - Monitoring + - Metrics - NVIDIA --- @@ -46,7 +46,7 @@ Overall, `dstack` collects three groups of metrics: | **Runs** | Run metrics include run counters for each user in each project. | | **Jobs** | A run consists of one or more jobs, each mapped to a container. Job metrics offer insights into execution time, cost, GPU model, NVIDIA DCGM telemetry, and more. | -For a full list of available metrics and labels, check out [Monitoring](../../docs/guides/monitoring.md). +For a full list of available metrics and labels, check out [Metrics](../../docs/guides/metrics.md). ??? info "NVIDIA" NVIDIA DCGM metrics are automatically collected for `aws`, `azure`, `gcp`, and `oci` backends, @@ -60,7 +60,7 @@ For a full list of available metrics and labels, check out [Monitoring](../../do only accessible through the UI and the [`dstack metrics`](dstack-metrics.md) CLI. !!! info "What's next?" - 1. See [Metrics](../../docs/guides/monitoring.md) + 1. See [Metrics](../../docs/guides/metrics.md) 1. Check [dev environments](../../docs/concepts/dev-environments.md), [tasks](../../docs/concepts/tasks.md), [services](../../docs/concepts/services.md), and [fleets](../../docs/concepts/fleets.md) diff --git a/docs/blog/posts/tpu-on-gcp.md b/docs/blog/posts/tpu-on-gcp.md index 8cdf1a051a..24534c7e76 100644 --- a/docs/blog/posts/tpu-on-gcp.md +++ b/docs/blog/posts/tpu-on-gcp.md @@ -50,7 +50,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm- === "Optimum TPU" -
+
```yaml type: service @@ -83,7 +83,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm- the official Docker image can be used instead of `dstackai/optimum-tpu:llama31`. === "vLLM" -
+
```yaml type: service diff --git a/docs/changelog/index.md b/docs/changelog/index.md deleted file mode 100644 index 05761ac57f..0000000000 --- a/docs/changelog/index.md +++ /dev/null @@ -1 +0,0 @@ -# Blog diff --git a/docs/docs/concepts/gateways.md b/docs/docs/concepts/gateways.md index 815eca4ae1..500d0d86cf 100644 --- a/docs/docs/concepts/gateways.md +++ b/docs/docs/concepts/gateways.md @@ -71,7 +71,7 @@ To delete a gateway, pass the gateway configuration to [`dstack delete`](../refe
```shell -$ dstack delete -f examples/deployment/gateway.dstack.yml +$ dstack delete -f examples/inference/gateway.dstack.yml ```
diff --git a/docs/docs/concepts/services.md b/docs/docs/concepts/services.md index 1f7b515b84..70e0b2d5bd 100644 --- a/docs/docs/concepts/services.md +++ b/docs/docs/concepts/services.md @@ -558,6 +558,6 @@ If one replica of a multi-replica service fails with retry enabled, 1. Read about [dev environments](dev-environments.md), [tasks](tasks.md), and [repos](repos.md) 2. Learn how to manage [fleets](fleets.md) 3. See how to set up [gateways](gateways.md) - 4. Check the [TGI :material-arrow-top-right-thin:{ .external }](../../examples/deployment/tgi/index.md){:target="_blank"}, - [vLLM :material-arrow-top-right-thin:{ .external }](../../examples/deployment/vllm/index.md){:target="_blank"}, and - [NIM :material-arrow-top-right-thin:{ .external }](../../examples/deployment/nim/index.md){:target="_blank"} examples + 4. Check the [TGI :material-arrow-top-right-thin:{ .external }](../../examples/inference/tgi/index.md){:target="_blank"}, + [vLLM :material-arrow-top-right-thin:{ .external }](../../examples/inference/vllm/index.md){:target="_blank"}, and + [NIM :material-arrow-top-right-thin:{ .external }](../../examples/inference/nim/index.md){:target="_blank"} examples diff --git a/docs/docs/guides/monitoring.md b/docs/docs/guides/metrics.md similarity index 100% rename from docs/docs/guides/monitoring.md rename to docs/docs/guides/metrics.md diff --git a/docs/examples.md b/docs/examples.md index 24881a7c09..128640b1ef 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -38,10 +38,10 @@ hide:
-## Distributed training +## Clusters
-

NCCL tests @@ -51,7 +51,7 @@ hide: Run multi-node NCCL tests with MPI

-

RCCL tests @@ -61,7 +61,7 @@ hide: Run multi-node RCCL tests with MPI

-

A3 Mega @@ -71,7 +71,7 @@ hide: Set up GCP A3 Mega clusters with optimized networking

-

A3 High @@ -83,10 +83,10 @@ hide:

-## Deployment +## Inference
-

SGLang @@ -95,7 +95,7 @@ hide: Deploy DeepSeek distilled models with SGLang

-

vLLM @@ -104,7 +104,7 @@ hide: Deploy Llama 3.1 with vLLM

-

TGI @@ -113,7 +113,7 @@ hide: Deploy Llama 4 with TGI

-

NIM @@ -122,7 +122,7 @@ hide: Deploy a DeepSeek distilled model with NIM

-

TensorRT-LLM diff --git a/docs/examples/deployment/nim/index.md b/docs/examples/clusters/a3high/index.md similarity index 100% rename from docs/examples/deployment/nim/index.md rename to docs/examples/clusters/a3high/index.md diff --git a/docs/examples/deployment/sglang/index.md b/docs/examples/clusters/a3mega/index.md similarity index 100% rename from docs/examples/deployment/sglang/index.md rename to docs/examples/clusters/a3mega/index.md diff --git a/docs/examples/deployment/tgi/index.md b/docs/examples/clusters/nccl-tests/index.md similarity index 100% rename from docs/examples/deployment/tgi/index.md rename to docs/examples/clusters/nccl-tests/index.md diff --git a/docs/examples/deployment/trtllm/index.md b/docs/examples/clusters/rccl-tests/index.md similarity index 100% rename from docs/examples/deployment/trtllm/index.md rename to docs/examples/clusters/rccl-tests/index.md diff --git a/docs/examples/deployment/vllm/index.md b/docs/examples/inference/nim/index.md similarity index 100% rename from docs/examples/deployment/vllm/index.md rename to docs/examples/inference/nim/index.md diff --git a/docs/examples/distributed-training/a3high-clusters/index.md b/docs/examples/inference/sglang/index.md similarity index 100% rename from docs/examples/distributed-training/a3high-clusters/index.md rename to docs/examples/inference/sglang/index.md diff --git a/docs/examples/distributed-training/a3mega-clusters/index.md b/docs/examples/inference/tgi/index.md similarity index 100% rename from docs/examples/distributed-training/a3mega-clusters/index.md rename to docs/examples/inference/tgi/index.md diff --git a/docs/examples/distributed-training/nccl-tests/index.md b/docs/examples/inference/trtllm/index.md similarity index 100% rename from docs/examples/distributed-training/nccl-tests/index.md rename to docs/examples/inference/trtllm/index.md diff --git a/docs/examples/distributed-training/rccl-tests/index.md b/docs/examples/inference/vllm/index.md similarity index 100% rename from docs/examples/distributed-training/rccl-tests/index.md rename to docs/examples/inference/vllm/index.md diff --git a/docs/overrides/main.html b/docs/overrides/main.html index 3ab9f7deff..51adf057be 100644 --- a/docs/overrides/main.html +++ b/docs/overrides/main.html @@ -117,9 +117,9 @@
Examples
- Distributed training Fine-tuning - Deployment + Clusters + Inference Accelerators LLMs Misc diff --git a/examples/accelerators/amd/README.md b/examples/accelerators/amd/README.md index 274eac5c76..062ab08923 100644 --- a/examples/accelerators/amd/README.md +++ b/examples/accelerators/amd/README.md @@ -11,7 +11,7 @@ Llama 3.1 70B in FP16 using [TGI :material-arrow-top-right-thin:{ .external }](h === "TGI" -
+
```yaml type: service @@ -44,7 +44,7 @@ Llama 3.1 70B in FP16 using [TGI :material-arrow-top-right-thin:{ .external }](h === "vLLM" -
+
```yaml type: service @@ -98,7 +98,7 @@ Llama 3.1 70B in FP16 using [TGI :material-arrow-top-right-thin:{ .external }](h > To speed up the `vLLM-ROCm` installation, we use a pre-built binary from S3. > You can find the task to build and upload the binary in - > [`examples/deployment/vllm/amd/` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/vllm/amd/){:target="_blank"}. + > [`examples/inference/vllm/amd/` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/vllm/amd/){:target="_blank"}. !!! info "Docker image" If you want to use AMD, specifying `image` is currently required. This must be an image that includes @@ -228,7 +228,7 @@ $ WANDB_API_KEY=... $ WANDB_PROJECT=... $ WANDB_NAME=axolotl-amd-llama31-train $ HUB_MODEL_ID=... -$ dstack apply -f examples/deployment/vllm/amd/.dstack.yml +$ dstack apply -f examples/inference/vllm/amd/.dstack.yml ```
@@ -236,8 +236,8 @@ $ dstack apply -f examples/deployment/vllm/amd/.dstack.yml ## Source code The source-code of this example can be found in -[`examples/deployment/tgi/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/tgi/amd){:target="_blank"}, -[`examples/deployment/vllm/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/vllm/amd){:target="_blank"}, +[`examples/inference/tgi/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/tgi/amd){:target="_blank"}, +[`examples/inference/vllm/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/vllm/amd){:target="_blank"}, [`examples/fine-tuning/axolotl/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/fine-tuning/axolotl/amd){:target="_blank"} and [`examples/fine-tuning/trl/amd` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/fine-tuning/trl/amd){:target="_blank"} diff --git a/examples/accelerators/intel/README.md b/examples/accelerators/intel/README.md index 531daf29be..b0fc4aab25 100644 --- a/examples/accelerators/intel/README.md +++ b/examples/accelerators/intel/README.md @@ -12,7 +12,7 @@ using [TGI on Gaudi :material-arrow-top-right-thin:{ .external }](https://github and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/HabanaAI/vllm-fork){:target="_blank"}. === "TGI" -
+
```yaml type: service @@ -60,7 +60,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/Haban === "vLLM" -
+
```yaml type: service @@ -162,7 +162,7 @@ Once the configuration is ready, run `dstack apply -f `.
```shell -$ dstack apply -f examples/deployment/vllm/.dstack.yml +$ dstack apply -f examples/inference/vllm/.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 ssh remote 152xCPU,1007GB,8xGaudi2:96GB yes $0 idle diff --git a/examples/accelerators/tpu/README.md b/examples/accelerators/tpu/README.md index 41c33b12a2..2307e89432 100644 --- a/examples/accelerators/tpu/README.md +++ b/examples/accelerators/tpu/README.md @@ -24,7 +24,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm- === "Optimum TPU" -
+
```yaml type: service @@ -56,7 +56,7 @@ and [vLLM :material-arrow-top-right-thin:{ .external }](https://github.com/vllm- the official Docker image can be used. === "vLLM" -
+
```yaml type: service @@ -179,8 +179,8 @@ Note, `v5litepod` is optimized for fine-tuning transformer-based models. Each co ## Source code The source-code of this example can be found in -[`examples/deployment/tgi/tpu` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/tgi/tpu){:target="_blank"}, -[`examples/deployment/vllm/tpu` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/vllm/tpu){:target="_blank"}, +[`examples/inference/tgi/tpu` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/tgi/tpu){:target="_blank"}, +[`examples/inference/vllm/tpu` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/vllm/tpu){:target="_blank"}, and [`examples/fine-tuning/optimum-tpu` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/fine-tuning/trl){:target="_blank"}. ## What's next? diff --git a/examples/distributed-training/a3high-clusters/README.md b/examples/clusters/a3high/README.md similarity index 99% rename from examples/distributed-training/a3high-clusters/README.md rename to examples/clusters/a3high/README.md index 52bbaa08b1..bae58d07d9 100644 --- a/examples/distributed-training/a3high-clusters/README.md +++ b/examples/clusters/a3high/README.md @@ -1,4 +1,4 @@ -# GCP A3 High clusters +# GCP A3 High This example shows how to set up a GCP A3 High cluster with [GPUDirect-TCPX](https://cloud.google.com/compute/docs/gpus/gpudirect) optimized NCCL communication and run [NCCL Tests](https://github.com/NVIDIA/nccl-tests) on it using `dstack`. diff --git a/examples/distributed-training/a3high-clusters/fleet.dstack.yml b/examples/clusters/a3high/fleet.dstack.yml similarity index 100% rename from examples/distributed-training/a3high-clusters/fleet.dstack.yml rename to examples/clusters/a3high/fleet.dstack.yml diff --git a/examples/distributed-training/a3high-clusters/nccl-tests.dstack.yml b/examples/clusters/a3high/nccl-tests.dstack.yml similarity index 100% rename from examples/distributed-training/a3high-clusters/nccl-tests.dstack.yml rename to examples/clusters/a3high/nccl-tests.dstack.yml diff --git a/examples/distributed-training/a3mega-clusters/README.md b/examples/clusters/a3mega/README.md similarity index 99% rename from examples/distributed-training/a3mega-clusters/README.md rename to examples/clusters/a3mega/README.md index c584ee3ad2..52fc9b5be6 100644 --- a/examples/distributed-training/a3mega-clusters/README.md +++ b/examples/clusters/a3mega/README.md @@ -1,4 +1,4 @@ -# GCP A3 Mega clusters +# GCP A3 Mega This example shows how to set up a GCP A3 Mega cluster with [GPUDirect-TCPXO](https://cloud.google.com/kubernetes-engine/docs/how-to/gpu-bandwidth-gpudirect-tcpx-autopilot) optimized NCCL communication and run [NCCL Tests](https://github.com/NVIDIA/nccl-tests) on it using `dstack`. diff --git a/examples/distributed-training/a3mega-clusters/fleet.dstack.yml b/examples/clusters/a3mega/fleet.dstack.yml similarity index 100% rename from examples/distributed-training/a3mega-clusters/fleet.dstack.yml rename to examples/clusters/a3mega/fleet.dstack.yml diff --git a/examples/distributed-training/a3mega-clusters/nccl-tests.dstack.yml b/examples/clusters/a3mega/nccl-tests.dstack.yml similarity index 100% rename from examples/distributed-training/a3mega-clusters/nccl-tests.dstack.yml rename to examples/clusters/a3mega/nccl-tests.dstack.yml diff --git a/examples/distributed-training/nccl-tests/.dstack.yml b/examples/clusters/nccl-tests/.dstack.yml similarity index 100% rename from examples/distributed-training/nccl-tests/.dstack.yml rename to examples/clusters/nccl-tests/.dstack.yml diff --git a/examples/distributed-training/nccl-tests/README.md b/examples/clusters/nccl-tests/README.md similarity index 100% rename from examples/distributed-training/nccl-tests/README.md rename to examples/clusters/nccl-tests/README.md diff --git a/examples/distributed-training/rccl-tests/.dstack.yml b/examples/clusters/rccl-tests/.dstack.yml similarity index 100% rename from examples/distributed-training/rccl-tests/.dstack.yml rename to examples/clusters/rccl-tests/.dstack.yml diff --git a/examples/distributed-training/rccl-tests/README.md b/examples/clusters/rccl-tests/README.md similarity index 100% rename from examples/distributed-training/rccl-tests/README.md rename to examples/clusters/rccl-tests/README.md diff --git a/examples/deployment/infinity/.dstack.yml b/examples/deployment/infinity/.dstack.yml deleted file mode 100644 index 66a5289a18..0000000000 --- a/examples/deployment/infinity/.dstack.yml +++ /dev/null @@ -1,13 +0,0 @@ -type: service -# This service deploys embeddings model and reranker model with Infinity. Learn more at https://dstack.ai/examples/infinity/ - -image: michaelf34/infinity:latest -env: - - INFINITY_MODEL_ID=BAAI/bge-small-en-v1.5;BAAI/bge-reranker-base; - - INFINITY_PORT=8000 -commands: - - infinity_emb v2 -port: 8000 - -resources: - gpu: 16GB diff --git a/examples/deployment/infinity/README.md b/examples/deployment/infinity/README.md deleted file mode 100644 index cfc0f409b9..0000000000 --- a/examples/deployment/infinity/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Infinity - -The following command a text embedding model as a service: - -```shell -dstack apply -f examples/deployment/infinity/.dstack.yml -``` - -See the configuration at [.dstack.yml](.dstack.yml). - -For more details, refer to [services](https://dstack.ai/docs/services). diff --git a/examples/deployment/lorax/.dstack.yml b/examples/deployment/lorax/.dstack.yml deleted file mode 100644 index fa3fbee6db..0000000000 --- a/examples/deployment/lorax/.dstack.yml +++ /dev/null @@ -1,14 +0,0 @@ -type: service -image: ghcr.io/predibase/lorax:latest - -env: - - HF_TOKEN - - MODEL_ID=mistralai/Mistral-7B-Instruct-v0.1 -commands: - - cd /usr/src - - lorax-launcher --port 8000 --model-id $MODEL_ID -port: 8000 - -resources: - gpu: 24GB - shm_size: 1GB diff --git a/examples/deployment/lorax/README.md b/examples/deployment/lorax/README.md deleted file mode 100644 index b1084ddd83..0000000000 --- a/examples/deployment/lorax/README.md +++ /dev/null @@ -1,14 +0,0 @@ -# LoRaX - -[LoRAX](https://github.com/predibase/lorax) allows serving multiple fine-tuned models optimized on -the same endpoint by dynamically loading and switching LoRA adapters. - -The following command deploys Mistral 7B Instruct as a base model via a service: - -```shell -dstack apply -f examples/deployment/lorax/.dstack.yml -``` - -See the configuration at [.dstack.yml](.dstack.yml). - -For more details, refer to [services](https://dstack.ai/docs/services). diff --git a/examples/deployment/ollama/.dstack.yml b/examples/deployment/ollama/.dstack.yml deleted file mode 100644 index 31d164c7f7..0000000000 --- a/examples/deployment/ollama/.dstack.yml +++ /dev/null @@ -1,19 +0,0 @@ -type: service -name: llama31 - -image: ollama/ollama -commands: - - ollama serve & - - sleep 3 - - ollama pull llama3.1 - - fg -port: 11434 -# Register the model -model: llama3.1 - -# Uncomment to leverage spot instances -#spot_policy: auto - -# Required resources -resources: - gpu: 24GB diff --git a/examples/deployment/ollama/README.md b/examples/deployment/ollama/README.md deleted file mode 100644 index 7bb833da21..0000000000 --- a/examples/deployment/ollama/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Ollama - -The following command deploys Mixtral 8x7B as a service: - -```shell -dstack apply . -f examples/deployment/ollama/serve.dstack.yml -``` - -See the configuration at [serve.dstack.yml](serve.dstack.yml). - -For more details, refer to [services](https://dstack.ai/docs/services). diff --git a/examples/deployment/nim/.dstack.yml b/examples/inference/nim/.dstack.yml similarity index 100% rename from examples/deployment/nim/.dstack.yml rename to examples/inference/nim/.dstack.yml diff --git a/examples/deployment/nim/README.md b/examples/inference/nim/README.md similarity index 91% rename from examples/deployment/nim/README.md rename to examples/inference/nim/README.md index 637279787e..cbfde9eac7 100644 --- a/examples/deployment/nim/README.md +++ b/examples/inference/nim/README.md @@ -24,7 +24,7 @@ This example shows how to deploy DeepSeek-R1-Distill-Llama-8B using [NVIDIA NIM Here's an example of a service that deploys DeepSeek-R1-Distill-Llama-8B using NIM. -
+
```yaml type: service @@ -65,7 +65,7 @@ To run a configuration, use the [`dstack apply`](https://dstack.ai/docs/referenc ```shell $ NGC_API_KEY=... -$ dstack apply -f examples/deployment/nim/.dstack.yml +$ dstack apply -f examples/inference/nim/.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vultr ewr 6xCPU, 60GB, 1xA100 (40GB) no $1.199 @@ -113,7 +113,7 @@ is available at `https://gateway./`. ## Source code The source-code of this example can be found in -[`examples/deployment/nim` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/nim){:target="_blank"}. +[`examples/inference/nim` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/nim){:target="_blank"}. ## What's next? diff --git a/examples/deployment/sglang/README.md b/examples/inference/sglang/README.md similarity index 96% rename from examples/deployment/sglang/README.md rename to examples/inference/sglang/README.md index 1d7b5df167..5fa896b2d5 100644 --- a/examples/deployment/sglang/README.md +++ b/examples/inference/sglang/README.md @@ -20,7 +20,7 @@ Here's an example of a service that deploys DeepSeek-R1-Distill-Llama 8B and 70B === "AMD" -
+
```yaml type: service @@ -47,7 +47,7 @@ Here's an example of a service that deploys DeepSeek-R1-Distill-Llama 8B and 70B === "NVIDIA" -
+
```yaml type: service diff --git a/examples/deployment/tgi/.dstack.yml b/examples/inference/tgi/.dstack.yml similarity index 100% rename from examples/deployment/tgi/.dstack.yml rename to examples/inference/tgi/.dstack.yml diff --git a/examples/deployment/tgi/README.md b/examples/inference/tgi/README.md similarity index 88% rename from examples/deployment/tgi/README.md rename to examples/inference/tgi/README.md index 5ed4c6b5e0..5a10637263 100644 --- a/examples/deployment/tgi/README.md +++ b/examples/inference/tgi/README.md @@ -24,7 +24,7 @@ This example shows how to deploy Llama 4 Scout with `dstack` using [HuggingFace Here's an example of a service that deploys [`Llama-4-Scout-17B-16E-Instruct` :material-arrow-top-right-thin:{ .external }](https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct){:target="_blank"} using TGI. -
+
```yaml type: service @@ -70,7 +70,7 @@ To run a configuration, use the [`dstack apply`](https://dstack.ai/docs/referenc ```shell $ HF_TOKEN=... -$ dstack apply -f examples/deployment/tgi/.dstack.yml +$ dstack apply -f examples/inference/tgi/.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vastai is-iceland 48xCPU, 128GB, 2xH200 (140GB) no $7.87 @@ -117,11 +117,11 @@ is available at `https://gateway./`. ## Source code The source-code of this example can be found in -[`examples/deployment/tgi` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/tgi). +[`examples/inference/tgi` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/tgi). ## What's next? 1. Check [services](https://dstack.ai/docs/services) -2. Browse the [Llama](https://dstack.ai/examples/llms/llama/), [vLLM](https://dstack.ai/examples/deployment/vllm/), [SgLang](https://dstack.ai/examples/deployment/sglang/) and [NIM](https://dstack.ai/examples/deployment/nim/) examples +2. Browse the [Llama](https://dstack.ai/examples/llms/llama/), [vLLM](https://dstack.ai/examples/inference/vllm/), [SgLang](https://dstack.ai/examples/inference/sglang/) and [NIM](https://dstack.ai/examples/inference/nim/) examples 3. See also [AMD](https://dstack.ai/examples/accelerators/amd/) and [TPU](https://dstack.ai/examples/accelerators/tpu/) diff --git a/examples/deployment/tgi/amd/.dstack.yml b/examples/inference/tgi/amd/.dstack.yml similarity index 100% rename from examples/deployment/tgi/amd/.dstack.yml rename to examples/inference/tgi/amd/.dstack.yml diff --git a/examples/deployment/tgi/tpu/.dstack.yml b/examples/inference/tgi/tpu/.dstack.yml similarity index 100% rename from examples/deployment/tgi/tpu/.dstack.yml rename to examples/inference/tgi/tpu/.dstack.yml diff --git a/examples/deployment/trtllm/README.md b/examples/inference/trtllm/README.md similarity index 94% rename from examples/deployment/trtllm/README.md rename to examples/inference/trtllm/README.md index 0db3077e7e..b8cdef0d45 100644 --- a/examples/deployment/trtllm/README.md +++ b/examples/inference/trtllm/README.md @@ -34,7 +34,7 @@ To use `trtllm-serve`, we first need to build the TensorRT-LLM Docker image from Here’s the task config that builds the image and pushes it using the provided Docker credentials. -
+
```yaml type: task @@ -70,7 +70,7 @@ To run it, pass the task configuration to `dstack apply`.
```shell -$ dstack apply -f examples/deployment/trtllm/build-image.dstack.yml +$ dstack apply -f examples/inference/trtllm/build-image.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 cudo ca-montreal-2 8xCPU, 25GB, (500.0GB) yes $0.1073 @@ -86,13 +86,13 @@ Provisioning... Below is the service configuration that deploys DeepSeek R1 using the built TensorRT-LLM image. -
+
```yaml type: service name: serve-r1 - # Specify the image built with `examples/deployment/trtllm/build-image.dstack.yml` + # Specify the image built with `examples/inference/trtllm/build-image.dstack.yml` image: dstackai/tensorrt_llm:9b931c0f6305aefa3660e6fb84a76a42c0eef167 env: - MAX_BATCH_SIZE=256 @@ -130,7 +130,7 @@ To run it, pass the configuration to `dstack apply`.
```shell -$ dstack apply -f examples/deployment/trtllm/serve-r1.dstack.yml +$ dstack apply -f examples/inference/trtllm/serve-r1.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vastai is-iceland 192xCPU, 2063GB, 8xH200 (141GB) yes $25.62 @@ -152,7 +152,7 @@ To deploy DeepSeek R1 Distill Llama 8B, follow the steps below. Here’s the task config that converts a Hugging Face model to a TensorRT-LLM checkpoint format and uploads it to S3 using the provided AWS credentials. -
+
```yaml type: task @@ -197,7 +197,7 @@ To run it, pass the configuration to `dstack apply`.
```shell -$ dstack apply -f examples/deployment/trtllm/convert-model.dstack.yml +$ dstack apply -f examples/inference/trtllm/convert-model.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vastai us-iowa 12xCPU, 85GB, 1xA100 (40GB) yes $0.66904 @@ -265,7 +265,7 @@ To run it, pass the configuration to `dstack apply`.
```shell -$ dstack apply -f examples/deployment/trtllm/build-model.dstack.yml +$ dstack apply -f examples/inference/trtllm/build-model.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vastai us-iowa 12xCPU, 85GB, 1xA100 (40GB) yes $0.66904 @@ -317,7 +317,7 @@ To run it, pass the configuration to `dstack apply`.
```shell -$ dstack apply -f examples/deployment/trtllm/serve-distill.dstack.yml +$ dstack apply -f examples/inference/trtllm/serve-distill.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 vastai us-iowa 12xCPU, 85GB, 1xA100 (40GB) yes $0.66904 @@ -366,7 +366,7 @@ is available at `https://gateway./`. ## Source code The source-code of this example can be found in -[`examples/deployment/trtllm` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/trtllm){:target="_blank"}. +[`examples/inference/trtllm` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/trtllm){:target="_blank"}. ## What's next? diff --git a/examples/deployment/trtllm/build-image.dstack.yml b/examples/inference/trtllm/build-image.dstack.yml similarity index 100% rename from examples/deployment/trtllm/build-image.dstack.yml rename to examples/inference/trtllm/build-image.dstack.yml diff --git a/examples/deployment/trtllm/build-model.dstack.yml b/examples/inference/trtllm/build-model.dstack.yml similarity index 100% rename from examples/deployment/trtllm/build-model.dstack.yml rename to examples/inference/trtllm/build-model.dstack.yml diff --git a/examples/deployment/trtllm/convert-model.dstack.yml b/examples/inference/trtllm/convert-model.dstack.yml similarity index 100% rename from examples/deployment/trtllm/convert-model.dstack.yml rename to examples/inference/trtllm/convert-model.dstack.yml diff --git a/examples/deployment/trtllm/serve-distill.dstack.yml b/examples/inference/trtllm/serve-distill.dstack.yml similarity index 100% rename from examples/deployment/trtllm/serve-distill.dstack.yml rename to examples/inference/trtllm/serve-distill.dstack.yml diff --git a/examples/deployment/trtllm/serve-r1.dstack.yml b/examples/inference/trtllm/serve-r1.dstack.yml similarity index 90% rename from examples/deployment/trtllm/serve-r1.dstack.yml rename to examples/inference/trtllm/serve-r1.dstack.yml index 9515d0d682..f5e9091720 100644 --- a/examples/deployment/trtllm/serve-r1.dstack.yml +++ b/examples/inference/trtllm/serve-r1.dstack.yml @@ -1,7 +1,7 @@ type: service name: serve-r1 -# Specify the image built with `examples/deployment/trtllm/build-image.dstack.yml` +# Specify the image built with `examples/inference/trtllm/build-image.dstack.yml` image: dstackai/tensorrt_llm:9b931c0f6305aefa3660e6fb84a76a42c0eef167 env: - MAX_BATCH_SIZE=256 diff --git a/examples/deployment/vllm/.dstack.yml b/examples/inference/vllm/.dstack.yml similarity index 100% rename from examples/deployment/vllm/.dstack.yml rename to examples/inference/vllm/.dstack.yml diff --git a/examples/deployment/vllm/README.md b/examples/inference/vllm/README.md similarity index 88% rename from examples/deployment/vllm/README.md rename to examples/inference/vllm/README.md index c9610ba527..75f986cb60 100644 --- a/examples/deployment/vllm/README.md +++ b/examples/inference/vllm/README.md @@ -23,7 +23,7 @@ This example shows how to deploy Llama 3.1 8B with `dstack` using [vLLM :materia Here's an example of a service that deploys Llama 3.1 8B using vLLM. -
+
```yaml type: service @@ -65,7 +65,7 @@ To run a configuration, use the [`dstack apply`](https://dstack.ai/docs/referenc
```shell -$ dstack apply -f examples/deployment/vllm/.dstack.yml +$ dstack apply -f examples/inference/vllm/.dstack.yml # BACKEND REGION RESOURCES SPOT PRICE 1 runpod CA-MTL-1 18xCPU, 100GB, A5000:24GB yes $0.12 @@ -113,12 +113,12 @@ is available at `https://gateway./`. ## Source code The source-code of this example can be found in -[`examples/deployment/vllm` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/deployment/vllm). +[`examples/inference/vllm` :material-arrow-top-right-thin:{ .external }](https://github.com/dstackai/dstack/blob/master/examples/inference/vllm). ## What's next? 1. Check [services](https://dstack.ai/docs/services) -2. Browse the [Llama 3.1](https://dstack.ai/examples/llms/llama31/), [TGI](https://dstack.ai/examples/deployment/tgi/) - and [NIM](https://dstack.ai/examples/deployment/nim/) examples +2. Browse the [Llama 3.1](https://dstack.ai/examples/llms/llama31/), [TGI](https://dstack.ai/examples/inference/tgi/) + and [NIM](https://dstack.ai/examples/inference/nim/) examples 3. See also [AMD](https://dstack.ai/examples/accelerators/amd/) and [TPU](https://dstack.ai/examples/accelerators/tpu/) diff --git a/examples/deployment/vllm/amd/.dstack.yml b/examples/inference/vllm/amd/.dstack.yml similarity index 100% rename from examples/deployment/vllm/amd/.dstack.yml rename to examples/inference/vllm/amd/.dstack.yml diff --git a/examples/deployment/vllm/amd/build-vllm.dstack.yml b/examples/inference/vllm/amd/build-vllm.dstack.yml similarity index 100% rename from examples/deployment/vllm/amd/build-vllm.dstack.yml rename to examples/inference/vllm/amd/build-vllm.dstack.yml diff --git a/examples/deployment/vllm/tpu/.dstack.yml b/examples/inference/vllm/tpu/.dstack.yml similarity index 100% rename from examples/deployment/vllm/tpu/.dstack.yml rename to examples/inference/vllm/tpu/.dstack.yml diff --git a/mkdocs.yml b/mkdocs.yml index 3ca396b153..50656b94d7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -50,6 +50,7 @@ theme: - navigation.tabs # - navigation.instant # - toc.integrate + # Comment to switch to sections in sidebar - navigation.sections # - navigation.expand - navigation.top @@ -95,9 +96,9 @@ plugins: 'examples/llms/llama31/index.md': 'examples/llms/llama/index.md' 'examples/llms/llama32/index.md': 'examples/llms/llama/index.md' 'docs/examples/accelerators/amd/index.md': 'examples/accelerators/amd/index.md' - 'docs/examples/deployment/nim/index.md': 'examples/deployment/nim/index.md' - 'docs/examples/deployment/vllm/index.md': 'examples/deployment/vllm/index.md' - 'docs/examples/deployment/tgi/index.md': 'examples/deployment/tgi/index.md' + 'docs/examples/deployment/nim/index.md': 'examples/inference/nim/index.md' + 'docs/examples/deployment/vllm/index.md': 'examples/inference/vllm/index.md' + 'docs/examples/deployment/tgi/index.md': 'examples/inference/tgi/index.md' 'providers.md': 'partners.md' 'backends.md': 'partners.md' 'blog/ambassador-program.md': 'blog/archive/ambassador-program.md' @@ -106,11 +107,19 @@ plugins: 'blog/data-centers-and-private-clouds.md': 'blog/posts/gpu-blocks-and-proxy-jump.md' 'blog/distributed-training-with-aws-efa.md': 'blog/posts/efa.md' 'blog/dstack-stats.md': 'blog/posts/dstack-metrics.md' - 'docs/concepts/metrics.md': 'docs/guides/monitoring.md' + 'docs/concepts/metrics.md': 'docs/guides/metrics.md' + 'docs/guides/monitoring.md': 'docs/guides/metrics.md' 'blog/nvidia-and-amd-on-vultr.md.md': 'blog/posts/nvidia-and-amd-on-vultr.md' - 'examples/misc/nccl-tests/index.md': 'examples/distributed-training/nccl-tests/index.md' - 'examples/misc/a3high-clusters/index.md': 'examples/distributed-training/a3high-clusters/index.md' - 'examples/misc/a3mega-clusters/index.md': 'examples/distributed-training/a3mega-clusters/index.md' + 'examples/misc/nccl-tests/index.md': 'examples/clusters/nccl-tests/index.md' + 'examples/misc/a3high-clusters/index.md': 'examples/clusters/a3high/index.md' + 'examples/misc/a3mega-clusters/index.md': 'examples/clusters/a3mega/index.md' + 'examples/distributed-training/nccl-tests/index.md': 'examples/clusters/nccl-tests/index.md' + 'examples/distributed-training/rccl-tests/index.md': 'examples/clusters/rccl-tests/index.md' + 'examples/deployment/nim/index.md': 'examples/inference/nim/index.md' + 'examples/deployment/vllm/index.md': 'examples/inference/vllm/index.md' + 'examples/deployment/tgi/index.md': 'examples/inference/tgi/index.md' + 'examples/deployment/sglang/index.md': 'examples/inference/sglang/index.md' + 'examples/deployment/trtllm/index.md': 'examples/inference/trtllm/index.md' - typeset - gen-files: scripts: # always relative to mkdocs.yml @@ -212,8 +221,8 @@ nav: - Gateways: docs/concepts/gateways.md - Guides: - Protips: docs/guides/protips.md + - Metrics: docs/guides/metrics.md - Server deployment: docs/guides/server-deployment.md - - Monitoring: docs/guides/monitoring.md - Plugins: docs/guides/plugins.md - Troubleshooting: docs/guides/troubleshooting.md - Reference: @@ -249,17 +258,17 @@ nav: - Fine-tuning: - TRL: examples/fine-tuning/trl/index.md - Axolotl: examples/fine-tuning/axolotl/index.md - - Distributed training: - - NCCL tests: examples/distributed-training/nccl-tests/index.md - - RCCL tests: examples/distributed-training/rccl-tests/index.md - - A3 Mega: examples/distributed-training/a3mega-clusters/index.md - - A3 High: examples/distributed-training/a3high-clusters/index.md + - Clusters: + - NCCL tests: examples/clusters/nccl-tests/index.md + - RCCL tests: examples/clusters/rccl-tests/index.md + - A3 Mega: examples/clusters/a3mega/index.md + - A3 High: examples/clusters/a3high/index.md - Deployment: - - SGLang: examples/deployment/sglang/index.md - - vLLM: examples/deployment/vllm/index.md - - TGI: examples/deployment/tgi/index.md - - NIM: examples/deployment/nim/index.md - - TensorRT-LLM: examples/deployment/trtllm/index.md + - SGLang: examples/inference/sglang/index.md + - vLLM: examples/inference/vllm/index.md + - TGI: examples/inference/tgi/index.md + - NIM: examples/inference/nim/index.md + - TensorRT-LLM: examples/inference/trtllm/index.md - LLMs: - Deepseek: examples/llms/deepseek/index.md - Llama: examples/llms/llama/index.md diff --git a/scripts/docs/gen_cli_reference.py b/scripts/docs/gen_cli_reference.py index ea35548b50..04db41df49 100644 --- a/scripts/docs/gen_cli_reference.py +++ b/scripts/docs/gen_cli_reference.py @@ -22,13 +22,16 @@ DISABLE_ENV = "DSTACK_DOCS_DISABLE_CLI_REFERENCE" +logger.info("Generating CLI reference...") + + @cache # TODO make caching work def call_dstack(command: str) -> str: return subprocess.check_output(shlex.split(command)).decode() def sub_help(match: re.Match) -> str: - logger.info("Generating help for `%s`", match.group(1)) + logger.debug("Generating CLI reference for `%s`", match.group(1)) try: output = call_dstack(match.group(1)) except subprocess.CalledProcessError: diff --git a/scripts/docs/gen_schema_reference.py b/scripts/docs/gen_schema_reference.py index 724a291d7a..acd4e722d8 100644 --- a/scripts/docs/gen_schema_reference.py +++ b/scripts/docs/gen_schema_reference.py @@ -19,6 +19,8 @@ FILE_PATTERN = "docs/reference/**.md" logger = logging.getLogger("mkdocs.plugins.dstack.schema") +logger.info("Generating schema reference...") + def get_type(annotation: Type) -> str: if get_origin(annotation) is Annotated: @@ -135,7 +137,7 @@ def generate_schema_reference( def sub_schema_reference(match: re.Match) -> str: - logger.info("Generating schema reference for `%s`", match.group(2)) + logger.debug("Generating schema reference for `%s`", match.group(2)) options = yaml.safe_load("\n".join(row[4:] for row in match.group(3).split("\n"))) logger.debug("Options: %s", options) return (