File tree Expand file tree Collapse file tree 1 file changed +2
-2
lines changed
demos/continuous_batching/structured_output Expand file tree Collapse file tree 1 file changed +2
-2
lines changed Original file line number Diff line number Diff line change @@ -21,7 +21,7 @@ There are no extra steps needed to use structured output. Whole behavior is trig
21
21
22
22
``` bash
23
23
mkdir models
24
- docker run --user $( id -u) :$( id -g) -d --device /dev/dri --group-add=$( stat -c " %g" /dev/dri/render* ) --rm -p 8000:8000 -v $( pwd) /models:/models:rw openvino/model_server:latest-gpu --source_model OpenVINO/Mistral-7B-Instruct-v0.3-int4-cw-ov --model_repository_path models --task text_generation --rest_port 8000 --target_device GPU --cache_size 2
24
+ docker run --user $( id -u) :$( id -g) -d --device /dev/dri --group-add=$( stat -c " %g" /dev/dri/render* | head -1 ) --rm -p 8000:8000 -v $( pwd) /models:/models:rw openvino/model_server:latest-gpu --source_model OpenVINO/Mistral-7B-Instruct-v0.3-int4-cw-ov --model_repository_path models --task text_generation --rest_port 8000 --target_device GPU --cache_size 2
25
25
```
26
26
:::
27
27
@@ -30,7 +30,7 @@ docker run --user $(id -u):$(id -g) -d --device /dev/dri --group-add=$(stat -c "
30
30
31
31
``` bash
32
32
mkdir models
33
- docker run --user $( id -u) :$( id -g) -d --device /dev/accel --group-add=$( stat -c " %g" /dev/dri/render* ) --rm -p 8000:8000 -v $( pwd) /models:/models:rw openvino/model_server:latest-gpu --source_model OpenVINO/Mistral-7B-Instruct-v0.3-int4-cw-ov --model_repository_path models --task text_generation --rest_port 8000 --target_device NPU --cache_size 2
33
+ docker run --user $( id -u) :$( id -g) -d --device /dev/accel --group-add=$( stat -c " %g" /dev/dri/render* | head -1 ) --rm -p 8000:8000 -v $( pwd) /models:/models:rw openvino/model_server:latest-gpu --source_model OpenVINO/Mistral-7B-Instruct-v0.3-int4-cw-ov --model_repository_path models --task text_generation --rest_port 8000 --target_device NPU --cache_size 2
34
34
```
35
35
:::
36
36
You can’t perform that action at this time.
0 commit comments