Skip to content

Commit a26d7d6

Browse files
ericcurtinericcurtin
committed
Add openaiurl flag to list and run commands
The openaiurl flag enables users to connect to external OpenAI-compatible API endpoints for both listing models and running chat interactions. When this flag is specified, the commands will bypass the local model runner and communicate directly with the provided endpoint. The list command now supports filtering models from external endpoints, and the run command supports both single prompt mode and interactive mode with external endpoints. Signed-off-by: Eric Curtin <[email protected]>
1 parent 3609fd7 commit a26d7d6

File tree

8 files changed

+169
-21
lines changed

8 files changed

+169
-21
lines changed

‎cmd/cli/commands/list.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
func newListCmd() *cobra.Command {
2323
var jsonFormat, openai, quiet bool
24+
var openaiURL string
2425
c := &cobra.Command{
2526
Use: "list [OPTIONS] [MODEL]",
2627
Aliases: []string{"ls"},
@@ -31,6 +32,46 @@ func newListCmd() *cobra.Command {
3132
return fmt.Errorf("--quiet flag cannot be used with --openai flag or OpenAI backend")
3233
}
3334

35+
// Handle --openaiurl flag for external OpenAI endpoints
36+
if openaiURL != "" {
37+
if quiet {
38+
return fmt.Errorf("--quiet flag cannot be used with --openaiurl flag")
39+
}
40+
ctx, err := desktop.NewContextForOpenAI(openaiURL)
41+
if err != nil {
42+
return fmt.Errorf("invalid OpenAI URL: %w", err)
43+
}
44+
client := desktop.New(ctx)
45+
models, err := client.ListOpenAI()
46+
if err != nil {
47+
return handleClientError(err, "Failed to list models from OpenAI endpoint")
48+
}
49+
var modelFilter string
50+
if len(args) > 0 {
51+
modelFilter = args[0]
52+
}
53+
if modelFilter != "" {
54+
filtered := models.Data[:0]
55+
for _, m := range models.Data {
56+
if matchesModelFilter(m.ID, modelFilter) {
57+
filtered = append(filtered, m)
58+
}
59+
}
60+
models.Data = filtered
61+
}
62+
if jsonFormat {
63+
output, err := formatter.ToStandardJSON(models)
64+
if err != nil {
65+
return err
66+
}
67+
fmt.Fprint(cmd.OutOrStdout(), output)
68+
return nil
69+
}
70+
// Display in table format with only MODEL NAME populated
71+
fmt.Fprint(cmd.OutOrStdout(), prettyPrintOpenAIModels(models))
72+
return nil
73+
}
74+
3475
// If we're doing an automatic install, only show the installation
3576
// status if it won't corrupt machine-readable output.
3677
var standaloneInstallPrinter standalone.StatusPrinter
@@ -56,6 +97,7 @@ func newListCmd() *cobra.Command {
5697
c.Flags().BoolVar(&jsonFormat, "json", false, "List models in a JSON format")
5798
c.Flags().BoolVar(&openai, "openai", false, "List models in an OpenAI format")
5899
c.Flags().BoolVarP(&quiet, "quiet", "q", false, "Only show model IDs")
100+
c.Flags().StringVar(&openaiURL, "openaiurl", "", "OpenAI-compatible API endpoint URL to list models from")
59101
return c
60102
}
61103

@@ -239,3 +281,24 @@ func appendRow(table *tablewriter.Table, tag string, model dmrm.Model) {
239281
model.Config.Size,
240282
})
241283
}
284+
285+
// prettyPrintOpenAIModels formats OpenAI model list in table format with only MODEL NAME populated
286+
func prettyPrintOpenAIModels(models dmrm.OpenAIModelList) string {
287+
// Sort models by ID
288+
sort.Slice(models.Data, func(i, j int) bool {
289+
return strings.ToLower(models.Data[i].ID) < strings.ToLower(models.Data[j].ID)
290+
})
291+
292+
var buf bytes.Buffer
293+
table := newTable(&buf)
294+
table.Header([]string{"MODEL NAME", "CREATED"})
295+
for _, model := range models.Data {
296+
table.Append([]string{
297+
model.ID,
298+
units.HumanDuration(time.Since(time.Unix(model.Created, 0))) + " ago",
299+
})
300+
}
301+
302+
table.Render()
303+
return buf.String()
304+
}

‎cmd/cli/commands/run.go‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,7 @@ func newRunCmd() *cobra.Command {
571571
var debug bool
572572
var colorMode string
573573
var detach bool
574+
var openaiURL string
574575

575576
const cmdArgs = "MODEL [PROMPT]"
576577
c := &cobra.Command{
@@ -585,10 +586,6 @@ func newRunCmd() *cobra.Command {
585586
}
586587
},
587588
RunE: func(cmd *cobra.Command, args []string) error {
588-
if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), asPrinter(cmd), debug); err != nil {
589-
return fmt.Errorf("unable to initialize standalone model runner: %w", err)
590-
}
591-
592589
model := args[0]
593590
prompt := ""
594591
argsLen := len(args)
@@ -621,6 +618,43 @@ func newRunCmd() *cobra.Command {
621618
}
622619
}
623620

621+
// Handle --openaiurl flag for external OpenAI endpoints
622+
if openaiURL != "" {
623+
if detach {
624+
return fmt.Errorf("--detach flag cannot be used with --openaiurl flag")
625+
}
626+
ctx, err := desktop.NewContextForOpenAI(openaiURL)
627+
if err != nil {
628+
return fmt.Errorf("invalid OpenAI URL: %w", err)
629+
}
630+
openaiClient := desktop.New(ctx)
631+
632+
if prompt != "" {
633+
// Single prompt mode
634+
useMarkdown := shouldUseMarkdown(colorMode)
635+
if err := openaiClient.ChatWithContext(cmd.Context(), model, prompt, nil, func(content string) {
636+
cmd.Print(content)
637+
}, useMarkdown); err != nil {
638+
return handleClientError(err, "Failed to generate a response")
639+
}
640+
cmd.Println()
641+
return nil
642+
}
643+
644+
// Interactive mode for external OpenAI endpoint
645+
if term.IsTerminal(int(os.Stdin.Fd())) {
646+
termenv.SetDefaultOutput(
647+
termenv.NewOutput(asPrinter(cmd), termenv.WithColorCache(true)),
648+
)
649+
return generateInteractiveWithReadline(cmd, openaiClient, model)
650+
}
651+
return generateInteractiveBasic(cmd, openaiClient, model)
652+
}
653+
654+
if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), asPrinter(cmd), debug); err != nil {
655+
return fmt.Errorf("unable to initialize standalone model runner: %w", err)
656+
}
657+
624658
// Check if this is an NVIDIA NIM image
625659
if isNIMImage(model) {
626660
// NIM images are handled differently - they run as Docker containers
@@ -733,6 +767,7 @@ func newRunCmd() *cobra.Command {
733767
c.Flags().BoolVar(&debug, "debug", false, "Enable debug logging")
734768
c.Flags().StringVar(&colorMode, "color", "no", "Use colored output (auto|yes|no)")
735769
c.Flags().BoolVarP(&detach, "detach", "d", false, "Load the model in the background without interaction")
770+
c.Flags().StringVar(&openaiURL, "openaiurl", "", "OpenAI-compatible API endpoint URL to chat with")
736771

737772
return c
738773
}

‎cmd/cli/desktop/context.go‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ type ModelRunnerContext struct {
9898
urlPrefix *url.URL
9999
// client is the model runner client.
100100
client DockerHttpClient
101+
// openaiPathPrefix is the path prefix for OpenAI-compatible endpoints.
102+
// For internal Docker Model Runner, this is "/engines/v1".
103+
// For external OpenAI-compatible endpoints, this is empty (the URL already includes the version path).
104+
openaiPathPrefix string
101105
}
102106

103107
// NewContextForMock is a ModelRunnerContext constructor exposed only for the
@@ -108,9 +112,10 @@ func NewContextForMock(client DockerHttpClient) *ModelRunnerContext {
108112
panic("error occurred while parsing known-good URL")
109113
}
110114
return &ModelRunnerContext{
111-
kind: types.ModelRunnerEngineKindDesktop,
112-
urlPrefix: urlPrefix,
113-
client: client,
115+
kind: types.ModelRunnerEngineKindDesktop,
116+
urlPrefix: urlPrefix,
117+
client: client,
118+
openaiPathPrefix: inference.InferencePrefix + "/v1",
114119
}
115120
}
116121

@@ -128,9 +133,26 @@ func NewContextForTest(endpoint string, client DockerHttpClient, kind types.Mode
128133
}
129134

130135
return &ModelRunnerContext{
131-
kind: kind,
132-
urlPrefix: urlPrefix,
133-
client: client,
136+
kind: kind,
137+
urlPrefix: urlPrefix,
138+
client: client,
139+
openaiPathPrefix: inference.InferencePrefix + "/v1",
140+
}, nil
141+
}
142+
143+
// NewContextForOpenAI creates a ModelRunnerContext for connecting to an external
144+
// OpenAI-compatible API endpoint. This is used when the --openaiurl flag is specified.
145+
func NewContextForOpenAI(endpoint string) (*ModelRunnerContext, error) {
146+
urlPrefix, err := url.Parse(endpoint)
147+
if err != nil {
148+
return nil, fmt.Errorf("invalid OpenAI endpoint URL: %w", err)
149+
}
150+
151+
return &ModelRunnerContext{
152+
kind: types.ModelRunnerEngineKindMobyManual,
153+
urlPrefix: urlPrefix,
154+
client: http.DefaultClient,
155+
openaiPathPrefix: "", // Empty prefix for external OpenAI-compatible endpoints
134156
}, nil
135157
}
136158

@@ -262,9 +284,10 @@ func DetectContext(ctx context.Context, cli *command.DockerCli, printer standalo
262284

263285
// Success.
264286
return &ModelRunnerContext{
265-
kind: kind,
266-
urlPrefix: urlPrefix,
267-
client: client,
287+
kind: kind,
288+
urlPrefix: urlPrefix,
289+
client: client,
290+
openaiPathPrefix: inference.InferencePrefix + "/v1",
268291
}, nil
269292
}
270293

@@ -289,6 +312,13 @@ func (c *ModelRunnerContext) Client() DockerHttpClient {
289312
return c.client
290313
}
291314

315+
// OpenAIPathPrefix returns the path prefix for OpenAI-compatible endpoints.
316+
// For internal Docker Model Runner, this returns the inference prefix.
317+
// For external OpenAI-compatible endpoints, this returns an empty string.
318+
func (c *ModelRunnerContext) OpenAIPathPrefix() string {
319+
return c.openaiPathPrefix
320+
}
321+
292322
func setUserAgent(client DockerHttpClient, userAgent string) {
293323
if httpClient, ok := client.(*http.Client); ok {
294324
transport := httpClient.Transport

‎cmd/cli/desktop/desktop.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ func (c *Client) List() ([]dmrm.Model, error) {
276276
}
277277

278278
func (c *Client) ListOpenAI() (dmrm.OpenAIModelList, error) {
279-
modelsRoute := inference.InferencePrefix + "/v1/models"
279+
modelsRoute := c.modelRunner.OpenAIPathPrefix() + "/models"
280280
body, err := c.listRaw(modelsRoute, "")
281281
if err != nil {
282282
return dmrm.OpenAIModelList{}, err
@@ -304,7 +304,7 @@ func (c *Client) Inspect(model string, remote bool) (dmrm.Model, error) {
304304
}
305305

306306
func (c *Client) InspectOpenAI(model string) (dmrm.OpenAIModel, error) {
307-
modelsRoute := inference.InferencePrefix + "/v1/models"
307+
modelsRoute := c.modelRunner.OpenAIPathPrefix() + "/models"
308308
rawResponse, err := c.listRaw(fmt.Sprintf("%s/%s", modelsRoute, model), model)
309309
if err != nil {
310310
return dmrm.OpenAIModel{}, err
@@ -398,7 +398,7 @@ func (c *Client) ChatWithContext(ctx context.Context, model, prompt string, imag
398398
return fmt.Errorf("error marshaling request: %w", err)
399399
}
400400

401-
completionsPath := inference.InferencePrefix + "/v1/chat/completions"
401+
completionsPath := c.modelRunner.OpenAIPathPrefix() + "/chat/completions"
402402

403403
resp, err := c.doRequestWithAuthContext(
404404
ctx,

‎cmd/cli/docs/reference/docker_model_list.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ options:
2626
experimentalcli: false
2727
kubernetes: false
2828
swarm: false
29+
- option: openaiurl
30+
value_type: string
31+
description: OpenAI-compatible API endpoint URL to list models from
32+
deprecated: false
33+
hidden: false
34+
experimental: false
35+
experimentalcli: false
36+
kubernetes: false
37+
swarm: false
2938
- option: quiet
3039
shorthand: q
3140
value_type: bool

‎cmd/cli/docs/reference/docker_model_run.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ options:
4141
experimentalcli: false
4242
kubernetes: false
4343
swarm: false
44+
- option: openaiurl
45+
value_type: string
46+
description: OpenAI-compatible API endpoint URL to chat with
47+
deprecated: false
48+
hidden: false
49+
experimental: false
50+
experimentalcli: false
51+
kubernetes: false
52+
swarm: false
4453
examples: |-
4554
### One-time prompt
4655

‎cmd/cli/docs/reference/model_list.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ List the models pulled to your local environment
99

1010
### Options
1111

12-
| Name | Type | Default | Description |
13-
|:----------------|:-------|:--------|:--------------------------------|
14-
| `--json` | `bool` | | List models in a JSON format |
15-
| `--openai` | `bool` | | List models in an OpenAI format |
16-
| `-q`, `--quiet` | `bool` | | Only show model IDs |
12+
| Name | Type | Default | Description |
13+
|:----------------|:---------|:--------|:-------------------------------------------------------|
14+
| `--json` | `bool` | | List models in a JSON format |
15+
| `--openai` | `bool` | | List models in an OpenAI format |
16+
| `--openaiurl` | `string` | | OpenAI-compatible API endpoint URL to list models from |
17+
| `-q`, `--quiet` | `bool` | | Only show model IDs |
1718

1819

1920
<!---MARKER_GEN_END-->

‎cmd/cli/docs/reference/model_run.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Run a model and interact with it using a submitted prompt or chat mode
1010
| `--color` | `string` | `no` | Use colored output (auto\|yes\|no) |
1111
| `--debug` | `bool` | | Enable debug logging |
1212
| `-d`, `--detach` | `bool` | | Load the model in the background without interaction |
13+
| `--openaiurl` | `string` | | OpenAI-compatible API endpoint URL to chat with |
1314

1415

1516
<!---MARKER_GEN_END-->

0 commit comments

Comments
 (0)