Skip to content

Commit 161b9ce

Browse files
ericcurtinericcurtin
authored
Merge pull request #530 from docker/openaiurl
Add openaiurl flag to list and run commands
2 parents 3609fd7 + a26d7d6 commit 161b9ce

File tree

8 files changed

+169
-21
lines changed

8 files changed

+169
-21
lines changed

‎cmd/cli/commands/list.go‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121

2222
func newListCmd() *cobra.Command {
2323
var jsonFormat, openai, quiet bool
24+
var openaiURL string
2425
c := &cobra.Command{
2526
Use: "list [OPTIONS] [MODEL]",
2627
Aliases: []string{"ls"},
@@ -31,6 +32,46 @@ func newListCmd() *cobra.Command {
3132
return fmt.Errorf("--quiet flag cannot be used with --openai flag or OpenAI backend")
3233
}
3334

35+
// Handle --openaiurl flag for external OpenAI endpoints
36+
if openaiURL != "" {
37+
if quiet {
38+
return fmt.Errorf("--quiet flag cannot be used with --openaiurl flag")
39+
}
40+
ctx, err := desktop.NewContextForOpenAI(openaiURL)
41+
if err != nil {
42+
return fmt.Errorf("invalid OpenAI URL: %w", err)
43+
}
44+
client := desktop.New(ctx)
45+
models, err := client.ListOpenAI()
46+
if err != nil {
47+
return handleClientError(err, "Failed to list models from OpenAI endpoint")
48+
}
49+
var modelFilter string
50+
if len(args) > 0 {
51+
modelFilter = args[0]
52+
}
53+
if modelFilter != "" {
54+
filtered := models.Data[:0]
55+
for _, m := range models.Data {
56+
if matchesModelFilter(m.ID, modelFilter) {
57+
filtered = append(filtered, m)
58+
}
59+
}
60+
models.Data = filtered
61+
}
62+
if jsonFormat {
63+
output, err := formatter.ToStandardJSON(models)
64+
if err != nil {
65+
return err
66+
}
67+
fmt.Fprint(cmd.OutOrStdout(), output)
68+
return nil
69+
}
70+
// Display in table format with only MODEL NAME populated
71+
fmt.Fprint(cmd.OutOrStdout(), prettyPrintOpenAIModels(models))
72+
return nil
73+
}
74+
3475
// If we're doing an automatic install, only show the installation
3576
// status if it won't corrupt machine-readable output.
3677
var standaloneInstallPrinter standalone.StatusPrinter
@@ -56,6 +97,7 @@ func newListCmd() *cobra.Command {
5697
c.Flags().BoolVar(&jsonFormat, "json", false, "List models in a JSON format")
5798
c.Flags().BoolVar(&openai, "openai", false, "List models in an OpenAI format")
5899
c.Flags().BoolVarP(&quiet, "quiet", "q", false, "Only show model IDs")
100+
c.Flags().StringVar(&openaiURL, "openaiurl", "", "OpenAI-compatible API endpoint URL to list models from")
59101
return c
60102
}
61103

@@ -239,3 +281,24 @@ func appendRow(table *tablewriter.Table, tag string, model dmrm.Model) {
239281
model.Config.Size,
240282
})
241283
}
284+
285+
// prettyPrintOpenAIModels formats OpenAI model list in table format with only MODEL NAME populated
286+
func prettyPrintOpenAIModels(models dmrm.OpenAIModelList) string {
287+
// Sort models by ID
288+
sort.Slice(models.Data, func(i, j int) bool {
289+
return strings.ToLower(models.Data[i].ID) < strings.ToLower(models.Data[j].ID)
290+
})
291+
292+
var buf bytes.Buffer
293+
table := newTable(&buf)
294+
table.Header([]string{"MODEL NAME", "CREATED"})
295+
for _, model := range models.Data {
296+
table.Append([]string{
297+
model.ID,
298+
units.HumanDuration(time.Since(time.Unix(model.Created, 0))) + " ago",
299+
})
300+
}
301+
302+
table.Render()
303+
return buf.String()
304+
}

‎cmd/cli/commands/run.go‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -571,6 +571,7 @@ func newRunCmd() *cobra.Command {
571571
var debug bool
572572
var colorMode string
573573
var detach bool
574+
var openaiURL string
574575

575576
const cmdArgs = "MODEL [PROMPT]"
576577
c := &cobra.Command{
@@ -585,10 +586,6 @@ func newRunCmd() *cobra.Command {
585586
}
586587
},
587588
RunE: func(cmd *cobra.Command, args []string) error {
588-
if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), asPrinter(cmd), debug); err != nil {
589-
return fmt.Errorf("unable to initialize standalone model runner: %w", err)
590-
}
591-
592589
model := args[0]
593590
prompt := ""
594591
argsLen := len(args)
@@ -621,6 +618,43 @@ func newRunCmd() *cobra.Command {
621618
}
622619
}
623620

621+
// Handle --openaiurl flag for external OpenAI endpoints
622+
if openaiURL != "" {
623+
if detach {
624+
return fmt.Errorf("--detach flag cannot be used with --openaiurl flag")
625+
}
626+
ctx, err := desktop.NewContextForOpenAI(openaiURL)
627+
if err != nil {
628+
return fmt.Errorf("invalid OpenAI URL: %w", err)
629+
}
630+
openaiClient := desktop.New(ctx)
631+
632+
if prompt != "" {
633+
// Single prompt mode
634+
useMarkdown := shouldUseMarkdown(colorMode)
635+
if err := openaiClient.ChatWithContext(cmd.Context(), model, prompt, nil, func(content string) {
636+
cmd.Print(content)
637+
}, useMarkdown); err != nil {
638+
return handleClientError(err, "Failed to generate a response")
639+
}
640+
cmd.Println()
641+
return nil
642+
}
643+
644+
// Interactive mode for external OpenAI endpoint
645+
if term.IsTerminal(int(os.Stdin.Fd())) {
646+
termenv.SetDefaultOutput(
647+
termenv.NewOutput(asPrinter(cmd), termenv.WithColorCache(true)),
648+
)
649+
return generateInteractiveWithReadline(cmd, openaiClient, model)
650+
}
651+
return generateInteractiveBasic(cmd, openaiClient, model)
652+
}
653+
654+
if _, err := ensureStandaloneRunnerAvailable(cmd.Context(), asPrinter(cmd), debug); err != nil {
655+
return fmt.Errorf("unable to initialize standalone model runner: %w", err)
656+
}
657+
624658
// Check if this is an NVIDIA NIM image
625659
if isNIMImage(model) {
626660
// NIM images are handled differently - they run as Docker containers
@@ -733,6 +767,7 @@ func newRunCmd() *cobra.Command {
733767
c.Flags().BoolVar(&debug, "debug", false, "Enable debug logging")
734768
c.Flags().StringVar(&colorMode, "color", "no", "Use colored output (auto|yes|no)")
735769
c.Flags().BoolVarP(&detach, "detach", "d", false, "Load the model in the background without interaction")
770+
c.Flags().StringVar(&openaiURL, "openaiurl", "", "OpenAI-compatible API endpoint URL to chat with")
736771

737772
return c
738773
}

‎cmd/cli/desktop/context.go‎

Lines changed: 39 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,10 @@ type ModelRunnerContext struct {
9898
urlPrefix *url.URL
9999
// client is the model runner client.
100100
client DockerHttpClient
101+
// openaiPathPrefix is the path prefix for OpenAI-compatible endpoints.
102+
// For internal Docker Model Runner, this is "/engines/v1".
103+
// For external OpenAI-compatible endpoints, this is empty (the URL already includes the version path).
104+
openaiPathPrefix string
101105
}
102106

103107
// NewContextForMock is a ModelRunnerContext constructor exposed only for the
@@ -108,9 +112,10 @@ func NewContextForMock(client DockerHttpClient) *ModelRunnerContext {
108112
panic("error occurred while parsing known-good URL")
109113
}
110114
return &ModelRunnerContext{
111-
kind: types.ModelRunnerEngineKindDesktop,
112-
urlPrefix: urlPrefix,
113-
client: client,
115+
kind: types.ModelRunnerEngineKindDesktop,
116+
urlPrefix: urlPrefix,
117+
client: client,
118+
openaiPathPrefix: inference.InferencePrefix + "/v1",
114119
}
115120
}
116121

@@ -128,9 +133,26 @@ func NewContextForTest(endpoint string, client DockerHttpClient, kind types.Mode
128133
}
129134

130135
return &ModelRunnerContext{
131-
kind: kind,
132-
urlPrefix: urlPrefix,
133-
client: client,
136+
kind: kind,
137+
urlPrefix: urlPrefix,
138+
client: client,
139+
openaiPathPrefix: inference.InferencePrefix + "/v1",
140+
}, nil
141+
}
142+
143+
// NewContextForOpenAI creates a ModelRunnerContext for connecting to an external
144+
// OpenAI-compatible API endpoint. This is used when the --openaiurl flag is specified.
145+
func NewContextForOpenAI(endpoint string) (*ModelRunnerContext, error) {
146+
urlPrefix, err := url.Parse(endpoint)
147+
if err != nil {
148+
return nil, fmt.Errorf("invalid OpenAI endpoint URL: %w", err)
149+
}
150+
151+
return &ModelRunnerContext{
152+
kind: types.ModelRunnerEngineKindMobyManual,
153+
urlPrefix: urlPrefix,
154+
client: http.DefaultClient,
155+
openaiPathPrefix: "", // Empty prefix for external OpenAI-compatible endpoints
134156
}, nil
135157
}
136158

@@ -262,9 +284,10 @@ func DetectContext(ctx context.Context, cli *command.DockerCli, printer standalo
262284

263285
// Success.
264286
return &ModelRunnerContext{
265-
kind: kind,
266-
urlPrefix: urlPrefix,
267-
client: client,
287+
kind: kind,
288+
urlPrefix: urlPrefix,
289+
client: client,
290+
openaiPathPrefix: inference.InferencePrefix + "/v1",
268291
}, nil
269292
}
270293

@@ -289,6 +312,13 @@ func (c *ModelRunnerContext) Client() DockerHttpClient {
289312
return c.client
290313
}
291314

315+
// OpenAIPathPrefix returns the path prefix for OpenAI-compatible endpoints.
316+
// For internal Docker Model Runner, this returns the inference prefix.
317+
// For external OpenAI-compatible endpoints, this returns an empty string.
318+
func (c *ModelRunnerContext) OpenAIPathPrefix() string {
319+
return c.openaiPathPrefix
320+
}
321+
292322
func setUserAgent(client DockerHttpClient, userAgent string) {
293323
if httpClient, ok := client.(*http.Client); ok {
294324
transport := httpClient.Transport

‎cmd/cli/desktop/desktop.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -276,7 +276,7 @@ func (c *Client) List() ([]dmrm.Model, error) {
276276
}
277277

278278
func (c *Client) ListOpenAI() (dmrm.OpenAIModelList, error) {
279-
modelsRoute := inference.InferencePrefix + "/v1/models"
279+
modelsRoute := c.modelRunner.OpenAIPathPrefix() + "/models"
280280
body, err := c.listRaw(modelsRoute, "")
281281
if err != nil {
282282
return dmrm.OpenAIModelList{}, err
@@ -304,7 +304,7 @@ func (c *Client) Inspect(model string, remote bool) (dmrm.Model, error) {
304304
}
305305

306306
func (c *Client) InspectOpenAI(model string) (dmrm.OpenAIModel, error) {
307-
modelsRoute := inference.InferencePrefix + "/v1/models"
307+
modelsRoute := c.modelRunner.OpenAIPathPrefix() + "/models"
308308
rawResponse, err := c.listRaw(fmt.Sprintf("%s/%s", modelsRoute, model), model)
309309
if err != nil {
310310
return dmrm.OpenAIModel{}, err
@@ -398,7 +398,7 @@ func (c *Client) ChatWithContext(ctx context.Context, model, prompt string, imag
398398
return fmt.Errorf("error marshaling request: %w", err)
399399
}
400400

401-
completionsPath := inference.InferencePrefix + "/v1/chat/completions"
401+
completionsPath := c.modelRunner.OpenAIPathPrefix() + "/chat/completions"
402402

403403
resp, err := c.doRequestWithAuthContext(
404404
ctx,

‎cmd/cli/docs/reference/docker_model_list.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ options:
2626
experimentalcli: false
2727
kubernetes: false
2828
swarm: false
29+
- option: openaiurl
30+
value_type: string
31+
description: OpenAI-compatible API endpoint URL to list models from
32+
deprecated: false
33+
hidden: false
34+
experimental: false
35+
experimentalcli: false
36+
kubernetes: false
37+
swarm: false
2938
- option: quiet
3039
shorthand: q
3140
value_type: bool

‎cmd/cli/docs/reference/docker_model_run.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,15 @@ options:
4141
experimentalcli: false
4242
kubernetes: false
4343
swarm: false
44+
- option: openaiurl
45+
value_type: string
46+
description: OpenAI-compatible API endpoint URL to chat with
47+
deprecated: false
48+
hidden: false
49+
experimental: false
50+
experimentalcli: false
51+
kubernetes: false
52+
swarm: false
4453
examples: |-
4554
### One-time prompt
4655

‎cmd/cli/docs/reference/model_list.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,11 +9,12 @@ List the models pulled to your local environment
99

1010
### Options
1111

12-
| Name | Type | Default | Description |
13-
|:----------------|:-------|:--------|:--------------------------------|
14-
| `--json` | `bool` | | List models in a JSON format |
15-
| `--openai` | `bool` | | List models in an OpenAI format |
16-
| `-q`, `--quiet` | `bool` | | Only show model IDs |
12+
| Name | Type | Default | Description |
13+
|:----------------|:---------|:--------|:-------------------------------------------------------|
14+
| `--json` | `bool` | | List models in a JSON format |
15+
| `--openai` | `bool` | | List models in an OpenAI format |
16+
| `--openaiurl` | `string` | | OpenAI-compatible API endpoint URL to list models from |
17+
| `-q`, `--quiet` | `bool` | | Only show model IDs |
1718

1819

1920
<!---MARKER_GEN_END-->

‎cmd/cli/docs/reference/model_run.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Run a model and interact with it using a submitted prompt or chat mode
1010
| `--color` | `string` | `no` | Use colored output (auto\|yes\|no) |
1111
| `--debug` | `bool` | | Enable debug logging |
1212
| `-d`, `--detach` | `bool` | | Load the model in the background without interaction |
13+
| `--openaiurl` | `string` | | OpenAI-compatible API endpoint URL to chat with |
1314

1415

1516
<!---MARKER_GEN_END-->

0 commit comments

Comments
 (0)