Skip to content

Commit 0f68b49

Browse files
ilopezlunaCopilot
ilopezluna
and
Copilot
authored
Package context size (#243)
* feat(builder): add FromModel function to create builders from existing model artifacts * feat(package): add support for packaging from existing models * feat(package): implement lightweight model writing for config-only modifications * make docs * Apply suggestion from @Copilot Co-authored-by: Copilot <[email protected]> * feat(package): refactor package model initialization and error handling * fix(store): update layer identification from DiffID to Digest * feat(builder): add originalLayers tracking and HasOnlyConfigChanges method * feat(package): enhance packaging options to include existing models * test(store): add tests for WriteLightweight method and index integrity * Apply suggestion from @Copilot Co-authored-by: Copilot <[email protected]> * feat(builder): enhance FromModel to return error on layer retrieval failure --------- Co-authored-by: Copilot <[email protected]>
1 parent b74fb0a commit 0f68b49

File tree

9 files changed

+855
-88
lines changed

9 files changed

+855
-88
lines changed

‎cmd/cli/commands/package.go‎

Lines changed: 152 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import (
1111
"path/filepath"
1212

1313
"github.com/docker/model-runner/pkg/distribution/builder"
14+
"github.com/docker/model-runner/pkg/distribution/distribution"
1415
"github.com/docker/model-runner/pkg/distribution/packaging"
1516
"github.com/docker/model-runner/pkg/distribution/registry"
1617
"github.com/docker/model-runner/pkg/distribution/tarball"
@@ -27,11 +28,12 @@ func newPackagedCmd() *cobra.Command {
2728
var opts packageOptions
2829

2930
c := &cobra.Command{
30-
Use: "package (--gguf <path> | --safetensors-dir <path>) [--license <path>...] [--context-size <tokens>] [--push] MODEL",
31-
Short: "Package a GGUF file or Safetensors directory into a Docker model OCI artifact.",
32-
Long: "Package a GGUF file or Safetensors directory into a Docker model OCI artifact, with optional licenses. The package is sent to the model-runner, unless --push is specified.\n" +
31+
Use: "package (--gguf <path> | --safetensors-dir <path> | --from <model>) [--license <path>...] [--context-size <tokens>] [--push] MODEL",
32+
Short: "Package a GGUF file, Safetensors directory, or existing model into a Docker model OCI artifact.",
33+
Long: "Package a GGUF file, Safetensors directory, or existing model into a Docker model OCI artifact, with optional licenses. The package is sent to the model-runner, unless --push is specified.\n" +
3334
"When packaging a sharded GGUF model, --gguf should point to the first shard. All shard files should be siblings and should include the index in the file name (e.g. model-00001-of-00015.gguf).\n" +
34-
"When packaging a Safetensors model, --safetensors-dir should point to a directory containing .safetensors files and config files (*.json, merges.txt). All files will be auto-discovered and config files will be packaged into a tar archive.",
35+
"When packaging a Safetensors model, --safetensors-dir should point to a directory containing .safetensors files and config files (*.json, merges.txt). All files will be auto-discovered and config files will be packaged into a tar archive.\n" +
36+
"When packaging from an existing model using --from, you can modify properties like context size to create a variant of the original model.",
3537
Args: func(cmd *cobra.Command, args []string) error {
3638
if len(args) != 1 {
3739
return fmt.Errorf(
@@ -42,16 +44,27 @@ func newPackagedCmd() *cobra.Command {
4244
)
4345
}
4446

45-
// Validate that either --gguf or --safetensors-dir is provided (mutually exclusive)
46-
if opts.ggufPath == "" && opts.safetensorsDir == "" {
47+
// Validate that exactly one of --gguf, --safetensors-dir, or --from is provided (mutually exclusive)
48+
sourcesProvided := 0
49+
if opts.ggufPath != "" {
50+
sourcesProvided++
51+
}
52+
if opts.safetensorsDir != "" {
53+
sourcesProvided++
54+
}
55+
if opts.fromModel != "" {
56+
sourcesProvided++
57+
}
58+
59+
if sourcesProvided == 0 {
4760
return fmt.Errorf(
48-
"Either --gguf or --safetensors-dir path is required.\n\n" +
61+
"One of --gguf, --safetensors-dir, or --from is required.\n\n" +
4962
"See 'docker model package --help' for more information",
5063
)
5164
}
52-
if opts.ggufPath != "" && opts.safetensorsDir != "" {
65+
if sourcesProvided > 1 {
5366
return fmt.Errorf(
54-
"Cannot specify both --gguf and --safetensors-dir. Please use only one format.\n\n" +
67+
"Cannot specify more than one of --gguf, --safetensors-dir, or --from. Please use only one source.\n\n" +
5568
"See 'docker model package --help' for more information",
5669
)
5770
}
@@ -134,6 +147,7 @@ func newPackagedCmd() *cobra.Command {
134147

135148
c.Flags().StringVar(&opts.ggufPath, "gguf", "", "absolute path to gguf file")
136149
c.Flags().StringVar(&opts.safetensorsDir, "safetensors-dir", "", "absolute path to directory containing safetensors files and config")
150+
c.Flags().StringVar(&opts.fromModel, "from", "", "reference to an existing model to repackage")
137151
c.Flags().StringVar(&opts.chatTemplatePath, "chat-template", "", "absolute path to chat template file (must be Jinja format)")
138152
c.Flags().StringArrayVarP(&opts.licensePaths, "license", "l", nil, "absolute path to a license file")
139153
c.Flags().StringArrayVar(&opts.dirTarPaths, "dir-tar", nil, "relative path to directory to package as tar (can be specified multiple times)")
@@ -147,65 +161,133 @@ type packageOptions struct {
147161
contextSize uint64
148162
ggufPath string
149163
safetensorsDir string
164+
fromModel string
150165
licensePaths []string
151166
dirTarPaths []string
152167
push bool
153168
tag string
154169
}
155170

156-
func packageModel(cmd *cobra.Command, opts packageOptions) error {
157-
var (
158-
target builder.Target
159-
err error
160-
)
161-
if opts.push {
162-
target, err = registry.NewClient(
163-
registry.WithUserAgent("docker-model-cli/" + desktop.Version),
164-
).NewTarget(opts.tag)
165-
} else {
166-
target, err = newModelRunnerTarget(desktopClient, opts.tag)
167-
}
168-
if err != nil {
169-
return err
170-
}
171+
// builderInitResult contains the result of initializing a builder from various sources
172+
type builderInitResult struct {
173+
builder *builder.Builder
174+
distClient *distribution.Client // Only set when building from existing model
175+
cleanupFunc func() // Optional cleanup function for temporary files
176+
}
177+
178+
// initializeBuilder creates a package builder from GGUF, Safetensors, or existing model
179+
func initializeBuilder(cmd *cobra.Command, opts packageOptions) (*builderInitResult, error) {
180+
result := &builderInitResult{}
181+
182+
if opts.fromModel != "" {
183+
// Get the model store path
184+
userHomeDir, err := os.UserHomeDir()
185+
if err != nil {
186+
return nil, fmt.Errorf("get user home directory: %w", err)
187+
}
188+
modelStorePath := filepath.Join(userHomeDir, ".docker", "models")
189+
if envPath := os.Getenv("MODELS_PATH"); envPath != "" {
190+
modelStorePath = envPath
191+
}
171192

172-
// Create package builder based on model format
173-
var pkg *builder.Builder
174-
if opts.ggufPath != "" {
193+
// Create a distribution client to access the model store
194+
distClient, err := distribution.NewClient(distribution.WithStoreRootPath(modelStorePath))
195+
if err != nil {
196+
return nil, fmt.Errorf("create distribution client: %w", err)
197+
}
198+
result.distClient = distClient
199+
200+
// Package from existing model
201+
cmd.PrintErrf("Reading model from store: %q\n", opts.fromModel)
202+
203+
// Get the model from the local store
204+
mdl, err := distClient.GetModel(opts.fromModel)
205+
if err != nil {
206+
return nil, fmt.Errorf("get model from store: %w", err)
207+
}
208+
209+
// Type assert to ModelArtifact - the Model from store implements both interfaces
210+
modelArtifact, ok := mdl.(types.ModelArtifact)
211+
if !ok {
212+
return nil, fmt.Errorf("model does not implement ModelArtifact interface")
213+
}
214+
215+
cmd.PrintErrf("Creating builder from existing model\n")
216+
result.builder, err = builder.FromModel(modelArtifact)
217+
if err != nil {
218+
return nil, fmt.Errorf("create builder from model: %w", err)
219+
}
220+
} else if opts.ggufPath != "" {
175221
cmd.PrintErrf("Adding GGUF file from %q\n", opts.ggufPath)
176-
pkg, err = builder.FromGGUF(opts.ggufPath)
222+
pkg, err := builder.FromGGUF(opts.ggufPath)
177223
if err != nil {
178-
return fmt.Errorf("add gguf file: %w", err)
224+
return nil, fmt.Errorf("add gguf file: %w", err)
179225
}
226+
result.builder = pkg
180227
} else {
181228
// Safetensors model from directory
182229
cmd.PrintErrf("Scanning directory %q for safetensors model...\n", opts.safetensorsDir)
183230
safetensorsPaths, tempConfigArchive, err := packaging.PackageFromDirectory(opts.safetensorsDir)
184231
if err != nil {
185-
return fmt.Errorf("scan safetensors directory: %w", err)
232+
return nil, fmt.Errorf("scan safetensors directory: %w", err)
186233
}
187234

188-
// Clean up temp config archive when done
235+
// Set up cleanup for temp config archive
189236
if tempConfigArchive != "" {
190-
defer os.Remove(tempConfigArchive)
237+
result.cleanupFunc = func() {
238+
os.Remove(tempConfigArchive)
239+
}
191240
}
192241

193242
cmd.PrintErrf("Found %d safetensors file(s)\n", len(safetensorsPaths))
194-
pkg, err = builder.FromSafetensors(safetensorsPaths)
243+
pkg, err := builder.FromSafetensors(safetensorsPaths)
195244
if err != nil {
196-
return fmt.Errorf("create safetensors model: %w", err)
245+
return nil, fmt.Errorf("create safetensors model: %w", err)
197246
}
198247

199248
// Add config archive if it was created
200249
if tempConfigArchive != "" {
201250
cmd.PrintErrf("Adding config archive from directory\n")
202251
pkg, err = pkg.WithConfigArchive(tempConfigArchive)
203252
if err != nil {
204-
return fmt.Errorf("add config archive: %w", err)
253+
return nil, fmt.Errorf("add config archive: %w", err)
205254
}
206255
}
256+
result.builder = pkg
257+
}
258+
259+
return result, nil
260+
}
261+
262+
func packageModel(cmd *cobra.Command, opts packageOptions) error {
263+
var (
264+
target builder.Target
265+
err error
266+
)
267+
if opts.push {
268+
target, err = registry.NewClient(
269+
registry.WithUserAgent("docker-model-cli/" + desktop.Version),
270+
).NewTarget(opts.tag)
271+
} else {
272+
target, err = newModelRunnerTarget(desktopClient, opts.tag)
273+
}
274+
if err != nil {
275+
return err
207276
}
208277

278+
// Initialize the package builder based on model format
279+
initResult, err := initializeBuilder(cmd, opts)
280+
if err != nil {
281+
return err
282+
}
283+
// Clean up any temporary files when done
284+
if initResult.cleanupFunc != nil {
285+
defer initResult.cleanupFunc()
286+
}
287+
288+
pkg := initResult.builder
289+
distClient := initResult.distClient
290+
209291
// Set context size
210292
if opts.contextSize > 0 {
211293
cmd.PrintErrf("Setting context size %d\n", opts.contextSize)
@@ -228,32 +310,48 @@ func packageModel(cmd *cobra.Command, opts packageOptions) error {
228310
}
229311
}
230312

231-
// Process directory tar archives
232-
if len(opts.dirTarPaths) > 0 {
233-
// Determine base directory for resolving relative paths
234-
var baseDir string
235-
if opts.safetensorsDir != "" {
236-
baseDir = opts.safetensorsDir
237-
} else {
238-
// For GGUF, use the directory containing the GGUF file
239-
baseDir = filepath.Dir(opts.ggufPath)
240-
}
313+
// Check if we can use lightweight repackaging (config-only changes from existing model)
314+
useLightweight := opts.fromModel != "" && pkg.HasOnlyConfigChanges()
241315

242-
processor := packaging.NewDirTarProcessor(opts.dirTarPaths, baseDir)
243-
tarPaths, cleanup, err := processor.Process()
244-
if err != nil {
245-
return err
316+
if useLightweight {
317+
cmd.PrintErrln("Creating lightweight model variant...")
318+
319+
// Get the model artifact with new config
320+
builtModel := pkg.Model()
321+
322+
// Write using lightweight method
323+
if err := distClient.WriteLightweightModel(builtModel, []string{opts.tag}); err != nil {
324+
return fmt.Errorf("failed to create lightweight model: %w", err)
246325
}
247-
defer cleanup()
248326

249-
for _, tarPath := range tarPaths {
250-
pkg, err = pkg.WithDirTar(tarPath)
327+
cmd.PrintErrln("Model variant created successfully")
328+
} else {
329+
// Process directory tar archives
330+
if len(opts.dirTarPaths) > 0 {
331+
// Determine base directory for resolving relative paths
332+
var baseDir string
333+
if opts.safetensorsDir != "" {
334+
baseDir = opts.safetensorsDir
335+
} else {
336+
// For GGUF, use the directory containing the GGUF file
337+
baseDir = filepath.Dir(opts.ggufPath)
338+
}
339+
340+
processor := packaging.NewDirTarProcessor(opts.dirTarPaths, baseDir)
341+
tarPaths, cleanup, err := processor.Process()
251342
if err != nil {
252-
return fmt.Errorf("add directory tar: %w", err)
343+
return err
344+
}
345+
defer cleanup()
346+
347+
for _, tarPath := range tarPaths {
348+
pkg, err = pkg.WithDirTar(tarPath)
349+
if err != nil {
350+
return fmt.Errorf("add directory tar: %w", err)
351+
}
253352
}
254353
}
255354
}
256-
257355
if opts.push {
258356
cmd.PrintErrln("Pushing model to registry...")
259357
} else {

‎cmd/cli/docs/reference/docker_model_package.yaml‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
command: docker model package
22
short: |
3-
Package a GGUF file or Safetensors directory into a Docker model OCI artifact.
3+
Package a GGUF file, Safetensors directory, or existing model into a Docker model OCI artifact.
44
long: |-
5-
Package a GGUF file or Safetensors directory into a Docker model OCI artifact, with optional licenses. The package is sent to the model-runner, unless --push is specified.
5+
Package a GGUF file, Safetensors directory, or existing model into a Docker model OCI artifact, with optional licenses. The package is sent to the model-runner, unless --push is specified.
66
When packaging a sharded GGUF model, --gguf should point to the first shard. All shard files should be siblings and should include the index in the file name (e.g. model-00001-of-00015.gguf).
77
When packaging a Safetensors model, --safetensors-dir should point to a directory containing .safetensors files and config files (*.json, merges.txt). All files will be auto-discovered and config files will be packaged into a tar archive.
8-
usage: docker model package (--gguf <path> | --safetensors-dir <path>) [--license <path>...] [--context-size <tokens>] [--push] MODEL
8+
When packaging from an existing model using --from, you can modify properties like context size to create a variant of the original model.
9+
usage: docker model package (--gguf <path> | --safetensors-dir <path> | --from <model>) [--license <path>...] [--context-size <tokens>] [--push] MODEL
910
pname: docker model
1011
plink: docker_model.yaml
1112
options:
@@ -39,6 +40,15 @@ options:
3940
experimentalcli: false
4041
kubernetes: false
4142
swarm: false
43+
- option: from
44+
value_type: string
45+
description: reference to an existing model to repackage
46+
deprecated: false
47+
hidden: false
48+
experimental: false
49+
experimentalcli: false
50+
kubernetes: false
51+
swarm: false
4252
- option: gguf
4353
value_type: string
4454
description: absolute path to gguf file

‎cmd/cli/docs/reference/model.md‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -5,29 +5,29 @@ Docker Model Runner
55

66
### Subcommands
77

8-
| Name | Description |
9-
|:------------------------------------------------|:-------------------------------------------------------------------------------|
10-
| [`df`](model_df.md) | Show Docker Model Runner disk usage |
11-
| [`inspect`](model_inspect.md) | Display detailed information on one model |
12-
| [`install-runner`](model_install-runner.md) | Install Docker Model Runner (Docker Engine only) |
13-
| [`list`](model_list.md) | List the models pulled to your local environment |
14-
| [`logs`](model_logs.md) | Fetch the Docker Model Runner logs |
15-
| [`package`](model_package.md) | Package a GGUF file or Safetensors directory into a Docker model OCI artifact. |
16-
| [`ps`](model_ps.md) | List running models |
17-
| [`pull`](model_pull.md) | Pull a model from Docker Hub or HuggingFace to your local environment |
18-
| [`push`](model_push.md) | Push a model to Docker Hub |
19-
| [`reinstall-runner`](model_reinstall-runner.md) | Reinstall Docker Model Runner (Docker Engine only) |
20-
| [`requests`](model_requests.md) | Fetch requests+responses from Docker Model Runner |
21-
| [`restart-runner`](model_restart-runner.md) | Restart Docker Model Runner (Docker Engine only) |
22-
| [`rm`](model_rm.md) | Remove local models downloaded from Docker Hub |
23-
| [`run`](model_run.md) | Run a model and interact with it using a submitted prompt or chat mode |
24-
| [`start-runner`](model_start-runner.md) | Start Docker Model Runner (Docker Engine only) |
25-
| [`status`](model_status.md) | Check if the Docker Model Runner is running |
26-
| [`stop-runner`](model_stop-runner.md) | Stop Docker Model Runner (Docker Engine only) |
27-
| [`tag`](model_tag.md) | Tag a model |
28-
| [`uninstall-runner`](model_uninstall-runner.md) | Uninstall Docker Model Runner (Docker Engine only) |
29-
| [`unload`](model_unload.md) | Unload running models |
30-
| [`version`](model_version.md) | Show the Docker Model Runner version |
8+
| Name | Description |
9+
|:------------------------------------------------|:------------------------------------------------------------------------------------------------|
10+
| [`df`](model_df.md) | Show Docker Model Runner disk usage |
11+
| [`inspect`](model_inspect.md) | Display detailed information on one model |
12+
| [`install-runner`](model_install-runner.md) | Install Docker Model Runner (Docker Engine only) |
13+
| [`list`](model_list.md) | List the models pulled to your local environment |
14+
| [`logs`](model_logs.md) | Fetch the Docker Model Runner logs |
15+
| [`package`](model_package.md) | Package a GGUF file, Safetensors directory, or existing model into a Docker model OCI artifact. |
16+
| [`ps`](model_ps.md) | List running models |
17+
| [`pull`](model_pull.md) | Pull a model from Docker Hub or HuggingFace to your local environment |
18+
| [`push`](model_push.md) | Push a model to Docker Hub |
19+
| [`reinstall-runner`](model_reinstall-runner.md) | Reinstall Docker Model Runner (Docker Engine only) |
20+
| [`requests`](model_requests.md) | Fetch requests+responses from Docker Model Runner |
21+
| [`restart-runner`](model_restart-runner.md) | Restart Docker Model Runner (Docker Engine only) |
22+
| [`rm`](model_rm.md) | Remove local models downloaded from Docker Hub |
23+
| [`run`](model_run.md) | Run a model and interact with it using a submitted prompt or chat mode |
24+
| [`start-runner`](model_start-runner.md) | Start Docker Model Runner (Docker Engine only) |
25+
| [`status`](model_status.md) | Check if the Docker Model Runner is running |
26+
| [`stop-runner`](model_stop-runner.md) | Stop Docker Model Runner (Docker Engine only) |
27+
| [`tag`](model_tag.md) | Tag a model |
28+
| [`uninstall-runner`](model_uninstall-runner.md) | Uninstall Docker Model Runner (Docker Engine only) |
29+
| [`unload`](model_unload.md) | Unload running models |
30+
| [`version`](model_version.md) | Show the Docker Model Runner version |
3131

3232

3333

0 commit comments

Comments
 (0)