Skip to content

Commit ef671b0

Browse files
committed
Fix inconsistent unit formatting in docker model ls output
Remove spaces between numbers and units in the PARAMETERS and SIZE columns to enable reliable column-based parsing with tools like awk. Changes: - Safetensors: Remove spaces from parameter units (K, M, B, T) - Safetensors: Use decimal units (MB, GB) matching 'docker images' style - GGUF: Add regex normalization to remove spaces from parser output - Update all tests to reflect new no-space formatting The formatting now produces consistent output like "361.82M" and "256MB" instead of "361.82 M" and "256.35 MiB", making the output parseable by standard column-based tools. Fixes #349
1 parent 7fdb650 commit ef671b0

File tree

4 files changed

+37
-13
lines changed

4 files changed

+37
-13
lines changed

pkg/distribution/internal/gguf/create.go

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package gguf
22

33
import (
44
"fmt"
5+
"regexp"
56
"strings"
67
"time"
78

@@ -55,10 +56,29 @@ func configFromFile(path string) types.Config {
5556
}
5657
return types.Config{
5758
Format: types.FormatGGUF,
58-
Parameters: strings.TrimSpace(gguf.Metadata().Parameters.String()),
59+
Parameters: normalizeUnitString(gguf.Metadata().Parameters.String()),
5960
Architecture: strings.TrimSpace(gguf.Metadata().Architecture),
6061
Quantization: strings.TrimSpace(gguf.Metadata().FileType.String()),
61-
Size: strings.TrimSpace(gguf.Metadata().Size.String()),
62+
Size: normalizeUnitString(gguf.Metadata().Size.String()),
6263
GGUF: extractGGUFMetadata(&gguf.Header),
6364
}
6465
}
66+
67+
var (
68+
// spaceBeforeUnitRegex matches one or more spaces between a number/decimal and a letter (unit)
69+
// Used to remove spaces between numbers and units (e.g., "16.78 M" -> "16.78M")
70+
// Pattern: digits/decimals, then whitespace, then letters (unit)
71+
spaceBeforeUnitRegex = regexp.MustCompile(`([0-9.]+)\s+([A-Za-z]+)`)
72+
)
73+
74+
// normalizeUnitString removes spaces between numbers and units for consistent formatting
75+
// Examples: "16.78 M" -> "16.78M", "256.35 MiB" -> "256.35MiB", "409M" -> "409M"
76+
func normalizeUnitString(s string) string {
77+
s = strings.TrimSpace(s)
78+
if len(s) == 0 {
79+
return s
80+
}
81+
// Remove space(s) between numbers/decimals and unit letters using regex
82+
// Pattern matches: number(s) or decimal, then whitespace, then letters (unit)
83+
return spaceBeforeUnitRegex.ReplaceAllString(s, "$1$2")
84+
}

pkg/distribution/internal/gguf/model_test.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ func TestGGUF(t *testing.T) {
3232
if cfg.Quantization != "Unknown" { // todo: testdata with a real value
3333
t.Fatalf("Unexpected quantization: got %s expected %s", cfg.Quantization, "Unknown")
3434
}
35-
if cfg.Size != "864 B" {
36-
t.Fatalf("Unexpected quantization: got %s expected %s", cfg.Quantization, "Unknown")
35+
if cfg.Size != "864B" {
36+
t.Fatalf("Unexpected size: got %s expected %s", cfg.Size, "864B")
3737
}
3838

3939
// Test GGUF metadata
@@ -118,8 +118,8 @@ func TestGGUFShards(t *testing.T) {
118118
if cfg.Quantization != "Unknown" { // todo: testdata with a real value
119119
t.Fatalf("Unexpected quantization: got %s expected %s", cfg.Quantization, "Unknown")
120120
}
121-
if cfg.Size != "864 B" {
122-
t.Fatalf("Unexpected quantization: got %s expected %s", cfg.Quantization, "Unknown")
121+
if cfg.Size != "864B" {
122+
t.Fatalf("Unexpected size: got %s expected %s", cfg.Size, "864B")
123123
}
124124

125125
// Test GGUF metadata

pkg/distribution/internal/safetensors/metadata.go

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"fmt"
77
"io"
88
"os"
9+
"strings"
910

1011
"github.com/docker/go-units"
1112
)
@@ -183,13 +184,16 @@ func (h *Header) ExtractMetadata() map[string]string {
183184
return metadata
184185
}
185186

186-
// formatParameters converts parameter count to human-readable format matching GGUF style
187-
// Returns format like "361.82 M" or "1.5 B" (space before unit, base 1000, where B = Billion)
187+
// formatParameters converts parameter count to human-readable format
188+
// Returns format like "361.82M" or "1.5B" (no space before unit, base 1000, where B = Billion)
188189
func formatParameters(params int64) string {
189-
return units.CustomSize("%.2f%s", float64(params), 1000.0, []string{"", " K", " M", " B", " T"})
190+
return units.CustomSize("%.2f%s", float64(params), 1000.0, []string{"", "K", "M", "B", "T"})
190191
}
191192

192-
// formatSize converts bytes to human-readable format
193+
// formatSize converts bytes to human-readable format matching Docker's style
194+
// Returns format like "256MB" (decimal units, no space, matching `docker images`)
193195
func formatSize(bytes int64) string {
194-
return units.HumanSizeWithPrecision(float64(bytes), 2)
196+
formatted := units.HumanSize(float64(bytes))
197+
// Remove space between number and unit to match Docker format (e.g., "256 MB" -> "256MB")
198+
return strings.ReplaceAll(formatted, " ", "")
195199
}

pkg/distribution/internal/safetensors/model_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func TestNewModel_WithMetadata(t *testing.T) {
9292
}
9393

9494
// Verify parameters (4096*4096 + 4096 = 16781312)
95-
expectedParams := "16.78 M"
95+
expectedParams := "16.78M"
9696
if config.Parameters != expectedParams {
9797
t.Errorf("Config.Parameters = %v, want %v", config.Parameters, expectedParams)
9898
}
@@ -220,7 +220,7 @@ func TestNewModel_NoMetadata(t *testing.T) {
220220
}
221221

222222
// Verify parameters (100*200 = 20000)
223-
expectedParams := "20.00 K"
223+
expectedParams := "20.00K"
224224
if config.Parameters != expectedParams {
225225
t.Errorf("Config.Parameters = %v, want %v", config.Parameters, expectedParams)
226226
}

0 commit comments

Comments
 (0)