@@ -25,6 +25,7 @@ import (
2525 "io"
2626 "os"
2727 "path/filepath"
28+ "strconv"
2829 "sync"
2930 "syscall"
3031 "time"
@@ -35,11 +36,12 @@ import (
3536 spec "github.com/opencontainers/image-spec/specs-go"
3637 ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3738 "github.com/sirupsen/logrus"
39+ "golang.org/x/sys/unix"
3840
3941 buildconfig "github.com/CloudNativeAI/modctl/pkg/backend/build/config"
4042 "github.com/CloudNativeAI/modctl/pkg/backend/build/hooks"
4143 "github.com/CloudNativeAI/modctl/pkg/backend/build/interceptor"
42- "github.com/CloudNativeAI/modctl/pkg/codec"
44+ pkgcodec "github.com/CloudNativeAI/modctl/pkg/codec"
4345 "github.com/CloudNativeAI/modctl/pkg/storage"
4446)
4547
@@ -142,43 +144,22 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
142144 return ocispec.Descriptor {}, fmt .Errorf ("failed to get relative path: %w" , err )
143145 }
144146
145- codec , err := codec .New (codec .TypeFromMediaType (mediaType ))
147+ codec , err := pkgcodec .New (pkgcodec .TypeFromMediaType (mediaType ))
146148 if err != nil {
147149 return ocispec.Descriptor {}, fmt .Errorf ("failed to create codec: %w" , err )
148150 }
149151
150- logrus .Infof ( "building file %s... "relPath )
152+ logrus .Debugf ( "builder: starting build layer for file %s"relPath )
151153
152154 // Encode the content by codec depends on the media type.
153155 reader , err := codec .Encode (path , workDirPath )
154156 if err != nil {
155157 return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
156158 }
157159
158- logrus .Infof ("calculating digest for %s..." , relPath )
159- // Calculate the digest of the encoded content.
160- hash := sha256 .New ()
161- size , err := io .Copy (hash , reader )
160+ reader , digest , size , err := computeDigestAndSize (mediaType , path , workDirPath , info , reader , codec )
162161 if err != nil {
163- return ocispec.Descriptor {}, fmt .Errorf ("failed to copy content to hash: %w" , err )
164- }
165-
166- digest := fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
167- logrus .Infof ("calculated digest for %s: %s" , relPath , digest )
168-
169- // Seek the reader to the beginning if supported,
170- // otherwise we needs to re-encode the content again.
171- if seeker , ok := reader .(io.ReadSeeker ); ok {
172- logrus .Infof ("seeking %s reader to beginning..." , relPath )
173- if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
174- return ocispec.Descriptor {}, fmt .Errorf ("failed to seek reader: %w" , err )
175- }
176- } else {
177- logrus .Infof ("%s reader is not seekable, re-encoding..." , relPath )
178- reader , err = codec .Encode (path , workDirPath )
179- if err != nil {
180- return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
181- }
162+ return ocispec.Descriptor {}, fmt .Errorf ("failed to compute digest and size: %w" , err )
182163 }
183164
184165 var (
@@ -213,24 +194,10 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
213194 applyDesc (& desc )
214195 }
215196
216- // Retrieve the file metadata.
217- metadata , err := getFileMetadata (path )
218- if err != nil {
219- return desc , fmt .Errorf ("failed to retrieve file metadata: %w" , err )
220- }
221-
222- metadataStr , err := json .Marshal (metadata )
223- if err != nil {
224- return desc , fmt .Errorf ("failed to marshal metadata: %w" , err )
225- }
226-
227- logrus .Infof ("retrieved file %s metadata: %s" , relPath , string (metadataStr ))
228-
229- // Apply the metadata to the descriptor annotation.
230- if desc .Annotations == nil {
231- desc .Annotations = make (map [string ]string )
197+ // Add file metadata to descriptor.
198+ if err := addFileMetadata (& desc , path , relPath ); err != nil {
199+ return desc , err
232200 }
233- desc .Annotations [modelspec .AnnotationFileMetadata ] = string (metadataStr )
234201
235202 return desc , nil
236203}
@@ -315,6 +282,109 @@ func buildModelConfig(modelConfig *buildconfig.Model, layers []ocispec.Descripto
315282 }, nil
316283}
317284
285+ // computeDigestAndSize computes the digest and size for the encoded content, using xattrs if available.
286+ func computeDigestAndSize (mediaType , path , workDirPath string , info os.FileInfo , reader io.Reader , codec pkgcodec.Codec ) (io.Reader , string , int64 , error ) {
287+ var digest string
288+ var size int64
289+
290+ if pkgcodec .IsRawMediaType (mediaType ) {
291+ // By default let's assume the mtime and size has changed.
292+ mtimeChanged := true
293+ sizeChanged := true
294+
295+ if mtime , err := getXattr (path , xattrMtimeKey (mediaType )); err == nil {
296+ if string (mtime ) == fmt .Sprintf ("%d" , info .ModTime ().UnixNano ()) {
297+ mtimeChanged = false
298+ }
299+ }
300+
301+ if sizeBytes , err := getXattr (path , xattrSizeKey (mediaType )); err == nil {
302+ if parsedSize , err := strconv .ParseInt (string (sizeBytes ), 10 , 64 ); err == nil {
303+ if parsedSize == info .Size () {
304+ sizeChanged = false
305+ }
306+ }
307+ }
308+
309+ if ! mtimeChanged && ! sizeChanged {
310+ // Check xattrs for cached digest and size.
311+ if sha256 , err := getXattr (path , xattrSha256Key (mediaType )); err == nil {
312+ digest = string (sha256 )
313+ logrus .Infof ("builder: retrieved sha256 hash from xattr for file %s [digest: %s]" , path , digest )
314+ }
315+
316+ if sizeBytes , err := getXattr (path , xattrSizeKey (mediaType )); err == nil {
317+ if parsedSize , err := strconv .ParseInt (string (sizeBytes ), 10 , 64 ); err == nil {
318+ size = parsedSize
319+ logrus .Infof ("builder: retrieved size from xattr for file %s [size: %d]" , path , size )
320+ }
321+ }
322+ }
323+ }
324+
325+ // Compute digest and size if not retrieved from xattrs.
326+ if digest == "" {
327+ logrus .Infof ("builder: calculating digest for file %s" , path )
328+ var err error
329+ hash := sha256 .New ()
330+ size , err = io .Copy (hash , reader )
331+ if err != nil {
332+ return reader , "" , 0 , fmt .Errorf ("failed to copy content to hash: %w" , err )
333+ }
334+ digest = fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
335+ logrus .Infof ("builder: calculated digest for file %s [digest: %s]" , path , digest )
336+
337+ // Reset reader
338+ reader , err = resetReader (reader , path , workDirPath , codec )
339+ if err != nil {
340+ return reader , "" , 0 , err
341+ }
342+
343+ // Store xattrs if raw media type.
344+ if pkgcodec .IsRawMediaType (mediaType ) {
345+ setXattr (path , xattrMtimeKey (mediaType ), fmt .Appendf ([]byte {}, "%d" , info .ModTime ().UnixNano ()))
346+ setXattr (path , xattrSha256Key (mediaType ), []byte (digest ))
347+ setXattr (path , xattrSizeKey (mediaType ), fmt .Appendf ([]byte {}, "%d" , size ))
348+ }
349+ }
350+
351+ return reader , digest , size , nil
352+ }
353+
354+ // resetReader resets the reader to the beginning or re-encodes if not seekable.
355+ func resetReader (reader io.Reader , path , workDirPath string , codec pkgcodec.Codec ) (io.Reader , error ) {
356+ if seeker , ok := reader .(io.ReadSeeker ); ok {
357+ logrus .Debugf ("builder: seeking reader to beginning for file %s" , path )
358+ if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
359+ return nil , fmt .Errorf ("failed to seek reader: %w" , err )
360+ }
361+ return reader , nil
362+ }
363+
364+ logrus .Debugf ("builder: reader not seekable, re-encoding file %s" , path )
365+ return codec .Encode (path , workDirPath )
366+ }
367+
368+ // addFileMetadata adds file metadata to the descriptor.
369+ func addFileMetadata (desc * ocispec.Descriptor , path , relPath string ) error {
370+ metadata , err := getFileMetadata (path )
371+ if err != nil {
372+ return fmt .Errorf ("failed to retrieve file metadata: %w" , err )
373+ }
374+
375+ metadataStr , err := json .Marshal (metadata )
376+ if err != nil {
377+ return fmt .Errorf ("failed to marshal metadata: %w" , err )
378+ }
379+ logrus .Infof ("builder: retrieved metadata for file %s [metadata: %s]" , relPath , string (metadataStr ))
380+
381+ if desc .Annotations == nil {
382+ desc .Annotations = make (map [string ]string )
383+ }
384+ desc .Annotations [modelspec .AnnotationFileMetadata ] = string (metadataStr )
385+ return nil
386+ }
387+
318388// splitReader splits the original reader into two readers.
319389func splitReader (original io.Reader ) (io.Reader , io.Reader ) {
320390 r1 , w1 := io .Pipe ()
@@ -368,3 +438,52 @@ func getFileMetadata(path string) (modelspec.FileMetadata, error) {
368438
369439 return metadata , nil
370440}
441+
442+ func xattrSha256Key (mediaType string ) string {
443+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
444+ // because the key may be unlimited under mac,
445+ // but on linux, in some cases, the user can only manipulate the user space.
446+ return fmt .Sprintf ("user.%s.sha256" , mediaType )
447+ }
448+
449+ func xattrSizeKey (mediaType string ) string {
450+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
451+ // because the key may be unlimited under mac,
452+ // but on linux, in some cases, the user can only manipulate the user space.
453+ return fmt .Sprintf ("user.%s.size" , mediaType )
454+ }
455+
456+ func xattrMtimeKey (mediaType string ) string {
457+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
458+ // because the key may be unlimited under mac,
459+ // but on linux, in some cases, the user can only manipulate the user space.
460+ return fmt .Sprintf ("user.%s.mtime" , mediaType )
461+ }
462+
463+ // getXattr retrieves an xattr value for a given key.
464+ func getXattr (path , key string ) ([]byte , error ) {
465+ var value []byte
466+ sz , err := unix .Getxattr (path , key , value )
467+ if err != nil {
468+ logrus .Warnf ("builder: failed to get xattr %s for file %s: %v" , key , path , err )
469+ return nil , err
470+ }
471+
472+ value = make ([]byte , sz )
473+ _ , err = unix .Getxattr (path , key , value )
474+ if err != nil {
475+ logrus .Warnf ("builder: failed to get xattr %s for file %s: %v" , key , path , err )
476+ return nil , err
477+ }
478+
479+ return value , nil
480+ }
481+
482+ // setXattr sets an xattr value for a given key.
483+ func setXattr (path , key string , value []byte ) {
484+ if err := unix .Setxattr (path , key , value , 0 ); err != nil {
485+ logrus .Warnf ("builder: failed to set xattr %s for file %s: %v" , key , path , err )
486+ } else {
487+ logrus .Infof ("builder: set xattr %s for file %s: %s" , key , path , string (value ))
488+ }
489+ }
0 commit comments