@@ -25,6 +25,7 @@ import (
2525 "io"
2626 "os"
2727 "path/filepath"
28+ "strconv"
2829 "sync"
2930 "syscall"
3031 "time"
@@ -35,11 +36,12 @@ import (
3536 spec "github.com/opencontainers/image-spec/specs-go"
3637 ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3738 "github.com/sirupsen/logrus"
39+ "golang.org/x/sys/unix"
3840
3941 buildconfig "github.com/CloudNativeAI/modctl/pkg/backend/build/config"
4042 "github.com/CloudNativeAI/modctl/pkg/backend/build/hooks"
4143 "github.com/CloudNativeAI/modctl/pkg/backend/build/interceptor"
42- "github.com/CloudNativeAI/modctl/pkg/codec"
44+ pkgcodec "github.com/CloudNativeAI/modctl/pkg/codec"
4345 "github.com/CloudNativeAI/modctl/pkg/storage"
4446)
4547
@@ -142,43 +144,22 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
142144 return ocispec.Descriptor {}, fmt .Errorf ("failed to get relative path: %w" , err )
143145 }
144146
145- codec , err := codec .New (codec .TypeFromMediaType (mediaType ))
147+ codec , err := pkgcodec .New (pkgcodec .TypeFromMediaType (mediaType ))
146148 if err != nil {
147149 return ocispec.Descriptor {}, fmt .Errorf ("failed to create codec: %w" , err )
148150 }
149151
150- logrus .Infof ( "building file %s... "relPath )
152+ logrus .Debugf ( "builder: starting build layer for file %s"relPath )
151153
152154 // Encode the content by codec depends on the media type.
153155 reader , err := codec .Encode (path , workDirPath )
154156 if err != nil {
155157 return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
156158 }
157159
158- logrus .Infof ("calculating digest for %s..." , relPath )
159- // Calculate the digest of the encoded content.
160- hash := sha256 .New ()
161- size , err := io .Copy (hash , reader )
160+ reader , digest , size , err := computeDigestAndSize (mediaType , path , workDirPath , info , reader , codec )
162161 if err != nil {
163- return ocispec.Descriptor {}, fmt .Errorf ("failed to copy content to hash: %w" , err )
164- }
165-
166- digest := fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
167- logrus .Infof ("calculated digest for %s: %s" , relPath , digest )
168-
169- // Seek the reader to the beginning if supported,
170- // otherwise we needs to re-encode the content again.
171- if seeker , ok := reader .(io.ReadSeeker ); ok {
172- logrus .Infof ("seeking %s reader to beginning..." , relPath )
173- if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
174- return ocispec.Descriptor {}, fmt .Errorf ("failed to seek reader: %w" , err )
175- }
176- } else {
177- logrus .Infof ("%s reader is not seekable, re-encoding..." , relPath )
178- reader , err = codec .Encode (path , workDirPath )
179- if err != nil {
180- return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
181- }
162+ return ocispec.Descriptor {}, fmt .Errorf ("failed to compute digest and size: %w" , err )
182163 }
183164
184165 var (
@@ -213,24 +194,10 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
213194 applyDesc (& desc )
214195 }
215196
216- // Retrieve the file metadata.
217- metadata , err := getFileMetadata (path )
218- if err != nil {
219- return desc , fmt .Errorf ("failed to retrieve file metadata: %w" , err )
220- }
221-
222- metadataStr , err := json .Marshal (metadata )
223- if err != nil {
224- return desc , fmt .Errorf ("failed to marshal metadata: %w" , err )
225- }
226-
227- logrus .Infof ("retrieved file %s metadata: %s" , relPath , string (metadataStr ))
228-
229- // Apply the metadata to the descriptor annotation.
230- if desc .Annotations == nil {
231- desc .Annotations = make (map [string ]string )
197+ // Add file metadata to descriptor.
198+ if err := addFileMetadata (& desc , path , relPath ); err != nil {
199+ return desc , err
232200 }
233- desc .Annotations [modelspec .AnnotationFileMetadata ] = string (metadataStr )
234201
235202 return desc , nil
236203}
@@ -315,6 +282,93 @@ func buildModelConfig(modelConfig *buildconfig.Model, layers []ocispec.Descripto
315282 }, nil
316283}
317284
285+ // computeDigestAndSize computes the digest and size for the encoded content, using xattrs if available.
286+ func computeDigestAndSize (mediaType , path , workDirPath string , info os.FileInfo , reader io.Reader , codec pkgcodec.Codec ) (io.Reader , string , int64 , error ) {
287+ var digest string
288+ var size int64
289+
290+ if pkgcodec .IsRawMediaType (mediaType ) {
291+ // Check xattrs for cached digest and size.
292+ if mtime , err := getXattr (path , xattrMtimeKey (mediaType )); err == nil {
293+ if string (mtime ) == fmt .Sprintf ("%d" , info .ModTime ().UnixNano ()) {
294+ if sha256 , err := getXattr (path , xattrSha256Key (mediaType )); err == nil {
295+ digest = string (sha256 )
296+ logrus .Infof ("builder: retrieved sha256 hash from xattr for file %s [digest: %s]" , path , digest )
297+ }
298+
299+ if sizeBytes , err := getXattr (path , xattrSizeKey (mediaType )); err == nil {
300+ if parsedSize , err := strconv .ParseInt (string (sizeBytes ), 10 , 64 ); err == nil {
301+ size = parsedSize
302+ logrus .Infof ("builder: retrieved size from xattr for file %s [size: %d]" , path , size )
303+ }
304+ }
305+ }
306+ }
307+ }
308+
309+ // Compute digest and size if not retrieved from xattrs.
310+ if digest == "" {
311+ logrus .Infof ("builder: calculating digest for file %s" , path )
312+ var err error
313+ hash := sha256 .New ()
314+ size , err = io .Copy (hash , reader )
315+ if err != nil {
316+ return reader , "" , 0 , fmt .Errorf ("failed to copy content to hash: %w" , err )
317+ }
318+ digest = fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
319+ logrus .Infof ("builder: calculated digest for file %s [digest: %s]" , path , digest )
320+
321+ // Reset reader
322+ reader , err = resetReader (reader , path , workDirPath , codec )
323+ if err != nil {
324+ return reader , "" , 0 , err
325+ }
326+
327+ // Store xattrs if raw media type.
328+ if pkgcodec .IsRawMediaType (mediaType ) {
329+ setXattr (path , xattrMtimeKey (mediaType ), fmt .Appendf ([]byte {}, "%d" , info .ModTime ().UnixNano ()))
330+ setXattr (path , xattrSha256Key (mediaType ), []byte (digest ))
331+ setXattr (path , xattrSizeKey (mediaType ), fmt .Appendf ([]byte {}, "%d" , size ))
332+ }
333+ }
334+
335+ return reader , digest , size , nil
336+ }
337+
338+ // resetReader resets the reader to the beginning or re-encodes if not seekable.
339+ func resetReader (reader io.Reader , path , workDirPath string , codec pkgcodec.Codec ) (io.Reader , error ) {
340+ if seeker , ok := reader .(io.ReadSeeker ); ok {
341+ logrus .Debugf ("builder: seeking reader to beginning for file %s" , path )
342+ if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
343+ return nil , fmt .Errorf ("failed to seek reader: %w" , err )
344+ }
345+ return reader , nil
346+ }
347+
348+ logrus .Debugf ("builder: reader not seekable, re-encoding file %s" , path )
349+ return codec .Encode (path , workDirPath )
350+ }
351+
352+ // addFileMetadata adds file metadata to the descriptor.
353+ func addFileMetadata (desc * ocispec.Descriptor , path , relPath string ) error {
354+ metadata , err := getFileMetadata (path )
355+ if err != nil {
356+ return fmt .Errorf ("failed to retrieve file metadata: %w" , err )
357+ }
358+
359+ metadataStr , err := json .Marshal (metadata )
360+ if err != nil {
361+ return fmt .Errorf ("failed to marshal metadata: %w" , err )
362+ }
363+ logrus .Infof ("builder: retrieved metadata for file %s [metadata: %s]" , relPath , string (metadataStr ))
364+
365+ if desc .Annotations == nil {
366+ desc .Annotations = make (map [string ]string )
367+ }
368+ desc .Annotations [modelspec .AnnotationFileMetadata ] = string (metadataStr )
369+ return nil
370+ }
371+
318372// splitReader splits the original reader into two readers.
319373func splitReader (original io.Reader ) (io.Reader , io.Reader ) {
320374 r1 , w1 := io .Pipe ()
@@ -368,3 +422,52 @@ func getFileMetadata(path string) (modelspec.FileMetadata, error) {
368422
369423 return metadata , nil
370424}
425+
426+ func xattrSha256Key (mediaType string ) string {
427+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
428+ // because the key may be unlimited under mac,
429+ // but on linux, in some cases, the user can only manipulate the user space.
430+ return fmt .Sprintf ("user.%s.sha256" , mediaType )
431+ }
432+
433+ func xattrSizeKey (mediaType string ) string {
434+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
435+ // because the key may be unlimited under mac,
436+ // but on linux, in some cases, the user can only manipulate the user space.
437+ return fmt .Sprintf ("user.%s.size" , mediaType )
438+ }
439+
440+ func xattrMtimeKey (mediaType string ) string {
441+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
442+ // because the key may be unlimited under mac,
443+ // but on linux, in some cases, the user can only manipulate the user space.
444+ return fmt .Sprintf ("user.%s.mtime" , mediaType )
445+ }
446+
447+ // getXattr retrieves an xattr value for a given key.
448+ func getXattr (path , key string ) ([]byte , error ) {
449+ var value []byte
450+ sz , err := unix .Getxattr (path , key , value )
451+ if err != nil {
452+ logrus .Warnf ("builder: failed to get xattr %s for file %s: %v" , key , path , err )
453+ return nil , err
454+ }
455+
456+ value = make ([]byte , sz )
457+ _ , err = unix .Getxattr (path , key , value )
458+ if err != nil {
459+ logrus .Warnf ("builder: failed to get xattr %s for file %s: %v" , key , path , err )
460+ return nil , err
461+ }
462+
463+ return value , nil
464+ }
465+
466+ // setXattr sets an xattr value for a given key.
467+ func setXattr (path , key string , value []byte ) {
468+ if err := unix .Setxattr (path , key , value , 0 ); err != nil {
469+ logrus .Warnf ("builder: failed to set xattr %s for file %s: %v" , key , path , err )
470+ } else {
471+ logrus .Infof ("builder: set xattr %s for file %s: %s" , key , path , string (value ))
472+ }
473+ }
0 commit comments