@@ -25,6 +25,7 @@ import (
2525 "io"
2626 "os"
2727 "path/filepath"
28+ "strconv"
2829 "sync"
2930 "syscall"
3031 "time"
@@ -35,6 +36,7 @@ import (
3536 spec "github.com/opencontainers/image-spec/specs-go"
3637 ocispec "github.com/opencontainers/image-spec/specs-go/v1"
3738 "github.com/sirupsen/logrus"
39+ "golang.org/x/sys/unix"
3840
3941 buildconfig "github.com/CloudNativeAI/modctl/pkg/backend/build/config"
4042 "github.com/CloudNativeAI/modctl/pkg/backend/build/hooks"
@@ -147,37 +149,91 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
147149 return ocispec.Descriptor {}, fmt .Errorf ("failed to create codec: %w" , err )
148150 }
149151
150- logrus .Infof ( "building file %s... "relPath )
152+ logrus .Debugf ( "builder: starting build layer for file %s"relPath )
151153
152154 // Encode the content by codec depends on the media type.
153155 reader , err := codec .Encode (path , workDirPath )
154156 if err != nil {
155157 return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
156158 }
157159
158- logrus .Infof ("calculating digest for %s..." , relPath )
159- // Calculate the digest of the encoded content.
160- hash := sha256 .New ()
161- size , err := io .Copy (hash , reader )
162- if err != nil {
163- return ocispec.Descriptor {}, fmt .Errorf ("failed to copy content to hash: %w" , err )
164- }
165-
166- digest := fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
167- logrus .Infof ("calculated digest for %s: %s" , relPath , digest )
160+ var (
161+ digest string
162+ size int64
163+ )
168164
169- // Seek the reader to the beginning if supported,
170- // otherwise we needs to re-encode the content again.
171- if seeker , ok := reader .(io.ReadSeeker ); ok {
172- logrus .Infof ("seeking %s reader to beginning..." , relPath )
173- if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
174- return ocispec.Descriptor {}, fmt .Errorf ("failed to seek reader: %w" , err )
165+ // Try to retrieve the sha256 hash from the file xatrrs.
166+ var xattrSha256 []byte
167+ if sz , err := unix .Getxattr (path , xattrSha256Key (mediaType ), xattrSha256 ); err != nil {
168+ logrus .Warnf ("builder: failed to get xattr sha256 for file %s: %v" , relPath , err )
169+ } else {
170+ xattrSha256 = make ([]byte , sz )
171+ if _ , err := unix .Getxattr (path , xattrSha256Key (mediaType ), xattrSha256 ); err != nil {
172+ logrus .Warnf ("builder: failed to get xattr sha256 for file %s: %v" , relPath , err )
173+ } else {
174+ digest = string (xattrSha256 )
175+ logrus .Infof ("builder: retrieved sha256 hash from xattr for file %s [digest: %s]" , relPath , digest )
175176 }
177+ }
178+
179+ // Try to retrieve the size from the file xatrrs.
180+ var xattrSize []byte
181+ if sz , err := unix .Getxattr (path , xattrSizeKey (mediaType ), xattrSize ); err != nil {
182+ logrus .Warnf ("builder: failed to get xattr size for file %s: %v" , relPath , err )
176183 } else {
177- logrus .Infof ("%s reader is not seekable, re-encoding..." , relPath )
178- reader , err = codec .Encode (path , workDirPath )
184+ xattrSize = make ([]byte , sz )
185+ if _ , err := unix .Getxattr (path , xattrSizeKey (mediaType ), xattrSize ); err != nil {
186+ logrus .Warnf ("builder: failed to get xattr size for file %s: %v" , relPath , err )
187+ } else {
188+ size , err = strconv .ParseInt (string (xattrSize ), 10 , 64 )
189+ if err != nil {
190+ logrus .Warnf ("Builder: failed to parse size from xattr for %s: %v" , relPath , err )
191+ return ocispec.Descriptor {}, fmt .Errorf ("failed to parse size from xattr: %w" , err )
192+ }
193+ logrus .Infof ("builder: retrieved size from xattr for file %s [size: %d]" , relPath , size )
194+ }
195+ }
196+
197+ // Calculate the digest of the encoded content as not retrieved from xattrs.
198+ if digest == "" {
199+ logrus .Infof ("builder: calculating digest for file %s" , relPath )
200+ // Calculate the digest of the encoded content.
201+ hash := sha256 .New ()
202+ size , err = io .Copy (hash , reader )
179203 if err != nil {
180- return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
204+ return ocispec.Descriptor {}, fmt .Errorf ("failed to copy content to hash: %w" , err )
205+ }
206+
207+ digest = fmt .Sprintf ("sha256:%x" , hash .Sum (nil ))
208+ logrus .Infof ("builder: calculated digest for file %s [digest: %s]" , relPath , digest )
209+
210+ // Seek the reader to the beginning if supported,
211+ // otherwise we needs to re-encode the content again.
212+ if seeker , ok := reader .(io.ReadSeeker ); ok {
213+ logrus .Debugf ("builder: seeking reader to beginning for file %s" , relPath )
214+ if _ , err := seeker .Seek (0 , io .SeekStart ); err != nil {
215+ return ocispec.Descriptor {}, fmt .Errorf ("failed to seek reader: %w" , err )
216+ }
217+ } else {
218+ logrus .Debugf ("builder: reader not seekable, re-encoding file %s" , relPath )
219+ reader , err = codec .Encode (path , workDirPath )
220+ if err != nil {
221+ return ocispec.Descriptor {}, fmt .Errorf ("failed to encode file: %w" , err )
222+ }
223+ }
224+
225+ // Set the digest to the xattrs.
226+ if err := unix .Setxattr (path , xattrSha256Key (mediaType ), []byte (digest ), 0 ); err != nil {
227+ logrus .Warnf ("builder: failed to set xattr digest for file %s: %v" , relPath , err )
228+ } else {
229+ logrus .Infof ("builder: setted xattr digest for file %s [digest: %s]" , relPath , digest )
230+ }
231+
232+ // Set the size to the xattrs.
233+ if err := unix .Setxattr (path , xattrSizeKey (mediaType ), []byte (strconv .FormatInt (size , 10 )), 0 ); err != nil {
234+ logrus .Warnf ("builder: failed to set xattr size for file %s: %v" , relPath , err )
235+ } else {
236+ logrus .Infof ("builder: setted xattr size for file %s [size: %d]" , relPath , size )
181237 }
182238 }
183239
@@ -224,7 +280,7 @@ func (ab *abstractBuilder) BuildLayer(ctx context.Context, mediaType, workDir, p
224280 return desc , fmt .Errorf ("failed to marshal metadata: %w" , err )
225281 }
226282
227- logrus .Infof ("retrieved file %s metadata: %s" , relPath , string (metadataStr ))
283+ logrus .Infof ("builder: retrieved metadata for file %s [ metadata: %s] " , relPath , string (metadataStr ))
228284
229285 // Apply the metadata to the descriptor annotation.
230286 if desc .Annotations == nil {
@@ -368,3 +424,17 @@ func getFileMetadata(path string) (modelspec.FileMetadata, error) {
368424
369425 return metadata , nil
370426}
427+
428+ func xattrSha256Key (mediaType string ) string {
429+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
430+ // because the key may be unlimited under mac,
431+ // but on linux, in some cases, the user can only manipulate the user space.
432+ return fmt .Sprintf ("user.%s.sha256" , mediaType )
433+ }
434+
435+ func xattrSizeKey (mediaType string ) string {
436+ // Uniformity between linux and mac platforms is simplified by adding the prefix 'user.',
437+ // because the key may be unlimited under mac,
438+ // but on linux, in some cases, the user can only manipulate the user space.
439+ return fmt .Sprintf ("user.%s.size" , mediaType )
440+ }
0 commit comments