Proper chaining for AMLFS flag (#3401)

* flag implementation & tests

* test backwards compat

* implement in sync, used bit masking

* add missing param to SMB validation

* fix remove test failures

* add check for mt posixStyle flag

* amlfs test

* Update common/unixStatAdapter.go

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* support poxisStyle in test framework

* add amlfs to stat, fix wrong type in test

* add assert

* fixed test

* fixed test

* fix typo

* proper chaining of flag

* extended test

* Delete C:\bin\azcopyTestFailureLogs/09685922-9c44-2b4e-67d8-f8b5386f872f-chunks/plans/09685922-9c44-2b4e-67d8-f8b5386f872f--00000.steV20

* Delete C:\bin\azcopyTestFailureLogs directory

* Delete C:\Users\wonwuakpa\Downloads\repro/repro/generated_data/00102d64-9711-4107-bd6a-531316909feb

* Delete C:\Users\wonwuakpa\Downloads\repro directory

* Delete e2etest/C:\bin\azcopyTestFailureLogs/4c7a3295-4699-4446-6b8d-95d5c4600e94-chunks/plans/4c7a3295-4699-4446-6b8d-95d5c4600e94--00000.steV20

* Delete e2etest/C:\bin\azcopyTestFailureLogs directory

---------

Co-authored-by: Gauri Lamunion <51212198+gapra-msft@users.noreply.github.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: 3 people

cmd/copy.go

@@ -307,6 +307,10 @@ func (raw *rawCopyCmdArgs) toCopyOptions(cmd *cobra.Command) (opts azcopy.CopyOp
 		return opts, err
 	}
 
+	if err = opts.PosixPropertiesStyle.Parse(raw.posixPropertiesStyle); err != nil {
+		return opts, err
+	}
+
 	opts.IncludePatterns = parsePatterns(raw.include)
 	opts.ExcludePatterns = parsePatterns(raw.exclude)
 	opts.ExcludePaths = parsePatterns(raw.excludePath)
@@ -585,6 +589,7 @@ func (raw *rawCopyCmdArgs) toOptions() (cooked CookedCopyCmdArgs, err error) {
 	return cooked, nil
 }
 
+// This method and ones it calls are legacy code not used in copy command flow (as of v10.32)
 func (raw rawCopyCmdArgs) cook() (cooked CookedCopyCmdArgs, err error) {
 	if cooked, err = raw.toOptions(); err != nil {
 		return cooked, err

cmd/setProperties.go

@@ -35,6 +35,7 @@ func (raw *rawCopyCmdArgs) setMandatoryDefaultsForSetProperties() {
 	raw.s2sInvalidMetadataHandleOption = common.DefaultInvalidMetadataHandleOption.String()
 	raw.forceWrite = common.EOverwriteOption.True().String()
 	raw.preserveOwner = common.PreserveOwnerDefault
+	raw.posixPropertiesStyle = common.EPosixPropertiesStyle.Standard().String()
 }
 
 func (cca *CookedCopyCmdArgs) checkIfChangesPossible() error {

common/unixStatAdapter.go

@@ -247,24 +247,27 @@ func ReadStatFromMetadata(metadata MetadataStore, contentLength int64) (UnixStat
 		s.groupGID = uint32(g)
 	}
 
-	// In cases, the permissions were uploaded in AMLFS style, determine what base to use
+	// We normalize the mode to have the same internal decimal representation for both posix styles.
+	// AMLFS (base 8), Standard (base 10)
 	if modeStr, ok := metadata.TryRead(POSIXModeMeta); ok {
 		modeBase := 10
 
 		// AMLFS stores permissions in octal and also sets AMLFS owner/group keys.
-		amlfsStyle := false
+
+		// Use multiple indicators to avoid misclassifying standard metadata as AMLFS.
+		amlfsStyle := 0
 		if _, ok := metadata.TryRead(AMLFSOwnerMeta); ok {
-			amlfsStyle = true
+			amlfsStyle++
 		}
 		if _, ok := metadata.TryRead(AMLFSGroupMeta); ok {
-			amlfsStyle = true
+			amlfsStyle++
 		}
 		// AMLFS formatter uses a leading 0 with %04o (e.g., "0755")
-		if strings.HasPrefix(*modeStr, "0") {
-			amlfsStyle = true
+		if len(*modeStr) > 1 && strings.HasPrefix(*modeStr, "0") {
+			amlfsStyle++
 		}
 
-		if amlfsStyle {
+		if amlfsStyle >= 1 {
 			modeBase = 8 // To persist AMLFS style formatting, we store in base 8
 		}
 
@@ -311,8 +314,8 @@ func ReadStatFromMetadata(metadata MetadataStore, contentLength int64) (UnixStat
 		s.accessTime = time.Unix(0, at)
 	}
 
-	// ModTime can come in either standard (nanoseconds) or AMLFS (formatted string) format
-	// It is stored internally as unix nanoseconds (Time.time type)
+	// ModTime can come in either Standard (nanoseconds) or AMLFS (formatted string)
+	// We normalize both formats to store modTime as unix nanoseconds (Time.time type)
 	if mtime, ok := metadata.TryRead(POSIXModTimeMeta); ok {
 		mt, err := strconv.ParseInt(*mtime, 10, 64)
 		if errors.Is(err, strconv.ErrSyntax) {

e2etest/declarativeScenario.go

@@ -668,7 +668,8 @@ func (s *scenario) validatePOSIXProperties(f *testObject, metadata map[string]*s
 		s.a.AssertNoErr(err, "reading stat from metadata")
 	}
 
-	s.a.Assert(f.verificationProperties.posixProperties.EquivalentToStatAdapter(adapter), equals(), "", "POSIX properties were mismatched")
+	s.a.Assert(f.verificationProperties.posixProperties.EquivalentToStatAdapter(adapter), equals(), "",
+		fmt.Sprintf("POSIX properties were mismatched for object %v", f.name))
 }
 
 func (s *scenario) validateSymlink(f *testObject, metadata map[string]*string) {

e2etest/declarativeTestFiles.go

@@ -178,6 +178,9 @@ type objectUnixStatContainer struct {
 
 	accessTime *time.Time
 	modTime    *time.Time
+
+	owner *uint32
+	group *uint32
 }
 
 func (o *objectUnixStatContainer) HasTimes() bool {
@@ -191,7 +194,9 @@ func (o *objectUnixStatContainer) Empty() bool {
 
 	return o.mode == nil &&
 		o.accessTime == nil &&
-		o.modTime == nil
+		o.modTime == nil &&
+		o.owner == nil &&
+		o.group == nil
 }
 
 func (o *objectUnixStatContainer) DeepCopy() *objectUnixStatContainer {
@@ -215,9 +220,20 @@ func (o *objectUnixStatContainer) DeepCopy() *objectUnixStatContainer {
 		out.modTime = &modTime
 	}
 
+	if o.owner != nil {
+		ownerId := *o.owner
+		out.owner = &ownerId
+	}
+
+	if o.group != nil {
+		groupId := *o.group
+		out.group = &groupId
+	}
+
 	return out
 }
 
+// EquivalentToStatAdapter validates the metadata values, not format
 func (o *objectUnixStatContainer) EquivalentToStatAdapter(s common.UnixStatAdapter) string {
 	if o == nil {
 		return "" // no comparison to make
@@ -226,8 +242,11 @@ func (o *objectUnixStatContainer) EquivalentToStatAdapter(s common.UnixStatAdapt
 	mismatched := make([]string, 0)
 	// only compare if we set it
 	if o.mode != nil {
-		if s.FileMode() != *o.mode {
-			mismatched = append(mismatched, "mode")
+		if (s.FileMode() != *o.mode) && // First do a straight comparison
+			fmt.Sprintf("%04o", uint32(s.FileMode())&0777) != fmt.Sprintf("%04o", *o.mode&0777) { // Compare just the permission bits
+			mismatched = append(mismatched, fmt.Sprintf("actual:%v mode expected:%v", strconv.FormatInt(int64(s.FileMode()), 10),
+				strconv.FormatUint(uint64(*o.mode), 10)))
+
 		}
 	}
 
@@ -243,6 +262,18 @@ func (o *objectUnixStatContainer) EquivalentToStatAdapter(s common.UnixStatAdapt
 		}
 	}
 
+	if o.owner != nil {
+		if *o.owner != s.Owner() {
+			mismatched = append(mismatched, "owner")
+		}
+	}
+
+	if o.group != nil {
+		if *o.group != s.Group() {
+			mismatched = append(mismatched, "group")
+		}
+	}
+
 	return strings.Join(mismatched, ", ")
 }
 
@@ -287,6 +318,24 @@ func (o *objectUnixStatContainer) AddToMetadata(metadata map[string]*string, sty
 		}
 	}
 
+	if o.owner != nil {
+		mask |= common.STATX_UID
+		if style == common.AMLFSPosixPropertiesStyle {
+			metadata[common.AMLFSOwnerMeta] = to.Ptr(strconv.FormatUint(uint64(*o.owner), 10))
+		} else {
+			metadata[common.POSIXOwnerMeta] = to.Ptr(strconv.FormatUint(uint64(*o.owner), 10))
+		}
+	}
+
+	if o.group != nil {
+		mask |= common.STATX_GID
+		if style == common.AMLFSPosixPropertiesStyle {
+			metadata[common.AMLFSGroupMeta] = to.Ptr(strconv.FormatUint(uint64(*o.owner), 10))
+		} else {
+			metadata[common.POSIXGroupMeta] = to.Ptr(strconv.FormatUint(uint64(*o.owner), 10))
+		}
+	}
+
 	metadata[common.LINUXStatxMaskMeta] = to.Ptr(strconv.FormatUint(uint64(mask), 10))
 }
 

e2etest/zt_preserve_posix_properties_test.go

@@ -4,6 +4,8 @@
 package e2etest
 
 import (
+	"os"
+	"strings"
 	"testing"
 	"time"
 
@@ -45,35 +47,79 @@ func TestPOSIX_SpecialFilesToBlob(t *testing.T) {
 	)
 }
 
-// Test that AMLFS style props are correctly uploaded to Blob storage
+// Test that AMLFS style props are correctly uploaded to & downloaded from Blob storage
 func TestPOSIX_UploadAMLFSStyle(t *testing.T) {
 	ptr := func(u uint32) *uint32 {
 		return &u
 	}
-	modTime, err := time.Parse(common.AMLFS_MOD_TIME_LAYOUT, "2026-01-02 15:04:05 -0700")
+	timeStr := "2026-01-02 15:04:05 -0700"
+	modTime, err := time.Parse(common.AMLFS_MOD_TIME_LAYOUT, timeStr)
 	a := assert.New(t)
 	a.NoError(err)
 
 	RunScenarios(
 		t,
 		eOperation.Copy(),
-		eTestFromTo.Other(common.EFromTo.LocalBlob(), common.EFromTo.BlobLocal()), // no blobblob since that's just metadata and we already test that
+		eTestFromTo.Other(common.EFromTo.LocalBlob(), common.EFromTo.BlobLocal(),
+			common.EFromTo.BlobBlob()),
 		eValidate.Auto(),
-		allCredentialTypes, // this relies upon a working source info provider; this validates appropriate creds are supplied to it.
+		allCredentialTypes,
 		anonymousAuthOnly,
 		params{
 			recursive:               true,
 			preservePOSIXProperties: true,
-			symlinkHandling:         common.ESymlinkHandlingType.Preserve(),
 			posixPropertiesStyle:    common.AMLFSPosixPropertiesStyle,
 		},
-		nil,
+		// Hook to validate AMLFS style is persisted to Blob
+		&hooks{
+			afterValidation: func(h hookHelper) {
+				if h.FromTo().To() != common.ELocation.Blob() {
+					return // Local destinations use native formatting
+				}
+				c := h.GetAsserter()
+				objects := h.GetDestination().getAllProperties(c)
+
+				var props *objectProperties
+				for key, p := range objects {
+					if strings.HasSuffix(key, "/b") || strings.HasSuffix(key, "b") {
+						props = p
+						break
+					}
+				}
+				a.NotNil(props, "could not find properties for file")
+
+				// check modtime format
+				mt, ok := props.nameValueMetadata[common.POSIXModTimeMeta]
+				c.Assert(ok, equals(), true, "mod time metadata is missing")
+				if ok {
+					_, err := time.Parse(common.AMLFS_MOD_TIME_LAYOUT, *mt)
+					c.AssertNoErr(err, "mod time is not in expected AMLFS format.")
+				}
+
+				_, ok = props.nameValueMetadata[common.AMLFSOwnerMeta]
+				c.Assert(ok, equals(), true, "AMLFS owner is missing")
+
+				_, ok = props.nameValueMetadata[common.AMLFSGroupMeta]
+				c.Assert(ok, equals(), true, "AMLFS group is missing")
+
+				val, ok := props.nameValueMetadata[common.POSIXModeMeta]
+				c.Assert(ok, equals(), true, "permissions metadata is missing.")
+				if ok {
+					c.Assert(strings.HasPrefix(*val, "0"), equals(), true, "permissions not in octal format")
+				}
+			},
+		},
 		testFiles{
 			defaultSize: "1K",
 			shouldTransfer: []interface{}{
 				folder(""),
-				f("a", with{posixProperties: objectUnixStatContainer{mode: ptr(common.S_IFREG | common.DEFAULT_FILE_PERM)}}),
-				f("b", with{posixProperties: objectUnixStatContainer{modTime: &modTime}}),
+				f("a", with{posixProperties: objectUnixStatContainer{
+					mode: ptr(common.S_IFREG | common.DEFAULT_FILE_PERM)}}),
+				f("b", with{posixProperties: objectUnixStatContainer{
+					modTime: &modTime,
+					owner:   ptr(uint32(os.Getuid())),
+					group:   ptr(uint32(os.Getgid())),
+					mode:    ptr(common.S_IFREG | common.DEFAULT_FILE_PERM)}}),
 			},
 		},
 		EAccountType.Standard(), EAccountType.Standard(), "",

ste/JobPartPlanFileName.go

@@ -217,6 +217,7 @@ func (jpfn JobPartPlanFileName) Create(order common.CopyJobPartOrderRequest) {
 		PreservePermissions:     order.PreservePermissions,
 		PreserveInfo:            order.PreserveInfo,
 		PreservePOSIXProperties: order.PreservePOSIXProperties,
+		PosixPropertiesStyle:    order.PosixPropertiesStyle,
 		// For S2S copy, per JobPartPlan info
 		S2SGetPropertiesInBackend:      order.S2SGetPropertiesInBackend,
 		S2SSourceChangeValidation:      order.S2SSourceChangeValidation,

ste/md5Comparer.go

@@ -42,8 +42,8 @@ var ErrMd5Mismatch = errors.New("the MD5 hash of the data, as we received it, di
 	"This means that either there is a data integrity error OR another tool has failed to keep the stored hash up to date. " +
 	"(NOTE In the specific case of downloading a Page Blob that has been used as a VM disk, the VM has probably changed the content since the hash was set. That's normal, and " +
 	"in that specific case you can simply disable the MD5 check. " +
-	"See the AzCopy options documentation for --check-md5.)" +
-	"Also see Notes section here https://docs.azure.cn/en-us/storage/common/storage-use-azcopy-blobs-download#download-a-blob")
+	"See the AzCopy options documentation for --check-md5.) " +
+	"Also see Notes section here https://learn.microsoft.com/en-us/storage/common/storage-use-azcopy-blobs-download#download-a-blob")
 
 // TODO: let's add an aka.ms link to the message, that gives more info
 const noMD5Stored = "no MD5 was stored in the Blob/File service against this file. So the downloaded data cannot be MD5-validated."

traverser/zc_filter.go

@@ -213,7 +213,7 @@ func (f *IncludeFilter) DoesPass(storedObject StoredObject) bool {
 	return false
 }
 
-// getEnumerationPreFilter returns a prefix, if any, which can be used service-side to pre-select
+// getEnumerationPreFilter returns a prefix, if any, which can be used service-side to preselect
 // things that will pass the filter. E.g. if there's exactly one include pattern, and it is
 // "foo*bar", then this routine will return "foo", since only things starting with "foo" can pass the filters.
 // Service side enumeration code can be given that prefix, to optimize the enumeration.