Add support for AMLFS style posix metadata

azcopy/syncEnumerator.go

@@ -189,6 +189,7 @@ func (s *syncer) initEnumerator(ctx context.Context, logLevel common.LogLevel, m
 		PreservePermissions:            s.opts.preservePermissions,
 		PreserveInfo:                   s.opts.preserveInfo,
 		PreservePOSIXProperties:        s.opts.preservePosixProperties,
+		PosixPropertiesStyle:           s.opts.posixPropertiesStyle,
 		S2SSourceChangeValidation:      true,
 		DestLengthValidation:           true,
 		S2SGetPropertiesInBackend:      true,

azcopy/syncOptions.go

@@ -38,6 +38,7 @@ type cookedSyncOptions struct {
 	includeDirectoryStubs   bool
 	preserveInfo            bool
 	preservePosixProperties bool
+	posixPropertiesStyle    common.PosixPropertiesStyle
 	forceIfReadOnly         bool
 	blockSize               int64
 	putBlobSize             int64
@@ -271,7 +272,8 @@ func (s *cookedSyncOptions) validateOptions() (err error) {
 			return err
 		}
 	} else {
-		err = PerformSMBSpecificValidation(s.fromTo, s.preservePermissions, s.preserveInfo, s.preservePosixProperties)
+		err = PerformSMBSpecificValidation(s.fromTo, s.preservePermissions, s.preserveInfo, s.preservePosixProperties,
+			s.posixPropertiesStyle)
 		if err != nil {
 			return err
 		}

azcopy/validationUtil.go

@@ -45,6 +45,7 @@ const (
 	PreservePermissionsFlag                   = "preserve-permissions"
 	PreserveInfoFlag                          = "preserve-info"
 	PreservePOSIXPropertiesIncompatibilityMsg = "to use the --preserve-posix-properties flag, both the source and destination must be POSIX-aware. Valid combinations are: Linux -> Blob, Blob -> Linux, or Blob -> Blob"
+	POSIXStyleMisuse                          = "to use --posix-properties-style flag, it has to be used with preserve-posix-properties. Please include this preserve flag in your AzCopy command"
 	DstShareDoesNotExists                     = "the destination file share does not exist; please create it manually with the required quota and settings before running the copy —refer to https://learn.microsoft.com/en-us/azure/storage/files/storage-how-to-create-file-share?tabs=azure-portal for SMB or https://learn.microsoft.com/en-us/azure/storage/files/storage-files-quick-create-use-linux for NFS."
 )
 
@@ -406,7 +407,8 @@ func PerformNFSSpecificValidation(fromTo common.FromTo,
 func PerformSMBSpecificValidation(fromTo common.FromTo,
 	preservePermissions common.PreservePermissionsOption,
 	preserveInfo bool,
-	preservePOSIXProperties bool) (err error) {
+	preservePOSIXProperties bool,
+	posixStyle common.PosixPropertiesStyle) (err error) {
 
 	if err = validatePreserveSMBPropertyOption(preserveInfo,
 		fromTo,
@@ -416,6 +418,10 @@ func PerformSMBSpecificValidation(fromTo common.FromTo,
 	if preservePOSIXProperties && !areBothLocationsPOSIXAware(fromTo) {
 		return errors.New(PreservePOSIXPropertiesIncompatibilityMsg)
 	}
+
+	if posixStyle.String() != "" && !preservePOSIXProperties {
+		return errors.New(POSIXStyleMisuse)
+	}
 	if err = validatePreserveSMBPropertyOption(preservePermissions.IsTruthy(),
 		fromTo,
 		PreservePermissionsFlag); err != nil {

cmd/copy.go

@@ -121,6 +121,10 @@ type rawCopyCmdArgs struct {
 	preserveSMBInfo bool
 	// Opt-in flag to persist additional POSIX properties
 	preservePOSIXProperties bool
+	// Opt-in flag to specify the style of POSIX properties. Used in-tandem with preservePosixProperties.
+	// Default is "standard"
+	// Using "amlfs" style will preserve properties to be compatible with Azure Managed Lustre File System
+	posixPropertiesStyle string
 	// Opt-in flag to preserve the blob index tags during service to service transfer.
 	s2sPreserveBlobTags bool
 	// Flag to enable Window's special privileges
@@ -529,6 +533,9 @@ func (raw *rawCopyCmdArgs) toOptions() (cooked CookedCopyCmdArgs, err error) {
 	} else {
 		cooked.preserveInfo = raw.preserveInfo && azcopy.AreBothLocationsSMBAware(cooked.FromTo)
 		cooked.preservePOSIXProperties = raw.preservePOSIXProperties
+		if err = cooked.posixPropertiesStyle.Parse(raw.posixPropertiesStyle); err != nil {
+			return cooked, err
+		}
 		cooked.preservePermissions = common.NewPreservePermissionsOption(raw.preservePermissions,
 			raw.preserveOwner,
 			cooked.FromTo)
@@ -693,6 +700,10 @@ type CookedCopyCmdArgs struct {
 	// Whether the user wants to preserve the POSIX properties ...
 	preservePOSIXProperties bool
 
+	// Specifies the style of POSIX properties preserved.
+	// Supported options: 'standard' (default) & 'amlfs' (Azure Managed Lustre File System)
+	posixPropertiesStyle common.PosixPropertiesStyle
+
 	// Whether to enable Windows special privileges
 	backupMode bool
 
@@ -1795,6 +1806,11 @@ Final Job Status: %v%s%s
 	cpCmd.PersistentFlags().BoolVar(&raw.preservePOSIXProperties, "preserve-posix-properties", false,
 		"False by default. 'Preserves' property info gleaned from stat or statx into object metadata.")
 
+	cpCmd.PersistentFlags().StringVar(&raw.posixPropertiesStyle, "posix-properties-style", "standard",
+		" `standard` by default. Use this flag to specify the style of POSIX properties to preserve. "+
+			"\n `amlfs` will preserve POSIX property metadata compatible with Azure Managed Lustre File System."+
+			"\n This flag must be used in-tandem with --preserve-posix-properties.")
+
 	cpCmd.PersistentFlags().BoolVar(&raw.preserveSymlinks, common.PreserveSymlinkFlagName, false,
 		"Preserve symbolic links when performing copy operations involving NFS resources or blob storages. "+
 			"If enabled, the symlink destination is stored as the blob content instead of uploading the file or folder it points to. "+

cmd/copyEnumeratorInit.go

@@ -59,6 +59,7 @@ func (cca *CookedCopyCmdArgs) initEnumerator(jobPartOrder common.CopyJobPartOrde
 	jobPartOrder.PreserveInfo = cca.preserveInfo
 	// We set preservePOSIXProperties if the customer has explicitly asked for this in transfer or if it is just a Posix-property only transfer
 	jobPartOrder.PreservePOSIXProperties = cca.preservePOSIXProperties || (cca.ForceWrite == common.EOverwriteOption.PosixProperties())
+	jobPartOrder.PosixPropertiesStyle = cca.posixPropertiesStyle
 
 	// Infer on download so that we get LMT and MD5 on files download
 	// On S2S transfers the following rules apply:

cmd/copyValidation.go

@@ -104,7 +104,8 @@ func (cooked *CookedCopyCmdArgs) validate() (err error) {
 	} else {
 		if err := azcopy.PerformSMBSpecificValidation(
 			cooked.FromTo, cooked.preservePermissions, cooked.preserveInfo,
-			cooked.preservePOSIXProperties); err != nil {
+			cooked.preservePOSIXProperties,
+			cooked.posixPropertiesStyle); err != nil {
 			return err
 		}
 

common/fe-ste-models.go

@@ -61,6 +61,7 @@ const (
 	RECOMMENDED_OBJECTS_COUNT = 10000000
 	WARN_MULTIPLE_PROCESSES   = "More than one AzCopy process is running. This is a non-blocking warning, AzCopy will continue the operation. \n But, it is best practice to run a single process per VM." +
 		"\nPlease terminate other instances." // This particular warning message does not abort the whole operation
+	AMLFS_MOD_TIME_LAYOUT = "2006-01-02 15:04:05 -0700"
 )
 
 ////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -1890,7 +1891,7 @@ func (sht *SymlinkHandlingType) Determine(Follow, Preserve bool) error {
 	return nil
 }
 
-// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
 
 var oncer = sync.Once{}
 
@@ -1900,3 +1901,33 @@ func WarnIfTooManyObjects() {
 			RECOMMENDED_OBJECTS_COUNT))
 	})
 }
+
+// //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+var EPosixPropertiesStyle = PosixPropertiesStyle(0)
+
+var StandardPosixPropertiesStyle = EPosixPropertiesStyle.Standard()
+var AMLFSPosixPropertiesStyle = EPosixPropertiesStyle.AMLFS()
+
+type PosixPropertiesStyle uint8
+
+// Standard means use the default POSIX properties type
+func (PosixPropertiesStyle) Standard() PosixPropertiesStyle {
+	return PosixPropertiesStyle(0)
+}
+
+// AMLFS means use the Azure Managed Lustre File System POSIX attributes for owner, group ID, mode and modtime
+func (PosixPropertiesStyle) AMLFS() PosixPropertiesStyle {
+	return PosixPropertiesStyle(1)
+}
+
+func (ppt PosixPropertiesStyle) String() string {
+	return enum.StringInt(ppt, reflect.TypeOf(ppt))
+}
+
+func (ppt *PosixPropertiesStyle) Parse(s string) error {
+	val, err := enum.ParseInt(reflect.TypeOf(ppt), s, true, true)
+	if err == nil {
+		*ppt = val.(PosixPropertiesStyle)
+	}
+	return err
+}

common/rpc-models.go

@@ -122,6 +122,7 @@ type CopyJobPartOrderRequest struct {
 	PreservePermissions            PreservePermissionsOption
 	PreserveInfo                   bool
 	PreservePOSIXProperties        bool
+	PosixPropertiesStyle           PosixPropertiesStyle
 	S2SGetPropertiesInBackend      bool
 	S2SSourceChangeValidation      bool
 	DestLengthValidation           bool

common/unixStatAdapter.go

@@ -1,6 +1,8 @@
 package common
 
 import (
+	"errors"
+	"fmt"
 	"os"
 	"strconv"
 	"time"
@@ -22,6 +24,8 @@ const ( // POSIX property metadata
 	POSIXSymlinkMeta       = "is_symlink"
 	POSIXOwnerMeta         = "posix_owner"
 	POSIXGroupMeta         = "posix_group"
+	AMLFSOwnerMeta         = "owner"
+	AMLFSGroupMeta         = "group"
 	POSIXModeMeta          = "permissions"
 	POSIXModTimeMeta       = "modtime"
 	LINUXAttributeMeta     = "linux_attribute"
@@ -50,6 +54,8 @@ var AllLinuxProperties = []string{
 	POSIXCTimeMeta,
 	POSIXModTimeMeta,
 	LINUXAttributeMeta,
+	AMLFSOwnerMeta,
+	AMLFSGroupMeta,
 }
 
 //goland:noinspection GoCommentStart
@@ -216,6 +222,14 @@ func ReadStatFromMetadata(metadata Metadata, contentLength int64) (UnixStatAdapt
 		s.ownerUID = uint32(o)
 	}
 
+	if owner, ok := TryReadMetadata(metadata, AMLFSOwnerMeta); ok {
+		o, err := strconv.ParseUint(*owner, 10, 32)
+		if err != nil {
+			return s, err
+		}
+		s.ownerUID = uint32(o)
+	}
+
 	if group, ok := TryReadMetadata(metadata, POSIXGroupMeta); ok {
 		g, err := strconv.ParseUint(*group, 10, 32)
 		if err != nil {
@@ -224,12 +238,36 @@ func ReadStatFromMetadata(metadata Metadata, contentLength int64) (UnixStatAdapt
 		s.groupGID = uint32(g)
 	}
 
-	if mode, ok := TryReadMetadata(metadata, POSIXModeMeta); ok {
-		m, err := strconv.ParseUint(*mode, 10, 32)
+	if group, ok := TryReadMetadata(metadata, AMLFSGroupMeta); ok {
+		g, err := strconv.ParseUint(*group, 10, 32)
 		if err != nil {
 			return s, err
 		}
+		s.groupGID = uint32(g)
+	}
 
+	// In cases, the permissions were uploaded in AMLFS style, determine what base to use
+	looksLikeOctal := func(s string) bool {
+		if len(s) < 1 || len(s) > 4 {
+			return false
+		}
+		for _, r := range s {
+			if r < '0' || r > '7' {
+				return false
+			}
+		}
+		return true
+	}
+
+	if modeStr, ok := TryReadMetadata(metadata, POSIXModeMeta); ok {
+		base := 10
+		if looksLikeOctal(*modeStr) {
+			base = 8
+		}
+		m, err := strconv.ParseUint(*modeStr, base, 32)
+		if err != nil {
+			return s, err
+		}
 		s.mode = uint32(m)
 	}
 
@@ -269,9 +307,16 @@ func ReadStatFromMetadata(metadata Metadata, contentLength int64) (UnixStatAdapt
 		s.accessTime = time.Unix(0, at)
 	}
 
+	// Always store ModTime in standard POSIX style
 	if mtime, ok := TryReadMetadata(metadata, POSIXModTimeMeta); ok {
 		mt, err := strconv.ParseInt(*mtime, 10, 64)
-		if err != nil {
+		if errors.Is(err, strconv.ErrSyntax) {
+			amlfsTime, err := time.Parse(AMLFS_MOD_TIME_LAYOUT, *mtime)
+			if err != nil {
+				return s, fmt.Errorf("could not parse metadata time: %w", err)
+			}
+			mt = amlfsTime.UnixNano()
+		} else if err != nil {
 			return s, err
 		}
 
@@ -342,11 +387,12 @@ func ClearStatFromBlobMetadata(metadata Metadata) {
 	}
 }
 
-func AddStatToBlobMetadata(s UnixStatAdapter, metadata Metadata) {
+func AddStatToBlobMetadata(s UnixStatAdapter, metadata Metadata, posixStyle PosixPropertiesStyle) {
 	if s == nil {
 		return
 	}
 
+	// applyMode extracts the file type (symlink, folder etc) from raw Unix file mode and adds the corresponding metadata
 	applyMode := func(mode os.FileMode) {
 		modes := map[uint32]string{
 			S_IFCHR:  POSIXCharDeviceMeta,
@@ -380,16 +426,30 @@ func AddStatToBlobMetadata(s UnixStatAdapter, metadata Metadata) {
 		}
 
 		if StatXReturned(mask, STATX_UID) {
-			TryAddMetadata(metadata, POSIXOwnerMeta, strconv.FormatUint(uint64(s.Owner()), 10))
+			if posixStyle == AMLFSPosixPropertiesStyle {
+				TryAddMetadata(metadata, AMLFSOwnerMeta, strconv.FormatUint(uint64(s.Owner()), 10))
+			} else {
+				TryAddMetadata(metadata, POSIXOwnerMeta, strconv.FormatUint(uint64(s.Owner()), 10))
+			}
 		}
 
 		if StatXReturned(mask, STATX_GID) {
-			TryAddMetadata(metadata, POSIXGroupMeta, strconv.FormatUint(uint64(s.Group()), 10))
+			if posixStyle == AMLFSPosixPropertiesStyle {
+				TryAddMetadata(metadata, AMLFSGroupMeta, strconv.FormatUint(uint64(s.Group()), 10))
+			} else {
+				TryAddMetadata(metadata, POSIXGroupMeta, strconv.FormatUint(uint64(s.Group()), 10))
+			}
 		}
 
 		if StatXReturned(mask, STATX_MODE) {
-			TryAddMetadata(metadata, POSIXModeMeta, strconv.FormatUint(uint64(s.FileMode()), 10))
-			applyMode(os.FileMode(s.FileMode()))
+			if posixStyle == AMLFSPosixPropertiesStyle {
+				permissions := fmt.Sprintf("%04o", uint64(s.FileMode()))                  // AMLFS uses octal
+				TryAddMetadata(metadata, POSIXModeMeta, permissions[len(permissions)-4:]) // only needs permission bits
+				applyMode(os.FileMode(s.FileMode()))
+			} else {
+				TryAddMetadata(metadata, POSIXModeMeta, strconv.FormatUint(uint64(s.FileMode()), 10))
+				applyMode(os.FileMode(s.FileMode()))
+			}
 		}
 
 		if StatXReturned(mask, STATX_INO) {
@@ -410,7 +470,11 @@ func AddStatToBlobMetadata(s UnixStatAdapter, metadata Metadata) {
 		}
 
 		if StatXReturned(mask, STATX_MTIME) {
-			TryAddMetadata(metadata, POSIXModTimeMeta, strconv.FormatInt(s.MTime().UnixNano(), 10))
+			if posixStyle == AMLFSPosixPropertiesStyle {
+				TryAddMetadata(metadata, POSIXModTimeMeta, s.MTime().Format(AMLFS_MOD_TIME_LAYOUT))
+			} else {
+				TryAddMetadata(metadata, POSIXModTimeMeta, strconv.FormatInt(s.MTime().UnixNano(), 10))
+			}
 		}
 
 		if StatXReturned(mask, STATX_CTIME) {