Refactor environment variable implementation

azcopy/client.go

@@ -87,13 +87,13 @@ func NewClient(opts ClientOptions) (Client, error) {
 	if common.AzcopyJobPlanFolder == "" {
 		panic("invalid state, AzcopyJobPlanFolder should not be an empty string")
 	}
-	cacheName := common.GetEnvironmentVariable(common.EEnvironmentVariable.LoginCacheName())
+	cacheName, _, specifiedCache := common.EEnvironmentVariable.LoginCacheName().Lookup()
 
 	c.oauthTokenManager = common.NewUserOAuthTokenManagerInstance(common.CredCacheOptions{
 		DPAPIFilePath: common.AzcopyJobPlanFolder,
-		KeyName:       common.Iff(cacheName != "", cacheName, oauthLoginSessionCacheKeyName),
+		KeyName:       common.Iff(specifiedCache, cacheName, oauthLoginSessionCacheKeyName),
 		ServiceName:   oauthLoginSessionCacheServiceName,
-		AccountName:   common.Iff(cacheName != "", cacheName, oauthLoginSessionCacheAccountName),
+		AccountName:   common.Iff(specifiedCache, cacheName, oauthLoginSessionCacheAccountName),
 	})
 	return c, nil
 }

azcopy/copy.go

@@ -184,11 +184,10 @@ func (c *Client) Copy(ctx context.Context, src, dest string, opts CopyOptions) (
 		err = t.redirectionTransfer(ctx)
 		return CopyResult{}, err
 	} else {
-
 		// Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value
 		if jobsAdmin.JobsAdmin != nil &&
 			(t.opts.fromTo.From().IsFile() || (t.opts.fromTo.To().IsFile() &&
-				common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "")) {
+				!common.EEnvironmentVariable.ConcurrencyValue().IsSet())) {
 			jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
 		}
 

azcopy/copyRedirection.go

@@ -22,7 +22,6 @@ package azcopy
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -53,13 +52,13 @@ func (e *transferExecutor) redirectionTransfer(ctx context.Context) error {
 // redirectionBlobUpload uploads data from os.stdin to a blob destination using piping (redirection).
 func (e *transferExecutor) redirectionBlobUpload(ctx context.Context) (err error) {
 	// Use the concurrency environment value
-	concurrencyEnvVar := common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue())
+	concurrencyEnvVar, lookupName, ok := common.EEnvironmentVariable.ConcurrencyValue().Lookup()
 
 	pipingUploadParallelism := PipingUploadParallelismDefault
-	if concurrencyEnvVar != "" {
+	if ok {
 		// handle when the concurrency value is AUTO
 		if concurrencyEnvVar == "AUTO" {
-			return errors.New("concurrency auto-tuning is not possible when using redirection transfers (AZCOPY_CONCURRENCY_VALUE = AUTO)")
+			return fmt.Errorf("concurrency auto-tuning is not possible when using redirection transfers (%s = AUTO)", lookupName)
 		}
 
 		// convert the concurrency value to
@@ -68,7 +67,7 @@ func (e *transferExecutor) redirectionBlobUpload(ctx context.Context) (err error
 
 		//handle the error if the conversion fails
 		if err != nil {
-			return fmt.Errorf("AZCOPY_CONCURRENCY_VALUE is not set to a valid value, an integer is expected (current value: %s): %w", concurrencyEnvVar, err)
+			return fmt.Errorf("%s is not set to a valid value, an integer is expected (current value: %s): %w", lookupName, concurrencyEnvVar, err)
 		}
 
 		pipingUploadParallelism = int(concurrencyValue) // Cast to Integer

azcopy/credentialUtil.go

@@ -62,16 +62,17 @@ var stashedEnvCredType = ""
 func GetCredTypeFromEnvVar() common.CredentialType {
 	rawVal := stashedEnvCredType
 	if stashedEnvCredType == "" {
-		rawVal = common.GetEnvironmentVariable(common.EEnvironmentVariable.CredentialType())
-		if rawVal == "" {
+		var ok bool
+		rawVal, _, ok = common.EEnvironmentVariable.CredentialType().Lookup()
+		if !ok {
 			return common.ECredentialType.Unknown()
 		}
 		stashedEnvCredType = rawVal
 	}
 
 	// Remove the env var after successfully fetching once,
 	// in case of env var is further spreading into child processes unexpectedly.
-	common.ClearEnvironmentVariable(common.EEnvironmentVariable.CredentialType())
+	common.EEnvironmentVariable.CredentialType().Clear()
 
 	// Try to get the value set.
 	var credType common.CredentialType
@@ -340,9 +341,7 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
 	}
 
 	if location == common.ELocation.S3() {
-		accessKeyID := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSAccessKeyID())
-		secretAccessKey := common.GetEnvironmentVariable(common.EEnvironmentVariable.AWSSecretAccessKey())
-		if accessKeyID == "" || secretAccessKey == "" {
+		if !common.EEnvironmentVariable.AWSAccessKeyID().IsSet() || !common.EEnvironmentVariable.AWSSecretAccessKey().IsSet() {
 			credType = common.ECredentialType.S3PublicBucket()
 			public = true
 			return
@@ -353,8 +352,7 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
 	}
 
 	if location == common.ELocation.GCP() {
-		googleAppCredentials := common.GetEnvironmentVariable(common.EEnvironmentVariable.GoogleAppCredentials())
-		if googleAppCredentials == "" {
+		if !common.EEnvironmentVariable.GoogleAppCredentials().IsSet() {
 			return common.ECredentialType.Unknown(), false, errors.New("GOOGLE_APPLICATION_CREDENTIALS environment variable must be set before using GCP transfer feature")
 		}
 		credType = common.ECredentialType.GoogleAppCredentials()
@@ -415,9 +413,7 @@ func doGetCredentialTypeForLocation(ctx context.Context, location common.Locatio
 	// BlobFS currently supports Shared key. Remove this piece of code, once
 	// we deprecate that.
 	if location == common.ELocation.BlobFS() {
-		name := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountName())
-		key := common.GetEnvironmentVariable(common.EEnvironmentVariable.AccountKey())
-		if name != "" && key != "" { // TODO: To remove, use for internal testing, SharedKey should not be supported from commandline
+		if common.EEnvironmentVariable.AccountName().IsSet() && common.EEnvironmentVariable.AccountKey().IsSet() { // TODO: To remove, use for internal testing, SharedKey should not be supported from commandline
 			credType = common.ECredentialType.SharedKey()
 			warnIfSharedKeyAuthForDatalake()
 		}

azcopy/output.go

@@ -201,7 +201,7 @@ func GetPerfDisplayText(perfDiagnosticStrings []string, constraint common.PerfCo
 }
 
 func shouldDisplayPerfStates() bool {
-	return common.GetEnvironmentVariable(common.EEnvironmentVariable.ShowPerfStates()) != ""
+	return common.EEnvironmentVariable.ShowPerfStates().IsSet()
 }
 
 // round api rounds up the float number after the decimal point.

cmd/copy.go

@@ -852,7 +852,7 @@ func (cca *CookedCopyCmdArgs) processCopyJobPartOrders() (err error) {
 	// Make AUTO default for Azure Files since Azure Files throttles too easily unless user specified concurrency value
 	if jobsAdmin.JobsAdmin != nil &&
 		(cca.FromTo.From().IsFile() || cca.FromTo.To().IsFile()) &&
-		common.GetEnvironmentVariable(common.EEnvironmentVariable.ConcurrencyValue()) == "" {
+		!common.EEnvironmentVariable.ConcurrencyValue().IsSet() {
 		jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
 	}
 

cmd/env.go

@@ -16,6 +16,7 @@ package cmd
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/Azure/azure-storage-azcopy/v10/common"
 	"github.com/spf13/cobra"
@@ -29,16 +30,35 @@ var envCmd = &cobra.Command{
 	Short: envCmdShortDescription,
 	Long:  envCmdLongDescription,
 	Run: func(cmd *cobra.Command, args []string) {
+		setDeprecatedVars := make(map[string]string)
+
 		for _, env := range common.VisibleEnvironmentVariables {
-			val := common.GetEnvironmentVariable(env)
-			if env.Hidden && !showSensitive {
+			// do a direct lookup, instead of calling indirect lookup since indirect can get deprecated values.
+			val, ok := os.LookupEnv(env.Name)
+
+			if env.Hidden && !showSensitive && ok {
 				val = "REDACTED"
+			} else if !ok {
+				val = "<Unset>"
+			}
+
+			// if this is a deprecated env var, don't display it unless we have a value. If we do display it, stash a warning to be displayed at the end, closest to the user's line of sight.
+			if env.ReplacedBy != "" {
+				if !ok {
+					continue // skip displaying it
+				} else {
+					setDeprecatedVars[env.Name] = env.ReplacedBy
+				}
 			}
 
 			glcm.Info(fmt.Sprintf("Name: %s\nCurrent Value: %s\nDescription: %s\n",
 				env.Name, val, env.Description))
 		}
 
+		for depName, newName := range setDeprecatedVars {
+			glcm.Info(fmt.Sprintf("Deprecated environment variable %s was set. Please check the new variable, %s, and migrate to it, as the deprecated variable may be removed in a future version.", depName, newName))
+		}
+
 		glcm.Exit(nil, EExitCode.Success())
 	},
 }

cmd/login.go

@@ -137,10 +137,10 @@ func (args rawLoginArgs) toOptions() (azcopy.LoginOptions, error) {
 	if err != nil {
 		return azcopy.LoginOptions{}, err
 	}
-	certificatePassword := common.GetEnvironmentVariable(common.EEnvironmentVariable.CertificatePassword())
-	clientSecret := common.GetEnvironmentVariable(common.EEnvironmentVariable.ClientSecret())
+	certificatePassword, _, certPasswordOK := common.EEnvironmentVariable.CertificatePassword().Lookup()
+	clientSecret, _, clientSecretOK := common.EEnvironmentVariable.ClientSecret().Lookup()
 
-	if certificatePassword != "" || clientSecret != "" {
+	if certPasswordOK || clientSecretOK {
 		glcm.Info(environmentVariableNotice)
 	}
 	return azcopy.LoginOptions{

cmd/root.go

@@ -240,14 +240,13 @@ func Initialize(isMigratedToLibrary, isBench, shouldWarn bool) (err error) {
 
 	// For benchmarking, try to autotune if possible, otherwise use the default values
 	if jobsAdmin.JobsAdmin != nil && isBench {
-		envVar := common.EEnvironmentVariable.ConcurrencyValue()
-		userValue := common.GetEnvironmentVariable(envVar)
-		if userValue == "" || userValue == "auto" {
+		userValue, varName, userSpecified := common.EEnvironmentVariable.ConcurrencyValue().Lookup()
+		if !userSpecified || userValue == "auto" {
 			jobsAdmin.JobsAdmin.SetConcurrencySettingsToAuto()
 		} else {
 			// Tell user that we can't actually auto tune, because configured value takes precedence
 			// This case happens when benchmarking with a fixed value from the env var
-			glcm.Info(fmt.Sprintf("Cannot auto-tune concurrency because it is fixed by environment variable %s", envVar.Name))
+			glcm.Info(fmt.Sprintf("Cannot auto-tune concurrency because it is fixed by environment variable %s", varName))
 		}
 	}
 	if !isMigratedToLibrary {

cmd/uihooks.go

@@ -237,8 +237,8 @@ func (lcm *lifecycleMgr) checkAndStartCPUProfiling() {
 	// the value AZCOPY_PROFILE_CPU indicates the path to save CPU profiling data.
 	// e.g. export AZCOPY_PROFILE_CPU="cpu.prof"
 	// For more details, please refer to https://golang.org/pkg/runtime/pprof/
-	cpuProfilePath := common.GetEnvironmentVariable(common.EEnvironmentVariable.ProfileCPU())
-	if cpuProfilePath != "" {
+	cpuProfilePath, _, ok := common.EEnvironmentVariable.ProfileCPU().Lookup()
+	if ok {
 		lcm.Info(fmt.Sprintf("pprof start CPU profiling, and saving profiling data to: %q", cpuProfilePath))
 		f, err := os.Create(cpuProfilePath)
 		if err != nil {
@@ -260,8 +260,8 @@ func (lcm *lifecycleMgr) checkAndTriggerMemoryProfiling() {
 	// the value AZCOPY_PROFILE_MEM indicates the path to save memory profiling data.
 	// e.g. export AZCOPY_PROFILE_MEM="mem.prof"
 	// For more details, please refer to https://golang.org/pkg/runtime/pprof/
-	memProfilePath := common.GetEnvironmentVariable(common.EEnvironmentVariable.ProfileMemory())
-	if memProfilePath != "" {
+	memProfilePath, _, ok := common.EEnvironmentVariable.ProfileMemory().Lookup()
+	if ok {
 		lcm.Info(fmt.Sprintf("pprof start memory profiling, and saving profiling data to: %q", memProfilePath))
 		f, err := os.Create(memProfilePath)
 		if err != nil {

cmd/zt_generic_service_traverser_test.go

@@ -321,7 +321,7 @@ func TestServiceTraverserWithWildcards(t *testing.T) {
 		gcpAccountURL.BucketName = "objectmatch*"
 		urlStr := gcpAccountURL.URL()
 
-		gcpServiceTraverser, err := traverser.NewGCPServiceTraverser(&urlStr, ctx, traverser.InitResourceTraverserOptions{})
+		gcpServiceTraverser, err := traverser.NewGCPServiceTraverser(&urlStr, common.EEnvironmentVariable.GoogleCloudProject().Value(), ctx, traverser.InitResourceTraverserOptions{})
 		a.Nil(err)
 
 		gcpDummyProcessor = dummyProcessor{}

common/ProxyLookupCache.go

@@ -43,7 +43,7 @@ func init() {
 		lookupMethod:    GetProxyFunc(),
 	}
 
-	ev := GetEnvironmentVariable(EEnvironmentVariable.CacheProxyLookup())
+	ev := EEnvironmentVariable.CacheProxyLookup().Value()
 	if strings.ToLower(ev) == "true" {
 		GlobalProxyLookup = c.getProxy
 	} else {

common/clientFactory.go

@@ -22,25 +22,26 @@ package common
 
 import (
 	"fmt"
+
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob"
 	"github.com/Azure/azure-sdk-for-go/sdk/storage/azdatalake"
 )
 
 func GetDatalakeSharedKeyCredential() (*azdatalake.SharedKeyCredential, error) {
-	name := GetEnvironmentVariable(EEnvironmentVariable.AccountName())
-	key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
+	name, _, nameOK := EEnvironmentVariable.AccountName().Lookup()
+	key, _, keyOK := EEnvironmentVariable.AccountKey().Lookup()
 	// If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables
-	if name == "" || key == "" {
+	if !nameOK || !keyOK {
 		return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential")
 	}
 	return azdatalake.NewSharedKeyCredential(name, key)
 }
 
 func GetBlobSharedKeyCredential() (*blob.SharedKeyCredential, error) {
-	name := GetEnvironmentVariable(EEnvironmentVariable.AccountName())
-	key := GetEnvironmentVariable(EEnvironmentVariable.AccountKey())
+	name, _, nameOK := EEnvironmentVariable.AccountName().Lookup()
+	key, _, keyOK := EEnvironmentVariable.AccountKey().Lookup()
 	// If the ACCOUNT_NAME and ACCOUNT_KEY are not set in environment variables
-	if name == "" || key == "" {
+	if !nameOK || !keyOK {
 		return nil, fmt.Errorf("ACCOUNT_NAME and ACCOUNT_KEY environment variables must be set before creating the SharedKey credential")
 	}
 	return blob.NewSharedKeyCredential(name, key)

common/credCache_windows.go

@@ -197,7 +197,7 @@ func (c *CredCache) saveTokenInternal(token OAuthTokenInfo) error {
 }
 
 func (c *CredCache) tokenFilePath() string {
-	if cacheFile := GetEnvironmentVariable(EEnvironmentVariable.LoginCacheName()); cacheFile != "" {
+	if cacheFile, _, ok := EEnvironmentVariable.LoginCacheName().Lookup(); ok {
 		return path.Join(c.dpapiFilePath, "/", cacheFile)
 	}
 

common/credentialFactory.go

@@ -43,9 +43,9 @@ func CreateS3Credential(ctx context.Context, credInfo CredentialInfo) (*credenti
 	case ECredentialType.S3PublicBucket():
 		return credentials.NewStatic("", "", "", credentials.SignatureAnonymous), nil
 	case ECredentialType.S3AccessKey():
-		accessKeyID := GetEnvironmentVariable(EEnvironmentVariable.AWSAccessKeyID())
-		secretAccessKey := GetEnvironmentVariable(EEnvironmentVariable.AWSSecretAccessKey())
-		sessionToken := GetEnvironmentVariable(EEnvironmentVariable.AwsSessionToken())
+		accessKeyID := EEnvironmentVariable.AWSAccessKeyID().Value()
+		secretAccessKey := EEnvironmentVariable.AWSSecretAccessKey().Value()
+		sessionToken := EEnvironmentVariable.AwsSessionToken().Value()
 
 		// create and return s3 credential
 		return credentials.NewStaticV4(accessKeyID, secretAccessKey, sessionToken), nil // S3 uses V4 signature
@@ -159,11 +159,11 @@ func GetCpkInfo(cpkInfo bool) (*blob.CPKInfo, error) {
 	}
 
 	// fetch EncryptionKey and EncryptionKeySHA256 from the environment variables
-	encryptionKey := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKey())
-	encryptionKeySHA256 := GetEnvironmentVariable(EEnvironmentVariable.CPKEncryptionKeySHA256())
+	encryptionKey, _, keyOk := EEnvironmentVariable.CPKEncryptionKey().Lookup()
+	encryptionKeySHA256, _, sha256Ok := EEnvironmentVariable.CPKEncryptionKeySHA256().Lookup()
 	encryptionAlgorithmAES256 := blob.EncryptionAlgorithmTypeAES256
 
-	if encryptionKey == "" || encryptionKeySHA256 == "" {
+	if !keyOk || !sha256Ok {
 		return nil, errors.New("fatal: failed to fetch cpk encryption key (" + EEnvironmentVariable.CPKEncryptionKey().Name +
 			") or hash (" + EEnvironmentVariable.CPKEncryptionKeySHA256().Name + ") from environment variables")
 	}