Resolve conflicts

Author: vmapetr

.npmrc

@@ -0,0 +1,3 @@
+registry=https://pkgs.dev.azure.com/mseng/PipelineTools/_packaging/PipelineTools_PublicPackages/npm/registry/
+
+always-auth=true

BuildConfigGen/EnsureUpdateModeVerifier.cs

@@ -33,6 +33,25 @@ public IEnumerable<string> GetVerifyErrors(bool skipContentCheck)
 
             if (!skipContentCheck)
             {
+                foreach(var r in RedirectedToTempl)
+                {
+                    string? sourceFile;
+                    string procesed = "";
+
+                    string? tempToKeep = null;
+                    sourceFile = r.Value;
+                    procesed = $"(processed temp={sourceFile}) ";
+                    tempToKeep = sourceFile;
+
+                    string? contentError = null;
+                    contentError = CompareFile(r, sourceFile, procesed, tempToKeep, contentError);
+
+                    if (contentError is not null)
+                    {
+                        yield return contentError;
+                    }
+                }
+
                 foreach (var r in CopiedFilesToCheck)
                 {
                     string? sourceFile;
@@ -49,30 +68,44 @@ public IEnumerable<string> GetVerifyErrors(bool skipContentCheck)
                         sourceFile = r.Value;
                     }
 
-                    FileInfo fi = new FileInfo(r.Key);
+                    string? contentError = null;
+                    contentError = CompareFile(r, sourceFile, procesed, tempToKeep, contentError);
 
-                    if (fi.Name.Equals("resources.resjson", StringComparison.OrdinalIgnoreCase))
+                    if (contentError is not null)
                     {
-                        // resources.resjson is generated by make.js and does not need to be verified
-                        // it can differ between configs if configs have different inputs (causes verifier to fail);
+                        yield return contentError;
                     }
-                    else
+                }
+            }
+        }
+
+        private string? CompareFile(KeyValuePair<string, string> r, string sourceFile, string procesed, string? tempToKeep, string? contentError)
+        {
+            FileInfo fi = new FileInfo(r.Key);
+
+            if (fi.Name.Equals("resources.resjson", StringComparison.OrdinalIgnoreCase))
+            {
+                // resources.resjson is generated by make.js and does not need to be verified
+                // it can differ between configs if configs have different inputs (causes verifier to fail);
+            }
+            else
+            {
+                if (Helpers.FilesEqual(sourceFile, r.Key))
+                {
+                    // if overwrite and content match, everything is good!  Verification passed.
+                }
+                else
+                {
+                    if (tempToKeep != null)
                     {
-                        if (Helpers.FilesEqual(sourceFile, r.Key))
-                        {
-                            // if overwrite and content match, everything is good!  Verification passed.
-                        }
-                        else
-                        {
-                            if (tempToKeep != null)
-                            {
-                                this.tempsToKeep.Add(tempToKeep!);
-                            }
-                            yield return $"Content doesn't match {r.Value} {procesed}to {r.Key} (overwrite=true).  Dest file doesn't match source.";
-                        }
+                        this.tempsToKeep.Add(tempToKeep!);
                     }
+
+                    contentError = $"Content doesn't match {r.Value} {procesed}to {r.Key} (overwrite=true).  Dest file doesn't match source.";
                 }
             }
+
+            return contentError;
         }
 
         public void CleanupTempFiles()

BuildConfigGen/Program.cs

@@ -192,7 +192,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                         }
 
                         UpdateMaxPatchForSprint(taskVersionInfo[t.Value.Name], currentSprint, ref maxPatchForCurrentSprint);
-                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping);
+                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping, checkGlobalVersion: false);
                     }
 
                     if (tasksNeedingUpdates.Count > 0)
@@ -201,11 +201,14 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                     }
 
                     // bump patch number for global if any tasks invalidated or if there is no existing global version
-                    if (taskVersionInfo.Values.Any(x => x.versionsUpdated.Any()) || globalVersion is null)
+                    bool anyTaskVersionUpdated = taskVersionInfo.Values.Any(x => x.versionsUpdated.Any());
+                    bool noCurrentGlobalVersion = globalVersion is null;
+                    bool maxPatchForCurrentSprintGreaterOrEqualToGlobalPatch = globalVersion is not null && maxPatchForCurrentSprint >= globalVersion.Patch;
+                    if (anyTaskVersionUpdated || noCurrentGlobalVersion || maxPatchForCurrentSprintGreaterOrEqualToGlobalPatch)
                     {
                         maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1;
 
-                        Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1");
+                        Console.WriteLine($"Global version: maxPatchForCurrentSprint = maxPatchForCurrentSprint + 1 anyTaskVersionUpdated={anyTaskVersionUpdated} noCurrentGlobalVersion={noCurrentGlobalVersion} maxPatchForCurrentSprintGreaterOrEqualToGlobalPatch={maxPatchForCurrentSprintGreaterOrEqualToGlobalPatch}");
                     }
 
                     Console.WriteLine($"Global version update: globalVersion = {globalVersion} maxPatchForCurrentSprint={maxPatchForCurrentSprint}");
@@ -218,7 +221,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                         IEnumerable<string> configsList = FilterConfigsForTask(configs, t);
                         HashSet<Config.ConfigRecord> targetConfigs = GetConfigRecords(configsList, writeUpdates);
                         UpdateVersionsForTask(t.Value.Name, taskVersionInfo[t.Value.Name], targetConfigs, currentSprint, globalVersionPath, globalVersion, generatedFolder);
-                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping);
+                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping, checkGlobalVersion: true);
                     }
                 }
 
@@ -264,7 +267,7 @@ private static void MainInner(string? task, string? configs, int? currentSprintN
                         HashSet<Config.ConfigRecord> targetConfigs = GetConfigRecords(configsList, writeUpdates);
 
                         UpdateVersionsGlobal(t.Value.Name, taskVersionInfo[t.Value.Name], targetConfigs, globalVersion);
-                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping);
+                        CheckForDuplicates(t.Value.Name, taskVersionInfo[t.Value.Name].configTaskVersionMapping, checkGlobalVersion: true);
                     }
                 }
 
@@ -335,15 +338,25 @@ private static IEnumerable<string> FilterConfigsForTask(string? configs, KeyValu
             return configsList;
         }
 
-        private static void CheckForDuplicates(string task, Dictionary<Config.ConfigRecord, TaskVersion> configTaskVersionMapping)
+        private static void CheckForDuplicates(string task, Dictionary<Config.ConfigRecord, TaskVersion> configTaskVersionMapping, bool checkGlobalVersion)
         {
-            var duplicateVersions = configTaskVersionMapping.GroupBy(x => x.Value).Select(x => new { version = x.Key, configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1);
+            var duplicateVersions = configTaskVersionMapping
+                .Where(x => checkGlobalVersion || !x.Key.useGlobalVersion)
+                .GroupBy(x => x.Value)
+                .Select(x => new { version = x.Key, hasGlobal = x.Where(x => x.Key.useGlobalVersion).Any(), configName = String.Join(",", x.Select(x => x.Key.name)), count = x.Count() }).Where(x => x.count > 1);
             if (duplicateVersions.Any())
             {
                 StringBuilder dupConfigsStr = new StringBuilder();
                 foreach (var x in duplicateVersions)
                 {
-                    dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}");
+                    if (x.hasGlobal)
+                    {
+                        dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}.  To fix, check-in globalversion.txt change generated by running 'node make.js build --task {task}   --includeLocalPackagesBuildConfig'");
+                    }
+                    else
+                    {
+                        dupConfigsStr.AppendLine($"task={task} version={x.version} specified in multiple configName={x.configName} config count={x.count}");
+                    }
                 }
 
                 throw new Exception(dupConfigsStr.ToString());
@@ -549,7 +562,7 @@ private static void MainUpdateTask(
 
                         if (config.enableBuildConfigOverrides)
                         {
-                            EnsureBuildConfigFileOverrides(config, taskTargetPath);
+                            EnsureBuildConfigFileOverrides(config, taskTargetPath, generatedFolder, task);
                         }
 
                         try
@@ -600,7 +613,7 @@ private static void MainUpdateTask(
 
                                 if (config.enableBuildConfigOverrides)
                                 {
-                                    CopyConfigOverrides(gitRootPath, taskTargetPath, taskOutput, config);
+                                    CopyConfigOverrides(gitRootPath, taskTargetPath, taskOutput, config, generatedFolder, task);
                                 }
 
                                 // if some files aren't present in destination, stop as following code assumes they're present and we'll just get a FileNotFoundException
@@ -620,7 +633,7 @@ private static void MainUpdateTask(
 
                             if (config.isNode)
                             {
-                                GetBuildConfigFileOverridePaths(config, taskTargetPath, out string configTaskPath, out string readmePath);
+                                GetBuildConfigFileOverridePaths(config, taskTargetPath, out string configTaskPath, out string readmePath, generatedFolder, task);
 
                                 string buildConfigPackageJsonPath = Path.Combine(taskTargetPath, buildConfigs, configTaskPath, "package.json");
 
@@ -781,15 +794,15 @@ private static bool HasTaskInputContainsPreprocessorInstructions(string gitRootP
             return hasPreprocessorDirectives;
         }
 
-        private static void EnsureBuildConfigFileOverrides(Config.ConfigRecord config, string taskTargetPath)
+        private static void EnsureBuildConfigFileOverrides(Config.ConfigRecord config, string taskTargetPath, string generatedFolder, string taskName)
         {
             if (!config.enableBuildConfigOverrides)
             {
                 throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides");
             }
 
             string path, readmeFile;
-            GetBuildConfigFileOverridePaths(config, taskTargetPath, out path, out readmeFile);
+            GetBuildConfigFileOverridePaths(config, taskTargetPath, out path, out readmeFile, generatedFolder, taskName);
 
             if (!Directory.Exists(path))
             {
@@ -799,7 +812,7 @@ private static void EnsureBuildConfigFileOverrides(Config.ConfigRecord config, s
             ensureUpdateModeVerifier!.WriteAllText(readmeFile, "Place files overridden for this config in this directory", suppressValidationErrorIfTargetPathDoesntExist: !Knob.Default.SourceDirectoriesMustContainPlaceHolders);
         }
 
-        private static void GetBuildConfigFileOverridePaths(Config.ConfigRecord config, string taskTargetPath, out string path, out string readmeFile)
+        private static void GetBuildConfigFileOverridePaths(Config.ConfigRecord config, string taskTargetPath, out string path, out string readmeFile, string generatedFolder, string taskName)
         {
             string directoryName = config.name;
 
@@ -813,19 +826,28 @@ private static void GetBuildConfigFileOverridePaths(Config.ConfigRecord config,
                 directoryName = config.overriddenDirectoryName;
             }
 
-            path = Path.Combine(taskTargetPath, buildConfigs, directoryName);
-            readmeFile = Path.Combine(taskTargetPath, buildConfigs, directoryName, filesOverriddenForConfigGoHereReadmeTxt);
+            if (config.useGlobalVersion)
+            {
+                // for global version, place artifacts in _generated (such as package-lock)
+                path = Path.Combine(generatedFolder, buildConfigs, taskName, directoryName);
+                readmeFile = Path.Combine(generatedFolder, buildConfigs, taskName, directoryName, filesOverriddenForConfigGoHereReadmeTxt);
+            }
+            else
+            {
+                path = Path.Combine(taskTargetPath, buildConfigs, directoryName);
+                readmeFile = Path.Combine(taskTargetPath, buildConfigs, directoryName, filesOverriddenForConfigGoHereReadmeTxt);
+            }
         }
 
-        private static void CopyConfigOverrides(string gitRootPath, string taskTargetPath, string taskOutput, Config.ConfigRecord config)
+        private static void CopyConfigOverrides(string gitRootPath, string taskTargetPath, string taskOutput, Config.ConfigRecord config, string generatedFolder, string taskName)
         {
             if (!config.enableBuildConfigOverrides)
             {
                 throw new Exception("BUG: should not get here: !config.enableBuildConfigOverrides");
             }
 
             string overridePathForBuildConfig;
-            GetBuildConfigFileOverridePaths(config, taskTargetPath, out overridePathForBuildConfig, out _);
+            GetBuildConfigFileOverridePaths(config, taskTargetPath, out overridePathForBuildConfig, out _, generatedFolder, taskName);
 
             bool doCopy;
             if (Knob.Default.SourceDirectoriesMustContainPlaceHolders)

BuildConfigGen/dev.sh

@@ -36,13 +36,11 @@ function detect_platform_and_runtime_id ()
     fi
 }
 
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPT_DIR=$1
 pushd "$SCRIPT_DIR"
 source "$SCRIPT_DIR/Misc/helpers.sh"
-
-DOTNETSDK_ROOT="$SCRIPT_DIR/_dotnetsdk"
-DOTNETSDK_VERSION="8.0.100"
-DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION"
+DOTNETSDK_INSTALLDIR=$2
+DOTNETSDK_VERSION=$3
 
 detect_platform_and_runtime_id
 echo "Current platform: $CURRENT_PLATFORM"
@@ -63,10 +61,8 @@ if [[ (! -d "${DOTNETSDK_INSTALLDIR}") || (! -e "${DOTNETSDK_INSTALLDIR}/.${DOTN
     # run dotnet-install.ps1 on windows, dotnet-install.sh on linux
     if [[ ("$CURRENT_PLATFORM" == "windows") ]]; then
         echo "Convert ${DOTNETSDK_INSTALLDIR} to Windows style path"
-        sdkinstallwindow_path=${DOTNETSDK_INSTALLDIR:1}
-        sdkinstallwindow_path=${sdkinstallwindow_path:0:1}:${sdkinstallwindow_path:1}
         architecture=$( echo $DETECTED_RUNTIME_ID | cut -d "-" -f2)
-        powershell -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "& \"./Misc/dotnet-install.ps1\" -Version ${DOTNETSDK_VERSION} -InstallDir \"${sdkinstallwindow_path}\" -Architecture ${architecture}  -NoPath; exit \$LastExitCode;" || checkRC dotnet-install.ps1
+        powershell -NoLogo -Sta -NoProfile -NonInteractive -ExecutionPolicy Unrestricted -Command "& \"./Misc/dotnet-install.ps1\" -Version ${DOTNETSDK_VERSION} -InstallDir \"${DOTNETSDK_INSTALLDIR}\" -Architecture ${architecture}  -NoPath; exit \$LastExitCode;" || checkRC dotnet-install.ps1
     else
         bash ./Misc/dotnet-install.sh --version ${DOTNETSDK_VERSION} --install-dir "${DOTNETSDK_INSTALLDIR}" --no-path || checkRC dotnet-install.sh
     fi

Tasks/AzureCLIV2/Strings/resources.resjson/en-US/resources.resjson

@@ -49,7 +49,7 @@
   "loc.messages.JS_InvalidFilePath": "Script file could not be found at specified script location: '%s'. Please verify the script exists at the specified path. If you want to use inline script, specify input `Script Location` as `inlineScript`.",
   "loc.messages.JS_InvalidErrorActionPreference": "Invalid ErrorActionPreference '%s'. The value must be one of: 'Stop', 'Continue', or 'SilentlyContinue'",
   "loc.messages.GlobalCliConfigAgentVersionWarning": "For agent version < 2.115.0, only global Azure CLI configuration can be used",
-  "loc.messages.UnacceptedScriptLocationValue": "%s is not a valid value for task input 'Script Location' (scriptLocation in YAML). Value can either be'inlineScript' or 'scriptPath'",
+  "loc.messages.UnacceptedScriptLocationValue": "%s is not a valid value for task input 'Script Location' (scriptLocation in YAML). Value can either be 'inlineScript' or 'scriptPath'",
   "loc.messages.ExpiredServicePrincipalMessageWithLink": "Secret expired, update service connection at %s See https://aka.ms/azdo-rm-workload-identity-conversion to learn more about conversion to secret-less service connections.",
   "loc.messages.ProxyConfig": "az tool is configured to use %s as proxy server",
   "loc.messages.FailedToRefreshAzSession": "The following error occurred while trying to refresh az-cli session: %s",

Tasks/AzureCLIV2/task.json

@@ -19,8 +19,8 @@
   "demands": [],
   "version": {
     "Major": 2,
-    "Minor": 247,
-    "Patch": 2
+    "Minor": 248,
+    "Patch": 0
   },
   "minimumAgentVersion": "2.0.0",
   "instanceNameFormat": "Azure CLI $(scriptPath)",

Tasks/AzureCLIV2/task.loc.json

@@ -19,8 +19,8 @@
   "demands": [],
   "version": {
     "Major": 2,
-    "Minor": 247,
-    "Patch": 2
+    "Minor": 248,
+    "Patch": 0
   },
   "minimumAgentVersion": "2.0.0",
   "instanceNameFormat": "ms-resource:loc.instanceNameFormat",

Tasks/AzureCloudPowerShellDeploymentV1/task.json

@@ -16,7 +16,7 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 1,
-        "Minor": 247,
+        "Minor": 248,
         "Patch": 0
     },
     "demands": [

Tasks/AzureCloudPowerShellDeploymentV1/task.loc.json

@@ -16,7 +16,7 @@
   "author": "Microsoft Corporation",
   "version": {
     "Major": 1,
-    "Minor": 247,
+    "Minor": 248,
     "Patch": 0
   },
   "demands": [

Tasks/AzureCloudPowerShellDeploymentV2/task.json

@@ -16,7 +16,7 @@
     "author": "Microsoft Corporation",
     "version": {
         "Major": 2,
-        "Minor": 247,
+        "Minor": 248,
         "Patch": 0
     },
     "demands": [