Merge remote-tracking branch 'origin/master' into Localization

Author: vmapetr

.azure-pipelines/get-pat.yml

@@ -0,0 +1,11 @@
+steps:
+- task: AzureCLI@2
+  inputs:
+    azureSubscription: ARM - WIF - manual
+    scriptType: pscore
+    scriptLocation: inlineScript
+    inlineScript: |
+      az account set --subscription $(SUBSCRIPTION_ID)
+      $accessToken = az account get-access-token --resource $(RESOURCE_ID) --query accessToken --output tsv
+      echo "##vso[task.setvariable variable=ACCESS_TOKEN;issecret=true]$accessToken"
+  displayName: Get Access Token

.azure-pipelines/pipeline.yml

@@ -111,6 +111,7 @@ extends:
             nugetMultiFeedWarnLevel: none
           timeoutInMinutes: 300
           steps:
+          - template: /.azure-pipelines/get-pat.yml@self
           - bash: |
               cd ./.azure-pipelines/scripts/
               npm install axios minimist
@@ -123,7 +124,7 @@ extends:
               node ./run-and-verify.js \
                   --projectUrl "$(CANARY_PROJECT_URL)" \
                   --pipelineId "$(CANARY_PIPELINE_ID)" \
-                  --token "$(CANARY_PAT)" \
+                  --token "$(ACCESS_TOKEN)" \
                   --templateParameters "{ \"branch\": \"${branch}\" }"
             displayName: Test Proxy Agent
 

docs/contribute.md

@@ -30,6 +30,25 @@ cd ./src
 ./dev.(sh/cmd) test # run unit tests before git commit/push
 ```
 
+## Debugging
+
+The agent can be run in debug mode by providing the parameter `--debug` to the `run` command.
+This will make the agent recognize the following environment variables:
+
+- `VSTSAGENT_DEBUG_TASK` - for remote debugging node-based pipeline tasks
+
+Note that all of these variables need to be defined on the node that is used to run the agent.
+Also, do not run production agents with this mode as it can cause pipelines to appear stuck.
+
+### `VSTSAGENT_DEBUG_TASK` environment variable
+
+When enabled, the agent will start the Node process with specific parameters. These parameters cause the process to wait for the debugger to attach before continuing with the execution of the pipeline task script. The value must be set to either:
+- Task `id`, which is an unique GUID identifier to be found in `task.json` definition of the task
+- Task `name` and major `version`, e.g. AzureCLIV2
+
+Only one task can be debugged at one time and all other tasks in the same pipeline will proceed as usual.
+If you wish to stop debugging this task either restart that agent without `--debug` option, or unset the variables from above.
+
 ## Editors
 
 [Using Visual Studio 2017](https://www.visualstudio.com/vs/)  

src/Agent.Worker/ContainerOperationProvider.cs

@@ -194,7 +194,7 @@ private async Task<string> GetAccessTokenUsingWorkloadIdentityFederation(IExecut
             Trace.Entering();
 
             var tenantId = string.Empty;
-            if(!registryEndpoint.Authorization?.Parameters?.TryGetValue(c_tenantId, out tenantId) ?? false)
+            if (!registryEndpoint.Authorization?.Parameters?.TryGetValue(c_tenantId, out tenantId) ?? false)
             {
                 throw new InvalidOperationException($"Could not read {c_tenantId}");
             }
@@ -708,7 +708,43 @@ private async Task StartContainerAsync(IExecutionContext executionContext, Conta
                     Func<string, string, string, string> addUserWithIdAndGroup;
                     Func<string, string, string> addUserToGroup;
 
+                    bool useShadowIfAlpine = false;
+
                     if (isAlpineBasedImage)
+                    {
+                        List<string> shadowInfoOutput = await DockerExec(executionContext, container.ContainerId, "apk list --installed | grep shadow");
+                        bool shadowPreinstalled = false;
+
+                        foreach (string shadowInfoLine in shadowInfoOutput)
+                        {
+                            if (shadowInfoLine.Contains("{shadow}", StringComparison.Ordinal))
+                            {
+                                Trace.Info("The 'shadow' package is preinstalled and therefore will be used.");
+                                shadowPreinstalled = true;
+                                break;
+                            }
+                        }
+
+                        bool userIdIsOutsideAdduserCommandRange = Int64.Parse(container.CurrentUserId) > 256000;
+
+                        if (userIdIsOutsideAdduserCommandRange && !shadowPreinstalled)
+                        {
+                            Trace.Info("User ID is outside the range of the 'adduser' command, therefore the 'shadow' package will be installed and used.");
+
+                            try
+                            {
+                                await DockerExec(executionContext, container.ContainerId, "apk add shadow");
+                            }
+                            catch (InvalidOperationException)
+                            {
+                                throw new InvalidOperationException(StringUtil.Loc("ApkAddShadowFailed"));
+                            }
+                        }
+
+                        useShadowIfAlpine = shadowPreinstalled || userIdIsOutsideAdduserCommandRange;
+                    }
+
+                    if (isAlpineBasedImage && !useShadowIfAlpine)
                     {
                         addGroup = (groupName) => $"addgroup {groupName}";
                         addGroupWithId = (groupName, groupId) => $"addgroup -g {groupId} {groupName}";
@@ -1009,7 +1045,7 @@ private async Task ContainerHealthcheck(IExecutionContext executionContext, Cont
             }
         }
 
-        private async Task<List<string>> DockerExec(IExecutionContext context, string containerId, string command, bool noExceptionOnError=false)
+        private async Task<List<string>> DockerExec(IExecutionContext context, string containerId, string command, bool noExceptionOnError = false)
         {
             Trace.Info($"Docker-exec is going to execute: `{command}`; container id: `{containerId}`");
             List<string> output = new List<string>();
@@ -1027,7 +1063,7 @@ private async Task<List<string>> DockerExec(IExecutionContext context, string co
             if (exitCode != 0)
             {
                 Trace.Error(message);
-                if(!noExceptionOnError)
+                if (!noExceptionOnError)
                 {
                     throw new InvalidOperationException(message);
                 }
@@ -1046,14 +1082,14 @@ private static void ThrowIfAlreadyInContainer()
         {
             if (PlatformUtil.RunningOnWindows)
             {
-                #pragma warning disable CA1416 // SupportedOSPlatform checks not respected in lambda usage
+#pragma warning disable CA1416 // SupportedOSPlatform checks not respected in lambda usage
                 // service CExecSvc is Container Execution Agent.
                 ServiceController[] scServices = ServiceController.GetServices();
                 if (scServices.Any(x => String.Equals(x.ServiceName, "cexecsvc", StringComparison.OrdinalIgnoreCase) && x.Status == ServiceControllerStatus.Running))
                 {
                     throw new NotSupportedException(StringUtil.Loc("AgentAlreadyInsideContainer"));
                 }
-                #pragma warning restore CA1416
+#pragma warning restore CA1416
             }
             else
             {

src/Misc/layoutbin/en-US/strings.json

@@ -27,6 +27,7 @@
   "AgentWithSameNameAlreadyExistInPool": "Pool {0} already contains an agent with name {1}.",
   "AllowContainerUserRunDocker": "Allow user '{0}' run any docker command without SUDO.",
   "AlreadyConfiguredError": "Cannot configure the agent because it is already configured. To reconfigure the agent, run 'config.cmd remove' or './config.sh remove' first.",
+  "ApkAddShadowFailed": "The user ID is outside the range of the 'adduser' command. The alternative command 'useradd' cannot be used because the 'shadow' package is not preinstalled and the attempt to install this package failed. Check network availability or use a docker image with the 'shadow' package preinstalled.",
   "ArgumentNeeded": "'{0}' has to be specified.",
   "ArtifactCustomPropertiesNotJson": "Artifact custom properties is not valid JSON: '{0}'",
   "ArtifactCustomPropertyInvalid": "Artifact custom properties must be prefixed with 'user-'. Invalid property: '{0}'",