Skip to content

Commit 21ea5f6

Browse files
mdh1418mdh1418
committed
[Helix] Adapt tests for helix nuances
UserEvents functional runtime tests differ from other runtime tests because it depends on OneCollect's Record-Trace tool to enable a userevents-based eventpipe session and to collect events. By design, Record-Trace requires elevated privileges, so these tests invoke a record-trace executable with sudo. When tests run on Helix, test artifacts are stripped of their permissions, so the test infrastructure was modified to give record-trace execute permissions (helix-extra-executables.list). Moreover, to avoid having one copy of record-trace per scenario, which in turn requires re-adding execute permissions for each, more modifications were added to copy over a single record-trace executable that would be used by all scenarios (OutOfProcess marker). Additionally, in Helix environments, TMPDIR is set to a helix specific temporary directory like /datadisks/disk1/work/<id>/t, and at this time, record-trace only scans /tmp/ for the runtime's diagnostic ports. So as a workaround, the tracee apps are spawned with TMPDIR set to /tmp. Lastly, the job steps to run tests on AzDO prevents restoring individual runtime test projects. Because record-trace is currently only resolvable through the dotnet-diagnostics-tests source, userevents_common.csproj was added to the group of projects restored at the beginning of copying native test components to restore Microsoft.OneCollect.RecordTrace.
1 parent 8b286c7 commit 21ea5f6

File tree

4 files changed

+82
-2
lines changed

4 files changed

+82
-2
lines changed

‎src/tests/Common/helixpublishwitharcade.proj‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,12 +412,28 @@
412412
<XUnitLogCheckerCommand>$(XUnitLogCheckerHelixPath)XUnitLogChecker$(ExeSuffix) $(XUnitLogCheckerArgs)</XUnitLogCheckerCommand>
413413
</PropertyGroup>
414414

415+
<ItemGroup>
416+
<_ExtraTestExecutablesListFiles Remote="@(_ExtraTestExecutablesListFiles)" />
417+
<_ExtraTestExecutablesListFiles Include="@(_MergedPayloadFiles)"
418+
Condition="$([System.String]::Copy('%(Identity)').ToLower().EndsWith('helix-extra-executables.list'))" />
419+
<_ExtraTestExecutables Remove="@(_ExtraTestExecutables)" />
420+
</ItemGroup>
421+
<ReadLinesFromFile File="%(_ExtraTestExecutablesListFiles.Identity)" Condition="'@(_ExtraTestExecutablesListFiles)' != ''">
422+
<Output TaskParameter="Lines" ItemName="_ExtraTestExecutables" />
423+
</ReadLinesFromFile>
424+
<ItemGroup>
425+
<_ExtraTestExecutables Remove="@(_ExtraTestExecutables)" Condition="'%(Identity)' == ''" />
426+
</ItemGroup>
427+
415428
<ItemGroup>
416429
<!-- We need to ensure that the test run script is marked as executable. -->
417430
<HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' == 'true'" Include="set TEST_HARNESS_STRIPE_TO_EXECUTE=.0.1" />
418431
<HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true'" Include="export TEST_HARNESS_STRIPE_TO_EXECUTE=.0.1" />
419432
<HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true'" Include="chmod +x $(_MergedWrapperRunScriptRelative)" />
420433

434+
<!-- Tests may depend on other executables. Copying files to Helix removes execute permissions, so mark them as executable as well. -->
435+
<HelixCommandLines Condition="'$(TestWrapperTargetsWindows)' != 'true' and Exists('$(TestBinDir)%(Identity)')" Include="@(_ExtraTestExecutables->'chmod +x %(Identity)')" />
436+
421437
<HelixCommandLines Include="$(_WorkaroundForNuGetMigrations)" />
422438

423439
<!-- Force assemblies to lazy-load for LLVM AOT test runs to enable using tests that fail at AOT time (and as a result can't be AOTd) -->

‎src/tests/build.proj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
<RestoreProjects Include="Common\XHarnessRunnerLibrary\XHarnessRunnerLibrary.csproj" />
4141
<RestoreProjects Include="Common\external\external.csproj" />
4242
<RestoreProjects Include="Common\ilasm\ilasm.ilproj" />
43+
<RestoreProjects Include="tracing\userevents\common\userevents_common.csproj" />
4344
</ItemGroup>
4445

4546
<ItemGroup>

‎src/tests/tracing/userevents/common/UserEventsTestRunner.cs‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,14 @@ public static int Run(
8787
traceeStartInfo.RedirectStandardOutput = true;
8888
traceeStartInfo.RedirectStandardError = true;
8989

90+
// record-trace currently only searches /tmp/ for diagnostic ports https://github.com/microsoft/one-collect/issues/183
91+
string diagnosticPortDir = "/tmp/";
92+
traceeStartInfo.Environment["TMPDIR"] = diagnosticPortDir;
93+
9094
Console.WriteLine($"Starting tracee process: {traceeStartInfo.FileName} {traceeStartInfo.Arguments}");
9195
using Process traceeProcess = Process.Start(traceeStartInfo);
92-
Console.WriteLine($"Tracee process started with PID: {traceeProcess.Id}");
96+
int traceePid = traceeProcess.Id;
97+
Console.WriteLine($"Tracee process started with PID: {traceePid}");
9398
traceeProcess.OutputDataReceived += (_, args) =>
9499
{
95100
if (!string.IsNullOrEmpty(args.Data))
@@ -115,6 +120,12 @@ public static int Run(
115120
}
116121
traceeProcess.WaitForExit(); // flush async output
117122

123+
// TMPDIR is configured on Helix, but the diagnostic port was created outside of helix's default temp datadisk path.
124+
// The diagnostic port should be automatically cleaned up when the tracee shutsdown, but just in case of an
125+
// abrupt exit, ensure cleanup to avoid leaving artifacts on helix machines.
126+
// When https://github.com/microsoft/one-collect/issues/183 is fixed, this and the above TMPDIR should be removed.
127+
CleanupTraceeDiagnosticPorts(diagnosticPortDir, traceePid);
128+
118129
if (!recordTraceProcess.HasExited)
119130
{
120131
// Until record-trace supports duration, the only way to stop it is to send SIGINT (ctrl+c)
@@ -144,6 +155,7 @@ public static int Run(
144155
if (!traceValidator(source))
145156
{
146157
Console.Error.WriteLine($"Trace file `{traceFilePath}` does not contain expected events.");
158+
UploadTraceFileFromHelix(traceFilePath, scenarioName);
147159
return -1;
148160
}
149161

@@ -169,5 +181,43 @@ private static string ResolveRecordTracePath(string userEventsScenarioDir)
169181
string recordTracePath = Path.Combine(commonDir, "record-trace");
170182
return recordTracePath;
171183
}
184+
185+
private static void CleanupTraceeDiagnosticPorts(string diagnosticPortDir, int traceePid)
186+
{
187+
try
188+
{
189+
string[] udsFiles = Directory.GetFiles(diagnosticPortDir, $"dotnet-diagnostic-{traceePid}-*-socket");
190+
foreach (string udsFile in udsFiles)
191+
{
192+
Console.WriteLine($"Deleting tracee diagnostic port UDS file: {udsFile}");
193+
File.Delete(udsFile);
194+
}
195+
}
196+
catch (Exception ex)
197+
{
198+
Console.Error.WriteLine($"Failed to cleanup tracee diagnostic ports: {ex}");
199+
}
200+
}
201+
202+
private static void UploadTraceFileFromHelix(string traceFilePath, string scenarioName)
203+
{
204+
var helixWorkItemDirectory = Environment.GetEnvironmentVariable("HELIX_WORKITEM_UPLOAD_ROOT");
205+
if (helixWorkItemDirectory != null && Directory.Exists(helixWorkItemDirectory))
206+
{
207+
var destPath = Path.Combine(helixWorkItemDirectory, $"{scenarioName}.nettrace");
208+
Console.WriteLine($"Uploading trace file to Helix work item directory: {destPath}");
209+
File.Copy(traceFilePath, destPath, overwrite: true);
210+
211+
try
212+
{
213+
Console.WriteLine($"Cleaning up original trace file: `{traceFilePath}`");
214+
File.Delete(traceFilePath);
215+
}
216+
catch (Exception ex)
217+
{
218+
Console.Error.WriteLine($"Failed to delete trace file: {ex}");
219+
}
220+
}
221+
}
172222
}
173223
}

‎src/tests/tracing/userevents/common/userevents_common.csproj‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,16 +13,29 @@
1313
<Compile Include="UserEventsRequirements.cs" />
1414
</ItemGroup>
1515

16-
<Target Name="CopyRecordTrace" BeforeTargets="Build" Condition="'$(TargetOS)' == 'linux' and ('$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'arm64')">
16+
<Target Name="CopyRecordTrace" BeforeTargets="Build;CopyAllNativeProjectReferenceBinaries" Condition="'$(TargetOS)' == 'linux' and ('$(TargetArchitecture)' == 'x64' or '$(TargetArchitecture)' == 'arm64')">
1717
<PropertyGroup>
1818
<_DestDir>$(TargetDir)</_DestDir>
1919
<_DestDir Condition="'$(_DestDir)' == ''">$(OutputPath)</_DestDir>
2020
<_RecordTraceSource>$(NuGetPackageRoot)microsoft.onecollect.recordtrace/$(MicrosoftOneCollectRecordTraceVersion)/runtimes/linux-$(TargetArchitecture)/native/record-trace</_RecordTraceSource>
21+
<_RecordTraceRelative Condition="$(BuildProjectRelativeDir) != ''">$(BuildProjectRelativeDir)record-trace</_RecordTraceRelative>
22+
<_RecordTraceRelative Condition="'$(_RecordTraceRelative)' == ''">$([System.IO.Path]::GetRelativePath('$(TestBinDir)', '$(_DestDir)record-trace'))</_RecordTraceRelative>
2123
</PropertyGroup>
2224

2325
<Copy SourceFiles="$(_RecordTraceSource)" DestinationFiles="$(_DestDir)record-trace" SkipUnchangedFiles="true" />
2426

2527
<!-- For local testing, ensure it has execute permissions -->
2628
<Exec Command="chmod +x '$(_DestDir)record-trace'" Condition="Exists('$(_DestDir)record-trace')" />
29+
30+
<!-- For Helix builds, artifacts copied over have their permissions reset. Add the executable to a list for helix to reapply execute permissions -->
31+
<WriteLinesToFile File="$(_DestDir)helix-extra-executables.list" Lines="$(_RecordTraceRelative)" Overwrite="true" />
32+
33+
<!-- Although this isn't a test project, userevents tests depend on record-trace, which in turn relies on helix-extra-executables.list to received execute permissions
34+
So to get the files included into Helix's _MergedPayloadFiles, we need to manually add an OutOfProcessTest marker. -->
35+
<WriteLinesToFile
36+
File="$(_DestDir)$(MSBuildProjectName).OutOfProcessTest"
37+
Lines="OutOfProcessTest"
38+
Overwrite="true"
39+
WriteOnlyWhenDifferent="true" />
2740
</Target>
2841
</Project>

0 commit comments

Comments
 (0)