Add hostservices translation support for clr 4 task host (dotnet#13154)

Attempt 2 for dotnet#13089

In scope of dotnet#12753 support for
translating HostServices in Out of Proc Task Host was added, but limited
to NET task host.

This change removes this limitation.

Author: YuliiaKovalova

src/Build.UnitTests/BackEnd/TaskHostConfiguration_Tests.cs

@@ -463,8 +463,14 @@ public void TestTranslationWithAppDomainSetup(byte[] configBytes)
 
             setup.SetConfigurationBytes(configBytes);
 
-            ((ITranslatable)config).Translate(TranslationHelpers.GetWriteTranslator());
-            INodePacket packet = TaskHostConfiguration.FactoryForDeserialization(TranslationHelpers.GetReadTranslator());
+            // Set version to 0 for CLR4 (Framework-to-Framework) communication which supports AppDomain.
+            ITranslator writeTranslator = TranslationHelpers.GetWriteTranslator();
+            writeTranslator.NegotiatedPacketVersion = 0;
+            ((ITranslatable)config).Translate(writeTranslator);
+
+            ITranslator readTranslator = TranslationHelpers.GetReadTranslator();
+            readTranslator.NegotiatedPacketVersion = 0;
+            INodePacket packet = TaskHostConfiguration.FactoryForDeserialization(readTranslator);
 
             TaskHostConfiguration deserializedConfig = packet as TaskHostConfiguration;
 

src/Build/BackEnd/Components/Communications/NodeProviderOutOfProcBase.cs

@@ -849,6 +849,12 @@ private void DrainPacketQueue(object state)
                                 NodePacketTypeExtensions.WriteVersion(writeStream, context._negotiatedPacketVersion);
                                 writeTranslator.NegotiatedPacketVersion = context._negotiatedPacketVersion;
                             }
+                            else if (!Handshake.IsHandshakeOptionEnabled(_handshakeOptions, HandshakeOptions.CLR2))
+                            {
+                                // CLR4 task hosts: set version to 0 to enable version-dependent fields.
+                                // CLR2 task hosts: leave as null (default) to skip version-dependent fields.
+                                writeTranslator.NegotiatedPacketVersion = 0;
+                            }
 
                             packet.Translate(writeTranslator);
 

src/Framework/BinaryTranslator.cs

@@ -119,7 +119,7 @@ public TranslationDirection Mode
             }
 
             /// <inheritdoc/>
-            public byte NegotiatedPacketVersion { get; set; }
+            public byte? NegotiatedPacketVersion { get; set; }
 
             /// <summary>
             /// Translates a boolean.
@@ -1006,7 +1006,7 @@ public TranslationDirection Mode
             }
 
             /// <inheritdoc/>
-            public byte NegotiatedPacketVersion { get; set; }
+            public byte? NegotiatedPacketVersion { get; set; }
 
             /// <summary>
             /// Translates a boolean.

src/Framework/ITranslator.cs

@@ -84,8 +84,12 @@ internal interface ITranslator : IDisposable
         /// from NodePacketTypeExtensions.PacketVersion when nodes are running different MSBuild versions.
         /// The negotiated version is used to conditionally serialize/deserialize fields that may
         /// not be supported by older packet versions.
+        /// Special values:
+        /// null: CLR2 (NET35) task host communication. Version-dependent fields are skipped because NET35 doesn't have them.
+        /// 0: The constant value for Framework-to-Framework (CLR4) task host communication. Supports HostServices, TargetName, ProjectFile, and AppDomain.
+        /// 2+: .NET task host communication with full support for version-dependent fields.
         /// </remarks>
-        byte NegotiatedPacketVersion { get; set; }
+        byte? NegotiatedPacketVersion { get; set; }
 
         /// <summary>
         /// Returns the current serialization mode.

src/Shared/INodePacket.cs

@@ -291,11 +291,9 @@ internal static class NodePacketTypeExtensions
         /// <summary>
         /// Defines the communication protocol version for node communication.
         /// 
-        /// Version 1: Introduced for the .NET Task Host protocol. This version
-        /// excludes the translation of appDomainConfig within TaskHostConfiguration
-        /// to maintain backward compatibility and reduce serialization overhead.
-        ///
-        /// Version 2: Adds support of HostServices and target name translation in TaskHostConfiguration.
+        /// null: CLR2 (NET35) task host. Version-dependent fields skipped (not compiled in NET35).
+        /// 0: The constant value for Framework-to-Framework (CLR4) task host. Supports HostServices, TargetName, ProjectFile.
+        /// 2+: .NET task host with full support for version-dependent fields.
         /// 
         /// When incrementing this version, ensure compatibility with existing
         /// task hosts and update the corresponding deserialization logic.

src/Shared/NodeEndpointOutOfProcBase.cs

@@ -721,6 +721,8 @@ private void RunReadLoop(
                                 ITranslator readTranslator = BinaryTranslator.GetReadTranslator(localReadPipe, _sharedReadBuffer);
 
                                 // parent sends a packet version that is already negotiated during handshake.
+                                // For Framework task hosts (CLR2/CLR4) without extended headers, defaults to 0.
+                                // For .NET task hosts, read from extended header (>= 1).
                                 readTranslator.NegotiatedPacketVersion = parentVersion;
                                 _packetFactory.DeserializeAndRoutePacket(0, packetType, readTranslator);
                             }

src/Shared/TaskHostConfiguration.cs

@@ -480,10 +480,10 @@ public void Translate(ITranslator translator)
             translator.TranslateCulture(ref _uiCulture);
 #if FEATURE_APPDOMAIN
             // The packet version is used to determine if the AppDomain configuration should be serialized.
-            // If the packet version is bigger then 0, it means the task host will running under .NET.
-            // Although MSBuild.exe runs under .NET Framework and has AppDomain support,
-            // we don't transmit AppDomain config when communicating with dotnet.exe (it is not supported in .NET 5+).
-            if (translator.NegotiatedPacketVersion == 0)
+            // null = CLR2 (NET35) task host - supports AppDomain
+            // 0 = CLR4 (NET472) task host - supports AppDomain
+            // We serialize AppDomain for Framework task hosts (null or 0), but not for .NET (>= 2).
+            if (translator.NegotiatedPacketVersion is null or 0)
             {
                 byte[] appDomainConfigBytes = null;
 
@@ -507,14 +507,17 @@ public void Translate(ITranslator translator)
             translator.Translate(ref _projectFileOfTask);
             translator.Translate(ref _taskName);
             translator.Translate(ref _taskLocation);
-            if (translator.NegotiatedPacketVersion >= 2)
+
+            // null = CLR2 (NET35) task hosts which don't have these fields compiled in.
+            // 0 = CLR4, 2+ = .NET - both support these fields.
+#if NET472 || NETCOREAPP
+            if (translator.NegotiatedPacketVersion.HasValue && translator.NegotiatedPacketVersion is 0 or >= 2)
             {
                 translator.Translate(ref _targetName);
                 translator.Translate(ref _projectFile);
-#if !NET35    
                 translator.Translate(ref _hostServices);
-#endif
             }
+#endif
 
             translator.Translate(ref _isTaskInputLoggingEnabled);
             translator.TranslateDictionary(ref _taskParameters, StringComparer.OrdinalIgnoreCase, TaskParameter.FactoryForDeserialization);