Skip tests using nuget from signed builds (#19111) (#19159)

Author: T-Gro

tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs

@@ -39,7 +39,7 @@ type DependencyManagerInteractiveTests() =
     let getErrors ((_value: Result<FsiValue option, exn>), (errors: FSharpDiagnostic[])) =
         errors
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``SmokeTest - #r nuget``() =
         let text = """
 #r @"nuget:Newtonsoft.Json, Version=9.0.1"
@@ -50,7 +50,7 @@ type DependencyManagerInteractiveTests() =
         Assert.Equal(typeof<int>, value.ReflectionType)
         Assert.Equal(0, value.ReflectionValue :?> int)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``SmokeTest - #r nuget package not found``() =
         let text = """
 #r @"nuget:System.Collections.Immutable.DoesNotExist, version=1.5.0"
@@ -259,7 +259,7 @@ TorchSharp.Tensor.LongTensor.From([| 0L .. 100L |]).Device
         ()
 
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Use Dependency Manager to restore packages with native dependencies, build and run script that depends on the results``() =
         // Skip test on arm64, because there is not an arm64 native library
         if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then
@@ -358,7 +358,7 @@ printfn ""%A"" result
         let value = opt.Value
         Assert.Equal(123, value.ReflectionValue :?> int32)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Use NativeResolver to resolve native dlls.``() =
         // Skip test on arm64, because there is not an arm64 native library
         if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then
@@ -442,7 +442,7 @@ printfn ""%A"" result
         let value = opt.Value
         Assert.Equal(123, value.ReflectionValue :?> int32)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Use AssemblyResolver to resolve assemblies``() =
         // Skip test on arm64, because there is not an arm64 native library
         if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then

tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs

@@ -168,7 +168,7 @@ stacktype.Name = "Stack"
         | Error(ex) -> Assert.IsAssignableFrom(typeof<FsiCompilationException>, ex)
 
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Script using System.Configuration succeeds``() =
         use script = new FSharpScript()
         let result, errors = script.Eval("""
@@ -246,7 +246,7 @@ System.Configuration.ConfigurationManager.AppSettings.Item "Environment" <- "LOC
 
 /// Native dll resolution is not implemented on desktop
 #if NETSTANDARD
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``ML - use assembly with native dependencies``() =
         // Skip test on arm64, because there is not an arm64 native library
         if RuntimeInformation.ProcessArchitecture = Architecture.Arm64 then
@@ -304,7 +304,7 @@ printfn "{@"%A"}" result
         | Ok(_) -> Assert.False(true, "expected a failure")
         | Error(ex) -> Assert.IsAssignableFrom(typeof<FsiCompilationException>, ex)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Eval script with invalid PackageName should fail immediately``() =
         use capture = new TestConsole.ExecutionCapture()
         use script = new FSharpScript(additionalArgs=[| |])
@@ -316,7 +316,7 @@ printfn "{@"%A"}" result
         Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("error NU1101:")), "Expect to error containing 'error NU1101:'")
         Assert.True( errors |> Seq.exists (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")), "Expect to error containing 'FSharp.Really.Not.A.Package'")
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Eval script with invalid PackageName should fail immediately and resolve one time only``() =
         use capture = new TestConsole.ExecutionCapture()
         use script = new FSharpScript(additionalArgs=[| |])
@@ -330,7 +330,7 @@ printfn "{@"%A"}" result
         Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.A.Package")) |> Seq.length))
         Assert.Equal(1, (errors |> Seq.filter (fun error -> error.Message.Contains("FSharp.Really.Not.Another.Package")) |> Seq.length))
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``FsharpPlus - report errors``() =
         let code = """
 #i "nuget:https://api.nuget.org/v3/index.json"
@@ -366,7 +366,7 @@ printTable [{|Age = 15; Weight = 88; Name = "Blahboolahboogaloo"|}]
         let value = opt.Value
         Assert.Equal(1, downcast value.ReflectionValue)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``ML - use assembly with ref dependencies``() =
         let code = """
 #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0"
@@ -383,7 +383,7 @@ tInput.Length
         let value = opt.Value
         Assert.Equal(4L, downcast value.ReflectionValue)
 
-    [<FSharp.Test.FactForNETCOREAPP>] // usessdkrefs is not a valid option for desktop compiler
+    [<FSharp.Test.FactForNETCOREAPPSkipOnSignedBuild>] // usessdkrefs is not a valid option for desktop compiler
     member _.``ML - use assembly with ref dependencies and without refing SMemory``() =
         let code = """
 #r "nuget:Microsoft.ML.OnnxTransformer,1.4.0"
@@ -399,7 +399,7 @@ tInput.Length
         let value = opt.Value
         Assert.Equal(4L, downcast value.ReflectionValue)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``System.Device.Gpio - Ensure we reference the runtime version of the assembly``() =
         let code = """
 #r "nuget:System.Device.Gpio, 1.0.0"
@@ -512,7 +512,7 @@ let x =
         Assert.False(foundInner)
 
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Script with nuget package that yields out of order dependencies works correctly``() =
         // regression test for: https://github.com/dotnet/fsharp/issues/9217
         let code = """
@@ -535,7 +535,7 @@ test pfloat "1.234"
         let value = opt.Value
         Assert.True(true = downcast value.ReflectionValue)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     member _.``Nuget package with method duplicates differing only in generic arity``() =
         // regression test for: https://github.com/dotnet/fsharp/issues/17796
         // Was an internal error
@@ -549,7 +549,7 @@ let add (col:IServiceCollection) =
         let _value,diag = script.Eval(code)
         Assert.Empty(diag)
 
-    [<Theory>]
+    [<FSharp.Test.TheorySkipOnSignedBuild>]
     [<InlineData("""#r "nuget:envdte,usepackagetargets=true" """, true, "")>]
     [<InlineData("""#r "nuget:envdte,usepackagetargets=false" """, true, "")>]
     [<InlineData("""#r "nuget:envdte,usepackagetargets=invalidvalue" """, false, "input.fsx (1,1)-(1,49) interactive error Invalid value for boolean 'usepackagetargets', valid values: true or false")>]

tests/FSharp.Compiler.Service.Tests/ProjectAnalysisTests.fs

@@ -5807,7 +5807,7 @@ let checkContentAsScript content =
 [<Collection(nameof NotThreadSafeResourceCollection)>]
 module ScriptClosureCacheUse =    
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``References from #r nuget are included in script project options`` () =
         let checkResults = checkContentAsScript """
     #i "nuget:https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"

tests/FSharp.Test.Utilities/Utilities.fs

@@ -39,6 +39,29 @@ type FactForDESKTOPAttribute() =
         do base.Skip <- "NETCOREAPP is not supported runtime for this kind of test, it is intended for DESKTOP only"
     #endif
 
+module SignedBuildSkip =
+    let isSignedBuild = System.Environment.GetEnvironmentVariable("SIGNTYPE") = "Real"
+    let skipMessage = "Test skipped on signed builds due to NuGet package restore restrictions"
+    
+    let skipIfSigned (attr: #FactAttribute) =
+        if isSignedBuild then
+            attr.Skip <- skipMessage
+
+type FactSkipOnSignedBuildAttribute() as this =
+    inherit FactAttribute()
+    do SignedBuildSkip.skipIfSigned this
+
+type TheorySkipOnSignedBuildAttribute() as this =
+    inherit TheoryAttribute()
+    do SignedBuildSkip.skipIfSigned this
+
+type FactForNETCOREAPPSkipOnSignedBuildAttribute() as this =
+    inherit FactAttribute()
+    do SignedBuildSkip.skipIfSigned this
+    #if !NETCOREAPP    
+    do base.Skip <- "Only NETCOREAPP is supported runtime for this kind of test."
+    #endif
+
 // This file mimics how Roslyn handles their compilation references for compilation testing
 module Utilities =
 

tests/fsharp/tests.fs

@@ -627,46 +627,46 @@ module CoreTests =
         | "" -> ()
         | diffs -> failwithf "'%s' and '%s' differ; %A" diffFileErr expectedFileErr diffs
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing`` () =
          runPrintingTest "--multiemit- --debug+" "output"
 
     // F# 5.0 changed some things printing output
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-langversion47`` () =
          runPrintingTest "--langversion:4.7" "output.47"
 
     // Output should not change with optimization off
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-optimizeoff`` () =
          runPrintingTest "--multiemit- --debug+ --optimize-" "output"
 
     // Legacy one-dynamic-assembly emit is the default for .NET Framework, which these tests are using
     // Turning that off enables multi-assembly-emit.  The printing test is useful for testing multi-assembly-emit
     // as it feeds in many incremental fragments into stdin of the FSI process.
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-multiemit`` () =
          runPrintingTest "--multiemit+ --debug+" "output.multiemit"
 
     // Multi-assembly-emit establishes some slightly different rules regarding internals, and this
     // needs to be tested with optimizations off.  The output should not change.
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-multiemit-optimizeoff`` () =
          runPrintingTest "--multiemit+ --debug+ --optimize-" "output.multiemit"
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-width-1000`` () =
          runPrintingTest "--use:preludePrintSize1000.fsx" "output.1000"
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-width-200`` () =  
          runPrintingTest "--use:preludePrintSize200.fsx" "output.200"
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-off`` () =
          runPrintingTest "--use:preludeShowDeclarationValuesFalse.fsx" "output.off"
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``printing-quiet`` () =
          runPrintingTest "--quiet" "output.quiet"
 
@@ -1645,77 +1645,77 @@ module RegressionTests =
     [<Fact>]
     let ``12383-FSC_OPTIMIZED`` () = singleTestBuildAndRun "regression/12383" FSC_OPTIMIZED
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``13219-bug-FSI`` () = singleTestBuildAndRun "regression/13219" FSI
 
     [<Fact>]
     let ``4715-optimized`` () =
         let cfg = testConfig "regression/4715"
         fsc cfg "%s -o:test.exe --optimize+" cfg.fsc_flags ["date.fs"; "env.fs"; "main.fs"]
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``multi-package-type-provider-test-FSI`` () = singleTestBuildAndRun "regression/13710" FSI
 
 #if NETCOREAPP
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.dll 64-bit fsc.dll .NET SDK generating optimized code`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY true)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.dll 64-bit .NET SDK generating debug code`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY false)
 
 #else
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, portable PDB`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY true)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating optimized code, full PDB`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY true)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code portable PDB`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY false)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fsc.exe 32-bit .NET Framework generating debug code, full PDB`` () =
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY false)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, portable PDB`` () = 
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with FSC = cfg.FSCANYCPU }
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY true)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``Large inputs 12322 fscAnyCpu.exe 64-bit .NET Framework generating optimized code, full PDB`` () = 
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with FSC = cfg.FSCANYCPU }
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:full " }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY true)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, portable PDB`` () = 
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with FSC = cfg.FSCANYCPU }
         let cfg = { cfg with fsc_flags = cfg.fsc_flags + " --debug:portable --define:PORTABLE_PDB" }
         singleTestBuildAndRunAux cfg (FSC_BUILDONLY false)
 
-    [<Fact>]
+    [<FSharp.Test.FactSkipOnSignedBuild>]
     let ``12322 fscAnyCpu.exe 64-bit .NET Framework generating debug code, full PDB`` () = 
         let cfg = testConfig "regression/12322"
         let cfg = { cfg with FSC = cfg.FSCANYCPU }