Various cabal-testsuite improvements (#10225)

* Improve bat scripts for CCompilerOverride

* Ensure Windows tests can cleanup the temp directory

* Implement `flaky` combinator

* Remove outdated tests

* Remove broken tests

These tests were testing for messages that were removed with the `cabal check` rework.

* Make `skip` and `broken` messages uniform

* Mark flaky tests

* Re-enable DeterministicTrivial

* Fix MacOS canonical paths

* Extend cabal-testsuite readme with `flaky`

* Skip non-terminating tests in Windows CI

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

Author: jasagredo

Cabal-tests/lib/Test/Utils/TempTestDir.hs

@@ -2,11 +2,12 @@
 
 module Test.Utils.TempTestDir
   ( withTestDir
+  , withTestDir'
   , removeDirectoryRecursiveHack
   ) where
 
 import Distribution.Compat.Internal.TempFile (createTempDirectory)
-import Distribution.Simple.Utils (warn)
+import Distribution.Simple.Utils (warn, TempFileOptions (..), defaultTempFileOptions)
 import Distribution.Verbosity
 
 import Control.Concurrent (threadDelay)
@@ -23,12 +24,26 @@ import qualified System.Info (os)
 -- | Much like 'withTemporaryDirectory' but with a number of hacks to make
 -- sure on windows that we can clean up the directory at the end.
 withTestDir :: (MonadIO m, MonadMask m) => Verbosity -> String -> (FilePath -> m a) -> m a
-withTestDir verbosity template action = do
-  systmpdir <- liftIO getTemporaryDirectory
+withTestDir verbosity template action = withTestDir' verbosity defaultTempFileOptions template action
+
+withTestDir' :: (MonadIO m, MonadMask m) => Verbosity -> TempFileOptions -> String -> (FilePath -> m a) -> m a
+withTestDir' verbosity tempFileOpts template action = do
+  systmpdir <-
+    -- MacOS returns /var/folders/... which is a symlink (/var -> /private/var),
+    -- so the test-suite struggles to make the build cwd-agnostic in particular
+    -- for the ShowBuildInfo tests. This canonicalizePath call makes it
+    -- /private/var/folders/... which will work.
+    liftIO $ canonicalizePath =<< getTemporaryDirectory
   bracket
     ( do { tmpRelDir <- liftIO $ createTempDirectory systmpdir template
          ; return $ systmpdir </> tmpRelDir } )
-    (liftIO . removeDirectoryRecursiveHack verbosity)
+    (liftIO
+      -- This ensures that the temp files are not deleted at the end of the test.
+      -- It replicates the behavior of @withTempDirectoryEx@.
+      . when (not (optKeepTempFiles tempFileOpts))
+      -- This is the bit that helps with Windows deleting all files.
+      . removeDirectoryRecursiveHack verbosity
+      )
     action
 
 -- | On Windows, file locks held by programs we run (in this case VCSs)

cabal-testsuite/PackageTests/Backpack/Includes2/cabal-internal.test.hs

@@ -2,7 +2,7 @@ import Test.Cabal.Prelude
 
 main = cabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
+    expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
         cabal "v2-build" ["exe"]
         withPlan $ do
             r <- runPlanExe' "I" "exe" []

cabal-testsuite/PackageTests/Backpack/Includes2/setup-external.test.hs

@@ -1,8 +1,7 @@
 import Test.Cabal.Prelude
 main = setupAndCabalTest $ do
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $ do
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $ do
     withPackageDb $ do
       withDirectory "mylib" $ setup_install_with_docs ["--ipid", "mylib-0.1.0.0"]
       withDirectory "mysql" $ setup_install_with_docs ["--ipid", "mysql-0.1.0.0"]

cabal-testsuite/PackageTests/Backpack/Includes2/setup-per-component.test.hs

@@ -2,8 +2,7 @@ import Test.Cabal.Prelude
 main = setupTest $ do
   -- No cabal test because per-component is broken with it
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
     withPackageDb $
       withDirectory "Includes2" $ do
         let setup_install' args = setup_install_with_docs args

cabal-testsuite/PackageTests/Backpack/Includes3/cabal-external.test.hs

@@ -1,8 +1,8 @@
 import Test.Cabal.Prelude
 
 main = cabalTest $ do
-    ghcVer <- isGhcVersion ">= 9.10"
     skipUnlessGhcVersion ">= 8.1"
+    ghcVer <- isGhcVersion ">= 9.10"
     skipIf "Windows + 9.10.1 (#10191)" (isWindows && ghcVer)
     withProjectFile "cabal.external.project" $ do
         cabal "v2-build" ["exe"]

cabal-testsuite/PackageTests/Backpack/Includes3/cabal-internal.test.hs

@@ -2,7 +2,7 @@ import Test.Cabal.Prelude
 
 main = cabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    expectBrokenIf isWindows 10191 $ withProjectFile "cabal.internal.project" $ do
+    expectBrokenIfWindowsCI 10191 $ withProjectFile "cabal.internal.project" $ do
         cabal "v2-build" ["exe"]
         withPlan $ do
             r <- runPlanExe' "I" "exe" []

cabal-testsuite/PackageTests/Backpack/Includes3/setup-external-ok.test.hs

@@ -3,8 +3,7 @@ import Data.List
 import qualified Data.Char as Char
 main = setupAndCabalTest $ do
   skipUnlessGhcVersion ">= 8.1"
-  ghc <- isGhcVersion "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*"
-  expectBrokenIf ghc 7987 $
+  expectBrokenIfGhc "== 9.0.2 || == 9.2.* || == 9.4.* || == 9.6.*" 7987 $
     withPackageDb $ do
       containers_id <- getIPID "containers"
       withDirectory "repo/sigs-0.1.0.0" $ setup_install_with_docs ["--ipid", "sigs-0.1.0.0"]
@@ -21,4 +20,3 @@ main = setupAndCabalTest $ do
       withDirectory "repo/exe-0.1.0.0" $ do
         setup_install []
         runExe' "exe" [] >>= assertOutputContains "fromList [(0,2),(2,4)]"
-

cabal-testsuite/PackageTests/Backpack/T4754/setup.test.hs

@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 main = setupAndCabalTest $ do
     skipUnlessGhcVersion ">= 8.1"
-    skipUnless "no profiling libs" =<< hasProfiledLibraries
+    skipIfNoProfiledLibraries
     setup "configure" ["--enable-profiling"]
     setup "build" []

cabal-testsuite/PackageTests/Backpack/T6385/cabal.test.hs

@@ -1,6 +1,6 @@
 import Test.Cabal.Prelude
 main =
-  cabalTest $ expectBrokenIf isWindows 10191 $ withShorterPathForNewBuildStore $ do
+  cabalTest $ expectBrokenIfWindows 10191 $ withShorterPathForNewBuildStore $ do
     skipUnlessGhcVersion ">= 8.1"
     withRepo "repo" $ do
       cabal "v2-build" ["T6385"]

cabal-testsuite/PackageTests/CCompilerOverride/custom-cc-clang.bat

@@ -1,11 +1,7 @@
 @echo OFF
 
-where /q clang.exe
-
-IF %ERRORLEVEL% EQU 0 (
-   call clang.exe -DNOERROR6 %*
-   EXIT /B %ERRORLEVEL%
-)
-
-ECHO "Cannot find C compiler"
-EXIT /B 1
+REM replace the libdir with the path to the compiler
+FOR /f "delims=" %%A in ('call ghc.exe --print-libdir') do set "var=%%A"
+setlocal EnableDelayedExpansion
+CALL !var:lib=mingw\bin\clang.exe! -DNOERROR6 %*
+EXIT /B %ERRORLEVEL%