Skip to content

Commit 69f2427

Browse files
cgrindelcgrindel
authored
chore: disable flaky test that fails with ExitFailure (-9) 
2 parents b2baa5f + cc87cfa commit 69f2427

File tree

2 files changed

+200
-134
lines changed

2 files changed

+200
-134
lines changed

‎.github/workflows/workflow.yaml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,8 @@ jobs:
130130
# Shutdown Bazel to free up memory
131131
# https://github.com/tweag/rules_haskell/issues/2089.
132132
bazel shutdown
133-
./bazel-ci-bin/tests/run-tests
133+
# Execute the tests dumping out memory/process information before and after each test.
134+
RHT_PRINT_MEMORY=true ./bazel-ci-bin/tests/run-tests
134135
bazel coverage //...
135136
136137
test-bindist:

‎rules_haskell_tests/tests/RunTests.hs

Lines changed: 198 additions & 133 deletions
Original file line numberDiff line numberDiff line change
@@ -6,153 +6,154 @@
66
import Control.Exception.Safe (bracket_)
77
import Data.Foldable (for_)
88
import Data.List (isInfixOf, sort)
9-
import System.Directory (copyFile)
9+
import GHC.Stack (HasCallStack)
10+
import System.Directory (copyFile, doesFileExist)
1011
import System.FilePath ((</>))
1112
import System.Info (os)
1213
import System.IO.Temp (withSystemTempDirectory)
1314
import System.Environment (lookupEnv)
15+
import System.Exit (ExitCode(..))
1416

1517
import qualified System.Process as Process
16-
import Test.Hspec.Core.Spec (SpecM)
17-
import Test.Hspec (context, hspec, it, describe, runIO)
18+
import Test.Hspec.Core.Spec (SpecM, SpecWith)
19+
import Test.Hspec (context, hspec, it, describe, runIO, around_, afterAll_)
1820

1921
import BinModule (b)
2022
import GenModule (a)
2123

2224
import IntegrationTesting
2325

2426
main :: IO ()
25-
main = hspec $ do
26-
it "bazel test" $ do
27-
assertSuccess (bazel ["test", "//..."])
28-
29-
it "bazel test prof" $ do
30-
ghcVersion <- lookupEnv "GHC_VERSION"
31-
32-
-- In .github/workflows/workflow.yaml we specify --test_tag_filters
33-
-- -dont_test_on_darwin. However, specifiying --test_tag_filters
34-
-- -requires_dynamic here alone would override that filter. So,
35-
-- we have to duplicate that filter here.
36-
let tagFilter | os == "darwin" = "-dont_test_on_darwin,-requires_dynamic,-skip_profiling" ++ (
37-
-- skip tests for specific GHC version, see https://github.com/tweag/rules_haskell/issues/2073
38-
maybe "" (",-dont_build_on_macos_with_ghc_" ++) ghcVersion)
39-
| otherwise = "-requires_dynamic,-skip_profiling"
40-
assertSuccess (bazel ["test", "-c", "dbg", "//...", "--build_tag_filters", tagFilter, "--test_tag_filters", tagFilter])
41-
42-
it "bazel build worker" $ do
43-
assertSuccess (bazel ["build", "@rules_haskell//tools/worker:bin"])
44-
45-
describe "stack_snapshot pinning" $
46-
it "handles packages in subdirectories correctly" $ do
47-
-- NOTE Keep in sync with
48-
-- .github/workflows/workflow.yaml
49-
let withBackup filename k =
50-
withSystemTempDirectory "bazel_backup" $ \tmp_dir -> do
51-
bracket_
52-
(copyFile filename (tmp_dir </> "backup"))
53-
(copyFile (tmp_dir </> "backup") filename)
54-
k
55-
-- Test that pinning works and produces buildable targets.
56-
-- Backup the lock file to avoid unintended changes when run locally.
57-
withBackup "stackage-pinning-test_snapshot.json" $ do
58-
assertSuccess (bazel ["run", "@stackage-pinning-test-unpinned//:pin"])
59-
assertSuccess (bazel ["build", "@stackage-pinning-test//:hspec"])
60-
61-
describe "repl" $ do
62-
it "for libraries" $ do
63-
assertSuccess (bazel ["run", "//tests/repl-targets:hs-lib-bad@repl", "--", "-ignore-dot-ghci", "-e", "1 + 2"])
64-
65-
it "for binaries" $ do
66-
assertSuccess (bazel ["run", "//tests/binary-indirect-cbits:binary-indirect-cbits@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
67-
68-
assertSuccess (bazel ["run", "//tests/repl-targets:hs-test-bad@repl", "--", "-ignore-dot-ghci", "-e", "1 + 2"])
69-
70-
it "with rebindable syntax" $ do
71-
let p' (stdout, _stderr) = lines stdout == ["True"]
72-
outputSatisfy p' (bazel ["run", "//tests/repl-targets:rebindable-syntax@repl", "--", "-ignore-dot-ghci", "-e", "check"])
73-
74-
it "sets classpath" $ do
75-
assertSuccess (bazel ["run", "//tests/java_classpath:java_classpath@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
76-
77-
-- Test `compiler_flags` from toolchain and rule for REPL
78-
it "compiler flags" $ do
79-
assertSuccess (bazel ["run", "//tests/repl-flags:compiler_flags@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
80-
81-
-- Test make variable expansion in `compiler_flags` and `repl_ghci_args`.
82-
describe "make variables" $ do
27+
main = hspec $ around_ printStatsHook $ do
28+
29+
describe "rules_haskell_tests" $ afterAll_ (shutdownBazel ".") $ do
30+
it "bazel test" $ do
31+
assertSuccess (bazel ["test", "//..."])
32+
33+
it "bazel test prof" $ do
34+
ghcVersion <- lookupEnv "GHC_VERSION"
35+
36+
-- In .github/workflows/workflow.yaml we specify --test_tag_filters
37+
-- -dont_test_on_darwin. However, specifiying --test_tag_filters
38+
-- -requires_dynamic here alone would override that filter. So,
39+
-- we have to duplicate that filter here.
40+
let tagFilter | os == "darwin" = "-dont_test_on_darwin,-requires_dynamic,-skip_profiling" ++ (
41+
-- skip tests for specific GHC version, see https://github.com/tweag/rules_haskell/issues/2073
42+
maybe "" (",-dont_build_on_macos_with_ghc_" ++) ghcVersion)
43+
| otherwise = "-requires_dynamic,-skip_profiling"
44+
assertSuccess (bazel ["test", "-c", "dbg", "//...", "--build_tag_filters", tagFilter, "--test_tag_filters", tagFilter])
45+
46+
it "bazel build worker" $ do
47+
assertSuccess (bazel ["build", "@rules_haskell//tools/worker:bin"])
48+
49+
describe "stack_snapshot pinning" $
50+
it "handles packages in subdirectories correctly" $ do
51+
-- NOTE Keep in sync with
52+
-- .github/workflows/workflow.yaml
53+
let withBackup filename k =
54+
withSystemTempDirectory "bazel_backup" $ \tmp_dir -> do
55+
bracket_
56+
(copyFile filename (tmp_dir </> "backup"))
57+
(copyFile (tmp_dir </> "backup") filename)
58+
k
59+
-- Test that pinning works and produces buildable targets.
60+
-- Backup the lock file to avoid unintended changes when run locally.
61+
withBackup "stackage-pinning-test_snapshot.json" $ do
62+
assertSuccess (bazel ["run", "@stackage-pinning-test-unpinned//:pin"])
63+
assertSuccess (bazel ["build", "@stackage-pinning-test//:hspec"])
64+
65+
describe "repl" $ do
66+
it "for libraries" $ do
67+
assertSuccess (bazel ["run", "//tests/repl-targets:hs-lib-bad@repl", "--", "-ignore-dot-ghci", "-e", "1 + 2"])
68+
69+
it "for binaries" $ do
70+
assertSuccess (bazel ["run", "//tests/binary-indirect-cbits:binary-indirect-cbits@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
71+
72+
assertSuccess (bazel ["run", "//tests/repl-targets:hs-test-bad@repl", "--", "-ignore-dot-ghci", "-e", "1 + 2"])
73+
74+
it "with rebindable syntax" $ do
75+
let p' (stdout, _stderr) = lines stdout == ["True"]
76+
outputSatisfy p' (bazel ["run", "//tests/repl-targets:rebindable-syntax@repl", "--", "-ignore-dot-ghci", "-e", "check"])
77+
78+
it "sets classpath" $ do
79+
assertSuccess (bazel ["run", "//tests/java_classpath:java_classpath@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
80+
81+
-- Test `compiler_flags` from toolchain and rule for REPL
8382
it "compiler flags" $ do
84-
assertSuccess (bazel ["run", "//tests/repl-make-variables:test-compiler-flags@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
85-
it "indirect repl flags" $ do
86-
assertSuccess (bazel ["run", "//tests/repl-make-variables:repl-indirect-flags", "--", "-ignore-dot-ghci", "-e", ":main"])
87-
it "direct repl flags" $ do
88-
assertSuccess (bazel ["run", "//tests/repl-make-variables:repl-direct-flags", "--", "-ignore-dot-ghci", "-e", ":main"])
89-
90-
-- Test `repl_ghci_args` from toolchain and rule for REPL
91-
it "repl flags" $ do
92-
assertSuccess (bazel ["run", "//tests/repl-flags:repl_flags@repl", "--", "-ignore-dot-ghci", "-e", "foo"])
93-
94-
it "fails on multiple definitions" $ do
95-
assertSuccess (bazel ["run", "//tests/repl-multiple-definition:repl", "--", "-ignore-dot-ghci", "-e", "final"])
96-
97-
describe "multi_repl" $ do
98-
it "loads transitive library dependencies" $ do
99-
let p' (stdout, _stderr) = lines stdout == ["tests/multi_repl/bc/src/BC/C.hs"]
100-
outputSatisfy p' (bazel ["run", "//tests/multi_repl:c_only_repl", "--", "-ignore-dot-ghci", "-e", ":show targets"])
101-
it "loads transitive source dependencies" $ do
102-
let p' (stdout, _stderr) = sort (lines stdout) == ["tests/multi_repl/a/src/A/A.hs","tests/multi_repl/bc/src/BC/B.hs","tests/multi_repl/bc/src/BC/C.hs"]
103-
outputSatisfy p' (bazel ["run", "//tests/multi_repl:c_multi_repl", "--", "-ignore-dot-ghci", "-e", ":show targets"])
104-
it "loads core library dependencies" $ do
105-
let p' (stdout, _stderr) = sort (lines stdout) == ["tests/multi_repl/core_package_dep/Lib.hs"]
106-
outputSatisfy p' (bazel ["run", "//tests/multi_repl:core_package_dep", "--", "-ignore-dot-ghci", "-e", ":show targets"])
107-
it "doesn't allow to manually load modules" $ do
108-
assertFailure (bazel ["run", "//tests/multi_repl:c_multi_repl", "--", "-ignore-dot-ghci", "-e", ":load BC.C", "-e", "c"])
109-
110-
describe "ghcide" $ do
111-
it "loads RunTests.hs" $
112-
assertSuccess (Process.proc "./.ghcide" ["tests/RunTests.hs"])
113-
it "loads module with module dependency" $
114-
assertSuccess (Process.proc "./.ghcide" ["tests/binary-with-lib/Main.hs"])
115-
116-
describe "failures" $ do
117-
-- Make sure not to include haskell_repl (@repl) or alias (-repl) targets
118-
-- in the query. Those would not fail under bazel test.
119-
all_failure_tests <- bazelQuery "kind('haskell_library|haskell_binary|haskell_test', //tests/failures/...) intersect attr('tags', 'manual', //tests/failures/...)"
120-
121-
for_ all_failure_tests $ \test -> do
122-
it test $ do
123-
assertFailure (bazel ["build", test])
124-
125-
context "known issues" $ do
126-
it "haskell_doc fails with plugins #1549" $
127-
-- https://github.com/tweag/rules_haskell/issues/1549
128-
assertFailure (bazel ["build", "//tests/haddock-with-plugin"])
129-
it "transitive re-exports do not work #1145" $
130-
-- https://github.com/tweag/rules_haskell/issues/1145
131-
assertFailure (bazel ["build", "//tests/package-reexport-transitive"])
132-
it "doctest failure with foreign import #1559" $
133-
-- https://github.com/tweag/rules_haskell/issues/1559
134-
assertFailure (bazel ["build", "//tests/haskell_doctest_ffi_1559:doctest-a"])
135-
136-
-- Test that the repl still works if we shadow some Prelude functions
137-
it "repl name shadowing" $ do
138-
let p (stdout, stderr) = not $ any ("error" `isInfixOf`) [stdout, stderr]
139-
outputSatisfy p (bazel ["run", "//tests/repl-name-conflicts:lib@repl", "--", "-ignore-dot-ghci", "-e", "stdin"])
140-
141-
it "Repl works with remote_download_toplevel" $ do
142-
let p (stdout, stderr) = not $ any ("error" `isInfixOf`) [stdout, stderr]
143-
withSystemTempDirectory "bazel_disk_cache" $ \tmp_disk_cache -> do
144-
assertSuccess $ bazel ["run", "//tests/multi_repl:c_only_repl", "--disk_cache=" <> tmp_disk_cache]
145-
assertSuccess $ bazel ["clean"]
146-
outputSatisfy p
147-
(bazel ["run", "//tests/multi_repl:c_only_repl", "--disk_cache=" <> tmp_disk_cache, "--remote_download_toplevel"])
148-
149-
it "bazel test examples" $ do
150-
assertSuccess $ (bazel ["build", "//..."]) { Process.cwd = Just "../examples" }
151-
assertSuccess $ (bazel ["test", "//..."]) { Process.cwd = Just "../examples" }
152-
153-
it "bazel test tutorial" $ do
154-
assertSuccess $ (bazel ["build", "//..."]) { Process.cwd = Just "../tutorial" }
155-
assertSuccess (bazel ["test", "//..."]) { Process.cwd = Just "../tutorial" }
83+
assertSuccess (bazel ["run", "//tests/repl-flags:compiler_flags@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
84+
85+
-- Test make variable expansion in `compiler_flags` and `repl_ghci_args`.
86+
describe "make variables" $ do
87+
it "compiler flags" $ do
88+
assertSuccess (bazel ["run", "//tests/repl-make-variables:test-compiler-flags@repl", "--", "-ignore-dot-ghci", "-e", ":main"])
89+
it "indirect repl flags" $ do
90+
assertSuccess (bazel ["run", "//tests/repl-make-variables:repl-indirect-flags", "--", "-ignore-dot-ghci", "-e", ":main"])
91+
it "direct repl flags" $ do
92+
assertSuccess (bazel ["run", "//tests/repl-make-variables:repl-direct-flags", "--", "-ignore-dot-ghci", "-e", ":main"])
93+
94+
-- Test `repl_ghci_args` from toolchain and rule for REPL
95+
it "repl flags" $ do
96+
assertSuccess (bazel ["run", "//tests/repl-flags:repl_flags@repl", "--", "-ignore-dot-ghci", "-e", "foo"])
97+
98+
it "fails on multiple definitions" $ do
99+
assertSuccess (bazel ["run", "//tests/repl-multiple-definition:repl", "--", "-ignore-dot-ghci", "-e", "final"])
100+
101+
describe "multi_repl" $ do
102+
it "loads transitive library dependencies" $ do
103+
let p' (stdout, _stderr) = lines stdout == ["tests/multi_repl/bc/src/BC/C.hs"]
104+
outputSatisfy p' (bazel ["run", "//tests/multi_repl:c_only_repl", "--", "-ignore-dot-ghci", "-e", ":show targets"])
105+
it "loads transitive source dependencies" $ do
106+
let p' (stdout, _stderr) = sort (lines stdout) == ["tests/multi_repl/a/src/A/A.hs","tests/multi_repl/bc/src/BC/B.hs","tests/multi_repl/bc/src/BC/C.hs"]
107+
outputSatisfy p' (bazel ["run", "//tests/multi_repl:c_multi_repl", "--", "-ignore-dot-ghci", "-e", ":show targets"])
108+
it "loads core library dependencies" $ do
109+
let p' (stdout, _stderr) = sort (lines stdout) == ["tests/multi_repl/core_package_dep/Lib.hs"]
110+
outputSatisfy p' (bazel ["run", "//tests/multi_repl:core_package_dep", "--", "-ignore-dot-ghci", "-e", ":show targets"])
111+
it "doesn't allow to manually load modules" $ do
112+
assertFailure (bazel ["run", "//tests/multi_repl:c_multi_repl", "--", "-ignore-dot-ghci", "-e", ":load BC.C", "-e", "c"])
113+
114+
describe "ghcide" $ do
115+
it "loads RunTests.hs" $
116+
assertSuccess (Process.proc "./.ghcide" ["tests/RunTests.hs"])
117+
it "loads module with module dependency" $
118+
assertSuccess (Process.proc "./.ghcide" ["tests/binary-with-lib/Main.hs"])
119+
120+
describe "failures" $ do
121+
-- Make sure not to include haskell_repl (@repl) or alias (-repl) targets
122+
-- in the query. Those would not fail under bazel test.
123+
all_failure_tests <- bazelQuery "kind('haskell_library|haskell_binary|haskell_test', //tests/failures/...) intersect attr('tags', 'manual', //tests/failures/...)"
124+
125+
for_ all_failure_tests $ \test -> do
126+
it test $ do
127+
assertFailure (bazel ["build", test])
128+
129+
context "known issues" $ do
130+
it "haskell_doc fails with plugins #1549" $
131+
-- https://github.com/tweag/rules_haskell/issues/1549
132+
assertFailure (bazel ["build", "//tests/haddock-with-plugin"])
133+
it "transitive re-exports do not work #1145" $
134+
-- https://github.com/tweag/rules_haskell/issues/1145
135+
assertFailure (bazel ["build", "//tests/package-reexport-transitive"])
136+
it "doctest failure with foreign import #1559" $
137+
-- https://github.com/tweag/rules_haskell/issues/1559
138+
assertFailure (bazel ["build", "//tests/haskell_doctest_ffi_1559:doctest-a"])
139+
140+
-- Test that the repl still works if we shadow some Prelude functions
141+
it "repl name shadowing" $ do
142+
let p (stdout, stderr) = not $ any ("error" `isInfixOf`) [stdout, stderr]
143+
outputSatisfy p (bazel ["run", "//tests/repl-name-conflicts:lib@repl", "--", "-ignore-dot-ghci", "-e", "stdin"])
144+
145+
-- GH2096: This test is flaky in CI using the MacOS GitHub runners. The flakiness is slowing
146+
-- development on other features. Disable this test until a satisfying solution is found.
147+
-- it "Repl works with remote_download_toplevel" $ do
148+
-- let p (stdout, stderr) = not $ any ("error" `isInfixOf`) [stdout, stderr]
149+
-- withSystemTempDirectory "bazel_disk_cache" $ \tmp_disk_cache -> do
150+
-- assertSuccess $ bazel ["run", "//tests/multi_repl:c_only_repl", "--disk_cache=" <> tmp_disk_cache]
151+
-- assertSuccess $ bazel ["clean"]
152+
-- outputSatisfy p
153+
-- (bazel ["run", "//tests/multi_repl:c_only_repl", "--disk_cache=" <> tmp_disk_cache, "--remote_download_toplevel"])
154+
155+
buildAndTest "../examples"
156+
buildAndTest "../tutorial"
156157

157158
-- * Bazel commands
158159

@@ -165,6 +166,70 @@ bazel args = Process.proc "bazel" args
165166
bazelQuery :: String -> SpecM a [String]
166167
bazelQuery q = lines <$> runIO (Process.readProcess "bazel" ["query", q] "")
167168

169+
-- | Shutdown Bazel
170+
shutdownBazel :: String -> IO ()
171+
shutdownBazel path = do
172+
-- Related to https://github.com/tweag/rules_haskell/issues/2089
173+
-- We experience intermittent "Exit Code: ExitFailure (-9)" errors. Shutdown
174+
-- Bazel when done executing tests for the workspace.
175+
assertSuccess (bazel ["shutdown"]) { Process.cwd = Just path }
176+
pure ()
177+
178+
buildAndTest :: HasCallStack => String -> SpecWith ()
179+
buildAndTest path = describe path $ afterAll_ (shutdownBazel path) $ do
180+
it "bazel build" $ do
181+
assertSuccess $ (bazel ["build", "//..."]) { Process.cwd = Just path }
182+
it "bazel test" $ do
183+
assertSuccess $ (bazel ["test", "//..."]) { Process.cwd = Just path }
184+
185+
-- * Print Memory Hooks
186+
187+
-- | Print memory information before and after each test
188+
-- Only perform the hook if RHT_PRINT_MEMORY is "true".
189+
printStatsHook :: IO () -> IO ()
190+
printStatsHook action = do
191+
rhtPrintMem <- lookupEnv "RHT_PRINT_MEMORY"
192+
case rhtPrintMem of
193+
Just "true" -> bracket_
194+
(printStats "=== BEFORE ===")
195+
(printStats "=== AFTER ===")
196+
action
197+
_ -> action
198+
199+
topPath :: String
200+
topPath = "/usr/bin/top"
201+
202+
dfPath :: String
203+
dfPath = "/bin/df"
204+
205+
-- | Print information about the computer state to debug intermittent failures
206+
-- Related to https://github.com/tweag/rules_haskell/issues/2089
207+
printStats :: String -> IO ()
208+
printStats msg = do
209+
-- Do not attempt to run top, if it does not exist.
210+
topExists <- doesFileExist topPath
211+
dfExists <- doesFileExist dfPath
212+
if topExists || dfExists then putStrLn msg else pure()
213+
if topExists then _printMemory else pure()
214+
if dfExists then _printDiskInfo else pure()
215+
216+
-- | Print information about the current memory state to debug intermittent failures
217+
-- Related to https://github.com/tweag/rules_haskell/issues/2089
218+
_printMemory :: IO ()
219+
_printMemory = do
220+
(exitCode, stdOut, stdErr) <- Process.readProcessWithExitCode topPath ["-l", "1", "-s", "0", "-o", "mem", "-n", "15"] ""
221+
case exitCode of
222+
ExitSuccess -> putStrLn stdOut
223+
ExitFailure _ -> putStrLn ("=== _printMemory failed ===\n" ++ stdErr)
224+
225+
-- | Print information about the disk drives to debug intermittent failures
226+
-- Related to https://github.com/tweag/rules_haskell/issues/2089
227+
_printDiskInfo :: IO ()
228+
_printDiskInfo = do
229+
(exitCode, stdOut, stdErr) <- Process.readProcessWithExitCode dfPath ["-H"] ""
230+
case exitCode of
231+
ExitSuccess -> putStrLn stdOut
232+
ExitFailure _ -> putStrLn ("=== _printDiskInfo failed ===\n" ++ stdErr)
168233

169234
-- Generated dependencies for testing the ghcide support
170235
_ghciIDE :: Int

0 commit comments

Comments
 (0)