Overhaul test harnesses and add tasty bounds.

Author: Patrick Thomson

cabal.project

@@ -14,8 +14,3 @@ source-repository-package
   type: git
   location: https://github.com/joshvera/proto3-wire.git
   tag: 84664e22f01beb67870368f1f88ada5d0ad01f56
-
-source-repository-package
-  type: git
-  location: https://github.com/rewinfrey/hspec-expectations-pretty-diff
-  tag: 94af5871c24ba319f7f72fefa53c1a4d074c9a29

semantic.cabal

@@ -274,7 +274,7 @@ library
   autogen-modules:     Paths_semantic
   other-modules:       Paths_semantic
   build-depends:       base >= 4.12 && < 5
-                     , ansi-terminal ^>= 0.8.2
+                     , ansi-terminal >= 0.8.2 && <1
                      , array ^>= 0.5.3.0
                      , attoparsec ^>= 0.13.2.2
                      , cmark-gfm == 0.1.8
@@ -384,7 +384,10 @@ test-suite test
                      , Glob
                      , hspec >= 2.6 && <3
                      , hspec-core >= 2.6 && <3
-                     , hspec-expectations-pretty-diff ^>= 0.7.2.5
+                     , hspec-expectations ^>= 0.8.2
+                     , tasty ^>= 1.2.3
+                     , tasty-golden ^>= 2.3.2
+                     , tasty-hspec ^>= 1.1.5.1
                      , HUnit ^>= 1.6.0.0
                      , leancheck >= 0.8 && <1
                      , temporary
@@ -400,7 +403,7 @@ test-suite parse-examples
                      , Glob
                      , hspec >= 2.4.1
                      , hspec-core
-                     , hspec-expectations-pretty-diff
+                     , hspec-expectations
 
 benchmark evaluation
   import:              haskell, executable-flags

test/Data/Term/Spec.hs

@@ -4,7 +4,7 @@ module Data.Term.Spec (spec) where
 import Data.Functor.Listable
 import Data.Term
 import Test.Hspec (Spec, describe, parallel)
-import Test.Hspec.Expectations.Pretty
+import Test.Hspec.Expectations
 import Test.Hspec.LeanCheck
 
 spec :: Spec

test/Diffing/Interpreter/Spec.hs

@@ -15,7 +15,7 @@ import Data.These
 import Diffing.Interpreter
 import qualified Data.Syntax as Syntax
 import Test.Hspec (Spec, describe, it, parallel)
-import Test.Hspec.Expectations.Pretty
+import Test.Hspec.Expectations
 import Test.Hspec.LeanCheck
 import Test.LeanCheck.Core
 import SpecHelpers ()

test/Integration/Spec.hs

@@ -14,7 +14,6 @@ import SpecHelpers
 
 import Test.Tasty
 import Test.Tasty.Golden
-import Test.Tasty.HUnit
 
 languages :: [FilePath]
 languages = ["go", "javascript", "json", "python", "ruby", "typescript", "tsx"]
@@ -27,6 +26,7 @@ testsForLanguage language = do
   let dir = "test/fixtures" </> language </> "corpus"
   let items = unsafePerformIO (examples dir)
   testGroup language (fmap testForExample items)
+{-# NOINLINE testsForLanguage #-}
 
 data Example = DiffExample { fileA :: FilePath, fileB :: FilePath, diffOutput :: FilePath }
              | ParseExample { file :: FilePath, parseOutput :: FilePath }
@@ -40,7 +40,12 @@ testForExample = \case
       (\ref new -> ["git", "diff", ref, new])
       diffOutput
       (BL.fromStrict <$> diffFilePaths ?session (Both fileA fileB))
-  ParseExample{file, parseOutput} -> testCase ("parses " <> file) (pure ())
+  ParseExample{file, parseOutput} ->
+    goldenVsStringDiff
+      ("parses " <> parseOutput)
+      (\ref new -> ["git", "diff", ref, new])
+      parseOutput
+      (parseFilePath ?session file >>= either throw (pure . BL.fromStrict))
 
 
 -- | Return all the examples from the given directory. Examples are expected to

test/Spec.hs

@@ -37,43 +37,70 @@ import qualified Semantic.Stat.Spec
 import Semantic.Config (defaultOptions, optionsLogLevel)
 import Semantic.Task (withOptions, TaskSession(..))
 import Test.Hspec
+import Test.Tasty as Tasty
+import Test.Tasty.Hspec as Tasty
+import Test.Tasty.Options as Tasty
+
+tests :: TaskSession -> [TestTree]
+tests session =
+  [ Integration.Spec.spec session
+  ]
+
+-- We can't bring this out of the IO monad until we divest
+-- from hspec, since testSpec operates in IO.
+allTests :: TaskSession -> IO TestTree
+allTests session = do
+  let nativeSpecs = tests session
+  asTastySpecs <- Tasty.testSpecs $ legacySpecs session
+  let allSpecs = nativeSpecs <> asTastySpecs
+  pure . Tasty.localOption Tasty.Success $ testGroup "semantic" allSpecs
+
+-- If you're writing new test modules, please don't add to this
+-- stanza: it is only there to prevent massive rewrites, and is
+-- converted into a Tasty TestTree in 'main'. (Quoth the tasty-hspec
+-- documentation: "hspec and tasty serve similar purposes; consider
+-- using one or the other.") Instead,
+legacySpecs :: TaskSession -> Spec
+legacySpecs args = do
+  describe "Semantic.Stat" Semantic.Stat.Spec.spec
+  parallel $ do
+    describe "Analysis.Go" (Analysis.Go.Spec.spec args)
+    describe "Analysis.PHP" (Analysis.PHP.Spec.spec args)
+    describe "Analysis.Python" (Analysis.Python.Spec.spec args)
+    describe "Analysis.Ruby" (Analysis.Ruby.Spec.spec args)
+    describe "Analysis.TypeScript" (Analysis.TypeScript.Spec.spec args)
+    describe "Assigning.Assignment" Assigning.Assignment.Spec.spec
+    describe "Control.Abstract.Evaluator" Control.Abstract.Evaluator.Spec.spec
+    describe "Data.Diff" Data.Diff.Spec.spec
+    describe "Data.Graph" Data.Graph.Spec.spec
+    describe "Data.Abstract.Path" Data.Abstract.Path.Spec.spec
+    describe "Data.Abstract.Name" Data.Abstract.Name.Spec.spec
+    describe "Data.Functor.Classes.Generic" Data.Functor.Classes.Generic.Spec.spec
+    describe "Data.Range" Data.Range.Spec.spec
+    describe "Data.Scientific" Data.Scientific.Spec.spec
+    describe "Data.Semigroup.App" Data.Semigroup.App.Spec.spec
+    describe "Data.Source" Data.Source.Spec.spec
+    describe "Data.Term" Data.Term.Spec.spec
+    describe "Diffing.Algorithm.RWS" Diffing.Algorithm.RWS.Spec.spec
+    describe "Diffing.Algorithm.SES" Diffing.Algorithm.SES.Spec.spec
+    describe "Diffing.Interpreter" Diffing.Interpreter.Spec.spec
+    describe "Graphing.Calls" Graphing.Calls.Spec.spec
+    describe "Numeric" Numeric.Spec.spec
+    describe "Rendering.TOC" Rendering.TOC.Spec.spec
+    describe "Reprinting.Spec" Reprinting.Spec.spec
+    describe "Rewriting.Go" Rewriting.Go.Spec.spec
+    describe "Rewriting.JSON" Rewriting.JSON.Spec.spec
+    describe "Rewriting.Python" Rewriting.Python.Spec.spec
+    describe "Tags.Spec" Tags.Spec.spec
+    describe "Semantic" Semantic.Spec.spec
+    describe "Semantic.CLI" Semantic.CLI.Spec.spec
+    describe "Semantic.IO" Semantic.IO.Spec.spec
+    describe "Parsing" Parsing.Spec.spec
+
 
 main :: IO ()
 main = do
-  withOptions defaultOptions { optionsLogLevel = Nothing } $ \ config logger statter -> hspec $ do
-    let args = TaskSession config "-" False logger statter
-    describe "Semantic.Stat" Semantic.Stat.Spec.spec
-    parallel $ do
-      describe "Analysis.Go" (Analysis.Go.Spec.spec args)
-      describe "Analysis.PHP" (Analysis.PHP.Spec.spec args)
-      describe "Analysis.Python" (Analysis.Python.Spec.spec args)
-      describe "Analysis.Ruby" (Analysis.Ruby.Spec.spec args)
-      describe "Analysis.TypeScript" (Analysis.TypeScript.Spec.spec args)
-      describe "Assigning.Assignment" Assigning.Assignment.Spec.spec
-      describe "Control.Abstract.Evaluator" Control.Abstract.Evaluator.Spec.spec
-      describe "Data.Diff" Data.Diff.Spec.spec
-      describe "Data.Graph" Data.Graph.Spec.spec
-      describe "Data.Abstract.Path" Data.Abstract.Path.Spec.spec
-      describe "Data.Abstract.Name" Data.Abstract.Name.Spec.spec
-      describe "Data.Functor.Classes.Generic" Data.Functor.Classes.Generic.Spec.spec
-      describe "Data.Range" Data.Range.Spec.spec
-      describe "Data.Scientific" Data.Scientific.Spec.spec
-      describe "Data.Semigroup.App" Data.Semigroup.App.Spec.spec
-      describe "Data.Source" Data.Source.Spec.spec
-      describe "Data.Term" Data.Term.Spec.spec
-      describe "Diffing.Algorithm.RWS" Diffing.Algorithm.RWS.Spec.spec
-      describe "Diffing.Algorithm.SES" Diffing.Algorithm.SES.Spec.spec
-      describe "Diffing.Interpreter" Diffing.Interpreter.Spec.spec
-      describe "Graphing.Calls" Graphing.Calls.Spec.spec
-      describe "Numeric" Numeric.Spec.spec
-      describe "Rendering.TOC" Rendering.TOC.Spec.spec
-      describe "Reprinting.Spec" Reprinting.Spec.spec
-      describe "Rewriting.Go" Rewriting.Go.Spec.spec
-      describe "Rewriting.JSON" Rewriting.JSON.Spec.spec
-      describe "Rewriting.Python" Rewriting.Python.Spec.spec
-      describe "Tags.Spec" Tags.Spec.spec
-      describe "Semantic" Semantic.Spec.spec
-      describe "Semantic.CLI" Semantic.CLI.Spec.spec
-      describe "Semantic.IO" Semantic.IO.Spec.spec
-      describe "Integration" (Integration.Spec.spec args)
-      describe "Parsing" Parsing.Spec.spec
+  withOptions defaultOptions { optionsLogLevel = Nothing } $ \ config logger statter ->
+    let session = TaskSession config "-" False logger statter
+    in allTests session >>= defaultMain
+

test/SpecHelpers.hs

@@ -71,7 +71,7 @@ import Data.Semigroup as X (Semigroup(..))
 import Control.Monad as X
 
 import Test.Hspec as X (Spec, SpecWith, context, describe, it, xit, parallel, pendingWith, around, runIO)
-import Test.Hspec.Expectations.Pretty as X
+import Test.Hspec.Expectations as X
 import Test.Hspec.LeanCheck as X
 import Test.LeanCheck as X