Add parallel parsing

The GHC API is no longer used to discover modules. Because of that,
modules can be parsed in parallel.

Author: martijnbastiaan

.github/workflows/ci.yml

@@ -71,23 +71,7 @@ jobs:
           - "9.2.8"
           - "9.0.2"
           - "8.10.7"
-          - "8.8.4"
-          - "8.6.5"
-          - "8.4.4"
         exclude:
-          # Windows gets stuck while running the testsuite, but this is not a
-          # doctest-parallel failure
-          - os: windows-latest
-            ghc: 8.8.4
-
-          # GHCUp fails to install newer GHCs on macOS
-          - os: macOS-latest
-            ghc: "8.8.4"
-          - os: macOS-latest
-            ghc: "8.6.5"
-          - os: macOS-latest
-            ghc: "8.4.4"
-
           # Newer macOSs don't have the right LLVM to compile our dependencies
           - os: macOS-latest
             ghc: "9.0.2"

CHANGES.markdown

@@ -1,6 +1,8 @@
 # 0.4
  * Account for `default-language` sections in Cabal files ([#85](https://github.com/martijnbastiaan/doctest-parallel/issues/85))
  * Add support for Cabal 3.14 ([#88](https://github.com/martijnbastiaan/doctest-parallel/pull/88))
+ * Add parallel parsing on Linux/macOS. The GHC API is now used to call the parser directly, which allows parallel parsing. On Windows, files will be parsed sequentially still due to the GHC API locking files. ([#85](https://github.com/martijnbastiaan/doctest-parallel/issues/89))
+ * Drop support for GHC 8.4, 8.6, and 8.8
 
 # 0.3.1.1
  * Add support for GHC 9.12 (loosened bounds in Hackage revision)

doctest-parallel.cabal

@@ -17,10 +17,7 @@ author:         Martijn Bastiaan <martijn@hmbastiaan.nl>
 maintainer:     Martijn Bastiaan <martijn@hmbastiaan.nl>
 build-type:     Simple
 tested-with:
-    GHC == 8.4.4
-  , GHC == 8.6.5
-  , GHC == 8.8.4
-  , GHC == 8.10.7
+    GHC == 8.10.7
   , GHC == 9.0.2
   , GHC == 9.2.8
   , GHC == 9.4.7
@@ -105,6 +102,7 @@ library
     , exceptions
     , filepath
     , ghc >=8.2 && <9.13
+    , ghc-exactprint
     , ghc-paths >=0.1.0.9
     , process
     , random >= 1.2

scripts/build_all.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+set -e
+
+GHCS=( "8.10.7" "9.0.2" "9.2.8" "9.4.8" "9.6.6" "9.8.1" "9.10.1" )
+
+cabal update
+
+for GHC in "${GHCS[@]}"
+do
+  echo "> cabal build all --with-compiler=ghc-$GHC --only-dependencies"
+  cabal build all --with-compiler=ghc-$GHC --only-dependencies
+done
+
+for GHC in "${GHCS[@]}"
+do
+  echo "> cabal build all --with-compiler=ghc-$GHC"
+  cabal build all --with-compiler=ghc-$GHC
+done

src/Data/List/Extra.hs

@@ -1,4 +1,4 @@
-module Data.List.Extra (trim) where
+module Data.List.Extra (trim, splitOn) where
 
 import Data.Char (isSpace)
 import Data.List (dropWhileEnd)
@@ -19,3 +19,21 @@ trimStart = dropWhile isSpace
 -- | Remove spaces from the end of a string, see 'trim'.
 trimEnd :: String -> String
 trimEnd = dropWhileEnd isSpace
+
+-- TODO: Use doctests after fixing: https://github.com/martijnbastiaan/doctest-parallel/issues/87
+
+-- | Break a list into pieces separated by the first argument, consuming the delimiter.
+--
+-- > splitOn '.' "A.B"
+-- ["A","B"]
+-- > splitOn '.' "A.B.C"
+-- ["A","B","C"]
+-- > splitOn '.' "."
+-- ["",""]
+-- > splitOn '.' ""
+-- [""]
+splitOn :: Eq a => a -> [a] -> [[a]]
+splitOn needle haystack =
+  case break (== needle) haystack of
+    (chunk, []) -> [chunk]
+    (chunk, _ : rest) -> chunk : splitOn needle rest

src/Test/DocTest.hs

@@ -37,7 +37,6 @@ import Panic
 import GHC.Utils.Panic
 #endif
 
-import Test.DocTest.Internal.Parse
 import Test.DocTest.Internal.Options
 import Test.DocTest.Internal.Runner
 import Test.DocTest.Internal.Nix (getNixGhciArgs)
@@ -110,17 +109,17 @@ isSuccess s = sErrors s == 0 && sFailures s == 0
 -- | Filter modules to be tested against a list of modules to be tested (specified
 -- by the user on the command line). If list is empty, test all modules. Throws
 -- and error if a non-existing module was specified.
-filterModules :: [ModuleName] -> [Module a] -> [Module a]
+filterModules :: [ModuleName] -> [ModuleName] -> [ModuleName]
 filterModules [] mods = mods
 filterModules wantedMods0 allMods0
   | (_:_) <- nonExistingMods = error ("Unknown modules specified: " <> show nonExistingMods)
   | otherwise = filter isSpecifiedMod allMods0
  where
   wantedMods1 = Set.fromList wantedMods0
-  allMods1 = Set.fromList (map moduleName allMods0)
+  allMods1 = Set.fromList allMods0
 
   nonExistingMods = Set.toList (wantedMods1 `Set.difference` allMods1)
-  isSpecifiedMod Module{moduleName} = moduleName `Set.member` wantedMods1
+  isSpecifiedMod nm = nm `Set.member` wantedMods1
 
 setSeed :: (?verbosity :: LogLevel) => ModuleConfig -> IO ModuleConfig
 setSeed cfg@ModuleConfig{cfgRandomizeOrder=True, cfgSeed=Nothing} = do
@@ -137,24 +136,19 @@ run lib Config{..} = do
 
   let
     implicitPrelude = DisableExtension ImplicitPrelude `notElem` libDefaultExtensions lib
-    (includeArgs, moduleArgs, otherGhciArgs) = libraryToGhciArgs lib
+    (includeArgs, allModules, otherGhciArgs) = libraryToGhciArgs lib
     evalGhciArgs = otherGhciArgs ++ ["-XNoImplicitPrelude"] ++ nixGhciArgs
-    parseGhcArgs = includeArgs ++ moduleArgs ++ otherGhciArgs ++ nixGhciArgs ++ cfgGhcArgs
+    parseGhcArgs = includeArgs ++ otherGhciArgs ++ nixGhciArgs ++ cfgGhcArgs
 
   let
     ?verbosity = cfgLogLevel
 
   modConfig <- setSeed cfgModuleConfig
 
-  -- Get examples from Haddock comments
-  Logging.log Verbose "Parsing comments.."
-  Logging.log Debug ("Calling GHC API with: " <> unwords parseGhcArgs)
-  allModules <- getDocTests parseGhcArgs
-
   -- Run tests
   Logging.log Verbose "Running examples.."
   let
     filteredModules = filterModules cfgModules allModules
-    filteredModulesMsg = intercalate ", " (map moduleName filteredModules)
+    filteredModulesMsg = intercalate ", " filteredModules
   Logging.log Debug ("Running examples in modules: " <> filteredModulesMsg)
-  runModules modConfig cfgThreads implicitPrelude evalGhciArgs filteredModules
+  runModules modConfig cfgThreads implicitPrelude parseGhcArgs evalGhciArgs filteredModules

src/Test/DocTest/Helpers.hs

@@ -31,7 +31,7 @@ import Distribution.PackageDescription
   ( GenericPackageDescription (condLibrary)
   , exposedModules, libBuildInfo, hsSourceDirs, defaultExtensions, package
   , packageDescription, condSubLibraries, includeDirs, autogenModules, ConfVar(..)
-  , defaultLanguage )
+  , defaultLanguage, BuildInfo (otherModules) )
 
 import Distribution.Compiler (CompilerFlavor(GHC))
 import Distribution.Pretty (prettyShow)
@@ -256,11 +256,12 @@ extractSpecificCabalLibrary maybeLibName pkgPath = do
       $ map compatPrettyShow
 #endif
       cSourceDirs
-    , libModules = exposedModules lib `rmList` autogenModules buildInfo
+    , libModules = modules `rmList` autogenModules buildInfo
     , libDefaultExtensions = defaultExtensions buildInfo
     , libDefaultLanguages = maybeToList (defaultLanguage buildInfo)
     }
    where
+    modules = otherModules buildInfo <> exposedModules lib
     buildInfo = libBuildInfo lib
     sourceDirs = hsSourceDirs buildInfo
     cSourceDirs = includeDirs buildInfo