Merge pull request #835 from phadej/auto-compare-typed-bench

Add AutoCompare and Typed benchmarks to suite

Author: phadej

benchmarks/aeson-benchmarks.cabal

@@ -113,19 +113,6 @@ executable aeson-benchmark-suite
     , time
     , vector
 
-  other-modules:
-    GitHub
-    Twitter
-    Twitter.Manual
-    Issue673
-
--- Benchmarks below are haven't been worked out yet
-
-executable aeson-benchmark-auto-compare
-  default-language: Haskell2010
-  main-is:          AutoCompare.hs
-  hs-source-dirs:   bench
-  ghc-options:      -Wall -O2 -rtsopts
   other-modules:
     Auto.G.BigProduct
     Auto.G.BigRecord
@@ -135,14 +122,22 @@ executable aeson-benchmark-auto-compare
     Auto.T.BigRecord
     Auto.T.BigSum
     Auto.T.D
+    AutoCompare
+    GitHub
+    Issue673
     Options
+    Twitter
+    Twitter.Generic
+    Twitter.Manual
+    Twitter.Options
+    Twitter.TH
+    Typed
+    Typed.Generic
+    Typed.Manual
+    Typed.TH
+    Utils
 
-  build-depends:
-      aeson-benchmarks
-    , base
-    , criterion
-    , deepseq
-    , template-haskell
+-- Benchmarks below are haven't been worked out yet
 
 executable aeson-benchmark-compare
   default-language: Haskell2010
@@ -154,7 +149,7 @@ executable aeson-benchmark-compare
     Compare.JsonBench
     Twitter
     Twitter.Manual
-    Typed.Common
+    Utils
 
   build-depends:
       aeson-benchmarks
@@ -164,6 +159,7 @@ executable aeson-benchmark-compare
     , bytestring
     , criterion              >=1.0
     , deepseq
+    , filepath
     , ghc-prim
     , text
 
@@ -191,36 +187,6 @@ executable aeson-benchmark-micro
     , ghc-prim
     , text
 
-executable aeson-benchmark-typed
-  default-language: Haskell2010
-  main-is:          Typed.hs
-  hs-source-dirs:   bench examples/src
-  ghc-options:      -Wall -O2 -rtsopts
-  other-modules:
-    Twitter
-    Twitter.Generic
-    Twitter.Manual
-    Twitter.Options
-    Twitter.TH
-    Typed.Common
-    Typed.Generic
-    Typed.Manual
-    Typed.TH
-
-  build-depends:
-      aeson-benchmarks
-    , base
-    , base-compat-batteries
-    , bytestring
-    , criterion              >=1.0
-    , deepseq
-    , ghc-prim
-    , text
-    , time
-
-  if impl(ghc <8.0)
-    build-depends: semigroups
-
 executable aeson-benchmark-compare-with-json
   default-language: Haskell2010
   main-is:          CompareWithJSON.hs

benchmarks/bench/AutoCompare.hs

@@ -1,7 +1,7 @@
 {-# LANGUAGE RankNTypes #-}
 {-# LANGUAGE ScopedTypeVariables #-}
 
-module Main (main) where
+module AutoCompare (benchmark, sanityCheck) where
 
 import Control.DeepSeq
 import Control.Monad
@@ -19,8 +19,8 @@ import qualified Auto.G.BigSum as G
 
 --------------------------------------------------------------------------------
 
-runBench :: IO ()
-runBench = defaultMain
+benchmark :: Benchmark
+benchmark = bgroup "AutoBench"
   [ compareBench "D" T.d G.d
   , compareBench "BigRecord" T.bigRecord G.bigRecord
   , compareBench "BigProduct" T.bigProduct G.bigProduct
@@ -66,8 +66,3 @@ check x = do
     decode_ s = case decode s of
       Just v -> fromJSON v
       Nothing -> fail ""
-
-main :: IO ()
-main = do
-  sanityCheck
-  runBench

benchmarks/bench/Compare.hs

@@ -11,7 +11,7 @@ import Criterion.Main
 import Data.BufferBuilder.Json
 import Twitter
 import Twitter.Manual ()
-import Typed.Common
+import Utils (readV)
 import qualified Data.Aeson as Aeson
 import qualified Compare.JsonBench as JsonBench
 
@@ -23,8 +23,8 @@ import Compare.JsonBuilder ()
 main :: IO ()
 main =
   defaultMain [
-     env (load "json-data/twitter100.json") $ \ ~(twtr :: Result) ->
-     bgroup "twitter" [
+     env (readV "json-data/twitter100.json") $ \ ~(twtr :: Result) ->
+     bgroup "CompareEncodeTwitter" [
          bench "aeson" $ nf Aeson.encode twtr
        , bench "buffer-builder" $ nf encodeJson twtr
 #ifdef MIN_VERSION_json_builder

benchmarks/bench/Compare/JsonBench.hs

@@ -19,7 +19,7 @@ import Criterion
 import Data.Aeson ((.:))
 import Data.Monoid ((<>))
 import Data.Text (Text)
-import Typed.Common (load)
+import Utils (readV)
 import qualified Control.Monad.Fail as Fail
 import qualified Data.Aeson as Aeson
 import qualified Data.BufferBuilder.Json as Json
@@ -328,9 +328,9 @@ instance JB.Value User where
 #endif
 
 benchmarks :: Benchmark
-benchmarks = env (load "json-data/buffer-builder.json") $
+benchmarks = env (readV "buffer-builder.json") $
     \ ~(parsedUserList :: [User]) ->
-    bgroup "json-bench" [
+    bgroup "CompareEncodeUserList" [
       bench "aeson" $ nf Aeson.encode parsedUserList
     , bench "buffer-builder" $ nf Json.encodeJson parsedUserList
 #ifdef MIN_VERSION_json_builder

benchmarks/bench/Suite.hs

@@ -10,11 +10,8 @@ import           Prelude.Compat
 import           Control.DeepSeq                (NFData)
 import           Criterion.Main                 (Benchmark, bench, bgroup,
                                                  defaultMain, env, nf, whnf)
-import           Data.Maybe                     (fromMaybe)
 import           Data.Proxy                     (Proxy (..))
 import           Data.Vector                    (Vector)
-import           System.Environment             (lookupEnv)
-import           System.FilePath                ((</>))
 
 import qualified Data.Aeson.Encoding.Builder    as Aeson.EB
 import qualified Data.Aeson.Parser.UnescapeFFI  as FFI
@@ -25,10 +22,12 @@ import qualified Data.ByteString.Char8          as BS8
 import qualified Data.ByteString.Lazy           as LBS
 import qualified Data.Text                      as T
 
+import qualified AutoCompare
 import qualified GitHub
 import qualified Issue673
-import qualified Twitter
-import qualified Twitter.Manual                 ()
+import qualified Typed
+
+import           Utils
 
 -------------------------------------------------------------------------------
 -- Decode bench
@@ -76,33 +75,21 @@ escapeBench = bgroup "Escape"
       , bench "pure" $ whnf Pure.unescapeText input
       ]
 
--------------------------------------------------------------------------------
--- Helpers
--------------------------------------------------------------------------------
-
-readS :: FilePath -> IO BS.ByteString
-readS fp = do
-    dataDir <- lookupEnv "AESON_BENCH_DATADIR"
-    BS.readFile $ fromMaybe "json-data" dataDir </> fp
-
-readL :: FilePath -> IO LBS.ByteString
-readL fp = do
-    dataDir <- lookupEnv "AESON_BENCH_DATADIR"
-    LBS.readFile $ fromMaybe "json-data" dataDir </> fp
-
 -------------------------------------------------------------------------------
 -- Main
 -------------------------------------------------------------------------------
 
 main :: IO ()
-main = defaultMain
-  [ bgroup "Examples"
-    [ bgroup "decode"
-      [ decodeBench "twitter100"    "twitter100.json"    (Proxy :: Proxy Twitter.Result)
-      , decodeBench "jp100"         "jp100.json"         (Proxy :: Proxy Twitter.Result)
-      , decodeBench "github-issues" "github-issues.json" (Proxy :: Proxy (Vector GitHub.Issue))
+main = do
+  AutoCompare.sanityCheck
+  defaultMain
+    [ bgroup "Examples"
+      [ bgroup "decode"
+        [ decodeBench "github-issues" "github-issues.json" (Proxy :: Proxy (Vector GitHub.Issue))
+        ]
       ]
+    , escapeBench
+    , Issue673.benchmark
+    , Typed.benchmark -- Twitter
+    , AutoCompare.benchmark -- compares Generic and TH
     ]
-  , escapeBench
-  , Issue673.benchmark
-  ]

benchmarks/bench/Typed.hs

@@ -1,20 +1,22 @@
 {-# LANGUAGE NoImplicitPrelude #-}
 
-module Main (main) where
+module Typed (benchmark) where
 
-import Prelude.Compat
+import           Criterion.Main
+import qualified Typed.Generic  as Generic
+import qualified Typed.Manual   as Manual
+import qualified Typed.TH       as TH
 
-import Criterion.Main
-import qualified Typed.Generic as Generic
-import qualified Typed.Manual as Manual
-import qualified Typed.TH as TH
-
-main :: IO ()
-main = defaultMain [
-    Generic.benchmarks
-  , Manual.benchmarks
-  , TH.benchmarks
-  , Generic.decodeBenchmarks
-  , Manual.decodeBenchmarks
-  , TH.decodeBenchmarks
+benchmark :: Benchmark
+benchmark = bgroup "Twitter"
+  [ bgroup "encode"
+    [ Generic.benchmarks
+    , Manual.benchmarks
+    , TH.benchmarks
+    ]
+  , bgroup "decode"
+    [ Generic.decodeBenchmarks
+    , Manual.decodeBenchmarks
+    , TH.decodeBenchmarks
+    ]
   ]

benchmarks/bench/Typed/Common.hs

@@ -8,7 +8,7 @@ import Data.Aeson hiding (Result)
 import Criterion
 import Data.ByteString.Lazy as L
 import Twitter.Generic
-import Typed.Common
+import Utils
 
 encodeDirect :: Result -> L.ByteString
 encodeDirect = encode
@@ -18,8 +18,8 @@ encodeViaValue = encode . toJSON
 
 benchmarks :: Benchmark
 benchmarks =
-  env ((,) <$> load "json-data/twitter100.json" <*> load "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "encodeGeneric" [
+  env ((,) <$> readV "twitter100.json" <*> readV "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "Generic" [
       bgroup "direct" [
         bench "twitter100" $ nf encodeDirect twitter100
       , bench "jp100"      $ nf encodeDirect jp100
@@ -35,8 +35,8 @@ decodeDirect = decode
 
 decodeBenchmarks :: Benchmark
 decodeBenchmarks =
-  env ((,) <$> L.readFile "json-data/twitter100.json" <*> L.readFile "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "decodeGeneric"
+  env ((,) <$> readL "twitter100.json" <*> readL "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "Generic"
     [ bgroup "direct"
       [ bench "twitter100" $ nf decodeDirect twitter100
       , bench "jp100"      $ nf decodeDirect jp100

benchmarks/bench/Typed/Generic.hs

@@ -8,7 +8,7 @@ import Data.Aeson hiding (Result)
 import Criterion
 import Data.ByteString.Lazy as L
 import Twitter.Manual
-import Typed.Common
+import Utils
 
 encodeDirect :: Result -> L.ByteString
 encodeDirect = encode
@@ -18,8 +18,8 @@ encodeViaValue = encode . toJSON
 
 benchmarks :: Benchmark
 benchmarks =
-  env ((,) <$> load "json-data/twitter100.json" <*> load "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "encodeManual" [
+  env ((,) <$> readV "twitter100.json" <*> readV "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "Manual" [
       bgroup "direct" [
         bench "twitter100" $ nf encodeDirect twitter100
       , bench "jp100"      $ nf encodeDirect jp100
@@ -35,8 +35,8 @@ decodeDirect = decode
 
 decodeBenchmarks :: Benchmark
 decodeBenchmarks =
-  env ((,) <$> L.readFile "json-data/twitter100.json" <*> L.readFile "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "decodeManual"
+  env ((,) <$> readL "twitter100.json" <*> readL "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "Manual"
     [ bgroup "direct"
       [ bench "twitter100" $ nf decodeDirect twitter100
       , bench "jp100"      $ nf decodeDirect jp100

benchmarks/bench/Typed/Manual.hs

@@ -8,7 +8,7 @@ import Data.Aeson hiding (Result)
 import Criterion
 import Data.ByteString.Lazy as L
 import Twitter.TH
-import Typed.Common
+import Utils
 
 encodeDirect :: Result -> L.ByteString
 encodeDirect = encode
@@ -18,8 +18,8 @@ encodeViaValue = encode . toJSON
 
 benchmarks :: Benchmark
 benchmarks =
-  env ((,) <$> load "json-data/twitter100.json" <*> load "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "encodeTH" [
+  env ((,) <$> readV "twitter100.json" <*> readV "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "TH" [
       bgroup "direct" [
         bench "twitter100" $ nf encodeDirect twitter100
       , bench "jp100"      $ nf encodeDirect jp100
@@ -35,8 +35,8 @@ decodeDirect = decode
 
 decodeBenchmarks :: Benchmark
 decodeBenchmarks =
-  env ((,) <$> L.readFile "json-data/twitter100.json" <*> L.readFile "json-data/jp100.json") $ \ ~(twitter100, jp100) ->
-  bgroup "decodeTH"
+  env ((,) <$> readL "twitter100.json" <*> readL "jp100.json") $ \ ~(twitter100, jp100) ->
+  bgroup "TH"
     [ bgroup "direct"
       [ bench "twitter100" $ nf decodeDirect twitter100
       , bench "jp100"      $ nf decodeDirect jp100