Fix benchmarks, run them on AppVeyor (#1033)

Since we only want to ensure that the benchmarks continue to
work, we use the following benchmark arguments to run them
as quickly as possible:

- `--quick` ensures that gauge runs only a single sample.
- `--min-duration=0` sets the number of iterations per sample to 1.
- `--include-first-iter` causes gauge to accept that one iteration
  instead of discarding it and running a second one.

This also removes the dhall-command benchmark:

This was a non-standard benchmark that failed when run
without input from stdin. To replace this profiling tool,
I have added instructions for profiling the main executables
to the README.

README.md

@@ -209,6 +209,24 @@ And each of them with `stack build <package-name>`, for example:
 $ stack build dhall-json
 ```
 
+## Profiling
+
+Say you want to profile the `dhall-to-json` executable.
+
+Build the containing package with profiling options:
+
+```console
+$ stack build --profile --library-profiling dhall-json
+```
+
+Run the executable on your input. For example:
+
+```console
+$ stack exec --profile --rts-options -p -- dhall-to-json <<< 'True && False'
+```
+
+This generates a `dhall-to-json.prof` file in your current directory.
+
 ## Build the website
 
 Building the website from source is currently only supported for Nix on Linux.

appveyor.yml

@@ -62,6 +62,8 @@ for:
     - stack test dhall-text
     - stack test dhall-bash
     # - stack test dhall-lsp-server # Disabled while the tests are broken.
+    - stack bench --benchmark-arguments "--quick --min-duration=0 --include-first-iter"
+    
 -
   matrix:
     only:

dhall/benchmark/dhall-command/Main.hs

@@ -1,20 +0,0 @@
-
-{-# LANGUAGE OverloadedStrings #-}
-
-module Main where
-
-import qualified Dhall.Main as Main
-import           Dhall.Binary (defaultStandardVersion)
-
-options :: Main.Options
-options = Main.Options
-  { Main.mode = Main.Default Nothing False False
-  , Main.explain = False
-  , Main.plain = False
-  , Main.ascii = False
-  , Main.standardVersion = defaultStandardVersion
-  }
-
-main :: IO ()
-main = do
-  Main.command options

dhall/benchmark/parser/Main.hs

@@ -38,7 +38,7 @@ listDirectory path = filter f <$> getDirectoryContents path
 type PreludeFiles = Map FilePath T.Text
 
 loadPreludeFiles :: IO PreludeFiles
-loadPreludeFiles = loadDirectory "Prelude"
+loadPreludeFiles = loadDirectory "./dhall-lang/Prelude"
     where
         loadDirectory :: FilePath -> IO PreludeFiles
         loadDirectory dir =

dhall/dhall.cabal

@@ -607,13 +607,3 @@ Benchmark deep-nested-large-record
         dhall                                         ,
         gauge                     >= 0.2.3    && < 0.3
     Default-Language: Haskell2010
-
-Benchmark dhall-command
-    Type: exitcode-stdio-1.0
-    Main-Is: Main.hs
-    Hs-Source-Dirs: benchmark/dhall-command
-    Build-Depends:
-        base                      >= 4        && < 5  ,
-        dhall
-    Default-Language: Haskell2010
-    ghc-options: -rtsopts -O2

nix/shared.nix

@@ -1,6 +1,15 @@
 { compiler ? "ghc843", coverage ? false, system ? builtins.currentSystem }:
 
 let
+  allDhallPackages = [
+    "dhall"
+    "dhall-bash"
+    "dhall-json"
+    "dhall-lsp-server"
+    "dhall-nix"
+    "dhall-text"
+  ];
+
   fetchNixpkgs = import ./fetchNixpkgs.nix;
 
   mass = function: names: haskellPackagesNew: haskellPackagesOld:
@@ -136,6 +145,9 @@ let
                     ++  pkgsNew.lib.optional (!(compiler == "ghcjs" || compiler == "ghc7103")) "dhall"
                     );
 
+                doBenchmarkExtension =
+                  mass pkgsNew.haskell.lib.doBenchmark allDhallPackages;
+
                 failOnAllWarningsExtension =
                   mass failOnAllWarnings [
                     "dhall"
@@ -212,6 +224,7 @@ let
                   [ (pkgsNew.haskell.lib.packagesFromDirectory { directory = ./.; })
                     extension
                     doCheckExtension
+                    doBenchmarkExtension
                     failOnAllWarningsExtension
                   ];
           }
@@ -573,9 +586,7 @@ let
       '';
     };
 
-  toShell = drv:
-    # Benchmark dependencies aren't added by default
-    (pkgs.haskell.lib.doBenchmark drv).env;
+  toShell = drv: drv.env;
 
   possibly-static = {
     dhall            = makeStaticIfPossible "dhall"           ;
@@ -619,13 +630,13 @@ in
 
     inherit (pkgs.releaseTools) aggregate;
 
-    shell-dhall            = toShell pkgs.haskell.packages."${compiler}".dhall           ;
-    shell-dhall-bash       = toShell pkgs.haskell.packages."${compiler}".dhall-bash      ;
-    shell-dhall-json       = toShell pkgs.haskell.packages."${compiler}".dhall-json      ;
-    shell-dhall-lsp-server = toShell pkgs.haskell.packages."${compiler}".dhall-lsp-server;
-    shell-dhall-nix        = toShell pkgs.haskell.packages."${compiler}".dhall-nix       ;
-    shell-dhall-text       = toShell pkgs.haskell.packages."${compiler}".dhall-text      ;
-    shell-dhall-try        = toShell pkgs.haskell.packages."${compiler}".dhall-try       ;
+    shell-dhall            = pkgs.haskell.packages."${compiler}".dhall.env           ;
+    shell-dhall-bash       = pkgs.haskell.packages."${compiler}".dhall-bash.env      ;
+    shell-dhall-json       = pkgs.haskell.packages."${compiler}".dhall-json.env      ;
+    shell-dhall-lsp-server = pkgs.haskell.packages."${compiler}".dhall-lsp-server.env;
+    shell-dhall-nix        = pkgs.haskell.packages."${compiler}".dhall-nix.env       ;
+    shell-dhall-text       = pkgs.haskell.packages."${compiler}".dhall-text.env      ;
+    shell-dhall-try        = pkgs.haskell.packages."${compiler}".dhall-try.env       ;
 
     image-dhall            = toDockerImage "dhall"           ;
     image-dhall-bash       = toDockerImage "dhall-bash"      ;