Merge pull request #768 from google-research/safe-names-dev

Recognize that "safe-names-dev" is the new "main"

Author: dougalm

.github/workflows/docs.yaml

@@ -10,11 +10,11 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
         include:
-          - os: ubuntu-18.04
-            install_deps: sudo apt-get install llvm-9-tools llvm-9-dev pkg-config
-            path_extension: /usr/lib/llvm-9/bin
+          - os: ubuntu-20.04
+            install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config
+            path_extension: /usr/lib/llvm-12/bin
 
     steps:
     - name: Checkout the repository

.github/workflows/haskell-ci.yaml

@@ -4,21 +4,21 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, safe-names-dev ]
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04, macos-latest]
+        os: [ubuntu-20.04, macos-latest]
         include:
           - os: macos-latest
-            install_deps: brew install llvm@9 pkg-config wget gzip
-            path_extension: $(brew --prefix llvm@9)/bin
-          - os: ubuntu-18.04
-            install_deps: sudo apt-get install llvm-9-tools llvm-9-dev pkg-config wget gzip
-            path_extension: /usr/lib/llvm-9/bin
+            install_deps: brew install llvm@12 pkg-config wget gzip
+            path_extension: $(brew --prefix llvm@12)/bin
+          - os: ubuntu-20.04
+            install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config wget gzip
+            path_extension: /usr/lib/llvm-12/bin
 
     steps:
     - name: Checkout the repository
@@ -52,8 +52,8 @@ jobs:
           $GITHUB_WORKSPACE/examples/t10k-images-idx3-ubyte
           $GITHUB_WORKSPACE/examples/t10k-labels-idx1-ubyte
 
-        key: ${{ runner.os }}-v2-${{ hashFiles('**/*.cabal', 'stack*.yaml') }}
-        restore-keys: ${{ runner.os }}-v2-
+        key: ${{ runner.os }}-v5-${{ hashFiles('**/*.cabal', 'stack*.yaml') }}
+        restore-keys: ${{ runner.os }}-v5-
 
     # This step is a workaround.
     # See issue for context: https://github.com/actions/cache/issues/445

.github/workflows/julia-ci.yaml

@@ -2,20 +2,20 @@ name: Julia tests
 
 on:
   push:
-    branches: [ main ]
+    branches: [ ]
   pull_request:
-    branches: [ main ]
+    branches: [ ]
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
         include:
-          - os: ubuntu-18.04
-            install_deps: sudo apt-get install llvm-9-tools llvm-9-dev pkg-config wget gzip
-            path_extension: /usr/lib/llvm-9/bin
+          - os: ubuntu-20.04
+            install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config wget gzip
+            path_extension: /usr/lib/llvm-12/bin
 
     steps:
     - name: Checkout the repository

.github/workflows/python-ci.yaml

@@ -4,18 +4,18 @@ on:
   push:
     branches: [ main ]
   pull_request:
-    branches: [ main ]
+    branches: [ main, safe-names-dev ]
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
         include:
-          - os: ubuntu-18.04
-            install_deps: sudo apt-get install llvm-9-tools llvm-9-dev pkg-config wget gzip
-            path_extension: /usr/lib/llvm-9/bin
+          - os: ubuntu-20.04
+            install_deps: sudo apt-get install llvm-12-tools llvm-12-dev pkg-config wget gzip
+            path_extension: /usr/lib/llvm-12/bin
 
     steps:
     - name: Checkout the repository

.gitignore

@@ -28,3 +28,5 @@ examples/export/array
 hie.yaml
 Manifest.toml
 julia/deps/build.log
+examples/t10k-images-idx3-ubyte
+examples/t10k-labels-idx1-ubyte

README.md

@@ -30,12 +30,12 @@ development. Expect monstrous bugs and razor-sharp edges!**
 ## Dependencies
 
   * Install [stack](https://www.haskellstack.org)
-  * Install LLVM 9
-    * Ubuntu/Debian: `apt-get install llvm-9-dev`
-    * macOS: `brew install llvm@9`
-      * Make sure `llvm@9` is on your `PATH` before building. Example: `export PATH="$(brew --prefix llvm@9)/bin:$PATH"`
-  * Install clang 9 (may be installed together with llvm)
-    * Ubuntu/Debian: `apt-get install clang-9`
+  * Install LLVM 12
+    * Ubuntu/Debian: `apt-get install llvm-12-dev`
+    * macOS: `brew install llvm@12`
+      * Make sure `llvm@12` is on your `PATH` before building. Example: `export PATH="$(brew --prefix llvm@12)/bin:$PATH"`
+  * Install clang 12 (may be installed together with llvm)
+    * Ubuntu/Debian: `apt-get install clang-12`
     * macOS: installs with llvm
   * Install libpng (often included by default in *nix platforms)
     * Ubuntu/Debian: `apt-get install libpng-dev`
@@ -85,6 +85,28 @@ manually. This will be ignored by git.
 
 This should work out of the box with Emacs' `lsp-haskell` package.
 
+### Building with Nix
+
+[Nix](https://nixos.org/) is a functional package manager and build system.
+
+To build with vanilla Nix:
+```bash
+$ nix-build
+```
+
+To build with flakes-enabled Nix:
+```bash
+$ nix build .#dex
+```
+The resulting `dex` binary should be in `result/bin/dex`.
+
+For development purposes, you can use a Nix environment with
+```bash
+$ nix-shell
+$ nix develop  # With flakes
+```
+and use `make` to use Stack to build Dex.
+
 ## Running
 
   * Traditional REPL: `dex repl`

default.nix

@@ -0,0 +1,72 @@
+{ pkgs ? import <nixpkgs> {},
+  llvm-hs-src ? pkgs.fetchFromGitHub {
+    owner = "llvm-hs";
+    repo = "llvm-hs";
+    rev = "llvm-12";
+    sha256 = "IG4Mh89bY+PtBJtzlXKYsPljfHP7OSQk03pV6fSmdRY=";
+  },
+  cudaPackage ? pkgs.cudaPackages.cudatoolkit_11,
+  cuda ? false,
+  optimized ? true,
+  live ? true,
+}:
+let
+  llvm-hs-pure = pkgs.haskellPackages.callCabal2nix "llvm-hs-pure" "${llvm-hs-src}/llvm-hs-pure" {
+  };
+  llvm-hs = (pkgs.haskellPackages.callCabal2nix "llvm-hs" "${llvm-hs-src}/llvm-hs" {
+    inherit llvm-hs-pure;
+  }).overrideAttrs (oldAttrs: rec {
+    buildInputs = oldAttrs.buildInputs ++ [
+      pkgs.llvm_12
+    ];
+  });
+  buildFlags = pkgs.lib.optionals optimized [
+    "-foptimized"
+  ] ++ pkgs.lib.optionals live [
+    "-flive"
+  ] ++ pkgs.lib.optionals cuda [
+    "-fcuda"
+    "--extra-include-dirs=${cudaPackage}/include"
+    "--extra-lib-dirs=${cudaPackage}/lib64/stubs"
+  ];
+  cxxFlags = [
+    "-fPIC"
+    "-std=c++11"
+    "-fno-exceptions"
+    "-fno-rtti"
+  ] ++ pkgs.lib.optional cuda "-DDEX_CUDA"
+    ++ pkgs.lib.optional live "-DDEX_LIVE";
+  buildRuntimeCommand =  ''
+      ${pkgs.clang_9}/bin/clang++ \
+      ${builtins.concatStringsSep " " cxxFlags} \
+      -c \
+      -emit-llvm  \
+      -I${pkgs.libpng}/include \
+      src/lib/dexrt.cpp  \
+      -o src/lib/dexrt.bc
+  '';
+in
+  # `callCabal2nix` converts `dex.cabal` into a Nix file and builds it.
+  # Before we do the Haskell build though, we need to first compile the Dex runtime
+  # so it's properly linked in when compiling Dex. Normally the makefile does this,
+  # so we instead sneak compiling the runtime in the configuration phase for the Haskell build.
+  (pkgs.haskellPackages.callCabal2nix "dex" ./. {
+    inherit llvm-hs;
+    inherit llvm-hs-pure;
+  }).overrideAttrs (attrs: {
+    configurePhase = ''
+    # Compile the Dex runtime
+    echo 'Compiling the Dex runtime...'
+    set -x
+    ${buildRuntimeCommand}
+    set +x
+    echo 'Done compiling the Dex runtime.'
+
+    # Run the Haskell configuration phase
+    ${attrs.configurePhase}
+    '';
+    configureFlags = builtins.concatStringsSep " " buildFlags;
+    buildInputs = attrs.buildInputs ++ (pkgs.lib.optional cuda
+      cudaPackage
+    );
+  })

dex.cabal

@@ -36,10 +36,10 @@ flag llvm-head
 
 flag foreign
   description:         Build the C interface to Dex
-  default:             True
+  default:             False
 
-flag safe-names
-  description:         Enables building with safe names checks
+flag debug
+  description:         Enable extra checks and stack trace printing (useful for developers)
   default:             False
 
 library dex-resources
@@ -51,25 +51,19 @@ library dex-resources
   default-extensions:  CPP
 
 library
-  exposed-modules:     Env, Syntax, Type, Inference,
-                       Parser, Util, Imp, Imp.Builder, Imp.Optimize,
-                       PPrint, Algebra, Parallelize, Optimize, Serialize
-                       Builder, Cat,
-                       Simplify, TopLevel,
-                       Autodiff, Interpreter, Logging, CUDA,
-                       LLVM.JIT, LLVM.Shims, JIT, LLVMExec, Export
-                       Err, LabeledItems, SourceRename,
-                       SaferNames.Name, SaferNames.LazyMap,
-                       SaferNames.Syntax, SaferNames.Bridge,
-                       SaferNames.PPrint, SaferNames.Parser,
-                       SaferNames.ResolveImplicitNames, SaferNames.SourceRename,
-                       SaferNames.Type, SaferNames.Builder, SaferNames.Inference,
-                       SaferNames.CheapReduction
+  exposed-modules:     Syntax, Cat, Util, PPrint, Serialize
+                       TopLevel, Interpreter, Logging, CUDA,
+                       LLVM.JIT, LLVM.Shims, JIT, LLVMExec,
+                       Err, LabeledItems, SourceRename, Name, Parser, MTL1,
+                       Type, Builder, Inference, CheapReduction, GenericTraversal,
+                       Simplify, Imp, Algebra, Linearize, Transpose,
+                       LLVM.HEAD.JIT
   if flag(live)
     exposed-modules:   Actor, RenderHtml, LiveOutput
   other-modules:       Paths_dex
   build-depends:       base, containers, mtl, bytestring,
-                       llvm-hs-pure, llvm-hs, transformers,
+                       llvm-hs-pure, llvm-hs, transformers, hashable, unordered-containers,
+                       exceptions, utf8-string, cryptonite,
                        -- Parsing
                        megaparsec, parser-combinators,
                        -- Text output
@@ -82,9 +76,8 @@ library
     build-depends:     warp, wai, blaze-html, http-types, cmark, binary
     cpp-options:       -DDEX_LIVE
     cxx-options:       -DDEX_LIVE
-  if flag(safe-names)
-    cpp-options:       -DDEX_SAFE_NAMES
-    cxx-options:       -DDEX_SAFE_NAMES
+  if flag(debug)
+    cpp-options:       -DDEX_DEBUG
   if !os(darwin)
     exposed-modules:   Resources
     hs-source-dirs:    src/resources
@@ -101,7 +94,8 @@ library
                        TupleSections, ScopedTypeVariables, LambdaCase, PatternSynonyms,
                        BlockArguments, GADTs, TypeOperators, DataKinds, KindSignatures
                        ConstraintKinds, FlexibleContexts, RankNTypes, QuantifiedConstraints,
-                       TypeOperators, MultiParamTypeClasses, FunctionalDependencies
+                       TypeOperators, MultiParamTypeClasses, FunctionalDependencies,
+                       GeneralizedNewtypeDeriving
   pkgconfig-depends:   libpng
   if flag(cuda)
     include-dirs:      /usr/local/cuda/include
@@ -112,21 +106,13 @@ library
     ghc-options:       -O3
   else
     ghc-options:       -O0
-  if flag(llvm-head)
-    cpp-options:       -DDEX_LLVM_VERSION=HEAD
-    build-depends:     mlir-hs
-    exposed-modules:   LLVM.HEAD.JIT,
-                       MLIR.Lower, MLIR.Eval
-  else
-    cpp-options:       -DDEX_LLVM_VERSION=9
-    exposed-modules:   LLVM.V9.JIT, LLVM.V9.Shims
 
 executable dex
   main-is:             dex.hs
   other-extensions:    OverloadedStrings
   build-depends:       dex, base, containers, haskeline, prettyprinter, mtl,
                        optparse-applicative, ansi-wl-pprint,
-                       unix, store, bytestring, directory
+                       unix, store, bytestring, directory, exceptions
   if os(darwin)
     build-depends:     dex-resources
   default-language:    Haskell2010

examples/ctc.dx

@@ -18,27 +18,26 @@ pass automatically.  That makes this code much shorter than
 most implementations.
 
 
-def interleave (blank:v) (labels: m=>v) : (m & (Fin 2))=>v =
+def interleave {m v} (blank:v) (labels: m=>v) : (m & (Fin 2))=>v =
   -- Turns "text" into "t e x t " by first pairing each letter with a blank,
   -- then flattening the pairs back into a single-index table.
   pairs = for i. [labels.i, blank]
   for (i, j). pairs.i.j
 
-def prepend (first: v) (seq: m=>v) : ({head:Unit | tail:m }=>v) =
+def prepend {m v} (first: v) (seq: m=>v) : ((Unit|m)=>v) =
   -- Concatenates a single element to the beginning of a sequence.
   for idx. case idx of
-    {| head = () |} -> first
-    {| tail = i  |} -> seq.i
+    Left () -> first
+    Right i -> seq.i
 
-def prepend_and_interleave (blank:v) (seq: m=>v) :
-  ({head:Unit | tail:(m & (Fin 2))}=>v) =
+def prepend_and_interleave {m v} (blank:v) (seq: m=>v) : ((Unit|(m & (Fin 2)))=>v) =
   -- Turns "text" into " t e x t".
   -- The output of this function has a slightly complicated output type, which
   -- has size 1 + 2 * (size m).
   interleaved = interleave blank seq
   prepend blank interleaved
 
-def clipidx (n:Type) -> (i:Int) : n =
+def clipidx (n:Type) [Ix n] (i:Int) : n =
   -- Returns element at 0 if less than zero.
   -- Ideally we could have an alternative
   -- to Fin that just clips the index at its bounds.
@@ -48,7 +47,8 @@ def logaddexp (x:Float) (y:Float) : Float =
   m = max x y
   m + ( log ( (exp (x - m) + exp (y - m))))
 
-def ctc [Eq vocab, Eq position, Eq time]
+def ctc {vocab time position}
+        [Eq vocab, Eq position, Eq time]
         (blank:  vocab)
         (logits: time=>vocab=>Float)
         (labels: position=>vocab)
@@ -102,7 +102,7 @@ def ctc [Eq vocab, Eq position, Eq time]
 
 '### Demo
 
-def randIdxNoZero (n:Type) -> (k:Key) : n =
+def randIdxNoZero (n:Type) [Ix n] (k:Key) : n =
   unif = rand k
   fromOrdinal n $ (1 + (FToI (floor ( unif * IToF ((size n) - 1)))))