Add FileCheck for unit testing (#3410)

Update SETUP.md including brief instructions on FileCheck.

Author: jackkoenig

.github/workflows/install-filecheck/action.yml

@@ -0,0 +1,27 @@
+name: Install FileCheck
+
+inputs:
+  version:
+    description: 'version to install'
+    required: false
+    default: 'FileCheck-16.0.6-test'
+
+runs:
+  using: composite
+  steps:
+    - id: cache-filecheck
+      uses: actions/cache@v3
+      with:
+        path: FileCheck
+        key: filecheck-${{ runner.os }}-${{ inputs.version }}
+
+    - shell: bash
+      if: steps.cache-filecheck.outputs.cache-hit != 'true'
+      run: |
+        mkdir -p filecheck/bin
+        wget -q https://github.com/jackkoenig/FileCheck/releases/download/${{ inputs.version }}/FileCheck-linux-x64
+        chmod +x FileCheck-linux-x64
+        mv FileCheck-linux-x64 filecheck/bin/FileCheck
+
+    - shell: bash
+      run: echo "$(pwd)/filecheck/bin" >> $GITHUB_PATH

.github/workflows/test.yml

@@ -58,6 +58,11 @@ jobs:
         uses: ./.github/workflows/build-slang
         with:
           version: ${{ inputs.slang }}
+      - name: Install FileCheck
+        run: |
+          sudo apt update
+          sudo apt install llvm-12-tools
+          echo "/usr/lib/llvm-12/bin" >> $GITHUB_PATH
       - name: Setup Java
         uses: actions/setup-java@v4
         with:

SETUP.md

@@ -0,0 +1,140 @@
+# Chisel Local Setup
+Instructions for setting up your environment to run Chisel locally.
+
+For a minimal setup, you only need to install [SBT (the Scala Build Tool)](http://www.scala-sbt.org), which will automatically fetch the appropriate version of Scala and Chisel based on on your project configuration.
+
+[Firtool](https://github.com/llvm/circt) is required to generate Verilog.
+
+[Verilator](https://www.veripool.org/wiki/verilator) is installation is required to simulate your Verilog designs.
+
+[FileCheck](https://llvm.org/docs/CommandGuide/FileCheck.html) is used by many of the unit tests.
+Please see the FileCheck link for documentation about FileCheck's syntax with examples.
+Alternatively you can search the tests for examples by searching for `CHECK:` which is used to tell FileCheck to attempt to match the following text in the generated output.
+
+## Ubuntu Linux
+
+1.  Install the JVM
+    ```bash
+    sudo apt-get install default-jdk
+    ```
+
+1.  Install sbt according to the instructions from [sbt download](https://www.scala-sbt.org/download.html).
+
+1.  Install Firtool
+
+    Choose whatever version is being [used in continuous integration](.github/workflows/install-circt/action.yml)
+    ```bash
+    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.56.1/circt-full-shared-linux-x64.tar.gz | tar -zx
+    ```
+    This will give you a directory called `firtool-1.56.1` containing the firtool binary, add this to your PATH as appropriate.
+    ```bash
+    export PATH=$PATH:$PWD/firtool-1.56.1/bin
+    ```
+    Alternatively, you can install the binary to a standard location by simply moving the binary (if you have root access).
+    ```bash
+    mv firtool-1.56.1/bin/firtool /usr/local/bin/
+    ```
+
+
+1.  Install Verilator.
+    We recommend relatively recent verilator
+    Follow these instructions to compile it from source.
+
+    1.  Install prerequisites (if not installed already):
+        ```bash
+        sudo apt-get install git make autoconf g++ flex bison
+        ```
+
+    2.  Clone the Verilator repository:
+        ```bash
+        git clone https://github.com/verilator/verilator
+        ```
+
+    3.  In the Verilator repository directory, check out a known good version:
+        ```bash
+        git pull
+        git checkout v5.004
+        ```
+
+    4.  In the Verilator repository directory, build and install:
+        ```bash
+        unset VERILATOR_ROOT # For bash, unsetenv for csh
+        autoconf # Create ./configure script
+        ./configure
+        make
+        sudo make install
+        ```
+
+1.  Install FileCheck.
+    FileCheck can usually be found in llvm-\*-tools packages, eg.
+    ```
+    sudo apt-get install llvm-12-tools
+    export PATH=$PATH:/usr/lib/llvm-12/bin
+    ```
+
+    You can alternatively download a statically-ish linked binary from https://github.com/jackkoenig/FileCheck
+    ```
+    mkdir filecheck
+    cd filecheck
+    wget -q https://github.com/jackkoenig/FileCheck/releases/download/FileCheck-16.0.6/FileCheck-linux-x64
+    mv FileCheck-linux-x64 FileCheck
+    chmod +x FileCheck
+    export PATH=$PATH:$PWD
+    ```
+    Similarly to firtool, you can install the binary to a more standard location by moving it.
+    ```
+    mv FileCheck /usr/local/bin
+    ```
+
+## Arch Linux
+1.  Install Verilator and SBT
+    ```bash
+    pacman -Sy verilator sbt
+    ```
+
+1. Install firtool
+
+    See the instructions for Ubuntu above, the firtool Ubuntu binary is a "many Linux" mostly statically linked binary.
+
+## Windows
+1.  [Download and install sbt for Windows](https://www.scala-sbt.org/download.html).
+
+Verilator does not appear to have native Windows support.
+However, Verilator works in [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) or in other Linux-compatible environments like Cygwin.
+
+There are no issues with generating Verilog from Chisel, which can be pushed to FPGA or ASIC tools.
+
+## Mac OS X
+1.  Install Verilator and SBT
+    ```bash
+    brew install sbt verilator
+    ```
+
+1.  Install firtool
+
+    ```bash
+    wget -q -O - https://github.com/llvm/circt/releases/download/firtool-1.56.1/circt-full-shared-macos-x64.tar.gz | tar -zx
+    ```
+    This will give you a directory called `firtool-1.56.1` containing the firtool binary, add this to your PATH as appropriate.
+    ```bash
+    export PATH=$PATH:$PWD/firtool-1.56.1/bin
+    ```
+    Alternatively, you can install the binary to a standard location by simply moving the binary.
+    ```bash
+    mv firtool-1.56.1/bin/firtool /usr/local/bin/
+    ```
+
+1.  Install FileCheck.
+    You can download a statically-ish linked binary from https://github.com/jackkoenig/FileCheck
+    ```
+    mkdir filecheck
+    cd filecheck
+    wget -q https://github.com/jackkoenig/FileCheck/releases/download/FileCheck-16.0.6/FileCheck-macos-x64
+    mv FileCheck-macos-x64 FileCheck
+    chmod +x FileCheck
+    export PATH=$PATH:$PWD
+    ```
+    Similarly to firtool, you can install the binary to a more standard location by moving it.
+    ```
+    mv FileCheck /usr/local/bin
+    ```

src/test/scala/chiselTests/ChiselEnum.scala

@@ -360,7 +360,7 @@ class IsOneOfTester extends BasicTester {
   stop()
 }
 
-class ChiselEnumSpec extends ChiselFlatSpec with Utils {
+class ChiselEnumSpec extends ChiselFlatSpec with Utils with FileCheck {
 
   behavior.of("ChiselEnum")
 
@@ -529,9 +529,11 @@ class ChiselEnumSpec extends ChiselFlatSpec with Utils {
       val out = IO(Output(MyEnum()))
       out := MyEnum(in)
     }
-    val (log, _) = grabLog(ChiselStage.emitCHIRRTL(new MyModule))
-    log should include("warn")
-    log should include("Casting non-literal UInt")
+    elaborateAndFileCheckOutAndErr(new MyModule)(
+      """| CHECK:      [W001] Casting non-literal UInt to [[enum:[a-zA-Z0-9_$.]+]].
+         | CHECK-SAME: You can use [[enum]].safe to cast without this warning.
+         |""".stripMargin
+    )
   }
 
   it should "NOT warn if the Enum is total" in {

src/test/scala/chiselTests/ChiselSpec.scala

@@ -211,6 +211,50 @@ trait ChiselRunners extends Assertions {
   }
 }
 
+trait FileCheck extends BeforeAndAfterEachTestData { this: Suite =>
+  import scala.Console.{withErr, withOut}
+
+  private def sanitize(n: String): String = n.replaceAll(" ", "_").replaceAll("\\W+", "")
+
+  private val testRunDir: os.Path = os.pwd / os.RelPath(BackendCompilationUtilities.TestDirectory)
+  private val suiteDir:   os.Path = testRunDir / sanitize(suiteName)
+  private var checkFile:  Option[os.Path] = None
+
+  override def beforeEach(testData: TestData): Unit = {
+    // TODO check that these are always available
+    val nameDir = suiteDir / sanitize(testData.name)
+    os.makeDir.all(nameDir)
+    checkFile = Some(nameDir / s"${sanitize(testData.text)}.check")
+    super.beforeEach(testData) // To be stackable, must call super.beforeEach
+  }
+
+  override def afterEach(testData: TestData): Unit = {
+    checkFile = None
+    super.afterEach(testData) // To be stackable, must call super.beforeEach
+  }
+
+  /** Run FileCheck on a String against some checks */
+  def fileCheckString(in: String, fileCheckArgs: String*)(check: String): Unit = {
+    // Filecheck needs the thing to check in a file
+    os.write.over(checkFile.get, check)
+    val extraArgs = os.Shellable(fileCheckArgs)
+    os.proc("FileCheck", checkFile.get, extraArgs).call(stdin = in)
+  }
+
+  /** Elaborate a Module to FIRRTL and check the FIRRTL with FileCheck */
+  def generateFirrtlAndFileCheck(t: => RawModule, fileCheckArgs: String*)(check: String): Unit = {
+    fileCheckString(ChiselStage.emitCHIRRTL(t), fileCheckArgs: _*)(check)
+  }
+
+  /** Elaborate a Module, capture the stdout and stderr, check stdout and stderr with FileCheck */
+  def elaborateAndFileCheckOutAndErr(t: => RawModule, fileCheckArgs: String*)(check: String): Unit = {
+    val outStream = new ByteArrayOutputStream()
+    withOut(outStream)(withErr(outStream)(ChiselStage.emitCHIRRTL(t)))
+    val result = outStream.toString
+    fileCheckString(outStream.toString, fileCheckArgs: _*)(check)
+  }
+}
+
 /** Spec base class for BDD-style testers. */
 abstract class ChiselFlatSpec extends AnyFlatSpec with ChiselRunners with Matchers
 

src/test/scala/chiselTests/DecoupledSpec.scala

@@ -6,7 +6,7 @@ import chisel3._
 import circt.stage.ChiselStage
 import chisel3.util.Decoupled
 
-class DecoupledSpec extends ChiselFlatSpec {
+class DecoupledSpec extends ChiselFlatSpec with FileCheck {
   "Decoupled() and Decoupled.empty" should "give DecoupledIO with empty payloads" in {
     ChiselStage.emitCHIRRTL(new Module {
       val io = IO(new Bundle {
@@ -19,21 +19,17 @@ class DecoupledSpec extends ChiselFlatSpec {
   }
 
   "Decoupled.map" should "apply a function to a wrapped Data" in {
-    val chirrtl = ChiselStage
-      .emitCHIRRTL(new Module {
-        val enq = IO(Flipped(Decoupled(UInt(8.W))))
-        val deq = IO(Decoupled(UInt(8.W)))
-        deq <> enq.map(_ + 1.U)
-      })
-
-    // Check for data assignment
-    chirrtl should include("""node _deq_map_bits_T = add(enq.bits, UInt<1>(0h1)""")
-    chirrtl should include("""node _deq_map_bits = tail(_deq_map_bits_T, 1)""")
-    chirrtl should include("""connect _deq_map.bits, _deq_map_bits""")
-    chirrtl should include("""connect deq, _deq_map""")
-
-    // Check for back-pressure (ready signal is driven in the opposite direction of bits + valid)
-    chirrtl should include("""connect enq.ready, _deq_map.ready""")
+    generateFirrtlAndFileCheck(new Module {
+      val enq = IO(Flipped(Decoupled(UInt(8.W))))
+      val deq = IO(Decoupled(UInt(8.W)))
+      deq <> enq.map(_ + 1.U)
+    })("""|CHECK: node [[node1:[a-zA-Z0-9_]+]] = add(enq.bits, UInt<1>(0h1))
+          |CHECK: node [[node2:[a-zA-Z0-9_]+]] = tail([[node1]], 1)
+          |CHECK: connect [[result:[a-zA-Z0-9_]+]].bits, [[node2]]
+          |# Check for back-pressure (ready signal is driven in the opposite direction of bits + valid)
+          |CHECK: connect enq.ready, [[result]].ready
+          |CHECK: connect deq, [[result]]
+          |""".stripMargin)
   }
 
   "Decoupled.map" should "apply a function to a wrapped Bundle" in {