Improve documentation of the nix code (#7321)

Author: zeme-wana

nix/README.md

@@ -1,29 +1,49 @@
 # DevEnv & CI - Maintenance Guide & Troubleshooting
 
-This document is intended both for maintainers of the nix code and for developers facing issues with their development environment or with the CI system.
+This document is intended both for maintainers of the nix code and for developers facing issues with their development environment or with the CI system. Use CTRL-F to look for relevant keywords.
 
-# Troubleshooting 
+### 1) `nix develop` fails to enter the shell, `cabal build` fails when it shouldn't.
 
+In general when facing any problem related to the nix shell or cabal failing to build when it shouldn't, the first step is to make sure you are using the latest shell from master: first exit the nix shell, then `git pull --rebase origin master`, then re-enter the nix shell (i.e. run `nix develop`). 
 
-<div style="background-color: #FFFF0033; padding: 5px;">
+If that fails, you might be facing a caching issue. In that case, try this before exiting and re-entering the nix shell:
 
-`nix develop` fails to enter the shell, `cabal build` fails when it shouldn't.
+`rm -r ~/.cabal/{store,packages} plutus-metatheory/_build dist dist-newstyle`
 
-</div>
+### 2) `nix develop` is updating the `flake.lock` file.
 
-In general when facing any problem related to the nix shell or cabal failing to build when it shouldn't, the first step is to make sure you are using the latest shell from master: first exit the nix shell, then `git pull --rebase origin master`, then re-enter the nix shell (i.e. run `nix develop`). 
+This should never happen, it is a bug in nix, and has been observed in version `2.26.1`.
+Downgrade or upgrade your nix installation to fix this issue. 
 
-If that fails, you might be facing a caching issue. In that case, try this before exiting and re-entering the nix shell:
+### 3) `cabal test all` fails
 
-```
-rm -r ~/.cabal/{store,packages} plutus-metatheory/_build dist dist-newstyle
-```
+Sometimes cabal needs a `cabal build all` before it can successfully execute a `cabal test all`.
 
-<div style="background-color: #FFFF0033; padding: 5px;">
+### 4) How to update `hackage`, `haskell.nix` and `CHaP`
 
-`nix develop` is updating the `flake.lock` file.
-</div>
+`nix flake update haskell-nix` updates [haskell.nix](https://github.com/input-output-hk/haskell.nix).
+This should be done infrequently as it is likely to break the nix code.
+If you just want new packages from `hackage` or `CHaP` instead, you can independently run:
+`nix flake update hackage CHaP`.
+Then you can change the `index-state` in `cabal.project`: you pick an arbitrary date, and if it's too new, `cabal` will error out and suggest the latest known date which you can copy-paste.
 
-This should never happen, it is a bug in nix, and has been observed in version `2.26.1`.
-Downgrade your nix installation to fix this issue. 
+### 5) How to change what gets build in CI
+
+Modify `nested-ci-jobs = {..}` in [./nix/outputs.nix](https://github.com/input-output-hk/haskell.nix).
+
+### 6) How to change what gets exposed in the flake outputs
+
+Modify `packages = {..}` in [./nix/outputs.nix](https://github.com/input-output-hk/haskell.nix).
+
+### 7) How to build fully static Haskell executables with Nix
+
+Look at `static-haskell-packages = {..}` in [./nix/outputs.nix](https://github.com/input-output-hk/haskell.nix).
+
+### 8) How to manage cross-compilation on Windows via Wine with Nix
+
+Look at `windows-hydra-jobs = {..}` in [./nix/outputs.nix](https://github.com/input-output-hk/haskell.nix).
+
+### 9) How to define build variants and cabal flags in the nix code
 
+The nix builds can be overridden inside [./nix/project.nix](https://github.com/input-output-hk/haskell.nix).
+New cabal flags and configuration options can be defined there.

nix/agda-tools.nix

@@ -2,6 +2,8 @@
 
 let
 
+  # Agda standard library pinned to v2.1.1.
+  # Used in: `nix/metatheory.nix` (as a build input) and `nix/shell.nix` (via agda-with-stdlib).
   agda-stdlib = agda-packages.standard-library.overrideAttrs (oldAtts: rec {
 
     version = "2.1.1";
@@ -26,6 +28,8 @@ let
     '';
   });
 
+  # Compose a tailored Agda toolchain and expose it via agdaPackages.
+  # Used in: `nix/metatheory.nix` (to build agda-with-stdlib-and-metatheory) and `nix/shell.nix`.
   # We want to keep control of which version of Agda we use, so we supply our own and override
   # the one from nixpkgs.
   #
@@ -68,6 +72,8 @@ let
       pkgs = frankenPkgs;
     };
 
+  # Patches Agda's build to compile interface files post-install.
+  # Used in: `agda-project` below via `modules`.
   # Agda is a huge pain. They have a special custom setup that compiles the
   # interface files for the Agda that ships with the compiler. These go in
   # the data files for the *library*, but they require the *executable* to
@@ -101,16 +107,21 @@ let
     '';
   };
 
+  # Default patch module (native toolchain).
   agda-project-module-patch-default =
     agda-project-module-patch { compiler-nix-name = "ghc"; };
 
+  # Patch module for static musl64 toolchain.
   agda-project-module-patch-musl64 =
     agda-project-module-patch { compiler-nix-name = "x86_64-unknown-linux-musl-ghc"; };
 
+  # The Agda executable from the built project.
   agda = agda-project.hsPkgs.Agda.components.exes.agda;
 
+  # The Agda mode executable.
   agda-mode = agda-project.hsPkgs.Agda.components.exes.agda-mode;
 
+  # Convenience wrapper providing `agda-with-stdlib` binary.
   agda-with-stdlib = pkgs.stdenv.mkDerivation {
     name = "agda-with-stdlib";
     phases = "installPhase";
@@ -121,6 +132,7 @@ let
     '';
   };
 
+  # The Agda hackage project used to produce the tools above.
   agda-project = pkgs.haskell-nix.hackage-project {
     name = "Agda";
     version = "2.7.0";
@@ -129,6 +141,7 @@ let
     modules = [ agda-project-module-patch-default ];
   };
 
+  # Path to the stdlib .agda-lib file for shell export.
   NIX_AGDA_STDLIB = "${agda-stdlib}/stadard-library.agda-lib";
 
 in

nix/build-latex-doc.nix

@@ -1,5 +1,8 @@
 { pkgs, lib, agda-tools }:
 
+# Builds a LaTeX/Agda-based document into a PDF.
+# Used in: `nix/latex-documents.nix` to define all document derivations.
+
 { name, description, src, output-pdf-name ? "*.pdf" }:
 
 pkgs.stdenv.mkDerivation {

nix/metatheory.nix

@@ -2,6 +2,7 @@
 
 let
 
+  # plutus-metatheory as an agda-package
   metatheory-agda-package = agda-tools.agda-packages.mkDerivation {
     name = "plutus-metatheory";
     pname = "plutus-metatheory";
@@ -13,6 +14,7 @@ let
     meta = { };
   };
 
+  # Tarball of the Agda library sources for distribution as an artifact.
   metatheory-agda-library = pkgs.stdenv.mkDerivation {
     name = "metatheory-agda-library";
     src = lib.cleanSource (self + /plutus-metatheory);
@@ -26,11 +28,14 @@ let
     '';
   };
 
+  # Developer helper: generates MAlonzo Haskell code from the metatheory.
+  # Used in: `nix/shell.nix` pre-commit hook and common tools.
   generate-malonzo-code = pkgs.writeShellScriptBin "generate-malonzo-code" ''
     cd "$(git rev-parse --show-toplevel)/plutus-metatheory"
     agda-with-stdlib --compile --ghc-dont-call-ghc src/Main.lagda.md
   '';
 
+  # Agda executable wrapper that includes both stdlib and the metatheory package.
   agda-with-stdlib-and-metatheory = pkgs.stdenv.mkDerivation {
     name = "agda-with-stdlib-and-metatheory";
     phases = "installPhase";

nix/pkgs.nix

@@ -1,5 +1,6 @@
 { inputs, system }:
 
+# Provides `pkgs` with project overlays and workarounds.
 import inputs.nixpkgs {
   inherit system;
   config = inputs.haskell-nix.config;

nix/project.nix

@@ -2,6 +2,7 @@
 { inputs, pkgs, lib, metatheory, r-with-packages, utils }:
 
 let
+  # Defines the Haskell project and its variants via haskell.nix.
   cabalProject = pkgs.haskell-nix.cabalProject' ({ config, pkgs, ... }:
     {
       name = "plutus";
@@ -30,6 +31,7 @@ let
       inputMap = { "https://chap.intersectmbo.org/" = inputs.CHaP; };
 
       sha256map = {
+        # We need one of these for each source-repository-package stanza in cabal.project.
         "https://github.com/jaccokrijnen/plutus-cert"."e814b9171398cbdfecdc6823067156a7e9fc76a3" =
           "0srqvx0b819b5crrbsa9hz2fnr50ahqizvvm0wdmyq2bbpk2rka7";
       };

nix/r-with-packages.nix

@@ -1,5 +1,8 @@
 { pkgs }:
 
+# R wrapper with required packages for benchmarking and analysis.
+# Add more R packages here as needed.
+# Used in: `nix/project.nix` (as build-tools) and `nix/shell.nix` common tools.
 pkgs.rWrapper.override {
   packages = [
     pkgs.rPackages.tidyverse

nix/shell.nix

@@ -2,6 +2,7 @@
 
 let
 
+  # Toolchain versions used in dev shells. Consumed by `project.shellFor`.
   tools = project.tools {
     # "latest" cabal would be 3.14.1.0 which breaks haddock generation.
     # TODO update cabal version once haddock generation is fixed upstream.
@@ -14,6 +15,7 @@ let
     stylish-haskell = "latest";
   };
 
+  # Pre-commit hooks for the repo. Injects into shell via shellHook.
   pre-commit-check = inputs.pre-commit-hooks.lib.${pkgs.system}.run {
     src = ../.;
     hooks = {
@@ -63,11 +65,13 @@ let
     };
   };
 
+  # Add extra Linux-only packages to the dev shell here.
   linux-pkgs = lib.optionals pkgs.hostPlatform.isLinux [
     pkgs.papi
     pkgs.util-linux
   ];
 
+  # Common packages/tools available in all shells.
   common-pkgs = [
     agda-tools.agda
     agda-tools.agda-with-stdlib
@@ -111,10 +115,12 @@ let
     pkgs.nodejs_20
   ];
 
+  # Locale archive setup for glibc hosts. Needed to fix cabal build issues on some hosts.
   locale-archive-hook =
     lib.optionalString (pkgs.stdenv.hostPlatform.libc == "glibc")
       "export LOCALE_ARCHIVE=${pkgs.glibcLocales}/lib/locale/locale-archive";
 
+  # Full developer shell with many tools.
   full-shell = project.shellFor {
     name = "plutus-shell-${project.args.compiler-nix-name}";
 
@@ -136,6 +142,7 @@ let
   };
 
 
+  # Lightweight shell with minimal tools.
   quick-shell = project.shellFor {
     name = "plutus-shell-${project.args.compiler-nix-name}";
     tools = { cabal = "latest"; };
@@ -147,6 +154,7 @@ let
   };
 
 
+  # Select shell by compiler used in the project variant.
   shell = {
     ghc967 = full-shell;
     ghc984 = quick-shell;

nix/utils.nix

@@ -2,6 +2,8 @@
 
 rec {
 
+  # Flattens a nested attribute set of derivations into name/value pairs.
+  # Used in: `nix/outputs.nix` (to build `flattened-ci-jobs` and `ciJobs`).
   flattenDerivationTree = separator: set:
     let
       recurse = name: name':
@@ -17,6 +19,8 @@ rec {
     assert lib.typeOf set == "set"; lib.listToAttrs (flatten "" set);
 
 
+  # Aggregates CI jobs which are required for Hydra to pass.
+  # Used in: `nix/outputs.nix` (to compute `hydra-required-job`).
   makeHydraRequiredJob = { self, pkgs }:
     let
       clean-jobs =
@@ -30,20 +34,27 @@ rec {
     };
 
 
+  # Retrieves the git revision from flake sourceInfo, or "unknown".
+  # Used in: `nix/project.nix` (adds `__GIT_REV__` CPP macro to builds).
+  # When the git tree is dirty, the sourceInfo attribute is missing from inputs.self.
   getSourceInfoRev = inputs:
     if inputs.self.sourceInfo ? rev then
       inputs.self.sourceInfo.rev
     else
       "unknown";
 
 
+  # Converts flake sourceInfo lastModifiedDate to ISO-8601, or empty string.
+  # Used in: `nix/project.nix` (adds `__GIT_COMMIT_DATE__` CPP macro).
+  # When the git tree is dirty, the sourceInfo attribute is missing from inputs.self.
   getSourceInfoLastModifiedDate = inputs:
     if inputs.self.sourceInfo ? lastModifiedDate then
       date_YYYYMMDDHHmmSS_ToIso8601 inputs.self.sourceInfo.lastModifiedDate
     else
       "";
 
 
+  # Helper to convert YYYYMMDDHHmmSS timestamps to ISO-8601.
   date_YYYYMMDDHHmmSS_ToIso8601 = ts:
     let
       year = lib.substring 0 4 ts;