diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml new file mode 100644 index 00000000000..51ccca3eece --- /dev/null +++ b/.github/workflows/nix.yml @@ -0,0 +1,66 @@ +name: Nix Build +on: [push, pull_request] + +permissions: + contents: read + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - uses: cachix/cachix-action@ad2ddac53f961de1989924296a1f236fcfbaa4fc # v15 + with: + name: nix-community + authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + continue-on-error: true + + - name: Build libfabric + run: nix build .#libfabric -L + + - name: Build all packages + run: | + nix build .#cassini-headers -L + nix build .#libcxi -L + nix build .#accel-config -L + + - name: Check flake + run: nix flake check + + devshell: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - uses: cachix/install-nix-action@08dcb3a5e62fa31e2da3d490afc4176ef55ecd72 # v30 + with: + nix_path: nixpkgs=channel:nixos-unstable + + - name: Test devShell + run: | + nix develop .#ci --command bash -c ' + ./autogen.sh + ./configure --prefix=$PWD/install \ + --enable-tcp \ + --enable-udp \ + --enable-rxm \ + --enable-shm \ + --enable-verbs \ + --enable-efa \ + --enable-psm3 + make -j$(nproc) + make install + ./install/bin/fi_info -l + ' + + - name: Upload build logs + if: failure() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 + with: + name: nix-devshell-config.log + path: config.log diff --git a/.gitignore b/.gitignore index 8f46ac5f6a6..48b9351ca04 100644 --- a/.gitignore +++ b/.gitignore @@ -96,3 +96,9 @@ prov/efa/docs/doxygen/latex # Clang's compilation database file and directory. /.cache /compile_commands.json + +# nix +result +result-man +.envrc +.direnv/ diff --git a/contrib/nix/lib/default.nix b/contrib/nix/lib/default.nix new file mode 100644 index 00000000000..d7290323708 --- /dev/null +++ b/contrib/nix/lib/default.nix @@ -0,0 +1,89 @@ +# Reusable library functions for libfabric Nix packaging +{ lib }: + +{ + # Combine CUDA header packages into a single derivation + # cuda_runtime.h needs headers from cuda_cccl, crt/ from cuda_nvcc, nvml.h from cuda_nvml_dev + mkCudaHeaders = pkgs: pkgs.symlinkJoin { + name = "cuda-headers-combined"; + paths = [ + pkgs.cudaPackages.cuda_cudart + pkgs.cudaPackages.cuda_cccl + pkgs.cudaPackages.cuda_nvcc + pkgs.cudaPackages.cuda_nvml_dev.include + ]; + }; + + # Get ROCm runtime if available on the platform (x86_64-linux only) + getRocmRuntime = pkgs: + if pkgs.stdenv.hostPlatform.isx86_64 + then pkgs.rocmPackages.clr + else null; + + # Conditionally return a value only on Linux + linuxOnly = pkgs: value: + if pkgs.stdenv.hostPlatform.isLinux then value else null; + + # Generate cross-compiled package attributes with a prefix + # packages: attrset of packages + # prefix: string prefix for attribute names (e.g., "cross-aarch64-linux") + # names: list of package names to include + mkCrossPackageAttrs = { packages, prefix, names }: + lib.listToAttrs (map (name: { + name = "${prefix}-${name}"; + value = packages.${name}; + }) names); + + # Create a cross-compilation dev shell + # Note: Linux-only deps (numactl, libnl, rdma-core) come via inputsFrom + mkCrossShell = { crossPkgs, crossPackages, arch }: + crossPkgs.mkShell { + name = "libfabric-cross-${arch}"; + packages = with crossPkgs; [ + autoconf + automake + libtool + pkg-config + libuuid + curl + ]; + inputsFrom = [ crossPackages.libfabric ]; + shellHook = '' + echo "Cross-compilation shell: ${arch}" + echo "Target: ${crossPkgs.stdenv.hostPlatform.config}" + ''; + }; + + # Dependencies that need native packages when cross-compiling + # Returns an attrset to merge with callPackage args + crossDepsForLibcxi = { nativePkgs, mkCudaHeaders }: + { + lm_sensors = nativePkgs.lm_sensors; + criterion = nativePkgs.criterion; + fuse = nativePkgs.fuse; + cuda_cudart = mkCudaHeaders nativePkgs; + level-zero = nativePkgs.level-zero; + rocm-runtime = + if nativePkgs.stdenv.hostPlatform.isx86_64 + then nativePkgs.rocmPackages.clr + else null; + }; + + crossDepsForLibfabric = { nativePkgs, mkCudaHeaders }: + { + numactl = nativePkgs.numactl; + lttng-ust = nativePkgs.lttng-ust; + rdma-core = nativePkgs.rdma-core; + liburing = nativePkgs.liburing; + libnl = nativePkgs.libnl; + # DSA/idxd is x86-only (uses immintrin.h) + idxd-config = null; + cuda_cudart = mkCudaHeaders nativePkgs; + gdrcopy = nativePkgs.cudaPackages.gdrcopy or null; + rocm-runtime = + if nativePkgs.stdenv.hostPlatform.isx86_64 + then nativePkgs.rocmPackages.clr + else null; + level-zero = nativePkgs.level-zero; + }; +} diff --git a/contrib/nix/pkgs/accel-config/default.nix b/contrib/nix/pkgs/accel-config/default.nix new file mode 100644 index 00000000000..f24506be169 --- /dev/null +++ b/contrib/nix/pkgs/accel-config/default.nix @@ -0,0 +1,64 @@ +{ + lib, + stdenv, + fetchFromGitHub, + autoreconfHook, + pkg-config, + libuuid, + json_c, + systemdLibs ? null, + kmod, +}: + +stdenv.mkDerivation rec { + pname = "accel-config"; + version = "4.1.9"; + + src = fetchFromGitHub { + owner = "intel"; + repo = "idxd-config"; + rev = "accel-config-v${version}"; + hash = "sha256-uV6cOha+g5fZBq+ucrNmfUW+3gQzX9BKGYvYaeCYv40="; + }; + + nativeBuildInputs = [ + autoreconfHook + pkg-config + ]; + + buildInputs = [ + libuuid + json_c + kmod + ] ++ lib.optionals (systemdLibs != null) [ + systemdLibs + ]; + + configureFlags = [ + "--disable-test" + "--disable-docs" + ] ++ lib.optionals (systemdLibs == null) [ + "--disable-systemd" + ]; + + # Create version.m4 which is normally generated from git, and make + # git-version-gen a no-op so it doesn't overwrite during build + postPatch = '' + echo "m4_define([GIT_VERSION], [${version}])" > version.m4 + cat > git-version-gen <<'EOF' + #!/bin/sh + exit 0 + EOF + chmod +x git-version-gen + ''; + + enableParallelBuilding = true; + + meta = with lib; { + description = "Utility for controlling and configuring Intel DSA and IAA accelerators"; + homepage = "https://github.com/intel/idxd-config"; + license = licenses.lgpl21Plus; + platforms = platforms.linux; + maintainers = with maintainers; [ ]; + }; +} diff --git a/contrib/nix/pkgs/cassini-headers/default.nix b/contrib/nix/pkgs/cassini-headers/default.nix new file mode 100644 index 00000000000..5741b7455f6 --- /dev/null +++ b/contrib/nix/pkgs/cassini-headers/default.nix @@ -0,0 +1,36 @@ +{ + lib, + stdenvNoCC, + fetchFromGitHub, +}: + +stdenvNoCC.mkDerivation rec { + pname = "cassini-headers"; + version = "13.0.0"; + + src = fetchFromGitHub { + owner = "HewlettPackard"; + repo = "shs-cassini-headers"; + rev = "release/shs-${version}"; + hash = "sha256-Fh8RZFAcqbEcbHCmSFNoIcbHP7uC/CovVdkgAA7SpNQ="; + }; + + dontBuild = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/include $out/share + cp -r include/* $out/include/ + # Copy share directory (contains csr_defs.json needed by libcxi build) + cp -r share/* $out/share/ + runHook postInstall + ''; + + meta = with lib; { + description = "Hardware definitions and C headers for HPE Cassini/Slingshot network interconnect"; + homepage = "https://github.com/HewlettPackard/shs-cassini-headers"; + license = with licenses; [ gpl2 bsd2 ]; + platforms = platforms.linux; + maintainers = with maintainers; [ ]; + }; +} diff --git a/contrib/nix/pkgs/cxi-driver-headers/default.nix b/contrib/nix/pkgs/cxi-driver-headers/default.nix new file mode 100644 index 00000000000..0356210e8cb --- /dev/null +++ b/contrib/nix/pkgs/cxi-driver-headers/default.nix @@ -0,0 +1,34 @@ +{ + lib, + stdenvNoCC, + fetchFromGitHub, +}: + +stdenvNoCC.mkDerivation rec { + pname = "cxi-driver-headers"; + version = "13.0.0"; + + src = fetchFromGitHub { + owner = "HewlettPackard"; + repo = "shs-cxi-driver"; + rev = "release/shs-${version}"; + hash = "sha256-kJNKuvg0x6gGMPlz1y5EiRReSOqx65dmmdwfqGpVptU="; + }; + + dontBuild = true; + + installPhase = '' + runHook preInstall + mkdir -p $out/include + cp -r include/* $out/include/ + runHook postInstall + ''; + + meta = with lib; { + description = "UAPI headers for HPE CXI driver (Slingshot)"; + homepage = "https://github.com/HewlettPackard/shs-cxi-driver"; + license = licenses.gpl2; + platforms = platforms.linux; + maintainers = with maintainers; [ ]; + }; +} diff --git a/contrib/nix/pkgs/libcxi/default.nix b/contrib/nix/pkgs/libcxi/default.nix new file mode 100644 index 00000000000..f756671a512 --- /dev/null +++ b/contrib/nix/pkgs/libcxi/default.nix @@ -0,0 +1,130 @@ +{ + lib, + stdenv, + fetchFromGitHub, + autoreconfHook, + pkg-config, + python3, + libuuid, + numactl, + cassini-headers, + cxi-driver-headers, + json_c, + libconfig, + libuv, + fuse, + libyaml, + libnl, + criterion, + lm_sensors ? null, + # GPU/hmem support + cuda_cudart ? null, + level-zero ? null, + rocm-runtime ? null, +}: + +stdenv.mkDerivation rec { + pname = "libcxi"; + version = "13.0.0"; + + src = fetchFromGitHub { + owner = "HewlettPackard"; + repo = "shs-libcxi"; + rev = "release/shs-${version}"; + hash = "sha256-zkVlIwrUKvtzuKwgk4yfAvmoehh5JSWFdZBeBINuFdU="; + }; + + nativeBuildInputs = [ + autoreconfHook + pkg-config + python3 + ]; + + buildInputs = [ + criterion + libuuid + numactl + cassini-headers + cxi-driver-headers + json_c + libconfig + libuv + fuse + libyaml + libnl + ] ++ lib.optionals (lm_sensors != null) [ + lm_sensors + ] ++ lib.optionals (cuda_cudart != null) [ + cuda_cudart + ] ++ lib.optionals (level-zero != null) [ + level-zero + ] ++ lib.optionals (rocm-runtime != null) [ + rocm-runtime + ]; + + configureFlags = [ + "--with-cassini-headers=${cassini-headers}" + "--without-systemd" + "--disable-tests" + # Set udev rules directory to output path + "--with-udevrulesdir=${placeholder "out"}/lib/udev/rules.d" + ] ++ lib.optionals (lm_sensors == null) [ + "--disable-libsensors" + ] ++ lib.optionals (cuda_cudart != null) [ + "--with-cuda=${cuda_cudart}" + ] ++ lib.optionals (level-zero != null) [ + "--with-ze=${level-zero}" + ] ++ lib.optionals (rocm-runtime != null) [ + "--with-rocm=${rocm-runtime}" + ]; + + postPatch = '' + # Fix path to csr_defs.json in the python script + substituteInPlace utils/cxi_dump_csrs.py \ + --replace-fail "/usr/share/cassini-headers/csr_defs.json" \ + "${cassini-headers}/share/cassini-headers/csr_defs.json" \ + --replace-fail "../cassini-headers/install/share/cassini-headers/csr_defs.json" \ + "${cassini-headers}/share/cassini-headers/csr_defs.json" + + # Fix shebangs (e.g., /usr/bin/env python3 -> /nix/store/.../python3) + patchShebangs utils/ + ''; + + # The configure script doesn't have --disable-libsensors, we need to patch + preConfigure = lib.optionalString (lm_sensors == null) '' + # Remove sensors check if lm_sensors is not available + substituteInPlace configure.ac --replace-fail \ + 'AC_CHECK_HEADERS([sensors/sensors.h], ,' \ + 'dnl AC_CHECK_HEADERS([sensors/sensors.h], ,' + ''; + + enableParallelBuilding = true; + doCheck = false; + + # Disable test building by patching Makefile.am + postAutoreconf = '' + # Remove tests from SUBDIRS and conditional test building + if [ -f Makefile.am ]; then + sed -i 's/SUBDIRS.*tests/SUBDIRS =/' Makefile.am || true + sed -i '/^if.*HAVE_CRITERION/,/^endif/d' Makefile.am || true + fi + ''; + + # Patch libtool to skip ldconfig during cross-compilation + # (ldconfig is a target binary that can't run on the host) + preBuild = '' + sed -i 's|finish_cmds=.*ldconfig.*|finish_cmds=""|' libtool || true + ''; + + postInstall = '' + rm -f $out/lib/*.la + ''; + + meta = with lib; { + description = "Low-level interface library for HPE Cassini high-speed NIC (Slingshot)"; + homepage = "https://github.com/HewlettPackard/shs-libcxi"; + license = with licenses; [ gpl2 bsd2 ]; + platforms = platforms.linux; + maintainers = with maintainers; [ ]; + }; +} diff --git a/contrib/nix/pkgs/libfabric/default.nix b/contrib/nix/pkgs/libfabric/default.nix new file mode 100644 index 00000000000..5fa352f9093 --- /dev/null +++ b/contrib/nix/pkgs/libfabric/default.nix @@ -0,0 +1,150 @@ +{ + lib, + stdenv, + pkg-config, + autoreconfHook, + libpsm2 ? null, + libuuid, + numactl ? null, + libnl ? null, + rdma-core ? null, + valgrind ? null, + cmocka, + lttng-ust ? null, + liburing ? null, + curl, + + # Source - can be overridden to use local source + src, + version ? "dev", + + # Optional dependencies + cuda_cudart ? null, + gdrcopy ? null, + rocm-runtime ? null, + level-zero ? null, + cassini-headers ? null, + cxi-driver-headers ? null, + libcxi ? null, + json_c ? null, + idxd-config ? null, + enableValgrind ? false, + enableLttng ? lttng-ust != null, + + # Builtin providers. + enableProvRxm ? true, + enableProvTcp ? true, + enableProvUdp ? true, + enableProvSockets ? true, + + # Real/hardware providers. + enableProvEfa ? stdenv.hostPlatform.isLinux, + # OPX requires Cornelis proprietary headers (hfi1dv.h) not in nixpkgs rdma-core + enableProvOpx ? false, + enableProvPsm2 ? stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86_64 && libpsm2 != null, + enableProvPsm3 ? stdenv.hostPlatform.isLinux && stdenv.hostPlatform.isx86_64, + enableProvUsnic ? stdenv.hostPlatform.isLinux, + enableProvVerbs ? stdenv.hostPlatform.isLinux, + enableProvShm ? stdenv.hostPlatform.isLinux, + enableProvCxi ? stdenv.hostPlatform.isLinux && libcxi != null && cassini-headers != null && cxi-driver-headers != null, + + # GPU memory support - enabled when packages are provided + enableHmemCuda ? stdenv.hostPlatform.isLinux && cuda_cudart != null, + enableCudaGDRCopy ? stdenv.hostPlatform.isLinux && enableHmemCuda && gdrcopy != null, + enableHmemRocr ? stdenv.hostPlatform.isLinux && rocm-runtime != null, + enableHmemZe ? stdenv.hostPlatform.isLinux && level-zero != null, +}: + +stdenv.mkDerivation { + pname = "libfabric"; + inherit src version; + + enableParallelBuilding = true; + + outputs = [ + "out" + "dev" + "man" + ]; + + nativeBuildInputs = [ + pkg-config + autoreconfHook + ]; + + buildInputs = [ + libuuid + curl + ] + ++ lib.optionals (numactl != null) [ numactl ] + ++ lib.optionals (libnl != null) [ libnl ] + ++ lib.optionals (rdma-core != null) [ rdma-core ] + ++ lib.optionals (liburing != null) [ liburing ] + ++ lib.optionals enableLttng [ lttng-ust ] + ++ lib.optionals enableProvPsm2 [ libpsm2 ] + ++ lib.optionals enableValgrind [ valgrind ] + ++ lib.optionals enableHmemCuda [ cuda_cudart ] + ++ lib.optionals enableCudaGDRCopy [ gdrcopy ] + ++ lib.optionals enableHmemRocr [ rocm-runtime ] + ++ lib.optionals enableHmemZe [ level-zero ] + ++ lib.optionals enableProvCxi [ + libcxi + cassini-headers + cxi-driver-headers + json_c + ] + ++ lib.optionals (enableProvShm && idxd-config != null) [ idxd-config ]; + + checkInputs = [ cmocka ]; + + # Disable CXI multinode tests - they require static linking against libfabric + # which doesn't work when CXI provider is built as a plugin (HAVE_CXI_DL=1) + # We use sed to remove the entire multinode test blocks (lines 55-94 approximately) + postPatch = lib.optionalString enableProvCxi '' + # Remove multinode test definitions from Makefile.include + # These tests link statically against libfabric which fails when CXI is a plugin + sed -i '/^# Stand-alone srun tests/,/^if HAVE_CRITERION/{ /^if HAVE_CRITERION/!d }' prov/cxi/Makefile.include + ''; + + configureFlags = [ + (lib.enableFeature enableProvEfa "efa") + (lib.enableFeature enableProvOpx "opx") + (lib.enableFeatureAs enableProvPsm2 "psm2" (lib.getLib libpsm2)) + (lib.enableFeature enableProvPsm3 "psm3") + (lib.enableFeature enableProvRxm "rxm") + (lib.enableFeature enableProvTcp "tcp") + (lib.enableFeature enableProvUdp "udp") + (lib.enableFeature enableProvUsnic "usnic") + (lib.enableFeature enableProvVerbs "verbs") + (lib.enableFeature enableProvShm "shm") + (lib.enableFeature enableProvCxi "cxi") + (lib.enableFeature enableProvSockets "sockets") + (lib.withFeatureAs enableValgrind "valgrind" valgrind) + # Use dlopen for GPU libs - avoids linking issues during cross-compilation + # and allows runtime detection of GPU support + (lib.withFeatureAs enableHmemCuda "cuda" (lib.getDev cuda_cudart)) + (lib.enableFeature enableHmemCuda "cuda-dlopen") + (lib.withFeatureAs enableCudaGDRCopy "gdrcopy" (lib.getDev gdrcopy)) + (lib.withFeatureAs enableHmemRocr "rocr" (lib.getDev rocm-runtime)) + (lib.withFeatureAs enableHmemZe "ze" (lib.getDev level-zero)) + (lib.withFeatureAs (enableProvShm && idxd-config != null) "idxd" ( + if idxd-config != null then idxd-config else null + )) + ] ++ lib.optionals (libnl != null) [ + "--with-libnl=${lib.getDev libnl}" + ]; + + meta = with lib; { + homepage = "https://ofiwg.github.io/libfabric/"; + description = "Open Fabric Interfaces"; + license = with licenses; [ + gpl2 + bsd2 + ]; + platforms = platforms.all; + maintainers = with maintainers; [ + bzizou + sielicki + ]; + }; +} diff --git a/contrib/nix/pkgs/xpmem/default.nix b/contrib/nix/pkgs/xpmem/default.nix new file mode 100644 index 00000000000..17ca470d982 --- /dev/null +++ b/contrib/nix/pkgs/xpmem/default.nix @@ -0,0 +1,48 @@ +{ + lib, + stdenv, + fetchFromGitHub, + autoreconfHook, + pkg-config, +}: + +stdenv.mkDerivation rec { + pname = "xpmem"; + version = "2.7.3-unstable-2024-09-11"; + + src = fetchFromGitHub { + owner = "hpc"; + repo = "xpmem"; + rev = "3bcab55479489fdd93847fa04c58ab16e9c0b3fd"; + hash = "sha256-MnuuUSgqjlrW0cfw3wZa2DA5MPgtjHwWsnD5OR/QMoo="; + }; + + nativeBuildInputs = [ + autoreconfHook + pkg-config + ]; + + # Only build the userspace library, not the kernel module + configureFlags = [ + "--disable-kernel-module" + ]; + + enableParallelBuilding = true; + + meta = with lib; { + description = "Cross-partition memory (XPMEM) userspace library"; + longDescription = '' + XPMEM is a Linux kernel module that enables a process to map the + memory of another process into its virtual address space. This + enables upper-level protocols such as MPI and SHMEM to perform + single-copy address-space to address-space transfers. + + This package only provides the userspace library. The kernel module + must be installed separately. + ''; + homepage = "https://github.com/hpc/xpmem"; + license = with licenses; [ gpl2Only lgpl21Only ]; + platforms = platforms.linux; + maintainers = with maintainers; [ ]; + }; +} diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000000..33567816e6b --- /dev/null +++ b/flake.lock @@ -0,0 +1,58 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1765835352, + "narHash": "sha256-XswHlK/Qtjasvhd1nOa1e8MgZ8GS//jBoTqWtrS1Giw=", + "rev": "a34fae9c08a15ad73f295041fec82323541400a9", + "revCount": 424, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.424%2Brev-a34fae9c08a15ad73f295041fec82323541400a9/019b25ac-f1ac-7ac9-baf4-5e130ad8baff/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1.424" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1767026758, + "narHash": "sha256-7fsac/f7nh/VaKJ/qm3I338+wAJa/3J57cOGpXi0Sbg=", + "rev": "346dd96ad74dc4457a9db9de4f4f57dab2e5731d", + "revCount": 918255, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/DeterminateSystems/nixpkgs-weekly/0.1.918255%2Brev-346dd96ad74dc4457a9db9de4f4f57dab2e5731d/019b6dc5-4c5a-7436-b8c4-28835e9ad1b3/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/DeterminateSystems/nixpkgs-weekly/0.1.918255" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1765674936, + "narHash": "sha256-k00uTP4JNfmejrCLJOwdObYC9jHRrr/5M/a/8L2EIdo=", + "owner": "nix-community", + "repo": "nixpkgs.lib", + "rev": "2075416fcb47225d9b68ac469a5c4801a9c4dd85", + "type": "github" + }, + "original": { + "owner": "nix-community", + "repo": "nixpkgs.lib", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000000..42b05e56bfb --- /dev/null +++ b/flake.nix @@ -0,0 +1,190 @@ +{ + description = "libfabric - Open Fabric Interfaces"; + + inputs = { + flake-parts.url = "https://flakehub.com/f/hercules-ci/flake-parts/0.1.424"; + nixpkgs.url = "https://flakehub.com/f/DeterminateSystems/nixpkgs-weekly/0.1.918255"; + }; + + outputs = inputs@{ flake-parts, nixpkgs, ... }: + flake-parts.lib.mkFlake { inherit inputs; } { + systems = [ "x86_64-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" ]; + + perSystem = { system, lib, pkgs, ... }: + let + # Import our reusable library + fabLib = import ./contrib/nix/lib { inherit lib; }; + + isLinux = pkgs.stdenv.hostPlatform.isLinux; + isDarwin = pkgs.stdenv.hostPlatform.isDarwin; + + # Local source filtered to only include relevant files + libfabricSrc = lib.fileset.toSource { + root = ./.; + fileset = lib.fileset.unions [ + ./src ./include ./prov ./util ./man ./fabtests + ./configure.ac ./Makefile.am ./autogen.sh ./config + (lib.fileset.fileFilter (file: lib.hasSuffix ".in" file.name) ./.) + (lib.fileset.fileFilter (file: lib.hasSuffix ".m4" file.name) ./.) + (lib.fileset.fileFilter (file: file.name == "Makefile.include" || file.name == "Makefile.am") ./.) + ]; + }; + + # Cross-compilation setup (Darwin -> Linux) + crossTargets = lib.optionalAttrs isDarwin { + aarch64-linux = { + crossPkgs = pkgs.pkgsCross.aarch64-multiplatform; + nativePkgs = import nixpkgs { system = "aarch64-linux"; config.allowUnfree = true; }; + }; + x86_64-linux = { + crossPkgs = pkgs.pkgsCross.gnu64; + nativePkgs = import nixpkgs { system = "x86_64-linux"; config.allowUnfree = true; }; + }; + }; + + # Build package set for a given pkgs + mkPackages = p: { isCross ? false, nativePkgs ? null }: + let + cudaHeaders = fabLib.mkCudaHeaders p; + rocmRuntime = fabLib.getRocmRuntime p; + + cassini-headers' = p.callPackage ./contrib/nix/pkgs/cassini-headers { }; + cxi-driver-headers' = p.callPackage ./contrib/nix/pkgs/cxi-driver-headers { }; + + libcxi' = p.callPackage ./contrib/nix/pkgs/libcxi ({ + cassini-headers = cassini-headers'; + cxi-driver-headers = cxi-driver-headers'; + cuda_cudart = cudaHeaders; + level-zero = p.level-zero; + rocm-runtime = rocmRuntime; + } // lib.optionalAttrs isCross (fabLib.crossDepsForLibcxi { + inherit nativePkgs; + mkCudaHeaders = fabLib.mkCudaHeaders; + })); + + accel-config' = p.callPackage ./contrib/nix/pkgs/accel-config { + systemdLibs = if isCross then nativePkgs.systemdLibs else p.systemdLibs or null; + }; + + xpmem' = p.callPackage ./contrib/nix/pkgs/xpmem { }; + in { + cassini-headers = cassini-headers'; + libcxi = libcxi'; + accel-config = accel-config'; + xpmem = xpmem'; + libfabric = p.callPackage ./contrib/nix/pkgs/libfabric ({ + src = libfabricSrc; + # Disable OPX - requires Cornelis proprietary headers (hfi1dv.h) not in nixpkgs + enableProvOpx = false; + # Linux-only dependencies + numactl = fabLib.linuxOnly p p.numactl; + libnl = fabLib.linuxOnly p p.libnl; + rdma-core = fabLib.linuxOnly p p.rdma-core; + liburing = fabLib.linuxOnly p p.liburing; + lttng-ust = fabLib.linuxOnly p p.lttng-ust; + libpsm2 = fabLib.linuxOnly p p.libpsm2; + cassini-headers = fabLib.linuxOnly p cassini-headers'; + cxi-driver-headers = fabLib.linuxOnly p cxi-driver-headers'; + libcxi = fabLib.linuxOnly p libcxi'; + json_c = fabLib.linuxOnly p p.json_c; + idxd-config = fabLib.linuxOnly p accel-config'; + cuda_cudart = fabLib.linuxOnly p cudaHeaders; + gdrcopy = fabLib.linuxOnly p (p.cudaPackages.gdrcopy or null); + rocm-runtime = fabLib.linuxOnly p p.rocmPackages.clr; + level-zero = fabLib.linuxOnly p p.level-zero; + } // lib.optionalAttrs isCross (fabLib.crossDepsForLibfabric { + inherit nativePkgs; + mkCudaHeaders = fabLib.mkCudaHeaders; + })); + }; + + # Build package sets + nativePackages = mkPackages pkgs { }; + + crossPackageSets = lib.mapAttrs (_: { crossPkgs, nativePkgs }: + mkPackages crossPkgs { isCross = true; inherit nativePkgs; } + ) crossTargets; + + # Package names to expose + packageNames = [ "libfabric" "cassini-headers" "libcxi" "accel-config" "xpmem" ]; + + # Development shell inputs + commonBuildInputs = with pkgs; [ + autoconf automake libtool pkg-config libuuid curl + ] ++ lib.optionals isLinux [ + numactl libnl rdma-core liburing lttng-ust json_c + libpsm2 ucx hwloc level-zero + nativePackages.cassini-headers nativePackages.libcxi + nativePackages.accel-config nativePackages.xpmem + ] ++ [ cmocka ]; + + devShellPackages = commonBuildInputs ++ (with pkgs; [ + clang-tools bear + ] ++ lib.optionals isLinux [ gdb valgrind ] + ++ lib.optionals isDarwin [ lldb ]); + + in { + _module.args.pkgs = import nixpkgs { + inherit system; + config.allowUnfree = true; + }; + + packages = { + inherit (nativePackages) libfabric; + default = nativePackages.libfabric; + } // lib.optionalAttrs isLinux { + inherit (nativePackages) cassini-headers libcxi accel-config xpmem; + } // lib.concatMapAttrs (arch: packages: + fabLib.mkCrossPackageAttrs { + inherit packages; + prefix = "cross-${arch}"; + names = packageNames; + } + ) crossPackageSets; + + devShells = { + default = pkgs.mkShell { + name = "libfabric-dev"; + packages = devShellPackages; + inputsFrom = [ nativePackages.libfabric ]; + shellHook = '' + echo "libfabric development shell (${system})" + echo "" + echo "Build commands:" + echo " ./autogen.sh - Generate configure script" + echo " ./configure - Configure the build" + echo " make - Build libfabric" + echo " make check - Run tests" + echo "" + '' + lib.optionalString isLinux '' + echo "Provider dependencies available:" + echo " - rdma-core (verbs, efa)" + echo " - libpsm2 (psm2)" + echo " - libcxi + cassini-headers (cxi)" + echo " - ucx (ucx)" + echo " - level-zero (ze/GPU)" + '' + lib.optionalString isDarwin '' + echo "Darwin build (limited providers: tcp, udp, rxm, sockets)" + echo "" + echo "Cross-compilation targets available:" + echo " nix build .#cross-aarch64-linux-libfabric" + echo " nix build .#cross-x86_64-linux-libfabric" + ''; + CFLAGS = "-g -O0"; + }; + ci = pkgs.mkShell { + name = "libfabric-ci"; + packages = commonBuildInputs; + }; + } // lib.mapAttrs' (arch: { crossPkgs, ... }: { + name = "cross-${arch}"; + value = fabLib.mkCrossShell { + inherit crossPkgs arch; + crossPackages = crossPackageSets.${arch}; + }; + }) crossTargets; + + checks = { inherit (nativePackages) libfabric; }; + }; + }; +}