experimenting with ty, a nix flake, envrc, and nextflow jobarrays

Author: nrminor

.envrc

@@ -0,0 +1 @@
+use flake

.gitignore

@@ -5,13 +5,16 @@
 !README.md
 !justfile
 !Dockerfile
+!Containerfile
 !pyproject.toml
+!flake.nix
 !pixi.lock
 !uv.lock
 !_quarto.yml
 !.pre-commit-config.yaml
 !refman.toml
 !data_manifest.yml
+!.envrc
 !.gitignore
 !.gitattributes
 !main.nf

Dockerfile ContainerfileDockerfile renamed to Containerfile

@@ -0,0 +1,71 @@
+{
+  description = "Reproducible dev shell for the `oneroof` bioinformatic processing pipeline";
+
+  inputs = {
+    nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+    flake-utils = "github:numtide/flake-utils";
+  };
+
+  outputs =
+    {
+      self,
+      nixpkgs,
+      flake-utils,
+      ...
+    }:
+    flake-utils.lib.eachDefaultSystem (
+      system:
+      let
+        pkgs = import nixpkgs {
+          inherit system;
+        };
+
+        dorado = pkgs.stdenv.mkDerivation {
+          name = "dorado";
+          src = pkgs.fetchurl {
+            url = "https://cdn.oxfordnanoportal.com/software/analysis/dorado-0.7.1-linux-x64.tar.gz";
+            sha256 = "<fill-me-in>"; # run nix with fake sha256 to get it
+          };
+
+          unpackPhase = "tar -xvf $src";
+          installPhase = ''
+            set -euo pipefail
+            mkdir -p $out
+            cp -r dorado-0.7.1-linux-x64/* $out/
+          '';
+        };
+
+      in
+      {
+        devShells.default = pkgs.mkShell {
+          name = "oneroof-env";
+
+          buildInputs = [
+            pkgs.stdenv
+            pkgs.gcc
+            pkgs.curl
+            pkgs.wget
+            pkgs.make
+            pkgs.git
+            pkgs.cmake
+            pkgs.libxml2
+            pkgs.libxslt
+            pkgs.libffi
+            pkgs.pixi
+            dorado
+          ];
+
+          shellHook = ''
+            export PATH=$PATH:${dorado}/bin:${dorado}/lib
+            export NXF_CACHE_DIR=/scratch
+            export NXF_HOME=/scratch
+            echo "Activating pixi..."
+            cd ~/  # or $HOME if you define it
+            pixi install --verbose --color=always --frozen
+            pixi shell hook >> ~/.bashrc
+
+          '';
+        };
+      }
+    );
+}

flake.nix

@@ -2,6 +2,8 @@ process GET_PRIMER_SEQS {
 
     /* */
 
+    array 1000
+
 	errorStrategy { task.attempt < 3 ? 'retry' : 'ignore' }
 	maxRetries 2
 
@@ -18,4 +20,4 @@ process GET_PRIMER_SEQS {
 	bedtools getfasta -fi ${refseq} -bed ${bed} > ${primer_combo}.fasta
 	"""
 
-}
+}

modules/bedtools.nf

@@ -6,6 +6,7 @@ process FILTER_WITH_CHOPPER {
     errorStrategy { task.attempt < 3 ? 'retry' : 'ignore' }
     maxRetries 2
 
+    array 1000
     cpus 4
 
     input:

modules/chopper.nf

@@ -5,6 +5,7 @@ process TRIM_ENDS_TO_PRIMERS {
     errorStrategy { task.attempt < 3 ? 'retry' : 'ignore' }
     maxRetries 2
 
+    array 1000
     cpus 3
 
     input:

modules/cutadapt.nf

@@ -2,6 +2,8 @@ process GET_PRIMER_PATTERNS {
 
     /* */
 
+    array 1000
+
 	errorStrategy { task.attempt < 3 ? 'retry' : 'ignore' }
 	maxRetries 2
 
@@ -21,4 +23,4 @@ process GET_PRIMER_PATTERNS {
 	-r ""
 	"""
 
-}
+}

modules/primer_patterns.nf

@@ -5,6 +5,7 @@ process FIND_COMPLETE_AMPLICONS {
 	errorStrategy { task.attempt < 3 ? 'retry' : 'ignore' }
 	maxRetries 2
 
+    array 1000
 	cpus 3
 
     input:

modules/seqkit.nf

@@ -109,6 +109,7 @@ dev-dependencies = [
     "quarto>=0.1.0",
     "ruff>=0.9.6",
     "tox>=4.18.1",
+    "ty>=0.0.0a8",
 ]
 compile-bytecode = true