Merge branch 'rel-1.4.0' into release

# Conflicts:
#	.github/workflows/build-containers.yml
#	modules/local/browser_aux/main.nf
#	modules/local/cache_builder/main.nf
#	modules/local/cath_tools/assign_cath_superfamilies/main.nf
#	modules/local/darwin_extract/main.nf
#	modules/local/domains/main.nf
#	modules/local/edgehog/main.nf
#	modules/local/fastoma_extract/main.nf
#	modules/local/fingerprints/main.nf
#	modules/local/go_import/main.nf
#	modules/local/h5_combine/main.nf
#	modules/local/hdf5import/main.nf
#	modules/local/hogprofile/main.nf
#	modules/local/hogprop/main.nf
#	modules/local/keywords/main.nf
#	modules/local/xref_fetch/main.nf
#	nextflow.config

Author: alpae

…/workflows/build-omadarwin-container.yml .github/workflows/build-containers.yml.github/workflows/build-omadarwin-container.yml renamed to .github/workflows/build-containers.yml .github/workflows/build-omadarwin-container.yml renamed to .github/workflows/build-containers.yml

@@ -1,11 +1,26 @@
-name: build omadarwin container and publish it to ghcr and dockerhub
+name: build containers and publish it to ghcr and dockerhub
 
 on:
   push:
 
 jobs:
-  docker:
+  build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: omabuild
+            context: containers/oma
+            platforms: linux/amd64,linux/arm64
+          - name: omadarwin
+            context: containers/darwin
+            platforms: linux/amd64,linux/arm64
+          - name: omabuild-rdf-py
+            context: containers/rdf/biosoda_py
+            platforms: linux/amd64,linux/arm64
+          - name: omabuild-rdf-java
+            context: containers/rdf/biosoda_java
+            platforms: linux/amd64,linux/arm64
     steps:
       - name: checkout repo
         uses: actions/checkout@v4
@@ -15,8 +30,8 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: |
-            dessimozlab/omadarwin
-            ghcr.io/dessimozlab/omadarwin
+            dessimozlab/${{ matrix.name }}
+            ghcr.io/dessimozlab/${{ matrix.name }}
           tags: |
             type=schedule
             type=ref,event=branch
@@ -25,7 +40,6 @@ jobs:
             type=semver,pattern={{major}}.{{minor}}
             type=edge
 
-
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v3
 
@@ -47,14 +61,31 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Build darwin-based build container
+      - name: Build ${{ matrix.name }} container
         uses: docker/build-push-action@v6
         with:
-          context: containers/darwin
-          platforms: linux/amd64,linux/arm64
+          context: ${{ matrix.context }}
+          platforms: ${{ matrix.platforms }}
           push: ${{ github.event_name != 'pull_request' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
           secrets: |
             "labgit=${{ secrets.LABGIT }}"
 
+  run-nextflow:
+    name: run nextflow pipeline with testdata
+    needs: build
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v4
+
+      - name: Install Nextflow
+        run: |
+          curl -s https://get.nextflow.io | bash
+          chmod +x nextflow
+          mv nextflow /usr/local/bin/
+
+      - name: Run Nextflow pipeline
+        run: nextflow run main.nf -profile docker,test

.github/workflows/build-omabuild-container.yml

@@ -70,7 +70,38 @@ process {
 
     withName: '.*:BUILD_HOG_H5'{
         cpus   = { 1 }
-        memory = { 500.MB + 6.B * orthoxml.size() * task.attempt }
+        memory = { 500.MB + 500.B * orthoxml.size() * (2*task.attempt-1) }
         time   = { 1.min * (Math.ceil( meta.nr_of_genomes / 3 ) + 10) * task.attempt }
     }
+
+    withName: ".*:ADD_PAIRWISE_ORTHOLOGS" {
+        cpus   = { meta.nr_of_genomes < 10 ? 2 : (meta.nr_of_genomes < 30 ? 4 : (meta.nr_of_genomes < 300 ? 6 : 12))  }
+        memory = { 3.GB + 10.B * (meta.max_nr_seqs_in_genome * meta.nr_of_genomes * 10 as Long) * task.attempt }
+        time   = { 1.min * (Math.ceil( meta.nr_of_genomes / 2 ) + 10) * task.attempt }
+    }
+
+    withName: "CONVERT_PROTEINS" {
+        cpus   = { 1 }
+        memory = { (400.MB + 12.B * dbpath.size()) * task.attempt }
+        time   = { (5.min + 1.sec * (genome.TotEntries as Long)/100) * task.attempt }
+    }
+
+    withName: "INFER_FINGERPRINTS" {
+        cpus   = { 1 }
+        memory = { (800.MB + meta.nr_of_amino_acids * 2.B) * task.attempt }
+        time   = { (10.min + Math.ceil(meta.nr_of_sequences / 100) * 1.sec) * (2 * task.attempt - 1)}
+    }
+
+    withName: "INFER_KEYWORDS" {
+        cpus   = { 1 }
+        memory = { (800.MB + meta.nr_of_sequences * 500.B) * task.attempt }
+        time   = { (10.min + Math.ceil(meta.nr_of_sequences / 500) * 1.sec) * (2 * task.attempt - 1)}
+    }
+
+    withName: "HMMER_HMMSEARCH" {
+        cpus   = { 4 }
+        memory = { 1.GB * (2*task.attempt-1) }
+        time   = { 2.h * (2*task.attempt-1)  }
+    }
+
 }

.github/workflows/run-pipeline-with-testdata.yml

@@ -30,6 +30,7 @@ params {
     hog_orthoxml               = "${projectDir}/testdata/fastoma/FastOMA_HOGs.orthoxml"
     genomes_dir                = "${projectDir}/testdata/fastoma/proteome"
     taxonomy_sqlite_path       = "${projectDir}/testdata/taxonomy.sqlite"
+    pfam_names_path            = "${projectDir}/testdata/Pfam-A.clans.stub.tsv.gz"
     xref_refseq                = "${projectDir}/assets/NO_FILE"
     go_gaf                     = "${projectDir}/testdata/fastoma/*.goa"
 }

assets/NO_FILE2

@@ -4,8 +4,6 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-omahdf5tordf = {git = "git+${LABGIT}/biosoda"}
-omamer = "*"
 edgehog = "*"
 hogprop = {git = "git+${LABGIT}/hogprop2"}
          #branch = "nf-convert",