removed gunzip, changes to ci test pipeline, ci changed to stub runs

Author: khersameesh24

.github/workflows/ci.yml

@@ -82,7 +82,12 @@ jobs:
       - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - name: "Run pipeline image mode with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
         continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }}
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results --mode image -stub
+
+      - name: "Run pipeline coordinate mode with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+        continue-on-error: ${{ matrix.NXF_VER == 'latest-everything' }}
+        run: |
+          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results --mode coordinate -stub

conf/modules.config

@@ -27,10 +27,6 @@ process {
         ]
     }
 
-    withName: GUNZIP {
-        ext.prefix = "transcripts.csv"
-    }
-
     withName: XENIUMRANGER_RESEGMENT {
         publishDir = [
             path: "${params.outdir}/xeniumranger/resegment",

modules.json

@@ -11,12 +11,6 @@
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/cellpose/cellpose.diff"
                     },
-                    "gunzip": {
-                        "branch": "master",
-                        "git_sha": "81880787133db07d9b4c1febd152c090eb8325dc",
-                        "installed_by": ["modules"],
-                        "patch": "modules/nf-core/gunzip/gunzip.diff"
-                    },
                     "multiqc": {
                         "branch": "master",
                         "git_sha": "7b50cb7be890e4b28cffb82e438cc6a8d7805d3f",
@@ -25,7 +19,8 @@
                     "untar": {
                         "branch": "master",
                         "git_sha": "05954dab2ff481bcb999f24455da29a5828af08d",
-                        "installed_by": ["modules"]
+                        "installed_by": ["modules"],
+                        "patch": "modules/nf-core/untar/untar.diff"
                     },
                     "unzip": {
                         "branch": "master",

modules/local/spatialconverter/parquet_to_csv/main.nf

@@ -6,10 +6,11 @@ process PARQUET_TO_CSV {
 
     input:
     tuple val(meta), path(transcripts)
+    val(extension)
 
     output:
-    tuple val(meta), path("*.csv"), emit: transcripts_csv
-    path("versions.yml")          , emit: versions
+    tuple val(meta), path("*.csv*"), emit: transcripts_csv
+    path("versions.yml")           , emit: versions
 
     when:
     task.ext.when == null || task.ext.when
@@ -23,7 +24,7 @@ process PARQUET_TO_CSV {
 
     stub:
     """
-    touch ${transcripts}
+    touch ${transcripts}.csv
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
         spatialconverter: "${task.version}"

modules/local/spatialconverter/parquet_to_csv/templates/parquet_to_csv.py

@@ -4,9 +4,17 @@
 
 if __name__ == '__main__':
     print("[START]")
+
+    ext = "${extension}" if "${extension}" else ".csv"
     df = pd.read_parquet("${transcripts}")
-    output="${transcripts}".replace(".parquet",".csv")
-    df.to_csv(f"{output}", index=False)
+
+    if ext == ".csv":
+        output="${transcripts}".replace(".parquet",".csv")
+        df.to_csv(f"{output}", index=False)
+
+    if ext == ".gz":
+        output="${transcripts}".replace(".parquet",".csv.gz")
+        df.to_csv(f"{output}", compression='gzip', index=False)
 
     #Output version information
     with open("versions.yml", "w") as f: