Merge pull request #245 from jeromekelleher/fix-dexplode-part-bug

Fix syncronous commands bug

Author: jeromekelleher

.github/workflows/ci.yml

@@ -48,6 +48,24 @@ jobs:
       - name: Run basic vcf2zarr example
         run: |
           python -m bio2zarr vcf2zarr convert tests/data/vcf/sample.vcf.gz sample.vcz -f
+      - name: Run two-pass vcf2zarr example
+        run: |
+          python -m bio2zarr vcf2zarr explode tests/data/vcf/sample.vcf.gz sample.icf -f
+          python -m bio2zarr vcf2zarr encode sample.icf sample.vcz -f
+      - name: Run distributed explode example
+        run: |
+          python -m bio2zarr vcf2zarr dexplode-init tests/data/vcf/sample.vcf.gz sample.icf 3 -f
+          python -m bio2zarr vcf2zarr dexplode-partition sample.icf 0
+          python -m bio2zarr vcf2zarr dexplode-partition sample.icf 1
+          python -m bio2zarr vcf2zarr dexplode-partition sample.icf 2
+          python -m bio2zarr vcf2zarr dexplode-finalise sample.icf
+      - name: Run distributed encode example
+        run: |
+          python -m bio2zarr vcf2zarr dencode-init sample.icf sample.vcz 3 -f --variants-chunk-size=3
+          python -m bio2zarr vcf2zarr dencode-partition sample.vcz 0
+          python -m bio2zarr vcf2zarr dencode-partition sample.vcz 1
+          python -m bio2zarr vcf2zarr dencode-partition sample.vcz 2
+          python -m bio2zarr vcf2zarr dencode-finalise sample.vcz
       - name: Run tests
         run: |
           pytest --cov=bio2zarr

bio2zarr/core.py

@@ -22,15 +22,6 @@
 
 numcodecs.blosc.use_threads = False
 
-# By default Tqdm creates a multiprocessing Lock to synchronise across processes,
-# which seems to cause some problems with leaked semaphores on certain combinations
-# of Mac and Python versions. We only access tqdm from the main process though,
-# so we don't need it and can override with a simpler threading Lock.
-# NOTE: this gets set multiple times to different locks as subprocesses are
-# spawned, but it doesn't matter because the only tqdm instance that is
-# used is the one in the main process.
-tqdm.tqdm.set_lock(threading.RLock())
-
 
 def display_number(x):
     ret = "n/a"
@@ -207,8 +198,11 @@ class ProgressConfig:
 
 
 def update_progress(inc):
-    with _progress_counter.get_lock():
-        _progress_counter.value += inc
+    # If the _progress_counter has not been set we are working in a
+    # synchronous non-progress tracking context
+    if _progress_counter is not None:
+        with _progress_counter.get_lock():
+            _progress_counter.value += inc
 
 
 def get_progress():