test: add test for big gzip compressed tables

.github/workflows/lint.yml

@@ -9,6 +9,10 @@ on:
 env:
   PY_COLORS: "1"
 
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -37,6 +41,5 @@ jobs:
             identify>2.6
 
       - name: pre-commit
-        shell: bash -l {0}
         run: |
           pre-commit run -a

.github/workflows/tests-external-cfitsio.yml

@@ -13,6 +13,10 @@ env:
   TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment"
   LATEST_CFITSIO_VER: "-4.6.3"
 
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -32,6 +36,9 @@ jobs:
           - { pyver: "3.8", npver: "1", cftsver: "-4.0.0"}
           # 4.1.0 is the first version for which tests pass for uint64
           - { pyver: "3.8", npver: "1", cftsver: "-4.1.0"}
+          # 4.4.0 is first version for which tests pass for compressed
+          # binary tables that exceed 2**32 bytes
+          - { pyver: "3.8", npver: "1", cftsver: "-4.4.0"}
           - { pyver: "3.8", npver: "1", cftsver: "latest"}
           - { pyver: "3.13", npver: "2.3.0", cftsver: "latest"}
 
@@ -64,7 +71,6 @@ jobs:
             setuptools
 
       - name: build external cfitsio
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
@@ -95,30 +101,29 @@ jobs:
           cd ..
 
       - name: test non-bundled build
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
           pip install ${PIP_OPTIONS} -e . \
             --config-settings="--global-option=--use-system-fitsio" \
             --config-settings="--global-option=--system-fitsio-includedir=$HOME/cfitsio-static-install/include" \
             --config-settings="--global-option=--system-fitsio-libdir=$HOME/cfitsio-static-install/lib"
-          pytest -vv fitsio
+
           python -c "import fitsio; assert not fitsio.cfitsio_has_bzip2_support()"
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
             python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()"
           else
             python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()"
           fi
 
+          pytest -vv fitsio
+
       - name: install bzip2 on linux
-        shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get install libbz2-dev
 
       - name: build external cfitsio w/ bzip2
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
@@ -149,7 +154,6 @@ jobs:
           cd ..
 
       - name: test non-bundled build w/ env vars w/ bzip2
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
@@ -160,10 +164,12 @@ jobs:
           export FITSIO_SYSTEM_FITSIO_INCLUDEDIR=$HOME/cfitsio-static-install/include
           export FITSIO_SYSTEM_FITSIO_LIBDIR=$HOME/cfitsio-static-install/lib
           pip install ${PIP_OPTIONS} -e .
-          pytest -vv fitsio
+
           python -c "import fitsio; assert fitsio.cfitsio_has_bzip2_support()"
           if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
             python -c "import fitsio; assert not fitsio.cfitsio_has_curl_support()"
           else
             python -c "import fitsio; assert fitsio.cfitsio_has_curl_support()"
           fi
+
+          pytest -vv fitsio

.github/workflows/tests.yml

@@ -12,6 +12,10 @@ env:
   # accessed at the C level from an unaligned location.
   TEST_CFLAGS: "-fsanitize=alignment -fno-sanitize-recover=alignment"
 
+defaults:
+  run:
+    shell: bash -leo pipefail {0}
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -60,7 +64,6 @@ jobs:
             setuptools
 
       - name: test bundled build
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
@@ -80,13 +83,11 @@ jobs:
           fi
 
       - name: install bzip2 and curl on linux
-        shell: bash -l {0}
         if: matrix.os == 'ubuntu-latest'
         run: |
           sudo apt-get install libbz2-dev curl libcurl4-openssl-dev
 
       - name: test install sdist .gz with no unit tests
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 
@@ -105,7 +106,6 @@ jobs:
           pip uninstall fitsio --yes
 
       - name: test sdist
-        shell: bash -l {0}
         run: |
           export CFLAGS="${CFLAGS} ${TEST_CFLAGS}"
 

fitsio/tests/test_image_compression.py

@@ -610,6 +610,69 @@ def test_image_mem_reopen_noop():
         assert np.array_equal(rimg, img)
 
 
+@pytest.mark.parametrize(
+    "coef",
+    [
+        1,
+        pytest.param(
+            2,
+            marks=pytest.mark.xfail(
+                condition=CFITSIO_VERSION < 4.4,
+                reason=(
+                    "Writing compressed binary tables exceeding "
+                    "2**32 bytes fails for cfitsio < 4.40!"
+                ),
+            ),
+        ),
+    ],
+)
+def test_image_compression_big_gzip(coef):
+    n1 = 50
+    n2 = 50
+    nHDU = 10
+    rng = np.random.RandomState(seed=10)
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tot = 0
+        pth = os.path.join(tmpdir, "test.fits.gz")
+        with FITS(pth, "rw", clobber=True) as out:
+            for i in range(nHDU):
+                out_list = []
+                out_names = []
+
+                out_names += ['A']
+                out_list += [rng.normal(size=n1 * n2 * coef * coef)]
+
+                out_names += ['B']
+                out_list += [np.ones(n1 * n2 * coef * coef)]
+
+                out_names += ['C']
+                out_list += [np.ones(n1 * n2 * coef * coef)]
+
+                out_names += ['D']
+                out_list += [np.ones(n1 * n2 * coef * coef)]
+
+                out_names += ['E']
+                out_list += [
+                    np.ones((n1 * n2 * coef * coef, n1 * n2 * coef * coef))
+                ]
+
+                tot += sum(
+                    out_val.itemsize * out_val.size for out_val in out_list
+                )
+
+                out.write(out_list, names=out_names)
+
+        print("wrote %0.2f MB" % (tot / 1e6), flush=True)
+        os.system(f"ls -lah {tmpdir}/test.fits.gz")
+
+        with FITS(pth, "r") as h:
+            print(h, flush=True)
+            assert len(h) == nHDU + 1
+            for k, name in zip([0, 1, -1], ["A", "B", "E"]):
+                assert np.array_equal(h[-1][name][:], out_list[k])
+
+
 @pytest.mark.parametrize("nan_value", [np.nan, np.inf, -np.inf])
 @pytest.mark.parametrize("dtype", [np.float32, np.float64])
 @pytest.mark.parametrize(