Install all dependencies with APT, reduce verbosity in Makefile build (#91)

* Install all dependencies with APT
Obsoletes conda for the Python dependencies.  Matplotlib
(python3-matplotlib in Ubuntu) does not appear to be needed at the
moment.  Capitalize the GitHub Action step names for consistency.

* Reduce verbose output from non-critical commands
Remove -v (--verbose) option from rsync and tar.  Do not store
intermediate downloads and elide seemingly redundant touch:es.  Note
that http://downloads.sf.net/cbflib/PyCifRW-4.3_rev_19Jun21.tar.gz
($(PY3CIFRWURL)) appears to be an uncompressed tarfile, despite its .gz
extension.

* Use all Docker CPU:s by default for CMake
Use e.g. --cpuset-cpus or --cpu-quota to externally limit CPU usage with
docker.  The analogous make -j may not work for the Makefile build,
because make is run recursively.

* Use UTF-8 locale inside container
Affects the regeneration of e.g. the SWIG interface files.  This is
arguably a buildsystem bug.

Author: hattne

.dev/docker/cmake_ctest.dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:24.04
 SHELL ["/bin/bash", "-c"]
+ENV LANG=C.UTF-8
 
 RUN mkdir /app
 COPY ./cbflib /app/cbflib
@@ -9,7 +10,7 @@ RUN apt-get update && \
 
 RUN cd /app/cbflib && \
   cmake . && \
-  cmake --build . --parallel 4
+  cmake --build . --parallel `nproc`
 
 RUN cd /app/cbflib && \
-  ctest --parallel 4
+  ctest --parallel `nproc`

.dev/docker/make_tests.dockerfile

@@ -1,29 +1,17 @@
 FROM ubuntu:24.04
 SHELL ["/bin/bash", "-c"]
+ENV LANG=C.UTF-8
 
 RUN mkdir /app
 COPY ./cbflib /app/cbflib
 
 RUN apt-get update && \
-  apt-get install -y bison build-essential git wget libjpeg-dev m4 automake libpcre2-dev liblzma-dev rsync gfortran libz-dev
-
-RUN mkdir -p ~/miniconda3 && \
-  wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda3/miniconda.sh && \
-  bash ~/miniconda3/miniconda.sh -b -u -p /usr/share/miniconda && \
-  rm ~/miniconda3/miniconda.sh
-
-RUN source /usr/share/miniconda/etc/profile.d/conda.sh && \
-  conda create -y -n build -c conda-forge python=3.11 python-build && \
-  conda create -y -n test -c conda-forge python=3.11 numpy matplotlib
+  apt-get install -y bison build-essential git wget libjpeg-dev m4 automake libpcre2-dev liblzma-dev python3-build python3-dev python3-numpy-dev python3-setuptools python3-venv rsync gfortran libz-dev
 
 RUN cd /app/cbflib && \
-  source /usr/share/miniconda/etc/profile.d/conda.sh && \
-  conda activate build && \
   make all
 
 RUN cd /app/cbflib && \
-  source /usr/share/miniconda/etc/profile.d/conda.sh && \
-  conda activate test && \
   make tests 2>&1 | tee test.out
 
 RUN ! grep -a ignored /app/cbflib/test.out

.github/workflows/cmake_ctest.yml

@@ -1,4 +1,4 @@
-name: Cmake build and tests
+name: CMake build and tests
 
 on:
   push:
@@ -18,14 +18,14 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: install extra dependencies
+    - name: Install extra dependencies
       run: |
         sudo apt-get update
         sudo apt-get install -y python3-numpy-dev
-    - name: build
+    - name: Build
       run: |
         cmake .
         cmake --build . --parallel `nproc`
-    - name: test
+    - name: Test
       run: |
         ctest --parallel `nproc`

.github/workflows/make_tests.yml

@@ -12,32 +12,22 @@ on:
   schedule:
     - cron: '0 20 * * 1'  # Monday 20:00 UTC
 
-
 jobs:
   build_test:
-
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-    - name: create conda environments
-      run: |
-        conda create -y -n build -c conda-forge python=3.11 python-build
-        conda create -y -n test -c conda-forge python=3.11 numpy matplotlib
-    - name: install extra dependencies
+    - name: Install extra dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install -y libjpeg-dev liblzma-dev
-    - name: build
+        sudo apt-get install -y libjpeg-dev liblzma-dev python3-build python3-dev python3-numpy-dev
+    - name: Build
       run: |
-        source /usr/share/miniconda/etc/profile.d/conda.sh
-        conda activate build
         make all
-    - name: test
+    - name: Test
       run: |
-        source /usr/share/miniconda/etc/profile.d/conda.sh
-        conda activate test
         make tests 2>&1 | tee test.out
-    - name : check test output
+    - name : Check test output
       run: |
         ! grep -a ignored test.out

Makefile

@@ -538,11 +538,6 @@ REGEX_INCLUDES ?=
 endif
 
 
-# Program to use to retrieve a URL
-
-DOWNLOAD ?= wget -N
-#DOWNLOAD   ?= curl -O -L
-
 # Flag to control symlinks versus copying
 
 SLFLAGS = --use_ln
@@ -1482,7 +1477,6 @@ $(SWIG_FORTRAN_KIT):    build_swig_fortran
 	git clone $(SWIG_FORTRAN_URL) $(SWIG_FORTRAN_KIT)
 	(export SWIG_FORTRAN_PREFIX=$(PWD);cd $(SWIG_FORTRAN_KIT); ./autogen.sh; \
 	./configure --prefix=$(F90CBF); make; make install) 
-	touch $(SWIG_FORTRAN_KIT)
 endif
 
 ifneq ($(CBFLIB_DONT_USE_LOCAL_SWIG),yes)
@@ -1496,7 +1490,6 @@ $(SWIG_KIT):    build_swig
 	git clone $(SWIG_URL) $(SWIG_KIT)
 	(export SWIG_PREFIX=$(PWD);cd $(SWIG_KIT); ./autogen.sh; \
 	./configure --prefix=$(SWIG_PREFIX); make; make install) 
-	touch $(SWIG_KIT)
 endif
 
 
@@ -1508,10 +1501,7 @@ build_py2cifrw:	$(M4)/Makefile.m4
 	touch build_py2cifrw
 $(PY2CIFRW):	build_py2cifrw
 	-rm -rf $(PY2CIFRW)
-	-rm -rf $(PY2CIFRW).tar.gz
-	$(DOWNLOAD) $(PY2CIFRWURL)
-	tar -xvf $(PY2CIFRW).tar.gz
-	-rm $(PY2CIFRW).tar.gz
+	wget -O - -nv $(PY2CIFRWURL) | tar -xzf -
 	(cd $(PY2CIFRW); \
 	PYTHONPATH=$(PY2CIFRW_PREFIX)/lib/python:$(PY2CIFRW_PREFIX)/lib64/python; export PYTHONPATH; \
 	mkdir -p $(PY2CIFRW_PREFIX)/lib/python/site-packages; \
@@ -1525,10 +1515,7 @@ build_py2ply:	$(M4)/Makefile.m4
 	touch build_py2ply
 $(PY2PLY):	build_py2ply
 	-rm -rf $(PY2PLY)
-	-rm -rf $(PY2PLY).tar.gz
-	$(DOWNLOAD) $(PY2PLYURL)
-	tar -xvf $(PY2PLY).tar.gz
-	-rm $(PY2PLY).tar.gz
+	wget -O - -nv $(PY2PLYURL) | tar -xzf -
 	(cd $(PY2PLY); \
 	PYTHONPATH=$(PY2CIFRW_PREFIX)/lib/python:$(PY2CIFRW_PREFIX)/lib64/python; export PYTHONPATH; \
 	mkdir -p $(PY2CIFRW_PREFIX)/lib/python/site-packages; \
@@ -1544,10 +1531,7 @@ build_py3cifrw: $(M4)/Makefile.m4
 	touch build_py3cifrw
 $(PY3CIFRW):    build_py3cifrw
 	-rm -rf $(PY3CIFRW)
-	-rm -rf $(PY3CIFRW).tar.gz
-	$(DOWNLOAD) $(PY3CIFRWURL)
-	tar -xvf $(PY3CIFRW).tar.gz
-	-rm $(PY3CIFRW).tar.gz
+	wget -O - -nv $(PY3CIFRWURL) | tar -xf -
 	(cd $(PY3CIFRW); \
 	PYTHONPATH=$(PY3CIFRW_PREFIX)/lib/python:$(PY3CIFRW_PREFIX)/lib64/python; export PYTHONPATH; \
 	mkdir -p $(PY3CIFRW_PREFIX)/lib/python/site-packages; \
@@ -1561,10 +1545,7 @@ build_py3ply:   $(M4)/Makefile.m4
 	touch build_py3ply
 $(PY3PLY):      build_py3ply
 	-rm -rf $(PY3PLY)
-	-rm -rf $(PY3PLY).tar.gz
-	$(DOWNLOAD) $(PY3PLYURL)
-	tar -xvf $(PY3PLY).tar.gz
-	-rm $(PY3PLY).tar.gz
+	wget -O - -nv $(PY3PLYURL) | tar -xzf -
 	(cd $(PY3PLY); \
 	PYTHONPATH=$(PY3CIFRW_PREFIX)/lib/python:$(PY3CIFRW_PREFIX)/lib64/python; export PYTHONPATH; \
 	mkdir -p $(PY3CIFRW_PREFIX)/lib/python/site-packages; \
@@ -1579,11 +1560,7 @@ endif
 ifneq ($(NUWEB_DEP),'')
 $(NUWEB_DEP):
 	-rm -rf $(NUWEB_DEP)
-	-rm -rf $(NUWEB_DEP).tar.gz
-	$(DOWNLOAD) $(NUWEB_URL)
-	tar -xvf $(NUWEB_DEP).tar.gz
-	touch $(NUWEB_DEP)
-	rm $(NUWEB_DEP).tar.gz
+	wget -O - -nv $(NUWEB_URL) | tar -xzf -
 
 $(NUWEB_DEP2): $(NUWEB_DEP)
 	(cd $(NUWEB_DEP); make nuweb; cp nuweb $(NUWEB_DEP2))
@@ -1598,19 +1575,15 @@ build_regex:    $(M4)/Makefile.m4
 	touch build_regex
 $(REGEX):   build_regex
 	-rm -rf $(REGEX)
-	-rm -rf $(REGEX).tar.gz
-	$(DOWNLOAD) $(REGEX_URL)
-	tar -xvf $(REGEX).tar.gz
-	touch $(REGEX)
-	-rm $(REGEX).tar.gz
+	wget -O - -nv $(REGEX_URL) | tar -xzf -
 	cp config.guess config.sub $(REGEX)
 	(cd $(REGEX); \
 	prefix=$(REGEX_PREFIX); export prefix; \
 	./configure --prefix=$(REGEX_PREFIX); make install)
 	@-cp $(REGEX_PREFIX)/include/pcre2posix.h $(REGEX_PREFIX)/include/regex.h
 $(REGEX)_INSTALL:   $(REGEX)
 	-rm -rf $(REGEX)_install
-	rsync -avz $(REGEX)/ $(REGEX)_install
+	rsync -az $(REGEX)/ $(REGEX)_install
 	(cd $(REGEX)_install; prefix=$(CBF_PREFIX); export prefix; \
 	make distclean; ./configure --prefix=$(CBF_PREFIX); make install )
 	@-cp $(CBF_PREFIX)/include/pcre2posix.h $(CBF_PREFIX)/include/regex.h
@@ -1622,17 +1595,13 @@ build_tiff:	$(M4)/Makefile.m4
 	touch build_tiff
 $(TIFF):	build_tiff config.guess config.sub
 	-rm -rf $(TIFF)
-	-rm -rf $(TIFF).tar.gz
-	$(DOWNLOAD) $(TIFF_URL)
-	tar -xvf $(TIFF).tar.gz
-	touch $(TIFF)
-	-rm $(TIFF).tar.gz
+	wget -O - -nv $(TIFF_URL) | tar -xzf -
 	cp config.guess config.sub $(TIFF)/config/
 	(cd $(TIFF); prefix=$(TIFF_PREFIX); export prefix; \
 	./configure --prefix=$(TIFF_PREFIX); make install)
 $(TIFF)_INSTALL:    $(TIFF)
 	-rm -rf $(TIFF)_install
-	rsync -avz $(TIFF)/  $(TIFF)_install
+	rsync -az $(TIFF)/  $(TIFF)_install
 	(cd $(TIFF)_install; make distclean; prefix=$(CBF_PREFIX); export prefix; \
 	./configure --prefix=$(CBF_PREFIX); make install)
 
@@ -1646,30 +1615,26 @@ build_hdf5:	$(M4)/Makefile.m4
 	touch build_hdf5
 $(HDF5):	build_hdf5 $(LIBAECdep)
 	-rm -rf $(HDF5)
-	-rm -rf $(HDF5).tar.gz
-	$(DOWNLOAD) $(HDF5_URL)
-	tar -xvf $(HDF5).tar.gz
+	wget -O - -nv $(HDF5_URL) | tar -xzf -
 	cp config.guess $(HDF5)/bin/config.guess
 	cp config.sub $(HDF5)/bin/config.sub
-	touch $(HDF5)
-	-rm $(HDF5).tar.gz
 	echo  "first level HDF5 install in "$(HDF5_PREFIX)
 	(cd $(ROOT)/$(HDF5); \
 	CFLAGS="$(CFLAGS)"; export CFLAGS; \
 	mkdir -p hdf5; prefix=$(ROOT)/$(HDF5)/hdf5; export prefix; \
 	./configure --prefix=$(ROOT)/$(HDF5)/hdf5 --enable-build-mode=production \
 	--enable-trace --enable-fortran --enable-using-memchecker --with-szlib=$(ROOT)  ;\
 	make install; \
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/bin/ $(HDF5_PREFIX)/bin; \
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/lib/ $(HDF5_PREFIX)/lib; \
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/include/ $(HDF5_PREFIX)/include; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/bin/ $(HDF5_PREFIX)/bin; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/lib/ $(HDF5_PREFIX)/lib; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/include/ $(HDF5_PREFIX)/include; \
 	cd $(HDF5_PREFIX)/bin; $(ROOT)/$(HDF5)/hdf5/bin/h5redeploy -force )
 $(HDF5)_INSTALL:    $(HDF5)
 	-rm -rf $(HDF5)_install
 	echo "final HDF5 install in "$(CBF_PREFIX)
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/bin/ $(CBF_PREFIX)/bin; \
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/lib/ $(CBF_PREFIX)/lib; \
-	rsync -avz $(ROOT)/$(HDF5)/hdf5/include/ $(CBF_PREFIX)/include; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/bin/ $(CBF_PREFIX)/bin; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/lib/ $(CBF_PREFIX)/lib; \
+	rsync -az $(ROOT)/$(HDF5)/hdf5/include/ $(CBF_PREFIX)/include; \
 	cd $(CBF_PREFIX)/bin; $(ROOT)/$(HDF5)/hdf5/bin/h5redeploy -force
 endif
 
@@ -1682,13 +1647,9 @@ build_libaec:	$(M4)/Makefile.m4
 $(LIBAEC):	build_libaec
 	mkdir -p $(SOLIB)
 	-rm -rf $(LIBAEC)
-	-rm -rf $(LIBAEC).tar.gz
-	$(DOWNLOAD) $(LIBAEC_URL)
-	tar -xvf $(LIBAEC).tar.gz
-	-rm $(LIBAEC).tar.gz
+	wget -O - -nv $(LIBAEC_URL) | tar -xzf -
 	(cd $(LIBAEC); mkdir -p build; cd build; ../configure --prefix=$(ROOT); make install; \
 	cp $(LIB)/libsz.so* $(LIB)/libaec.so* $(SOLIB))
-	touch $(LIBAEC)
 endif
 
 ifneq ($(CBFLIB_DONT_USE_LZ4),yes)
@@ -1701,10 +1662,7 @@ $(LZ4): $(HDF5)	build_lz4
 	mkdir -p $(SOLIB)
 	-rm -rf $(LZ4)
 ifneq ($(MSYS2),yes)
-	-rm -rf $(LZ4).tar.gz
-	$(DOWNLOAD) $(LZ4_URL)
-	tar -xvf $(LZ4).tar.gz
-	-rm $(LZ4).tar.gz
+	wget -O - -nv $(LZ4_URL) | tar -xzf -
 	(cp $(LZ4include)/lz4.h $(INCLUDE); \
 	$(CC) $(CFLAGS) $(SOWCFLAGS) $(INCLUDES) $(WARNINGS) -c $(LZ4src)/lz4.c -o lz4.o; \
 	$(CC) $(CFLAGS) $(SOCFLAGS) $(INCLUDES) $(WARNINGS) -c $(LZ4src)/h5zlz4.c -o h5zlz4.o; \
@@ -1714,7 +1672,6 @@ else
 	git clone $(LZ4_URL)
 	(cd $(LZ4); mkdir build; cd build; cmake .. -G 'MSYS Makefiles' -DENABLE_LZ4_PLUGIN="yes"; make all; cp plugins/* $(SOLIB))
 endif 
-	touch $(LZ4)
 endif
 
 
@@ -1730,7 +1687,6 @@ $(BSHUF): $(HDF5)  build_BSHUF $(LZ4dep)
 	git clone $(BSHUF_URL)
 	(cd $(BSHUF); git submodule update --init; $(PYTHON3) -m build --config-setting=install \
           -C--h5plugin -C--h5plugin-dir=../solib -C--zstd -C--user) 
-	touch $(BSHUF)
 endif
 
 ifneq ($(CBFLIB_DONT_USE_ZSTD),yes)
@@ -1750,7 +1706,6 @@ $(ZSTD): $(HDF5)  build_ZSTD
 	$(CC) -shared zstd_h5plugin.o $(HDF5SOLIBS_LOCAL) $(HDF5SOLIBS_SYSTEM) -lzstd \
 	    -o $(SOLIB)/$(ZSTDFILTER).so; \
 	rm zstd_h5plugin.o)
-	touch $(ZSTD)
 endif
 
 ifneq ($(CBFLIB_DONT_USE_CQRLIB),yes)
@@ -1772,7 +1727,6 @@ cqrlib $(SOLIB)/libcqr.so $(LIB)lbcqr.a:  build_CQRLIB
 ifneq ($(RANLIB),)
 	$(RANLIB) $(LIB)/libcqr.a
 endif
-	touch cqrlib
 endif
 
 
@@ -1952,7 +1906,6 @@ $(PY2CBF)/xmas/readmarheader.py \
 $(PY2CBF)/xmas/xmasheaders.py   \
 $(PY2CBF)/xmas/xmas_cif_template.cif : $(NUWEB_DEP) $(NUWEB_DEP2) $(PY2CBF)/pycbf.w
 	(cd $(PY2CBF); $(NUWEB) pycbf.w )
-	touch $(PY2CBF)/py2setup_py.m4
 
 $(PY2CBF)/_py2cbf.$(PY2CBFEXT):	$(PY2CBF)  shared \
 	$(PY2CBF)/py2setup.py                    \
@@ -2024,7 +1977,6 @@ $(PY3CBF)/xmas/readmarheader.py \
 $(PY3CBF)/xmas/xmasheaders.py   \
 $(PY3CBF)/xmas/xmas_cif_template.cif: $(NUWEB_DEP) $(NUWEB_DEP2) $(PY3CBF)/pycbf.w
 	(cd $(PY3CBF); $(NUWEB) pycbf.w )
-	touch $(PY3CBF)/py3setup_py.m4
 
 
 $(PY3CBF)/_pycbf.$(PY3CBFEXT):	$(PY3CBF)  shared \
@@ -2484,16 +2436,10 @@ $(BIN)/cbf_testxfelread: $(LIB)/libcbf.a $(EXAMPLES)/cbf_testxfelread.c $(EXAMPL
 #
 
 $(DATADIRI):	$(M4)/Makefile.m4
-	(cd ..; $(DOWNLOAD) $(DATAURLI))
-	(cd ..; tar -zxvf CBFlib_$(VERSION)_Data_Files_Input.tar.gz)
-	touch $(DATADIRI)
-	-(cd ..; rm CBFlib_$(VERSION)_Data_Files_Input.tar.gz)
+	wget -O - -nv $(DATAURLI) | tar -C .. -xzf -
 
 $(DATADIRO):	$(M4)/Makefile.m4
-	(cd ..; $(DOWNLOAD) $(DATAURLO))
-	(cd ..; tar -zxvf CBFlib_$(VERSION)_Data_Files_Output.tar.gz)
-	touch $(DATADIRO)
-	-(cd ..; rm CBFlib_$(VERSION)_Data_Files_Output.tar.gz)
+	wget -O - -nv $(DATAURLO) | tar -C .. -xzf -
 
 # Input Data Files 
 
@@ -3174,7 +3120,7 @@ tar:   $(DOCUMENTS) $(SOURCE) $(SRC)/cbf.stx $(HEADERS) $(M4FILES)\
 	README.html README Makefile \
 	$(JPEGS)
 	-/bin/rm -f CBFlib.tar*
-	tar cvBf CBFlib.tar \
+	tar -cf CBFlib.tar \
 	$(DOCUMENTS) $(SOURCE) $(SRC)/cbf.stx $(HEADERS) $(M4FILES)\
 	$(EXAMPLES) \
 	README.html README Makefile \