bump PartMC submodule version to >v2.8.0 (and as a result add hdf5 as submodule + perl as build-time dependency) (#353)

Co-authored-by: Jeffrey Curtis <[email protected]>

Author: slayoo

.github/workflows/tests+pypi.yml

@@ -142,7 +142,7 @@ jobs:
       - run: |
           unset CI
           python -m build 2>&1 | tee build.log
-          exit `fgrep -i warning build.log | grep -v "WARNING setuptools_scm" | wc -l`
+          exit `fgrep -i warning build.log | grep -v "WARNING setuptools_scm" | grep -v "-warnings" | grep -v "All Warnings are enabled" | wc -l`
 
       - if: startsWith(matrix.platform, 'macos-') && matrix.python-version == '3.11'
         run: |
@@ -206,10 +206,10 @@ jobs:
           GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python -m pytest --durations=10 -v -s -We -p no:unraisableexception gitmodules/devops_tests
 
 ### uncomment to gain ssh access in case of failure
-#      - if: ${{ failure() }}
-#        uses: mxschmitt/action-tmate@v3
-#        with:
-#          limit-access-to-actor: true
+      - if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+        with:
+          limit-access-to-actor: true
 
   dist_check:
     runs-on: ubuntu-latest

.gitmodules

@@ -53,3 +53,8 @@
 [submodule "gitmodules/optional"]
 	path = gitmodules/optional
 	url = https://github.com/TartanLlama/optional.git
+	shallow = true
+[submodule "gitmodules/hdf5"]
+	path = gitmodules/hdf5
+	url = https://github.com/HDFGroup/hdf5.git
+	shallow = true

CMakeLists.txt

@@ -132,6 +132,9 @@ set(camp_SOURCES
 )
 add_prefix(gitmodules/camp/src/ camp_SOURCES)
 
+set(hdf5_GENERATED_HEADERS H5Edefin.h H5Einit.h H5Epubgen.h H5Eterm.h H5version.h H5overflow.h)
+add_prefix(gitmodules/hdf5/src/ hdf5_GENERATED_HEADERS)
+
 set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcompound.c denum.c daux.c
   dvlen.c nc.c dfile.c dnotnc4.c dstring.c nclist.c nchashmap.c dinstance_intern.c dtype.c dgroup.c
   nclistmgr.c dattget.c dattinq.c dvarinq.c dfilter.c derror.c doffsets.c datt.c dattput.c dcopy.c drc.c
@@ -140,13 +143,21 @@ set(netcdf_c_dispatch_SOURCES dvar.c ddim.c dvarput.c dvarget.c ddispatch.c dcom
 )
 add_prefix(gitmodules/netcdf-c/libdispatch/ netcdf_c_dispatch_SOURCES)
 
+set(netcdf_c_libhdf5_SOURCES hdf5dispatch.c hdf5create.c hdf5open.c hdf5file.c 
+  hdf5attr.c hdf5debug.c hdf5dim.c hdf5filter.c hdf5grp.c hdf5internal.c hdf5set_format_compatibility.c
+  hdf5type.c hdf5var.c nc4hdf.c nc4info.c nc4mem.c nc4memcb.c
+)
+add_prefix(gitmodules/netcdf-c/libhdf5/ netcdf_c_libhdf5_SOURCES)
+
 set(netcdf_c_lib_SOURCES nc_initialize.c)
 add_prefix(gitmodules/netcdf-c/liblib/ netcdf_c_lib_SOURCES)
 
 set(netcdf_c_src_SOURCES nc3dispatch.c nc3internal.c dim.c var.c ncio.c v1hpg.c memio.c posixio.c)
 add_prefix(gitmodules/netcdf-c/libsrc/ netcdf_c_src_SOURCES)
 
-set(netcdf_c_src4_SOURCES nc4internal.c ncindex.c nc4cache.c nc4dispatch.c nc4type.c nc4grp.c nc4var.c)
+set(netcdf_c_src4_SOURCES nc4internal.c ncindex.c nc4cache.c nc4dispatch.c nc4type.c nc4grp.c nc4var.c
+  ncfunc.c nc4dim.c nc4filters.c nc4attr.c
+)
 add_prefix(gitmodules/netcdf-c/libsrc4/ netcdf_c_src4_SOURCES)
 
 set(netcdf_f_SOURCES typeSizes.F90 module_netcdf_nf_data.F90 module_netcdf_nc_data.F90
@@ -176,7 +187,7 @@ set(partmclib_SOURCES condense_solver.c aero_state.F90 integer_varray.F90 intege
   gas_state.F90 coagulation.F90 exact_soln.F90 coagulation_dist.F90 coag_kernel.F90 spec_line.F90 
   rand.F90 aero_particle.F90 aero_particle_array.F90 mpi.F90 netcdf.F90 aero_info.F90 
   aero_info_array.F90 nucleate.F90 condense.F90 fractal.F90 chamber.F90 camp_interface.F90
-  photolysis.F90
+  photolysis.F90 aero_component.F90
 )
 add_prefix(gitmodules/partmc/src/ partmclib_SOURCES)
 list(APPEND partmclib_SOURCES src/spec_file_pypartmc.F90 src/sys.F90)
@@ -321,13 +332,38 @@ execute_process(
   OUTPUT_FILE ${CMAKE_BINARY_DIR}/include/camp/version.h
 )
 
+### HDF5 ###########################################################################################
+
+set(HDF5_EXTERNALLY_CONFIGURED 1)
+set(BUILD_STATIC_LIBS ON)
+set(HDF5_BUILD_HL_LIB ON)
+set(BUILD_SHARED_LIBS OFF)
+set(BUILD_TESTING OFF)
+set(HDF5_BUILD_TOOLS OFF)
+set(HDF5_BUILD_EXAMPLES OFF)
+set(HDF5_ENABLE_Z_LIB_SUPPORT OFF)
+set(HDF5_ENABLE_SZIP_SUPPORT OFF)
+
+add_subdirectory(${CMAKE_SOURCE_DIR}/gitmodules/hdf5)
+
+foreach(file ${hdf5_GENERATED_HEADERS})
+  execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} "-c" "import re;open(1,'wb').write(re.sub(b'\\r',b'',open(0,'rb').read()))"
+    INPUT_FILE ${CMAKE_SOURCE_DIR}/${file}
+    OUTPUT_FILE ${CMAKE_BINARY_DIR}/${file}
+  )
+  file(REMOVE ${CMAKE_SOURCE_DIR}/${file})
+endforeach()
+
 ### netCDF #########################################################################################
 
 file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/netcdf)
 string(CONCAT cmd
   "import sys; print(''.join([line for line in sys.stdin"
   " if ('VERSION' in line and line.strip().startswith('SET'))"
   " or 'CHUNK_' in line"
+  " or '_CHUNKS_' in line"
+  " or '_CACHE_SIZE' in line"
   " or line.strip().startswith('CHECK_INCLUDE_FILE')"
   " or line.strip().startswith('CHECK_TYPE_SIZE')"
   " or line.strip().startswith('CHECK_FUNCTION_EXISTS')"
@@ -349,6 +385,7 @@ LIST(APPEND netcdf_c_SOURCES ${netcdf_c_lib_SOURCES})
 LIST(APPEND netcdf_c_SOURCES ${netcdf_c_dispatch_SOURCES})
 LIST(APPEND netcdf_c_SOURCES ${netcdf_c_src_SOURCES})
 LIST(APPEND netcdf_c_SOURCES ${netcdf_c_src4_SOURCES})
+LIST(APPEND netcdf_c_SOURCES ${netcdf_c_libhdf5_SOURCES})
 foreach (f ${m4_SOURCES})
   set(tmp ${CMAKE_BINARY_DIR}/netcdf/${f}.c)
   add_custom_command(
@@ -380,15 +417,22 @@ target_include_directories(netcdf_clib PRIVATE
   ${CMAKE_BINARY_DIR}/netcdf
   ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/include
   ${CMAKE_SOURCE_DIR}/gitmodules/netcdf-c/libsrc
+  ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/src
+  ${CMAKE_BINARY_DIR}/gitmodules/hdf5/src
+  ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/hl/src
+  ${CMAKE_SOURCE_DIR}/gitmodules/hdf5/src/H5FDsubfiling
 )
 target_compile_definitions(netcdf_clib PRIVATE
   HAVE_CONFIG_H
   USE_NETCDF4
+  USE_HDF5
 )
 target_compile_options(netcdf_clib PRIVATE
     $<$<C_COMPILER_ID:GNU>:-Wno-unused-result -Wno-pointer-to-int-cast -Wno-int-to-pointer-cast>
     $<$<C_COMPILER_ID:AppleClang>:-Wno-pointer-to-int-cast -Wno-int-to-void-pointer-cast>
 )
+target_link_libraries(netcdf_clib PRIVATE hdf5-static)
+target_link_libraries(netcdf_clib PRIVATE hdf5_hl-static)
 
 ### netCDF-FORTRAN #################################################################################
 
@@ -435,7 +479,7 @@ add_dependencies(partmclib ${SUNDIALS_items})
 string(CONCAT 
   cmd
   "import sys;"
-  "print(''.join([f'#define {line.split()[3]} {line.split()[5]}\\n' for line in sys.stdin if 'parameter ::' in line]))"
+  "print(''.join([line if line.startswith('#') else f'#define {line.split()[3]} {line.split()[5]}\\n' for line in sys.stdin if 'parameter ::' in line or line.startswith('#')]))"
 )
 include(CheckFunctionExists)
 foreach(file aero_data;gas_data;output)

MANIFEST.in

@@ -122,6 +122,9 @@ include gitmodules/netcdf-c/include/nchttp.h
 include gitmodules/netcdf-c/include/netcdf_mem.h
 include gitmodules/netcdf-c/include/netcdf_filter.h
 include gitmodules/netcdf-c/include/netcdf_aux.h
+include gitmodules/netcdf-c/include/hdf5internal.h
+include gitmodules/netcdf-c/include/hdf5dispatch.h
+include gitmodules/netcdf-c/include/ncdimscale.h
 include gitmodules/netcdf-c/libdispatch/utf8proc_data.c
 include gitmodules/netcdf-c/libdispatch/utf8proc.h
 include gitmodules/netcdf-c/libdispatch/dvar.c
@@ -188,6 +191,12 @@ include gitmodules/netcdf-c/libsrc4/nc4dispatch.c
 include gitmodules/netcdf-c/libsrc4/nc4type.c
 include gitmodules/netcdf-c/libsrc4/nc4grp.c
 include gitmodules/netcdf-c/libsrc4/nc4var.c
+include gitmodules/netcdf-c/libsrc4/nc4dim.c
+include gitmodules/netcdf-c/libsrc4/nc4attr.c
+include gitmodules/netcdf-c/libsrc4/nc4filters.c
+include gitmodules/netcdf-c/libsrc4/ncfunc.c
+
+graft gitmodules/netcdf-c/libhdf5
 
 include gitmodules/netcdf-fortran/COPYRIGHT
 include gitmodules/netcdf-fortran/CMakeExtras/MatchNetCDFFortranTypes.cmake
@@ -230,3 +239,12 @@ include gitmodules/netcdf-fortran/fortran/netcdf_text_variables.F90
 include gitmodules/netcdf-fortran/fortran/netcdf_expanded.F90
 include gitmodules/netcdf-fortran/fortran/netcdf4_eightbyte.F90
 include gitmodules/netcdf-fortran/fortran/netcdf4_func.F90
+
+include gitmodules/hdf5/COPYING
+include gitmodules/hdf5/CMakeLists.txt
+include gitmodules/hdf5/*.cmake
+graft gitmodules/hdf5/src
+graft gitmodules/hdf5/bin
+graft gitmodules/hdf5/config
+include gitmodules/hdf5/hl/CMakeLists.txt
+graft gitmodules/hdf5/hl/src

README.md

@@ -289,7 +289,7 @@ MOSAIC_HOME=<<PATH_TO_MOSAIC_LIB>> pip install --force-reinstall --no-binary=PyP
 ```
 
 - Q: Why `pip install PyPartMC` triggers compilation on my brand new Apple machine, while it quickly downloads and installs binary packages when executed on older Macs, Windows or Linux?    
-  A: We are not yet providing binary wheels on PyPI for Apple-silicon (arm64) machines. Cross-compilation with gfortran is only supported with experimental unofficial builds [and is tricky](https://github.com/iains/gcc-12-branch/issues/23), while Github Actions ARM64 virtual machines are [costly](https://docs.github.com/en/billing/managing-billing-for-github-actions/about-billing-for-github-actions#minute-multipliers).  
+  A: We are providing binary wheels on PyPI for Apple-silicon (arm64) machines for selected macOS version made available by Github. In case the macOS version you are using is newer, compilation from source is triggered.  
 
 - Q: Why some of the constructors expect data to be passed as **lists of single-entry dictionaries** instead of multi-element dictionaries?    
   A: This is intentional and related with PartMC relying on the order of elements within spec-file input; while Python dictionaries preserve ordering (insertion order), JSON format does not, and we intend to make these data structures safe to be [de]serialized using JSON.   
@@ -301,6 +301,9 @@ import PyPartMC
 PyPartMC.__versions_of_build_time_dependencies__['PartMC']
 ```
 
+- Q: Why m4 and perl are required at compile time?    
+  A: PyPartMC includes parts of netCDF and HDF5 codebases which depend on m4 and perl, respectively, for generating source files before compilation.
+
 ## Troubleshooting 
 
 #### Common installation issues 

gitmodules/hdf5

@@ -7,6 +7,7 @@ add_executable(main
   $ENV{PARTMC_HOME}/src/spec_line.F90
   $ENV{PARTMC_HOME}/src/util.F90
   $ENV{PARTMC_HOME}/src/constants.F90
+  $ENV{PARTMC_HOME}/src/aero_component.F90
   $ENV{PARTMC_HOME}/src/aero_data.F90
   $ENV{PARTMC_HOME}/src/aero_mode.F90
   $ENV{PARTMC_HOME}/src/aero_dist.F90

gitmodules/partmc

@@ -20,7 +20,7 @@ program main
   call spec_file_close(f_aero_data)
 
   call spec_file_open("aero_dist.dat", f_aero_dist)
-  call spec_file_read_aero_dist(f_aero_dist, aero_data, aero_dist)
+  call spec_file_read_aero_dist(f_aero_dist, aero_data, .false., aero_dist)
   call spec_file_close(f_aero_dist)
 
   call aero_state_zero(aero_state)
@@ -29,7 +29,7 @@ program main
     AERO_STATE_WEIGHT_NUMMASS_SOURCE)
   call aero_state_set_n_part_ideal(aero_state, dble(n_part))
   call aero_state_add_aero_dist_sample(aero_state, aero_data, &
-    aero_dist, 1d0, 0d0, .true., .true., n_part_add)
+    aero_dist, 1d0, 1d0, 0d0, .true., .true., n_part_add)
 
   num_concs = aero_state_num_concs(aero_state, aero_data)
   masses = aero_state_masses(aero_state, aero_data)

readme_fortran/CMakeLists.txt

@@ -38,7 +38,7 @@ subroutine f_aero_dist_from_json(ptr_c, aero_data_ptr_c) bind(C)
     call c_f_pointer(ptr_c, aero_dist)
     call c_f_pointer(aero_data_ptr_c, aero_data_ptr_f)
 
-    call spec_file_read_aero_dist(file, aero_data_ptr_f, aero_dist)
+    call spec_file_read_aero_dist(file, aero_data_ptr_f, .false., aero_dist)
   end subroutine
 
   subroutine f_aero_dist_n_mode(ptr_c, n_mode) bind(C)