Initial proof-of-concept for dynamic HiPO loading in highspy

Author: mathgeekcoder

CMakeLists.txt

@@ -93,6 +93,15 @@ if (PYTHON_BUILD_SETUP)
   set(BUILD_TESTING OFF)
 endif()
 
+option(HIPO "Build HIPO" OFF)
+option(HIPO_PYTHON_BUILD "Build HiPO Python shared library" OFF)
+message(STATUS "Build HiPO Python: ${HIPO_PYTHON_BUILD}")
+if (HIPO_PYTHON_BUILD)
+  set(BUILD_CXX OFF)
+  set(BUILD_TESTING OFF)
+  set(HIPO ON)
+endif()
+
 include(CMakeDependentOption)
 
 CMAKE_DEPENDENT_OPTION(ALL_TESTS "Build all tests" OFF "BUILD_TESTING;BUILD_CXX" OFF)
@@ -146,7 +155,7 @@ endif()
   # message(STATUS "CUPLDP with Nvidia is only supported on Linux at the moment. Using CPU version.")
 # endif()
 
-option(HIPO "Build HIPO" OFF)
+
 message(STATUS "Build HIPO: ${HIPO}")
 option(BUILD_OPENBLAS "Build OpenBLAS" OFF)
 message(STATUS "Build OpenBLAS: ${BUILD_OPENBLAS}")
@@ -850,6 +859,7 @@ else(FAST_BUILD)
   endif()
 
   include(python-highs)
+  include(hipo-python)
 
   option(USE_DOTNET_STD_21 "Use .Net Standard 2.1 support" ON)
   message(STATUS ".Net: Use .Net Framework 2.1 support: ${USE_DOTNET_STD_21}")

cmake/hipo-python.cmake

@@ -0,0 +1,132 @@
+# HiPO Python build module
+# Similar to python-highs.cmake but builds the HiPO shared library
+
+if (NOT HIPO_PYTHON_BUILD)
+  return()
+endif()
+
+include(sources-python)
+
+# could use subset of sources, but this is easier to maintain
+set(sources_python ${highs_sources_python} 
+                   ${cupdlp_sources_python} 
+                   ${ipx_sources_python} 
+                   ${basiclu_sources_python})
+
+set(headers_python ${highs_headers_python} 
+                   ${cupdlp_headers_python} 
+                   ${ipx_headers_python} 
+                   ${basiclu_headers_python})
+
+# Create shared library
+add_library(highs_hipo SHARED 
+    ${sources_python}
+    ${hipo_sources}
+    ${factor_highs_sources}
+    ${hipo_util_sources}
+    ${hipo_orderings_sources}
+
+    ${headers_python}
+    ${hipo_headers}
+    ${factor_highs_headers}
+    ${hipo_util_headers}
+    ${hipo_orderings_headers}
+)
+
+target_include_directories(highs_hipo PRIVATE 
+    ${include_dirs_python} 
+)
+
+target_compile_definitions(highs_hipo PRIVATE HIPO_LIBRARY_BUILD)
+
+# Dependencies
+find_package(ZLIB)
+if(ZLIB_FOUND)
+    target_link_libraries(highs_hipo PRIVATE ZLIB::ZLIB)
+endif()
+
+
+if (NOT USE_CMAKE_FIND_BLAS)
+    if(APPLE)
+        target_link_libraries(highs_hipo PRIVATE "-framework Accelerate")
+        target_compile_definitions(highs_hipo PRIVATE HIPO_USES_APPLE_BLAS)
+    elseif(WIN32)
+    if(TARGET OpenBLAS::OpenBLAS)
+        target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        target_link_libraries(highs_hipo PRIVATE OpenBLAS::OpenBLAS)
+    elseif(OPENBLAS_LIB)
+        target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        message(STATUS "Linking against OpenBLAS via raw library: ${OPENBLAS_LIB}")
+        target_link_libraries(highs_hipo PRIVATE ${OPENBLAS_LIB})
+        target_include_directories(highs_hipo PRIVATE ${OPENBLAS_INCLUDE_DIR})
+    elseif(BUILD_OPENBLAS)
+        target_link_libraries(highs_hipo PRIVATE openblas)
+        target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+    else()
+        message(FATAL_ERROR "OpenBLAS not found on Windows.")
+    endif()
+
+    target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+    else()
+        # LINUX
+        if(BLAS_LIB)
+            target_link_libraries(highs_hipo PRIVATE "${BLAS_LIB}" cblas)
+        elseif(OPENBLAS_LIB)
+            target_link_libraries(highs_hipo PRIVATE "${OPENBLAS_LIB}")
+            target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        elseif(BUILD_OPENBLAS)
+            target_link_libraries(highs_hipo PRIVATE openblas)
+            target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        else()
+            message(FATAL_ERROR "No BLAS library available")
+        endif(BLAS_LIB)
+    endif(APPLE)
+else()
+
+    if (WIN32 AND TARGET OpenBLAS::OpenBLAS)
+        target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        target_link_libraries(highs_hipo PRIVATE OpenBLAS::OpenBLAS)
+    else()
+        target_link_libraries(highs_hipo PRIVATE BLAS::BLAS)
+
+        string(TOLOWER "${BLAS_LIBRARIES}" blas_lower)
+        if(blas_lower MATCHES "openblas")
+            target_compile_definitions(highs_hipo PRIVATE HIPO_USES_OPENBLAS)
+        elseif(blas_lower MATCHES "accelerate")
+            target_compile_definitions(highs_hipo PRIVATE HIPO_USES_APPLE_BLAS)
+        endif()
+    endif()
+endif()
+
+if(MSVC)
+  target_compile_options(highs_hipo PRIVATE "/bigobj")
+endif()
+
+if (NOT MSVC) 
+  target_compile_options(highs_hipo PRIVATE "-ftemplate-depth=2048")
+endif()
+
+
+# Set library properties
+set_target_properties(highs_hipo PROPERTIES
+    OUTPUT_NAME "highs_hipo"
+    POSITION_INDEPENDENT_CODE ON
+    CXX_VISIBILITY_PRESET hidden
+    C_VISIBILITY_PRESET hidden
+    VISIBILITY_INLINES_HIDDEN ON
+)
+
+if(WIN32)
+    set_target_properties(highs_hipo PROPERTIES WINDOWS_EXPORT_ALL_SYMBOLS OFF)
+elseif(APPLE)
+    set_target_properties(highs_hipo PROPERTIES INSTALL_RPATH "@loader_path")
+else()
+    set_target_properties(highs_hipo PROPERTIES INSTALL_RPATH "$ORIGIN")
+endif()
+
+# Install to Python package directory
+install(TARGETS highs_hipo
+    LIBRARY DESTINATION highspy_hipo
+    RUNTIME DESTINATION highspy_hipo
+    ARCHIVE DESTINATION highspy_hipo
+)

cmake/sources-python.cmake

@@ -283,7 +283,8 @@ set(highs_sources_python
     highs/util/HighsUtils.cpp
     highs/util/HSet.cpp
     highs/util/HVectorBase.cpp
-    highs/util/stringutil.cpp)
+    highs/util/stringutil.cpp
+    highs/lp_data/DynamicHipoLoader.cpp)
 
 set(highs_headers_python
     extern/pdqsort/pdqsort.h
@@ -321,6 +322,7 @@ set(highs_headers_python
     highs/lp_data/HighsSolve.h
     highs/lp_data/HighsStatus.h
     highs/lp_data/HStruct.h
+    highs/lp_data/DynamicHipoLoader.h
     highs/mip/feasibilityjump.hh
     highs/mip/HighsCliqueTable.h
     highs/mip/HighsConflictPool.h
@@ -444,4 +446,155 @@ set(highs_headers_python
     highs/util/HVectorBase.h
     highs/util/stringutil.h
     highs/Highs.h
-  )
+  )
+
+
+  set(hipo_sources 
+    highs/ipm/hipo/ipm/CurtisReidScaling.cpp 
+    highs/ipm/hipo/ipm/IpmData.cpp 
+    highs/ipm/hipo/ipm/FactorHiGHSSolver.cpp 
+    highs/ipm/hipo/ipm/Control.cpp
+    highs/ipm/hipo/ipm/Iterate.cpp 
+    highs/ipm/hipo/ipm/LogHighs.cpp
+    highs/ipm/hipo/ipm/Model.cpp
+    highs/ipm/hipo/ipm/Refine.cpp
+    highs/ipm/hipo/ipm/Solver.cpp
+    highs/ipm/hipo/HipoCApi.cpp)
+
+set(hipo_headers
+    highs/ipm/hipo/ipm/CurtisReidScaling.h
+    highs/ipm/hipo/ipm/IpmData.h
+    highs/ipm/hipo/ipm/FactorHiGHSSolver.h
+    highs/ipm/hipo/ipm/Parameters.h
+    highs/ipm/hipo/ipm/Control.h
+    highs/ipm/hipo/ipm/Info.h
+    highs/ipm/hipo/ipm/Iterate.h
+    highs/ipm/hipo/ipm/LinearSolver.h
+    highs/ipm/hipo/ipm/LogHighs.h
+    highs/ipm/hipo/ipm/Model.h
+    highs/ipm/hipo/ipm/Options.h
+    highs/ipm/hipo/ipm/Solver.h
+    highs/ipm/hipo/ipm/Status.h
+    highs/ipm/hipo/HipoCApi.h)
+
+set(factor_highs_sources
+    highs/ipm/hipo/factorhighs/Analyse.cpp
+    highs/ipm/hipo/factorhighs/CallAndTimeBlas.cpp
+    highs/ipm/hipo/factorhighs/CliqueStack.cpp
+    highs/ipm/hipo/factorhighs/DataCollector.cpp
+    highs/ipm/hipo/factorhighs/DenseFactHybrid.cpp
+    highs/ipm/hipo/factorhighs/DenseFactKernel.cpp
+    highs/ipm/hipo/factorhighs/DgemmParallel.cpp
+    highs/ipm/hipo/factorhighs/FactorHiGHS.cpp
+    highs/ipm/hipo/factorhighs/Factorise.cpp
+    highs/ipm/hipo/factorhighs/FormatHandler.cpp
+    highs/ipm/hipo/factorhighs/HybridHybridFormatHandler.cpp
+    highs/ipm/hipo/factorhighs/HybridSolveHandler.cpp
+    highs/ipm/hipo/factorhighs/KrylovMethodsIpm.cpp
+    highs/ipm/hipo/factorhighs/Numeric.cpp
+    highs/ipm/hipo/factorhighs/SolveHandler.cpp
+    highs/ipm/hipo/factorhighs/Swaps.cpp
+    highs/ipm/hipo/factorhighs/Symbolic.cpp)
+
+set(factor_highs_headers
+    highs/ipm/hipo/factorhighs/Analyse.h
+    highs/ipm/hipo/factorhighs/CallAndTimeBlas.h
+    highs/ipm/hipo/factorhighs/CliqueStack.h
+    highs/ipm/hipo/factorhighs/DataCollector.h
+    highs/ipm/hipo/factorhighs/DenseFact.h
+    highs/ipm/hipo/factorhighs/DgemmParallel.h
+    highs/ipm/hipo/factorhighs/FactorHiGHS.h
+    highs/ipm/hipo/factorhighs/FactorHiGHSSettings.h
+    highs/ipm/hipo/factorhighs/Factorise.h
+    highs/ipm/hipo/factorhighs/FormatHandler.h
+    highs/ipm/hipo/factorhighs/HybridHybridFormatHandler.h
+    highs/ipm/hipo/factorhighs/HybridSolveHandler.h
+    highs/ipm/hipo/factorhighs/KrylovMethodsIpm.h
+    highs/ipm/hipo/factorhighs/Numeric.h
+    highs/ipm/hipo/factorhighs/ReturnValues.h
+    highs/ipm/hipo/factorhighs/SolveHandler.h
+    highs/ipm/hipo/factorhighs/Swaps.h
+    highs/ipm/hipo/factorhighs/Symbolic.h
+    highs/ipm/hipo/factorhighs/Timing.h)
+
+set(hipo_util_sources
+    highs/ipm/hipo/auxiliary/Auxiliary.cpp
+    highs/ipm/hipo/auxiliary/KrylovMethods.cpp
+    highs/ipm/hipo/auxiliary/Log.cpp
+    highs/ipm/hipo/auxiliary/VectorOperations.cpp)
+
+set(hipo_util_headers
+    highs/ipm/hipo/auxiliary/Auxiliary.h
+    highs/ipm/hipo/auxiliary/IntConfig.h
+    highs/ipm/hipo/auxiliary/KrylovMethods.h
+    highs/ipm/hipo/auxiliary/Log.h
+    highs/ipm/hipo/auxiliary/mycblas.h
+    highs/ipm/hipo/auxiliary/VectorOperations.h)
+
+set(hipo_orderings_sources
+    extern/amd/amd_1.c
+    extern/amd/amd_2.c
+    extern/amd/amd_aat.c
+    extern/amd/amd_control.c
+    extern/amd/amd_defaults.c
+    extern/amd/amd_info.c
+    extern/amd/amd_order.c
+    extern/amd/amd_post_tree.c
+    extern/amd/amd_postorder.c
+    extern/amd/amd_preprocess.c
+    extern/amd/amd_valid.c
+    extern/amd/SuiteSparse_config.c
+    extern/metis/GKlib/error.c
+    extern/metis/GKlib/mcore.c
+    extern/metis/GKlib/memory.c
+    extern/metis/GKlib/random.c
+    extern/metis/libmetis/auxapi.c
+    extern/metis/libmetis/balance.c
+    extern/metis/libmetis/bucketsort.c
+    extern/metis/libmetis/coarsen.c
+    extern/metis/libmetis/compress.c
+    extern/metis/libmetis/contig.c
+    extern/metis/libmetis/fm.c
+    extern/metis/libmetis/gklib.c
+    extern/metis/libmetis/graph.c
+    extern/metis/libmetis/initpart.c
+    extern/metis/libmetis/mcutil.c
+    extern/metis/libmetis/mmd.c
+    extern/metis/libmetis/ometis.c
+    extern/metis/libmetis/options.c
+    extern/metis/libmetis/refine.c
+    extern/metis/libmetis/separator.c
+    extern/metis/libmetis/sfm.c
+    extern/metis/libmetis/srefine.c
+    extern/metis/libmetis/util.c
+    extern/metis/libmetis/wspace.c
+    extern/rcm/rcm.cpp)
+
+set(hipo_orderings_headers
+    extern/amd/amd_internal.h
+    extern/amd/amd.h
+    extern/amd/SuiteSparse_config.h
+    extern/metis/GKlib/gk_arch.h
+    extern/metis/GKlib/gk_defs.h
+    extern/metis/GKlib/gk_macros.h
+    extern/metis/GKlib/gk_mkblas.h
+    extern/metis/GKlib/gk_mkmemory.h
+    extern/metis/GKlib/gk_mkpqueue.h
+    extern/metis/GKlib/gk_mkrandom.h
+    extern/metis/GKlib/gk_mksort.h
+    extern/metis/GKlib/gk_ms_inttypes.h
+    extern/metis/GKlib/gk_ms_stat.h
+    extern/metis/GKlib/gk_ms_stdint.h
+    extern/metis/GKlib/gk_proto.h
+    extern/metis/GKlib/gk_struct.h
+    extern/metis/GKlib/gk_types.h
+    extern/metis/GKlib/GKlib.h
+    extern/metis/libmetis/defs.h
+    extern/metis/libmetis/gklib_defs.h
+    extern/metis/libmetis/macros.h
+    extern/metis/libmetis/metislib.h
+    extern/metis/libmetis/proto.h
+    extern/metis/libmetis/stdheaders.h
+    extern/metis/libmetis/struct.h
+    extern/metis/metis.h
+    extern/rcm/rcm.h)

cmake/sources.cmake

@@ -185,7 +185,8 @@ set(hipo_sources
     ipm/hipo/ipm/LogHighs.cpp
     ipm/hipo/ipm/Model.cpp
     ipm/hipo/ipm/Refine.cpp
-    ipm/hipo/ipm/Solver.cpp)
+    ipm/hipo/ipm/Solver.cpp
+    ipm/hipo/HipoCApi.cpp)
 
 set(hipo_headers
     ipm/hipo/ipm/CurtisReidScaling.h
@@ -200,7 +201,8 @@ set(hipo_headers
     ipm/hipo/ipm/Model.h
     ipm/hipo/ipm/Options.h
     ipm/hipo/ipm/Solver.h
-    ipm/hipo/ipm/Status.h)
+    ipm/hipo/ipm/Status.h
+    ipm/hipo/HipoCApi.h)
 
 set(factor_highs_sources
     ipm/hipo/factorhighs/Analyse.cpp
@@ -436,7 +438,8 @@ set(highs_sources
     util/HighsUtils.cpp
     util/HSet.cpp
     util/HVectorBase.cpp
-    util/stringutil.cpp)
+    util/stringutil.cpp
+    lp_data/DynamicHipoLoader.cpp)
 
 # add catch header?
 set(highs_headers
@@ -477,6 +480,7 @@ set(highs_headers
     lp_data/HighsSolve.h
     lp_data/HighsStatus.h
     lp_data/HStruct.h
+    lp_data/DynamicHipoLoader.h
     mip/feasibilityjump.hh
     mip/HighsCliqueTable.h
     mip/HighsConflictPool.h