Merge pull request #322 from Distributive-Network/caleb/build/overhaul

Improved Build System

Author: philippedistributive

.git-blame-ignore-revs

@@ -0,0 +1,8 @@
+# chore(linting): lint all python files
+aae30e864449442cf0b04e94f8a242b1b667de9a
+
+# chore(linting): lint all JavaScript files
+16dc3153b3cb684ca72445ed058babc8f5d97f42
+
+# chore(linting): lint all C++ files
+58cd4b45777b046f03a63255c1d93e289e1cab5e

.github/workflows/test-and-publish.yaml

@@ -9,16 +9,38 @@ on:
   workflow_call:
   workflow_dispatch:
     inputs:
-      debug_enabled:
-        type: boolean
-        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+      debug_enabled_os:
+        type: choice
+        description: Optionally, choose an OS to run the build with SSH debugging on (https://github.com/fawazahmed0/action-debug)
         required: false
-        default: false
-      dump_cores:
-        type: boolean
-        description: 'Include core dumps in CI artifacts'
+        options:
+        - ''
+        - 'ubuntu-20.04'
+        - 'macos-12'
+        - 'macos-13'
+        - 'macos-14'
+        - 'windows-2019'
+      debug_enabled_python:
+        type: choice
+        description: Choose a Python version to run the build with SSH debugging on
         required: false
-        default: false
+        options:
+        - ''
+        - '3.8'
+        - '3.9'
+        - '3.10'
+        - '3.11'
+        - '3.12'
+      build_type:
+        type: choice
+        description: 'Choose the build type to use'
+        required: false
+        default: 'Debug'
+        options:
+        - 'Debug'
+        - 'Profile'
+        - 'DRelease'
+        - 'Release'
   pull_request:
 
 env:
@@ -100,11 +122,10 @@ jobs:
       fail-fast: false
       matrix:
         # The lowest supported version is Ubuntu 20.04 + Python 3.8 or macOS 12 + Python 3.9
-        os: [ 'ubuntu-20.04', 'macos-12', 'windows-2019', 'macos-14' ]
+        os: [ 'ubuntu-20.04', 'macos-12', 'macos-13', 'macos-14', 'windows-2019' ]
         python_version: [ '3.8', '3.9', '3.10', '3.11', '3.12' ]
         exclude:
           # macOS 12 comes with Python 3.9 by default, so we drop ci support for Python 3.8 on macOS
-          # FIXME: We can't build on macOS 11 for now because our prebuilt `uncrustify` binary requires macOS 12
           - os: 'macos-12'
             python_version: '3.8'
           # actions/setup-python: The version '3.8'/'3.9' with architecture 'arm64' was not found for macOS.
@@ -165,7 +186,8 @@ jobs:
       - name: Build wheel
         run: |
           echo $(poetry run python --version)
-          poetry build --format=wheel
+          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
+          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry build --format=wheel
           ls -lah ./dist/
       - name: Upload wheel as CI artifacts
         uses: actions/upload-artifact@v3
@@ -179,7 +201,7 @@ jobs:
           name: docs-${{ github.run_id }}-${{ github.sha }}
           path: ./build/docs/html/
       - name: Set cores to get stored in /cores
-        if: ${{ matrix.os != 'windows-2019' && github.event_name == 'workflow_dispatch' && inputs.dump_cores }}
+        if: ${{ matrix.os != 'windows-2019' }}
         # TODO (Caleb Aikens) figure out how to get Windows core dumps
         run: |
           sudo mkdir -p /cores
@@ -196,7 +218,8 @@ jobs:
           # TODO (Caleb Aikens) figure out how to get Windows core dumps
             ulimit -c unlimited
           fi
-          poetry run python -m pip install --force-reinstall --verbose ./dist/*
+          WORKFLOW_BUILD_TYPE=${{ inputs.build_type }}
+          BUILD_TYPE=${WORKFLOW_BUILD_TYPE:-"Debug"} poetry run python -m pip install --force-reinstall --verbose ./dist/*
           poetry run python -m pytest tests/python
       - name: Run JS tests (peter-jr)
         if: ${{ (success() || failure()) }}
@@ -206,20 +229,18 @@ jobs:
             ulimit -c unlimited
           fi
           poetry run bash ./peter-jr ./tests/js/
+      - name: SSH debug session
+        if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled_os == matrix.os && inputs.debug_enabled_python == matrix.python_version}}
+        uses: fawazahmed0/action-debug@main
+        with:
+          credentials: "admin:admin"
       - name: Upload core dumps as CI artifacts
         uses: actions/upload-artifact@v3
-        if: ${{ matrix.os != 'windows-2019' && github.event_name == 'workflow_dispatch' && inputs.dump_cores }}
+        if: ${{ matrix.os != 'windows-2019' && failure() }}
         # TODO (Caleb Aikens) figure out how to get Windows core dumps
         with:
           name: cores-${{ matrix.os }}-${{ matrix.python_version }}
           path: /cores
-      # Enable tmate debugging of manually-triggered workflows if the input option was provided
-      - name: SSH debug session
-        if: ${{ (success() || failure()) && github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
-        uses: mxschmitt/action-tmate@v3
-        with:
-          detached: true
-          limit-access-to-actor: true
   sdist:
     runs-on: ubuntu-20.04
     steps:
@@ -236,7 +257,7 @@ jobs:
       - name: Build source distribution (sdist) file
         run: |
           poetry self add "poetry-dynamic-versioning[plugin]"
-          poetry build --format=sdist
+          BUILD_DOCS=1 BUILD_TYPE=Release poetry build --format=sdist
           ls -lah ./dist/
       - name: Upload sdist as CI artifacts
         uses: actions/upload-artifact@v3

.gitignore

@@ -11,6 +11,10 @@ firefox-*/
 __pycache__
 Testing/Temporary
 _spidermonkey_install
+uncrustify-*.tar.gz
+uncrustify-*/
+uncrustify
+*.uncrustify
 __pycache__/*
 dist
 *.so

CMakeLists.txt

@@ -17,23 +17,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_EXTENSIONS OFF)
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
-# Add an external; appends to `PYTHONMONKEY_EXTERNAL_FILES` in the parent scope.
-function(pythonmonkey_add_external PYTHONMONKEY_EXTERNAL)
-  add_subdirectory("cmake/externals/${PYTHONMONKEY_EXTERNAL}")
-  set(PYTHONMONKEY_EXTERNAL_FILE "cmake/externals/${PYTHONMONKEY_EXTERNAL}/CMakeLists.txt")
-  source_group(
-    TREE "${CMAKE_CURRENT_SOURCE_DIR}/cmake/externals/${PYTHONMONKEY_EXTERNAL}"
-    PREFIX "Externals\\${PYTHONMONKEY_EXTERNAL}"
-    FILES "${PYTHONMONKEY_EXTERNAL_FILE}"
-  )
-  list(APPEND PYTHONMONKEY_EXTERNAL_FILES "${PYTHONMONKEY_EXTERNAL_FILE}")
-
-  set(PYTHONMONKEY_EXTERNAL_FILES ${PYTHONMONKEY_EXTERNAL_FILES} PARENT_SCOPE)
-endfunction()
-
-file (GLOB SOURCE_FILES "src/*.cc" "src/internalBinding/*.cc") # Find all C++ files in the src directory
-file (GLOB HEADER_FILES "include/*.hh") # Find all header files in the include directory
-file (GLOB PYTHON_FILES "python/*.cc" "python/*.hh") # Find all the python bindings in the python directory
+file (GLOB_RECURSE HEADER_FILES "include/*.hh") # Find all header files in the include directory and below
+file (GLOB_RECURSE SOURCE_FILES "src/*.cc") # Find all C++ files in the src directory and below
+
 
 include_directories(${CMAKE_CURRENT_LIST_DIR})
 
@@ -43,8 +29,50 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
   ### Code block from: https://cliutils.gitlab.io/modern-cmake/chapters/projects/submodule.html
   include(FetchContent)
 
-  SET(COMPILE_FLAGS "-ggdb -Ofast -fno-rtti") # optimize but also emit debug symbols
-  SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS} $ENV{EXTRA_CMAKE_CXX_FLAGS}")
+  if (WIN32)
+    SET(COMPILE_FLAGS "/GR- /W0")
+
+    SET(OPTIMIZED "/O2")
+    SET(UNOPTIMIZED "/Od")
+    SET(KEEP_SYMBOLS "/DEBUG:FULL")
+    SET(STRIP_SYMBOLS "/DEBUG:NONE")
+    SET(PROFILE "/PROFILE")
+  else()
+    SET(COMPILE_FLAGS "-fno-rtti -Wno-invalid-offsetof")
+
+    SET(OPTIMIZED "-Ofast -DNDEBUG")
+    SET(UNOPTIMIZED "-O0")
+    SET(KEEP_SYMBOLS "-ggdb")
+    SET(STRIP_SYMBOLS "-s")
+    SET(PROFILE "-pg")
+  endif()
+  SET(PROFILE_FLAGS  "${UNOPTIMIZED} ${KEEP_SYMBOLS} ${PROFILE}")
+  SET(DEBUG_FLAGS    "${UNOPTIMIZED} ${KEEP_SYMBOLS}")
+  SET(DRELEASE_FLAGS   "${OPTIMIZED} ${KEEP_SYMBOLS}")
+  SET(RELEASE_FLAGS    "${OPTIMIZED} ${STRIP_SYMBOLS}")
+
+  if(GENERATOR_IS_MULTI_CONFIG)
+    set(CMAKE_CONFIGURATION_TYPES "Profile;Debug;DRelease;Release" CACHE STRING "" FORCE)
+    string(APPEND COMPILE_FLAGS "$<$<CONFIG:Profile>:${PROFILE_FLAGS}> $<$<CONFIG:Debug>:${DEBUG_FLAGS}> $<$<CONFIG:DRelease>:${DRELEASE_FLAGS}> $<$<CONFIG:Release>:${RELEASE_FLAGS}>")
+  else()
+    set_property(CACHE PM_BUILD_TYPE PROPERTY HELPSTRING "Choose the type of build")
+    set_property(CACHE PM_BUILD_TYPE PROPERTY STRINGS "Profile;Debug;DRelease;Release")
+    if(PM_BUILD_TYPE STREQUAL "Profile")
+      list(APPEND COMPILE_FLAGS "${PROFILE_FLAGS}")
+    elseif(PM_BUILD_TYPE STREQUAL "Debug")
+      list(APPEND COMPILE_FLAGS "${DEBUG_FLAGS}")
+    elseif(PM_BUILD_TYPE STREQUAL "DRelease")
+      list(APPEND COMPILE_FLAGS "${DRELEASE_FLAGS}")
+    else()  #Release build
+      message("PM_BUILD_TYPE not detected or invalid value, defaulting to Release build.")
+      set(PM_BUILD_TYPE Release CACHE STRING "" FORCE)
+      list(APPEND COMPILE_FLAGS "${RELEASE_FLAGS}") 
+    endif()
+    message("PythonMonkey build type is: ${PM_BUILD_TYPE}")
+    list(JOIN COMPILE_FLAGS " " COMPILE_FLAGS)
+  endif()
+  
+  SET(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} ${COMPILE_FLAGS}")
 
   set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake/modules)
   if(APPLE)
@@ -55,36 +83,35 @@ if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME)
     set(PYTHONLIBS_VERSION_STRING ${Python_VERSION})
     set(PYTHON_INCLUDE_DIR ${Python_INCLUDE_DIRS})
     set(PYTHON_LIBRARIES ${Python_LIBRARIES})
-    message("Apple - Using Python:${Python_VERSION_MAJOR} - Libraries:${PYTHON_LIBRARIES} - IncludeDirs: ${PYTHON_INCLUDE_DIR}")
   elseif(UNIX)
     find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED)
     set(Python_FIND_VIRTUALENV FIRST) # (require cmake >= v3.15 and this is the default) use the Python version configured by pyenv if available
     set(PYTHON_LIBRARIES ${Python_LIBRARIES})
     set(PYTHON_INCLUDE_DIR ${Python_INCLUDE_DIRS})
-    message("Linux - Using Python:${Python_VERSION_MAJOR}.${Python_VERSION_MINOR} - Libraries:${PYTHON_LIBRARIES} - IncludeDirs: ${PYTHON_INCLUDE_DIR}")
     find_package(SpiderMonkey REQUIRED)
   elseif(WIN32)
-    find_package(PythonInterp 3.8 REQUIRED)
-    find_package(PythonLibs 3.8   REQUIRED)
+    find_package(Python 3.8 COMPONENTS Interpreter Development REQUIRED)
+    set(Python_FIND_VIRTUALENV FIRST) # (require cmake >= v3.15 and this is the default) use the Python version configured by pyenv if available
+    set(PYTHON_LIBRARIES ${Python_LIBRARIES})
+    set(PYTHON_INCLUDE_DIR ${Python_INCLUDE_DIRS})
     find_package(SpiderMonkey REQUIRED)
-    set(PYTHONLIBS_VERSION_STRING $ENV{PY_VERSION})
   endif()
-  include_directories(${PYTHON_INCLUDE_DIRS})
+  message("${CMAKE_SYSTEM_NAME} - Using Python:${Python_VERSION} - Libraries:${Python_LIBRARIES} - IncludeDirs: ${Python_INCLUDE_DIRS}")
+  include_directories(${Python_INCLUDE_DIRS})
   include_directories(${SPIDERMONKEY_INCLUDE_DIR})
 
   # Add doxygen if this is the main app
-  find_package(Doxygen)
-  if(Doxygen_FOUND)
+  option(BUILD_DOCS "Build documentation" OFF)
+  if(BUILD_DOCS)
+    find_package(Doxygen)
+    if(Doxygen_FOUND)
       add_subdirectory(cmake/docs)
-  else()
+    else()
       message(STATUS "Doxygen not found. Not building docs.")
+    endif()
   endif()
 
 endif()  
 
 # Add compiled folder directories
 add_subdirectory(src)
-
-pythonmonkey_add_external("uncrustify")
-pythonmonkey_add_external("autopep8")
-add_subdirectory(cmake/format)

Makefile

@@ -4,24 +4,37 @@
 # @date         March 2024
 #
 
-BUILD  = debug
+BUILD = Debug	# (case-insensitive) Release, DRelease, Debug, or Profile
+DOCS = false
+VERBOSE = true
 PYTHON = python3
-RUN    = poetry run
+RUN = poetry run
 
-PYTHON_BUILD_ENV = VERBOSE=1 EXTRA_CMAKE_CXX_FLAGS="$(EXTRA_CMAKE_CXX_FLAGS)"
 OS_NAME := $(shell uname -s)
 
 ifeq ($(OS_NAME),Linux)
-CPU_COUNT=$(shell cat /proc/cpuinfo  | grep -c processor)
-MAX_JOBS=10
+CPU_COUNT = $(shell cat /proc/cpuinfo  | grep -c processor)
+MAX_JOBS = 10
 CPUS := $(shell test $(CPU_COUNT) -lt $(MAX_JOBS) && echo $(CPU_COUNT) || echo $(MAX_JOBS))
 PYTHON_BUILD_ENV += CPUS=$(CPUS)
 endif
 
-EXTRA_CMAKE_CXX_FLAGS = -Wno-invalid-offsetof $(JOBS)
+ifeq ($(BUILD),Profile)
+PYTHON_BUILD_ENV += BUILD_TYPE=Profile
+else ifeq ($(BUILD),Debug)
+PYTHON_BUILD_ENV += BUILD_TYPE=Debug
+else ifeq ($(BUILD),DRelease)
+PYTHON_BUILD_ENV += BUILD_TYPE=DRelease
+else # Release build
+PYTHON_BUILD_ENV += BUILD_TYPE=Release
+endif
+
+ifeq ($(DOCS),true)
+PYTHON_BUILD_ENV += BUILD_DOCS=1
+endif
 
-ifeq ($(BUILD),debug)
-EXTRA_CMAKE_CXX_FLAGS += -O0
+ifeq ($(VERBOSE),true)
+PYTHON_BUILD_ENV += VERBOSE=1
 endif
 
 .PHONY: build test all clean debug
@@ -37,10 +50,9 @@ all:	build test
 clean:
 	rm -rf build/src/CMakeFiles/pythonmonkey.dir
 	rm -f build/src/pythonmonkey.so
-	rm -f python/pythonmonkey.so
+	rm -f python/pythonmonkey/pythonmonkey.so
 
 debug:
-	@echo EXTRA_CMAKE_CXX_FLAGS=$(EXTRA_CMAKE_CXX_FLAGS)
 	@echo JOBS=$(JOBS)
 	@echo CPU_COUNT=$(CPU_COUNT)
-	@echo OS_NAME=$(OS_NAME)
+	@echo OS_NAME=$(OS_NAME)

README.md

@@ -78,7 +78,12 @@ Read this if you want to build a local version.
     - [Poetry](https://python-poetry.org/docs/#installation)
     - [poetry-dynamic-versioning](https://github.com/mtkennerly/poetry-dynamic-versioning)
 
-2. Run `poetry install`. This command automatically compiles the project and installs the project as well as dependencies into the poetry virtualenv.
+2. Run `poetry install`. This command automatically compiles the project and installs the project as well as dependencies into the poetry virtualenv. If you would like to build the docs, set the `BUILD_DOCS` environment variable, like so: `BUILD_DOCS=1 poetry install`.
+PythonMonkey supports multiple build types, which you can build by setting the `BUILD_TYPE` environment variable, like so: `BUILD_TYPE=Debug poetry install`. The build types are (case-insensitive):
+- `Release`: stripped symbols, maximum optimizations (default)
+- `DRelease`: same as `Release`, except symbols are not stripped
+- `Debug`: minimal optimizations
+- `Profile`: same as `Debug`, except profiling is enabled 
 
 If you are using VSCode, you can just press <kbd>Ctrl</kbd> + <kbd>Shift</kbd> + <kbd>B</kbd> to [run build task](https://code.visualstudio.com/docs/editor/tasks#_custom-tasks) - We have [the `tasks.json` file configured for you](.vscode/tasks.json).