Feature/add fuzzing (#799)

Add fuzz testing 

* Also improves benchmark setup
* Add fuzzing workflow and enable benchmarking options in CI configurations
* Add documentation for building benchmarks, fuzz testing, and running tests
* Remove C++ flag for macos, resulted in undefined symbols
* Increase celix error buffer size for fuzz testing
* Eliminate clang warnings -Werror,-Wvla-cxx-extension
* Refactor `benchmark` dependency to `test_requires` when `enable_benchmarking` is set.
* Fix formatting issue in README under "C Patterns" section.
* Update to actions/cache 4.3.0
* Add asan and ubsan for fuzz testing builds
* Enable ubsan with asan for linux ci

---------

Co-authored-by: PengZheng <howtofly@gmail.com>

Author: pnoltes

.github/workflows/fuzzing.yml

@@ -0,0 +1,59 @@
+name: Celix Fuzzing
+
+on:
+  push:
+  pull_request:
+  schedule:
+    - cron: '0 3 * * *'
+
+jobs:
+  fuzz-utils:
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0
+      - name: Set up Python
+        uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c #v4.9.1
+        with:
+          python-version: '3.x'
+      - name: Set Compiler Environment Variables
+        run: |
+          echo "CC=clang" >> $GITHUB_ENV
+          echo "CXX=clang++" >> $GITHUB_ENV
+      - name: Install Conan
+        run: pip install conan
+      - name: Cache Conan
+        uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 #v4.3.0
+        with:
+          path: ~/.conan2/p
+          key: ${{ runner.os }}-conan-${{ hashFiles('conanfile.py', 'libs/utils/**') }}
+          restore-keys: |
+            ${{ runner.os }}-conan-
+      - name: Setup Conan Profile
+        run: |
+          conan profile detect 
+      - name: Conan install
+        run: conan install . --output-folder=build --build=missing -o "celix/*:build_utils=True" -o "celix/*:enable_fuzzing=True" -o "celix/*:enable_address_sanitizer=True" -o "celix/*:enable_undefined_sanitizer=True"
+      - name: Conan build
+        run: conan build . --output-folder=build -o "celix/*:build_utils=True" -o "celix/*:enable_fuzzing=True"  -o "celix/*:enable_address_sanitizer=True" -o "celix/*:enable_undefined_sanitizer=True" -o "celix/*:celix_err_buffer_size=5120"
+      - name: Set fuzzer run time
+        id: set-runtime
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "FUZZ_TIME=600" >> ${GITHUB_ENV}
+          else
+            echo "FUZZ_TIME=30" >> ${GITHUB_ENV}
+          fi
+      - name: Run properties fuzzer
+        run: |
+          source build/conanrun.sh
+          ./build/libs/utils/fuzzing/celix_properties_fuzzer -max_total_time=$FUZZ_TIME ./build/libs/utils/fuzzing/properties_corpus
+      - name: Run version fuzzer
+        run: |
+          source build/conanrun.sh
+          ./build/libs/utils/fuzzing/celix_version_fuzzer -max_total_time=$FUZZ_TIME ./build/libs/utils/fuzzing/version_corpus
+      - name: Run filter fuzzer
+        run: |
+          source build/conanrun.sh
+          ./build/libs/utils/fuzzing/celix_filter_fuzzer -max_total_time=$FUZZ_TIME ./build/libs/utils/fuzzing/filter_corpus

.github/workflows/macos.yml

@@ -56,6 +56,7 @@ jobs:
         env:
           CONAN_BUILD_OPTIONS: |
             -o celix/*:enable_testing=True
+            -o celix/*:enable_benchmarking=True
             -o celix/*:enable_address_sanitizer=True
             -o celix/*:build_all=True
             -o celix/*:enable_cmake_warning_tests=True
@@ -79,7 +80,7 @@ jobs:
         uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c #v3.3.0
       - name: Install dependencies
         run: |
-          brew install lcov jansson rapidjson libzip ccache ninja openssl@1.1
+          brew install lcov jansson rapidjson libzip ccache ninja openssl@1.1 google-benchmark
       - name: Prepare ccache timestamp
         id: ccache_cache_timestamp
         run: |
@@ -95,6 +96,7 @@ jobs:
         env:
           BUILD_OPTIONS: |
             -DENABLE_TESTING=ON
+            -DENABLE_BENCHMARKING=ON
             -DENABLE_ADDRESS_SANITIZER=ON
             -DENABLE_TESTING_ON_CI=ON
             -DCMAKE_BUILD_TYPE=Release

.github/workflows/ubuntu.yml

@@ -77,7 +77,9 @@ jobs:
           CXX: ${{ matrix.compiler[1] }}
           CONAN_BUILD_OPTIONS: |
             -o celix:enable_testing=True
+            -o celix:enable_benchmarking=True
             -o celix:enable_address_sanitizer=True
+            -o celix:enable_undefined_sanitizer=True
             -o celix:build_all=True
             -o celix:enable_cmake_warning_tests=True
             -o celix:enable_testing_on_ci=True
@@ -120,6 +122,7 @@ jobs:
           libzip-dev \
           libjansson-dev \
           libcurl4-openssl-dev \
+          libbenchmark-dev \
           default-jdk \
           cmake \
           libffi-dev \
@@ -145,6 +148,7 @@ jobs:
         BUILD_OPTIONS: |
           -DBUILD_EXPERIMENTAL=ON
           -DENABLE_TESTING=ON
+          -DENABLE_BENCHMARKING=ON
           -DRSA_JSON_RPC=ON
           -DRSA_REMOTE_SERVICE_ADMIN_SHM_V2=ON
           -DENABLE_TESTING_ON_CI=ON

CMakeLists.txt

@@ -168,6 +168,8 @@ endif ()
 option(ENABLE_CMAKE_WARNING_TESTS "Enable cmake warning tests to test warning prints" OFF)
 option(ENABLE_TESTING_ON_CI "Whether to enable testing on CI. This influence allowed timing errors during unit tests" OFF)
 option(ENABLE_DEPRECATED_WARNINGS "Enable compiler warnings for usage of deprecated functionality" OFF)
+option(ENABLE_FUZZING "Enable fuzz testing, using LibFuzzer" OFF) #Note support for LibFuzzer is built-in for Clang
+option(ENABLE_BENCHMARKING "Enable benchmarking, using Google Benchmark" OFF)
 
 if (NOT ENABLE_DEPRECATED_WARNINGS)
     set(CMAKE_C_FLAGS "-Wno-deprecated-declarations ${CMAKE_C_FLAGS}")

cmake/celix_project/CelixProject.cmake

@@ -25,13 +25,17 @@ mark_as_advanced(CLEAR ENABLE_UNDEFINED_SANITIZER)
 mark_as_advanced(CLEAR ENABLE_THREAD_SANITIZER)
 
 if (ENABLE_ADDRESS_SANITIZER)
+    set(UBSAN_SAN "")
+    if (ENABLE_UNDEFINED_SANITIZER)
+        set(UBSAN_SAN "undefined,")
+    endif ()
     if("${CMAKE_C_COMPILER_ID}" MATCHES "Clang")
         set(CMAKE_C_FLAGS "-DCELIX_ASAN_ENABLED ${CMAKE_C_FLAGS}")
-        set(CMAKE_C_FLAGS "-shared-libasan -fsanitize=address -fno-omit-frame-pointer ${CMAKE_C_FLAGS}")
-        set(CMAKE_CXX_FLAGS "-shared-libasan -fsanitize=address -fno-omit-frame-pointer ${CMAKE_CXX_FLAGS}")
+        set(CMAKE_C_FLAGS "-shared-libasan -fsanitize=${UBSAN_SAN}address -fno-omit-frame-pointer ${CMAKE_C_FLAGS}")
+        set(CMAKE_CXX_FLAGS "-shared-libasan -fsanitize=${UBSAN_SAN}address -fno-omit-frame-pointer ${CMAKE_CXX_FLAGS}")
         if (APPLE)
-            set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=address ${CMAKE_EXE_LINKER_FLAGS}")
-            set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=address ${CMAKE_SHARED_LINKER_FLAGS}")
+            set(CMAKE_EXE_LINKER_FLAGS "-fsanitize=${UBSAN_SAN}address ${CMAKE_EXE_LINKER_FLAGS}")
+            set(CMAKE_SHARED_LINKER_FLAGS "-fsanitize=${UBSAN_SAN}address ${CMAKE_SHARED_LINKER_FLAGS}")
         else ()
             # Fix a linux clang deficiency where the ASan runtime library is not found automatically
             # Find the ASan runtime library path and set RPATH
@@ -56,19 +60,15 @@ if (ENABLE_ADDRESS_SANITIZER)
         endif ()
     elseif ("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU")
         set(CMAKE_C_FLAGS "-DCELIX_ASAN_ENABLED ${CMAKE_C_FLAGS}")
-        set(CMAKE_C_FLAGS "-lasan -fsanitize=address -fno-omit-frame-pointer ${CMAKE_C_FLAGS}")
-        set(CMAKE_CXX_FLAGS "-lasan -fsanitize=address -fno-omit-frame-pointer ${CMAKE_CXX_FLAGS}")
+        set(CMAKE_C_FLAGS "-lasan -fsanitize=${UBSAN_SAN}address -fno-omit-frame-pointer ${CMAKE_C_FLAGS}")
+        set(CMAKE_CXX_FLAGS "-lasan -fsanitize=${UBSAN_SAN}address -fno-omit-frame-pointer ${CMAKE_CXX_FLAGS}")
     else ()
         message(WARNING "Address sanitizer is not supported for ${CMAKE_C_COMPILER_ID}")
     endif ()
-endif()
-
-if (ENABLE_UNDEFINED_SANITIZER)
+elseif (ENABLE_UNDEFINED_SANITIZER)
     set(CMAKE_C_FLAGS "-fsanitize=undefined ${CMAKE_C_FLAGS}")
     set(CMAKE_CXX_FLAGS "-fsanitize=undefined ${CMAKE_CXX_FLAGS}")
-endif()
-
-if (ENABLE_THREAD_SANITIZER)
+elseif (ENABLE_THREAD_SANITIZER)
     set(CMAKE_C_FLAGS "-fsanitize=thread ${CMAKE_C_FLAGS}")
     set(CMAKE_CXX_FLAGS "-fsanitize=thread ${CMAKE_CXX_FLAGS}")
 endif()

cmake/cmake_celix/BundlePackaging.cmake

@@ -287,23 +287,23 @@ function(add_celix_bundle)
     #########################################################
 
     ###### Packaging the bundle using using jar or zip and a content dir. Configuring dependencies ######
-    if (JAR_COMMAND)
+    if (ZIP_COMMAND)
+        file(MAKE_DIRECTORY ${BUNDLE_CONTENT_DIR}) #Note needed because working_directory is bundle content dir
         add_custom_command(OUTPUT ${BUNDLE_FILE}
-                COMMAND ${CMAKE_COMMAND} -E make_directory ${BUNDLE_CONTENT_DIR}
                 COMMAND ${CMAKE_COMMAND} -E copy_if_different ${BUNDLE_GEN_DIR}/MANIFEST.json ${BUNDLE_CONTENT_DIR}/META-INF/MANIFEST.json
-                COMMAND ${JAR_COMMAND} ${CELIX_JAR_COMMAND_ARGUMENTS} ${BUNDLE_FILE} -C ${BUNDLE_CONTENT_DIR} .
+                COMMAND ${ZIP_COMMAND} ${CELIX_ZIP_COMMAND_ARGUMENTS} ${BUNDLE_FILE} *
                 COMMENT "Packaging ${BUNDLE_TARGET_NAME}"
                 DEPENDS ${BUNDLE_TARGET_NAME} "$<TARGET_PROPERTY:${BUNDLE_TARGET_NAME},BUNDLE_DEPEND_TARGETS>" ${BUNDLE_GEN_DIR}/MANIFEST.json
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                WORKING_DIRECTORY ${BUNDLE_CONTENT_DIR}
         )
-    elseif (ZIP_COMMAND)
-        file(MAKE_DIRECTORY ${BUNDLE_CONTENT_DIR}) #Note needed because working_directory is bundle content dir
+    elseif (JAR_COMMAND)
         add_custom_command(OUTPUT ${BUNDLE_FILE}
+                COMMAND ${CMAKE_COMMAND} -E make_directory ${BUNDLE_CONTENT_DIR}
                 COMMAND ${CMAKE_COMMAND} -E copy_if_different ${BUNDLE_GEN_DIR}/MANIFEST.json ${BUNDLE_CONTENT_DIR}/META-INF/MANIFEST.json
-                COMMAND ${ZIP_COMMAND} ${CELIX_ZIP_COMMAND_ARGUMENTS} ${BUNDLE_FILE} *
+                COMMAND ${JAR_COMMAND} ${CELIX_JAR_COMMAND_ARGUMENTS} ${BUNDLE_FILE} -C ${BUNDLE_CONTENT_DIR} .
                 COMMENT "Packaging ${BUNDLE_TARGET_NAME}"
                 DEPENDS ${BUNDLE_TARGET_NAME} "$<TARGET_PROPERTY:${BUNDLE_TARGET_NAME},BUNDLE_DEPEND_TARGETS>" ${BUNDLE_GEN_DIR}/MANIFEST.json
-                WORKING_DIRECTORY ${BUNDLE_CONTENT_DIR}
+                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
         )
     else ()
         message(FATAL_ERROR "A jar or zip command is needed to jar/zip bundles")
@@ -934,21 +934,21 @@ function(install_celix_bundle)
     set(BUNDLE_FILE_INSTALL "${BUNDLE_FILE}.install")
     get_target_property(BUNDLE_FILE_NAME ${BUNDLE} "BUNDLE_FILE_NAME")
     get_target_property(BUNDLE_GEN_DIR ${BUNDLE} "BUNDLE_GEN_DIR")
-    if (JAR_COMMAND)
+    if (ZIP_COMMAND)
         install(CODE
                 "execute_process(
                 COMMAND ${CMAKE_COMMAND} -E copy_if_different ${BUNDLE_GEN_DIR}/MANIFEST.json META-INF/MANIFEST.json
-                COMMAND ${JAR_COMMAND} ${CELIX_JAR_COMMAND_ARGUMENTS} ${BUNDLE_FILE_INSTALL} -C ${BUNDLE_CONTENT_DIR} .
-                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+                COMMAND ${ZIP_COMMAND} ${CELIX_ZIP_COMMAND_ARGUMENTS} ${BUNDLE_FILE_INSTALL} . -i *
+                WORKING_DIRECTORY ${BUNDLE_CONTENT_DIR}
             )"
                 COMPONENT ${BUNDLE}
         )
-    elseif (ZIP_COMMAND)
+    elseif (JAR_COMMAND)
         install(CODE
                 "execute_process(
                 COMMAND ${CMAKE_COMMAND} -E copy_if_different ${BUNDLE_GEN_DIR}/MANIFEST.json META-INF/MANIFEST.json
-                COMMAND ${ZIP_COMMAND} ${CELIX_ZIP_COMMAND_ARGUMENTS} ${BUNDLE_FILE_INSTALL} . -i *
-                WORKING_DIRECTORY ${BUNDLE_CONTENT_DIR}
+                COMMAND ${JAR_COMMAND} ${CELIX_JAR_COMMAND_ARGUMENTS} ${BUNDLE_FILE_INSTALL} -C ${BUNDLE_CONTENT_DIR} .
+                WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
             )"
                 COMPONENT ${BUNDLE}
         )

conanfile.py

@@ -47,6 +47,8 @@ class CelixConan(ConanFile):
         "enable_address_sanitizer": False,
         "enable_undefined_sanitizer": False,
         "enable_thread_sanitizer": False,
+        "enable_fuzzing": False,
+        "enable_benchmarking": False,
         "install_find_modules": False,
         "build_all": False,
         "build_http_admin": False,
@@ -130,6 +132,9 @@ def validate(self):
         if self.options.build_rsa_discovery_zeroconf and self.settings.os != "Linux":
             raise ConanInvalidConfiguration("Celix build_rsa_discovery_zeroconf is only supported for Linux")
 
+        if self.options.enable_fuzzing and self.settings.compiler != "clang" and self.settings.compiler != "apple-clang":
+            raise ConanInvalidConfiguration("Celix enable_fuzzing=True requires the 'clang' compiler")
+
         self.validate_config_option_is_positive_number("celix_err_buffer_size")
         self.validate_config_option_is_positive_number("celix_utils_max_strlen")
         self.validate_config_option_is_positive_number("celix_properties_optimization_string_buffer_size")
@@ -145,12 +150,18 @@ def package_id(self):
         del self.info.options.enable_testing_on_ci
         del self.info.options.enable_ccache
         del self.info.options.enable_deprecated_warnings
+        del self.info.options.enable_testing
+        del self.info.options.enable_benchmarking
+        del self.info.options.enable_fuzzing
+        del self.info.options.enable_code_coverage
 
     def build_requirements(self):
         if self.options.enable_testing:
             self.test_requires("gtest/1.10.0")
         if self.options.enable_ccache:
             self.build_requires("ccache/4.7.4")
+        if self.options.enable_benchmarking:
+            self.test_requires("benchmark/[>=1.6.2]")
 
     def configure(self):
         # copy options to options, fill in defaults if not set
@@ -308,6 +319,8 @@ def configure(self):
             self.options['openssl'].shared = True
         if self.options.enable_testing:
             self.options['gtest'].shared = True
+        if self.options.enable_benchmarking:
+            self.options['benchmark'].shared = True
         if (self.options.build_rsa_discovery_common
                 or (self.options.build_rsa_remote_service_admin_dfi and self.options.enable_testing)):
             self.options['libxml2'].shared = True

documents/README.md

@@ -81,7 +81,6 @@ bundles contains binaries depending on the stdlibc++ library.
 
 * Building
   * [Building and Installing Apache Celix](building/README.md)
-  * [Building and Developing Apache Celix with CLion](building/dev_celix_with_clion.md)
 * C Patterns
   * [Apache Celix C Patterns](c_patterns.md)
 * Utils

documents/building/README.md

@@ -237,3 +237,10 @@ cmake -DCMAKE_BUILD_TYPE=RelWithDebInfo ../libs/pushstreams
 make -j
 sudo make install
 ```
+
+# Further Reading
+
+- [Building with CLion](dev_celix_with_clion.md)
+- [Building and Running Tests](testing.md)
+- [Fuzz Testing](fuzz_testing.md)
+- [Building and Running Benchmarks](benchmarks.md)

documents/building/benchmarks.md

@@ -0,0 +1,74 @@
+---
+title: Benchmarks in Apache Celix
+---
+
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+   
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+
+# Benchmarks in Apache Celix
+
+This document describes how to build and run benchmarks for Apache Celix.
+
+## Building Benchmarks
+
+Benchmarks can be built using the CMake option `ENABLE_BENCHMARKING`
+The Apache Celix benchmarks uses Google benchmark library. 
+
+To build benchmarks run:
+
+```sh
+cmake -B build -DENABLE_BENCHMARKING=ON
+cmake --build build
+```
+
+## Benchmarks
+
+The following benchmark executables are available after building the utils and framework benchmarks:
+
+**Utils Benchmarks:**
+- `build/libs/utils/benchmark/celix_filter_benchmark`
+- `build/libs/utils/benchmark/celix_long_hashmap_benchmark`
+- `build/libs/utils/benchmark/celix_string_hashmap_benchmark`
+- `build/libs/utils/benchmark/celix_utils_benchmark`
+
+**Framework Benchmarks:**
+- `build/libs/framework/benchmark/celix_framework_benchmark`
+
+Paths may vary depending on your configuration and enabled options.
+
+## Running Benchmarks
+
+Benchmark executables are located in the `build` directory, typically under the relevant bundle or library subdirectory. To run a benchmark:
+
+```sh
+./build/libs/utils/benchmarks/celix_utils_benchmark
+## Command-Line Options
+The benchmark executables accept standard Google Benchmark command-line options. 
+For example, to run only benchmarks matching a specific pattern and output results in JSON format:
+
+```bash
+./build/libs/utils/benchmark/celix_filter_benchmark --benchmark_filter=complexFilter --benchmark_format=json
+```
+
+Replace `celix_utils_benchmark` and the filter pattern as needed. To see a list of supported command-line flags, run the benchmark executable with the `--help` option:
+
+```bash
+./build/libs/utils/benchmarks/./celix_filter_benchmark --help
+```
+
+This will display all available Google Benchmark options.