Sample Onsets - move from Ruby FFI to external binary

Previously we called the sample onset detection functionality in the aubio library via a Ruby -> C FFI bridge. This approach unfortunately appears to cause problems when executing in a "hardened runtime" on macOS Big Sur. This patch moves this FFI-based Ruby gem to a static binary which uses stdout to return the onset values.

Onset times should remain identical as the binary has been tuned in the same way as the original Ruby gem:

# [:window_size]     1024
# [:hop_size]        512
# [:onset_threshold] 0.3
# [:minioi_ms]       12.0 (ms)

Author: samaaron

app/external/aubio/CMakeLists.txt

@@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.12)
 
 project(aubio-5)
 set(SOURCE_ROOT ${CMAKE_CURRENT_LIST_DIR}/src)
+set(EXAMPLES_ROOT ${CMAKE_CURRENT_LIST_DIR}/examples)
 
 set(SOURCES
     ${SOURCE_ROOT}/exports.def
@@ -122,11 +123,23 @@ set(SOURCES
     ${SOURCE_ROOT}/utils/windll.c
 )
 
-add_library(${PROJECT_NAME} SHARED ${SOURCES} ${RESOURCES}) # Win32 ignored on non-windows
+add_library(${PROJECT_NAME} STATIC ${SOURCES} ${RESOURCES}) # Win32 ignored on non-windows
 target_include_directories(${PROJECT_NAME}
     PRIVATE
     src
     ${LIBSNDFILE_INCLUDE_DIR}
+    )
+
+add_executable(aubio_onset
+    ${EXAMPLES_ROOT}/aubioonset.c
+    ${EXAMPLES_ROOT}/utils.c)
+
+target_link_libraries(aubio_onset PRIVATE ${PROJECT_NAME})
+
+target_include_directories(aubio_onset
+    PRIVATE
+    src
+    ${LIBSNDFILE_INCLUDE_DIR}
 )
 
 # 'lib' is appended to the library name automatically on most non-Windows platforms
@@ -206,6 +219,7 @@ target_compile_definitions(${PROJECT_NAME}
     -DHAVE_SWRESAMPLE
     -DHAVE_MEMCPY_HACKS
     -DHAVE_SNDFILE
+    -DHAVE_CONFIG
     #-DHAVE_ACCELERATE
     #-DHAVE_INTEL_IPP (needs intel lib)
     #-DHAVE_SAMPLERATE (needs extra lib in path)

app/external/aubio/examples/config.h

@@ -0,0 +1,17 @@
+#pragma once
+
+#define HAVE_STDLIB_H 1
+#define HAVE_STDIO_H 1
+#define HAVE_STRING_H 1
+#define HAVE_MATH_H 1
+#define HAVE_ERRNO_H 1
+#define HAVE_LIMITS_H 1
+#define HAVE_STDARG_H 1
+#define HAVE_MEMCPY_HACKS 1
+#define HAVE_C99_VARARGS_MACROS 1
+
+#ifdef WIN32
+#define HAVE_WIN_HACKS 1
+#else
+#define HAVE_UNISTD_H 1
+#endif

app/external/aubio/examples/utils.c

@@ -38,13 +38,22 @@ int usejack = 0;
 char_t *sink_uri = NULL;
 char_t *source_uri = NULL;
 // general stuff
+
+// Use defaults as found in the original aubio Ruby gem
+// to match existing behaviour:
+// [:window_size]     1024
+// [:hop_size]        512
+// [:onset_threshold] 0.3
+// [:minioi_ms]       12.0 (ms)
+
+
 uint_t samplerate = 0;
-uint_t buffer_size = 512;
-uint_t hop_size = 256;
+uint_t buffer_size = 1024;
+uint_t hop_size = 512;
 // onset stuff
 char_t * onset_method = "default";
-smpl_t onset_threshold = 0.0; // will be set if != 0.
-smpl_t onset_minioi = 0.0; // will be set if != 0.
+smpl_t onset_threshold = 0.3; // will be set if != 0.
+smpl_t onset_minioi = 0.012; // will be set if != 0.
 // pitch stuff
 char_t * pitch_unit = "default";
 char_t * pitch_method = "default";

app/external/linux_build_externals.sh

@@ -12,10 +12,6 @@ cmake -DERLANG_INCLUDE_PATH=${ERLANG_INCLUDE_PATH} -G "Unix Makefiles" ..
 
 echo "Building sp_midi..."
 cmake --build . --target sp_midi
-
-if [ "$1" = "--build-aubio" ]; then
-  echo "Building aubio..."
-  cmake --build . --target aubio
-fi
+cmake --build . --target aubio
 
 cd "${SCRIPT_DIR}"

app/external/mac_build_externals.sh

@@ -11,11 +11,7 @@ cmake -G "Unix Makefiles" -D ERLANG_INCLUDE_PATH="${SCRIPT_DIR}/../../prebuilt/m
 
 echo "Building sp_midi..."
 cmake  --build . --target sp_midi
-
-
-if [ "$1" = "--build-aubio" ]; then
-  echo "Building aubio..."
-  cmake --build . --target aubio
-fi
+echo "Building aubio onset..."
+cmake --build . --target aubio
 
 cd "${SCRIPT_DIR}"

app/linux-prebuild.sh

@@ -3,22 +3,17 @@ set -e # Quit script on error
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 echo "Warning: Unix build scripts are still a work in progress!"
 
-# Build external dependencies
-if [ "$1" = "--build-aubio" ]; then
-  "${SCRIPT_DIR}/external/linux_build_externals.sh" --build-aubio
-else
-  "${SCRIPT_DIR}/external/linux_build_externals.sh"
-fi
+
+
+"${SCRIPT_DIR}/external/linux_build_externals.sh"
+
 
 # Install dependencies to server
 echo "Copying external dependencies to the server..."
 mkdir -p "${SCRIPT_DIR}/server/erlang/sonic_pi_server/priv/"
 cp ${SCRIPT_DIR}/external/build/sp_midi-prefix/src/sp_midi-build/*.so ${SCRIPT_DIR}/server/erlang/sonic_pi_server/priv/
 
-if [ "$1" = "--build-aubio" ]; then
-  mkdir -p "${SCRIPT_DIR}/server/native/lib"
-  cp "${SCRIPT_DIR}/external/build/aubio-prefix/src/aubio-build/libaubio-5.so" "${SCRIPT_DIR}/server/native/lib/"
-fi
+cp "${SCRIPT_DIR}/external/build/aubio-prefix/src/aubio-build/aubio_onset" "${SCRIPT_DIR}/server/native/"
 
 #dont remove ruby-aubio-prerelease, as needed in linux build
 #it is removed in the windows-prebuild

app/mac-prebuild.sh

@@ -2,16 +2,25 @@
 set -e # Quit script on error
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
+# Check to see if we have a bundled Ruby and if so, use that
+# Otherwise use system ruby
 
-# Build external dependencies
-if [ "$1" = "--without-aubio" ]; then
-    "${SCRIPT_DIR}/external/mac_build_externals.sh"
+BUNDLED_RUBY="${SCRIPT_DIR}/server/native/ruby/bin/ruby"
+if [ -f "$BUNDLED_RUBY" ]; then
+    echo "Found bundled Ruby: ${BUNDLED_RUBY}"
+    RUBY=$BUNDLED_RUBY
 else
-    "${SCRIPT_DIR}/external/mac_build_externals.sh" --build-aubio
-    mkdir -p "${SCRIPT_DIR}/server/native/lib"
-    cp "${SCRIPT_DIR}/external/build/aubio-prefix/src/aubio-build/libaubio-5.dylib" "${SCRIPT_DIR}/server/native/lib/"
+    echo "Using system Ruby"
+    RUBY=ruby
 fi
 
+
+# Build external dependencies
+"${SCRIPT_DIR}/external/mac_build_externals.sh"
+# mkdir -p "${SCRIPT_DIR}/server/native/lib"
+ cp "${SCRIPT_DIR}/external/build/aubio-prefix/src/aubio-build/aubio_onset" "${SCRIPT_DIR}/server/native/"
+
+
 # Install dependencies to server
 echo "Copying external dependencies to the server..."
 mkdir -p "${SCRIPT_DIR}/server/erlang/sonic_pi_server/priv/"
@@ -33,16 +42,15 @@ mv supercollider/extra-plugins/* supercollider/plugins/
 rm -rf supercollider/extra-plugins
 
 echo "Compiling native ruby extensions..."
-ruby "${SCRIPT_DIR}/server/ruby/bin/compile-extensions.rb"
+$RUBY "${SCRIPT_DIR}/server/ruby/bin/compile-extensions.rb"
 
 echo "Translating tutorial..."
-#assumes linux uses system ruby
-#so dont use prefix server/native/ruby/bin/ruby, as unnecessary to set this up
-ruby "${SCRIPT_DIR}/server/ruby/bin/i18n-tool.rb" -t
+
+$RUBY "${SCRIPT_DIR}/server/ruby/bin/i18n-tool.rb" -t
 
 echo "Generating docs for the Qt GUI..."
 cp "${SCRIPT_DIR}/gui/qt/utils/ruby_help.tmpl" "${SCRIPT_DIR}/gui/qt/utils/ruby_help.h"
-ruby "${SCRIPT_DIR}/server/ruby/bin/qt-doc.rb" -o "${SCRIPT_DIR}/gui/qt/utils/ruby_help.h"
+$RUBY "${SCRIPT_DIR}/server/ruby/bin/qt-doc.rb" -o "${SCRIPT_DIR}/gui/qt/utils/ruby_help.h"
 
 echo "Updating GUI translation files..."
 # Use lrelease on PATH if available otherwise assume Qt was installed via homebrew

app/server/ruby/core.rb

@@ -50,10 +50,6 @@
 
 $:.unshift "#{File.expand_path("../rb-native", __FILE__)}/#{ruby_api}/"
 
-## Add aubio native library to ENV if not present (the aubio library needs to be told the location)
-native_lib_path = File.expand_path("../../native/", __FILE__)
-ENV["AUBIO_LIB"] ||= Dir[native_lib_path + "/lib/libaubio*.{*dylib,so*,dll}"].first
-
 module SonicPi
   module Core
     class ThreadLocal

app/server/ruby/lib/sonicpi/samplebuffer.rb

@@ -14,7 +14,6 @@
 require_relative "buffer"
 require_relative "util"
 require_relative "sox"
-require 'aubio'
 
 
 module SonicPi
@@ -87,14 +86,31 @@ def info
     end
 
     def onset_data
-      return @aubio_onset_data if @aubio_onset_data
+     return @aubio_onset_data if @aubio_onset_data
       @aubio_sem.synchronize do
-        return @aubio_onset_data if @aubio_onset_data
+       return @aubio_onset_data if @aubio_onset_data
         __no_kill_block do
-          aubio_file = Aubio.open(@path, {sample_rate: sample_rate})
-          native_onsets = aubio_file.onsets.to_a.ring
-          aubio_file.close
-          @aubio_onset_data = native_onsets
+
+          # These are the aubio defaults set by old gem and now
+          # hard-coded into the aubio_onset binary: (this was worth
+          # maintaining to preserve backwards compatibility. Might also
+          # be nice to let users tweak these values in the future)
+
+          # [:window_size]     1024
+          # [:hop_size]        512
+          # [:onset_threshold] 0.3
+          # [:minioi_ms]       12.0 (ms)
+
+          begin
+            aubio_onsets_command = "'#{aubio_onset_path}' '#{@path}'"
+            onsets_str = `#{aubio_onsets_command}`
+            onsets = onsets_str.split.map(&:to_f)
+          rescue Exception => e
+            log_exception e
+            onsets = []
+          end
+
+          @aubio_onset_data = onsets.ring
         end
       end
       return @aubio_onset_data
@@ -106,7 +122,7 @@ def onsets(stretch=1)
       @aubio_sem.synchronize do
         return @aubio_onsets[stretch] if @aubio_onsets[stretch]
         onset_times = data.map do |el|
-          [1, (el[:s].to_f / duration)].min * stretch
+          [1, (el / duration)].min * stretch
         end
         @aubio_onsets[stretch] = onset_times
       end

app/server/ruby/lib/sonicpi/util.rb

@@ -181,7 +181,7 @@ def raspberry_pi_400?
      def raspberry_pi_400_64?
       os == :raspberry && @@raspberry_pi_400_64
     end
-    
+
    def unify_tilde_dir(path)
       if os == :windows
         path
@@ -417,6 +417,15 @@ def native_path
       File.absolute_path("#{server_path}/native/")
     end
 
+    def aubio_onset_path
+      case os
+      when :windows
+        File.absolute_path("#{native_path}/aubio_onset.exe")
+      else
+        File.absolute_path("#{native_path}/aubio_onset")
+      end
+    end
+
     def sox_path
       File.join(native_path, "sox", __exe_fix("sox"))
     end