diff --git a/.github/workflows/bindings-ruby.yml b/.github/workflows/bindings-ruby.yml
index 63f7f61533c..680862fb764 100644
--- a/.github/workflows/bindings-ruby.yml
+++ b/.github/workflows/bindings-ruby.yml
@@ -1,55 +1,11 @@
 name: Bindings Tests (Ruby)
+
 on:
   push:
-    paths:
-      - bindings/ruby/**
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/common.h
-      - examples/common.cpp
-      - examples/common-whisper.h
-      - examples/common-whisper.cpp
-      - examples/stb_vorbis.c
-      - examples/miniaudio.h
+    branches:
+      - master
   pull_request:
-    paths:
-      - bindings/ruby/**
-      - src/**/*.c
-      - src/**/*.cpp
-      - src/**/*.h
-      - src/**/*.m
-      - src/**/*.metal
-      - include/**/*.c
-      - include/**/*.cpp
-      - include/**/*.h
-      - include/**/*.m
-      - include/**/*.metal
-      - ggml/**/*.c
-      - ggml/**/*.cpp
-      - ggml/**/*.h
-      - ggml/**/*.m
-      - ggml/**/*.metal
-      - scripts/get-flags.mk
-      - examples/common.h
-      - examples/common.cpp
-      - examples/common-whisper.h
-      - examples/common-whisper.cpp
-      - examples/stb_vorbis.c
-      - examples/miniaudio.h
+    types: [opened, synchronize, reopened]
 
 jobs:
   ubuntu-22:
@@ -60,6 +16,6 @@ jobs:
     steps:
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.1'
+          ruby-version: '3.2'
       - uses: actions/checkout@v4
       - run: rake test
diff --git a/bindings/ruby/.gitignore b/bindings/ruby/.gitignore
index e04a90a9c69..e93e6facdeb 100644
--- a/bindings/ruby/.gitignore
+++ b/bindings/ruby/.gitignore
@@ -1,3 +1,6 @@
 LICENSE
 pkg/
 lib/whisper.*
+ext/sources/*
+!ext/sources/CMakeGraphVizOptions.cmake
+ext/mkmf.log
diff --git a/bindings/ruby/Rakefile b/bindings/ruby/Rakefile
index 0d52e88a31a..bc6f843369b 100644
--- a/bindings/ruby/Rakefile
+++ b/bindings/ruby/Rakefile
@@ -3,11 +3,15 @@ require "bundler/gem_tasks"
 require "rake/testtask"
 require_relative "extsources"
 
+SOURCES_DIR = "ext/sources"
+
 SOURCES = FileList[]
 
 EXTSOURCES.each do |src|
   basename = src.pathmap("%f")
-  dest = basename == "LICENSE" ? basename : src.pathmap("%{../..,ext}p")
+  dest = basename == "LICENSE" ? basename
+                               : src.pathmap("%{\\.\\./\\.\\.,#{SOURCES_DIR}}p")
+                                    .pathmap("%{\\.\\./javascript,#{SOURCES_DIR}/bindings/javascript}p")
   dir = dest.pathmap("%d")
   file src
   directory dir
@@ -18,7 +22,6 @@ EXTSOURCES.each do |src|
 end
 
 CLEAN.include SOURCES
-CLEAN.include FileList["ext/**/*.o", "ext/**/*.metal", "ext/**/*.tmp", "ext/whisper.{so,bundle,dll}"]
 
 SRC = FileList["ext/*.{c,cpp,h}"]
 
@@ -36,6 +39,20 @@ file "ext/Makefile" => SRC + ["ext/extconf.rb"] + SOURCES do |t|
     ruby "extconf.rb"
   end
 end
+if File.exist? "ext/Makefile"
+  task :make_clean do
+    cd "ext" do
+      sh "make", "clean"
+    end
+  end
+  task clean: :make_clean
+  task :make_distclean do
+    cd "ext" do
+      sh "make", "distclean"
+    end
+  end
+  task clobber: :make_distclean
+end
 
 file SO_FILE => "ext/Makefile" do |t|
   chdir "ext" do
diff --git a/bindings/ruby/ext/extconf.rb b/bindings/ruby/ext/extconf.rb
index d9d6fc060b7..29ebbc604c5 100644
--- a/bindings/ruby/ext/extconf.rb
+++ b/bindings/ruby/ext/extconf.rb
@@ -1,212 +1,25 @@
-require 'mkmf'
-
-# need to use c++ compiler flags
-$CXXFLAGS << ' -std=c++17'
-
-$LDFLAGS << ' -lstdc++'
-
-# Set to true when building binary gems
-if enable_config('static-stdlib', false)
-  $LDFLAGS << ' -static-libgcc -static-libstdc++'
-end
-
-if enable_config('march-tune-native', false)
-  $CFLAGS << ' -march=native -mtune=native'
-  $CXXFLAGS << ' -march=native -mtune=native'
-end
-
-if ENV['WHISPER_METAL']
-  $GGML_METAL ||= true
-  $DEPRECATE_WARNING ||= true
-end
-
-$UNAME_S = `uname -s`.chomp
-$UNAME_P = `uname -p`.chomp
-$UNAME_M = `uname -m`.chomp
-
-if $UNAME_S == 'Darwin'
-  unless ENV['GGML_NO_METAL']
-    $GGML_METAL ||= true
-  end
-  $GGML_NO_OPENMP ||= true
-end
-
-if $GGML_METAL
-  $GGML_METAL_EMBED_LIBRARY = true
-end
-
-$MK_CPPFLAGS = '-Iggml/include -Iggml/src -Iggml/src/ggml-cpu -Iinclude -Isrc -Iexamples -DGGML_USE_CPU'
-$MK_CFLAGS   = '-std=c11   -fPIC'
-$MK_CXXFLAGS = '-std=c++17 -fPIC'
-$MK_NVCCFLAGS = '-std=c++17'
-$MK_LDFLAGS = ''
-
-$OBJ_GGML = []
-$OBJ_WHISPER = []
-$OBJ_COMMON = []
-$OBJ_SDL = []
-
-$MK_CPPFLAGS << ' -D_XOPEN_SOURCE=600'
-
-if $UNAME_S == 'Linux'
-  $MK_CPPFLAGS << ' -D_GNU_SOURCE'
-end
-
-if $UNAME_S == 'Darwin'
-  $MK_CPPFLAGS << ' -D_DARWIN_C_SOURCE'
-end
-
-if ENV['WHISPER_DEBUG']
-  $MK_CFLAGS    << ' -O0 -g'
-  $MK_CXXFLAGS  << ' -O0 -g'
-  $MK_LDFLAGS   << ' -g'
-  $MK_NVCCFLAGS << ' -O0 -g'
-else
-  $MK_CPPFLAGS   << ' -DNDEBUG'
-  $MK_CFLAGS     << ' -O3'
-  $MK_CXXFLAGS   << ' -O3'
-  $MK_NVCCFLAGS  << ' -O3'
-end
-
-$WARN_FLAGS =
-  ' -Wall' <<
-  ' -Wextra' <<
-  ' -Wpedantic' <<
-  ' -Wcast-qual' <<
-  ' -Wno-unused-function'
-
-$MK_CFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wshadow' <<
-  ' -Wstrict-prototypes' <<
-  ' -Wpointer-arith' <<
-  ' -Wmissing-prototypes' <<
-  ' -Werror=implicit-int' <<
-  ' -Werror=implicit-function-declaration'
-
-$MK_CXXFLAGS <<
-  $WARN_FLAGS <<
-  ' -Wmissing-declarations' <<
-  ' -Wmissing-noreturn'
-
-unless `#{cc_command} #{$LDFLAGS} -Wl,-v 2>&1`.chomp.include? 'dyld-1015.7'
-  $MK_CPPFLAGS << ' -DHAVE_BUGGY_APPLE_LINKER'
-end
-
-if %w[Linux Darwin FreeBSD NetBSD OpenBSD Haiku].include? $UNAME_S
-  $MK_CFLAGS   << ' -pthread'
-  $MK_CXXFLAGS << ' -pthread'
-end
-
-unless $_WIN32
-  $DSO_EXT = '.so'
-else
-  $DSO_EXT = '.dll'
-end
-
-unless ENV['RISCV']
-  if %w[x86_64 i686 amd64].include? $UNAME_M
-    $HOST_CXXFLAGS ||= ''
-
-    $MK_CFLAGS     << ' -march=native -mtune=native'
-    $HOST_CXXFLAGS << ' -march=native -mtune=native'
-  end
-else
-  $MK_CFLAGS   << ' -march=rv64gcv -mabi=lp64d'
-  $MK_CXXFLAGS << ' -march=rv64gcv -mabi=lp64d'
-end
-
-unless ENV['GGML_NO_ACCELERATE']
-  if $UNAME_S == 'Darwin'
-    $MK_CPPFLAGS << ' -DGGML_USE_ACCELERATE -DGGML_USE_BLAS -DGGML_BLAS_USE_ACCELERATE'
-    $MK_CPPFLAGS << ' -DACCELERATE_NEW_LAPACK'
-    $MK_CPPFLAGS << ' -DACCELERATE_LAPACK_ILP64'
-    $MK_LDFLAGS  << ' -framework Accelerate'
-    $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-  end
-end
-
-if ENV['GGML_OPENBLAS']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if ENV['GGML_OPENBLAS64']
-  $MK_CPPFLAGS << " -DGGML_USE_BLAS #{`pkg-config --cflags-only-I openblas64`.chomp}"
-  $MK_CFLAGS   << " #{`pkg-config --cflags-only-other openblas64)`.chomp}"
-  $MK_LDFLAGS  << " #{`pkg-config --libs openblas64`}"
-  $OBJ_GGML    << 'ggml/src/ggml-blas/ggml-blas.o'
-end
-
-if $GGML_METAL
-  $MK_CPPFLAGS << ' -DGGML_USE_METAL'
-  $MK_LDFLAGS  << ' -framework Foundation -framework Metal -framework MetalKit'
-  $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal.o'
-
-  if ENV['GGML_METAL_NDEBUG']
-    $MK_CPPFLAGS << ' -DGGML_METAL_NDEBUG'
-  end
-
-  if $GGML_METAL_EMBED_LIBRARY
-    $MK_CPPFLAGS << ' -DGGML_METAL_EMBED_LIBRARY'
-    $OBJ_GGML    << 'ggml/src/ggml-metal/ggml-metal-embed.o'
-  end
-end
-
-$OBJ_GGML <<
-  'ggml/src/ggml.o' <<
-  'ggml/src/ggml-alloc.o' <<
-  'ggml/src/ggml-backend.o' <<
-  'ggml/src/ggml-backend-reg.o' <<
-  'ggml/src/ggml-opt.o' <<
-  'ggml/src/ggml-quants.o' <<
-  'ggml/src/ggml-threading.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-cpp.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-aarch64.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-hbm.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-quants.o' <<
-  'ggml/src/ggml-cpu/ggml-cpu-traits.o' <<
-  'ggml/src/ggml-cpu/unary-ops.o' <<
-  'ggml/src/ggml-cpu/binary-ops.o' <<
-  'ggml/src/ggml-cpu/vec.o' <<
-  'ggml/src/ggml-cpu/ops.o'
-
-$OBJ_WHISPER <<
-  'src/whisper.o' <<
-  'examples/common.o' <<
-  'examples/common-whisper.o'
-
-$objs = $OBJ_GGML + $OBJ_WHISPER + $OBJ_COMMON + $OBJ_SDL
-$objs <<
-  "ruby_whisper.o" <<
-  "ruby_whisper_context.o" <<
-  "ruby_whisper_transcribe.o" <<
-  "ruby_whisper_params.o" <<
-  "ruby_whisper_error.o" <<
-  "ruby_whisper_segment.o" <<
-  "ruby_whisper_model.o"
-
-$CPPFLAGS  = "#{$MK_CPPFLAGS} #{$CPPFLAGS}"
-$CFLAGS    = "#{$CPPFLAGS} #{$MK_CFLAGS} #{$GF_CFLAGS} #{$CFLAGS}"
-$BASE_CXXFLAGS = "#{$MK_CXXFLAGS} #{$CXXFLAGS}"
-$CXXFLAGS  = "#{$BASE_CXXFLAGS} #{$HOST_CXXFLAGS} #{$GF_CXXFLAGS} #{$CPPFLAGS}"
-$NVCCFLAGS = "#{$MK_NVCCFLAGS} #{$NVCCFLAGS}"
-$LDFLAGS   = "#{$MK_LDFLAGS} #{$LDFLAGS}"
-
-create_makefile('whisper')
-
-File.open 'Makefile', 'a' do |file|
-  file.puts 'include scripts/get-flags.mk'
-  file.puts 'include cpu.mk'
-
-  if $GGML_METAL
-    file.puts 'include metal.mk'
-
-    if $GGML_METAL_EMBED_LIBRARY
-      file.puts 'include metal-embed.mk'
-    end
-  end
+require "mkmf"
+
+# TODO: options such as CoreML
+
+cmake = find_executable("cmake") || abort
+
+prefix = File.join("build", "whisper.cpp.dot")
+system cmake, "-S", "sources", "-B", "build", "--graphviz", prefix, "-D", "BUILD_SHARED_LIBS=OFF", exception: true
+libs = Dir["#{prefix}.*"].collect {|file|
+  "lib" + file.sub("#{prefix}.", "") + ".a"
+}
+
+$INCFLAGS << " -Isources/include -Isources/ggml/include -Isources/examples"
+$LOCAL_LIBS << " " << libs.join(" ")
+$cleanfiles << " build" << libs.join(" ")
+
+create_makefile "whisper" do |conf|
+  conf << <<~EOF
+    $(TARGET_SO): #{libs.join(" ")}
+    #{libs.join(" ")}: cmake-targets
+    cmake-targets:
+    #{"\t"}#{cmake} -S sources -B build -D BUILD_SHARED_LIBS=OFF -D CMAKE_ARCHIVE_OUTPUT_DIRECTORY=#{__dir__} -D CMAKE_POSITION_INDEPENDENT_CODE=ON
+    #{"\t"}#{cmake} --build build --config Release --target common whisper
+  EOF
 end
diff --git a/bindings/ruby/ext/sources/CMakeGraphVizOptions.cmake b/bindings/ruby/ext/sources/CMakeGraphVizOptions.cmake
new file mode 100644
index 00000000000..746c14bb6fb
--- /dev/null
+++ b/bindings/ruby/ext/sources/CMakeGraphVizOptions.cmake
@@ -0,0 +1,8 @@
+set(GRAPHVIZ_EXECUTABLES FALSE)
+set(GRAPHVIZ_STATIC_LIBS TRUE)
+set(GRAPHVIZ_SHARED_LIBS FALSE)
+set(GRAPHVIZ_MODULE_LIBS FALSE)
+set(GRAPHVIZ_INTERFACE_LIBS FALSE)
+set(GRAPHVIZ_OBJECT_LIBS FALSE)
+set(GRAPHVIZ_UNKNOWN_LIBS FALSE)
+set(GRAPHVIZ_GENERATE_DEPENDERS FALSE)
diff --git a/bindings/ruby/extsources.rb b/bindings/ruby/extsources.rb
index 1dc900d41b7..6e528182a25 100644
--- a/bindings/ruby/extsources.rb
+++ b/bindings/ruby/extsources.rb
@@ -1,6 +1,35 @@
-require "yaml"
+ignored_dirs = %w[
+  .devops
+  examples/wchess/wchess.wasm
+  examples/whisper.android
+  examples/whisper.android.java
+  examples/whisper.objc
+  examples/whisper.swiftui
+  grammars
+  models
+  samples
+  scripts
+]
+ignored_files = %w[
+  AUTHORS
+  LICENSE
+  Makefile
+  README.md
+  README_sycl.md
+  .gitignore
+  .gitmodules
+  whisper.nvim
+  twitch.sh
+  yt-wsp.sh
+]
 
-sources = `git ls-files -z ../..`.split("\x0")
-paths = YAML.load_file("../../.github/workflows/bindings-ruby.yml")[true]["push"]["paths"]
-paths.delete "bindings/ruby/**"
-EXTSOURCES = (Dir.glob(paths, base: "../..").collect {|path| "../../#{path}"} << "../../LICENSE") & sources
+EXTSOURCES =
+  `git ls-files -z ../..`.split("\x0")
+    .select {|file|
+      basename = File.basename(file)
+
+      ignored_dirs.all? {|dir| !file.start_with?("../../#{dir}")} &&
+        !ignored_files.include?(basename) &&
+        (file.start_with?("../..") || file.start_with?("../javascript")) &&
+        (!file.start_with?("../../.github/") || basename == "bindings-ruby.yml")
+    }
diff --git a/bindings/ruby/tests/helper.rb b/bindings/ruby/tests/helper.rb
index a182319d95f..25c43d21ea2 100644
--- a/bindings/ruby/tests/helper.rb
+++ b/bindings/ruby/tests/helper.rb
@@ -2,6 +2,13 @@
 require "whisper"
 require_relative "jfk_reader/jfk_reader"
 
+puts "ls"
+puts `ls`
+puts "cat ext/Makefile"
+puts `cat ext/Makefile`
+puts "nm lib/whisper.so"
+puts `nm lib/whisper.so`
+
 class TestBase < Test::Unit::TestCase
   AUDIO = File.join(__dir__, "..", "..", "..", "samples", "jfk.wav")
 
diff --git a/bindings/ruby/tests/test_package.rb b/bindings/ruby/tests/test_package.rb
index 33c2b37e532..c4a74b046c0 100644
--- a/bindings/ruby/tests/test_package.rb
+++ b/bindings/ruby/tests/test_package.rb
@@ -25,6 +25,8 @@ def test_install
       Dir.mktmpdir do |dir|
         system "gem", "install", "--install-dir", dir.shellescape, "--no-document", "pkg/#{filename.shellescape}", exception: true
         assert_path_exist File.join(dir, "gems/whispercpp-#{version}/lib", basename)
+        assert_path_exist File.join(dir, "gems/whispercpp-#{version}/LICENSE")
+        assert_path_not_exist File.join(dir, "gems/whispercpp-#{version}/ext/build")
       end
     end
   end
diff --git a/bindings/ruby/whispercpp.gemspec b/bindings/ruby/whispercpp.gemspec
index 308019582be..da00c1cade9 100644
--- a/bindings/ruby/whispercpp.gemspec
+++ b/bindings/ruby/whispercpp.gemspec
@@ -15,7 +15,8 @@ Gem::Specification.new do |s|
                 if s.extra_rdoc_files.include?(basename)
                   basename
                 else
-                  file.sub("../..", "ext")
+                  file.sub("../..", "ext/sources")
+                      .sub("../javascript", "ext/sources/bindings/javascript")
                 end
               }