diff --git a/infra/base-images/base-builder-ruby/Dockerfile b/infra/base-images/base-builder-ruby/Dockerfile
index 9daa535ff9dc..14f88112e8c9 100644
--- a/infra/base-images/base-builder-ruby/Dockerfile
+++ b/infra/base-images/base-builder-ruby/Dockerfile
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder
 
 RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
 
-RUN install_ruby.sh
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
-
-RUN gem update --system 3.5.11
+RUN /usr/local/bin/install_ruby.sh
+RUN /usr/local/bin/gem update --system 3.5.11
 
 # Install ruzzy
 WORKDIR $SRC/ruzzy
diff --git a/infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile b/infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile
index ba307b9feff6..23b31b13b1f9 100644
--- a/infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile
+++ b/infra/base-images/base-builder-ruby/ubuntu-20-04.Dockerfile
@@ -18,10 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-20-04
 
 RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
 
-RUN install_ruby.sh
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
-
-RUN gem update --system 3.5.11
+RUN /usr/local/bin/install_ruby.sh
+RUN /usr/local/bin/gem update --system 3.5.11
 
 # Install ruzzy
 WORKDIR $SRC/ruzzy
diff --git a/infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile b/infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile
index 0123839a6ec4..36fa2b41e467 100644
--- a/infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile
+++ b/infra/base-images/base-builder-ruby/ubuntu-24-04.Dockerfile
@@ -18,8 +18,8 @@ FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-24-04
 
 RUN git clone https://github.com/trailofbits/ruzzy.git $SRC/ruzzy
 
-RUN install_ruby.sh
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
+RUN /usr/local/bin/install_ruby.sh
+RUN /usr/local/bin/gem update --system 3.5.11
 
 RUN gem update --system 3.5.11
 
diff --git a/infra/base-images/base-builder/install_ruby.sh b/infra/base-images/base-builder/install_ruby.sh
index bec6f2d17ece..d9e443cdf26c 100755
--- a/infra/base-images/base-builder/install_ruby.sh
+++ b/infra/base-images/base-builder/install_ruby.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -eux
 # Copyright 2024 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,14 +15,19 @@
 #
 ################################################################################
 
-apt update
-apt install -y lsb-release software-properties-common gnupg2 binutils xz-utils libyaml-dev
-gpg2 --keyserver keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+echo "Starting ruby installation"
+RUBY_VERSION=3.3.1
+RUBY_DEPS="binutils xz-utils libyaml-dev libffi-dev zlib1g-dev"
+apt update && apt install -y $RUBY_DEPS
+curl -O https://cache.ruby-lang.org/pub/ruby/3.3/ruby-$RUBY_VERSION.tar.gz
+tar -xvf ruby-$RUBY_VERSION.tar.gz
+cd ruby-$RUBY_VERSION
+./configure
+make -j$(nproc)
+make install
+cd ../
 
-curl -sSL https://get.rvm.io > ruby_installation.sh
-chmod +x ruby_installation.sh
-bash ruby_installation.sh stable
+# Clean up the sources.
+rm -rf ./ruby-$RUBY_VERSION ruby-$RUBY_VERSION.tar.gz
 
-. /etc/profile.d/rvm.sh
-
-rvm install ruby-3.3.1
+echo "Finished installing ruby"
\ No newline at end of file
diff --git a/infra/base-images/base-runner/Dockerfile b/infra/base-images/base-runner/Dockerfile
index 43fe476743ad..5f5f455f11f1 100644
--- a/infra/base-images/base-runner/Dockerfile
+++ b/infra/base-images/base-runner/Dockerfile
@@ -109,14 +109,14 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
 COPY install_javascript.sh /
 RUN /install_javascript.sh && rm /install_javascript.sh
 
-# Copy built ruby and ruzzy from builder
-COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
-COPY --from=base-ruby /install/ruzzy /install/ruzzy
-COPY ruzzy /usr/local/bin/ruzzy
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
-# RubyGems installation directory
-ENV GEM_HOME="$OUT/fuzz-gem"
-ENV GEM_PATH="/install/ruzzy"
+# Copy built ruby. It is up to the fuzzing harnesses
+# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
+# on how the harnesses are packaged.
+COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
+COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
+COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
+COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
+
 
 # Do this last to make developing these files easier/faster due to caching.
 COPY bad_build_check \
diff --git a/infra/base-images/base-runner/ubuntu-20-04.Dockerfile b/infra/base-images/base-runner/ubuntu-20-04.Dockerfile
index 436826942658..c421cb21a003 100644
--- a/infra/base-images/base-runner/ubuntu-20-04.Dockerfile
+++ b/infra/base-images/base-runner/ubuntu-20-04.Dockerfile
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
 COPY install_javascript.sh /
 RUN /install_javascript.sh && rm /install_javascript.sh
 
-# Copy built ruby and ruzzy from builder
-COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
-COPY --from=base-ruby /install/ruzzy /install/ruzzy
-COPY ruzzy /usr/bin/ruzzy
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
-# RubyGems installation directory
-ENV GEM_HOME="$OUT/fuzz-gem"
-ENV GEM_PATH="/install/ruzzy"
+# Copy built ruby. It is up to the fuzzing harnesses
+# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
+# on how the harnesses are packaged.
+COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
+COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
+COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
+COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
 
 # Do this last to make developing these files easier/faster due to caching.
 COPY bad_build_check \
diff --git a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile
index e1ac15e2a17f..2e252a25ffd8 100644
--- a/infra/base-images/base-runner/ubuntu-24-04.Dockerfile
+++ b/infra/base-images/base-runner/ubuntu-24-04.Dockerfile
@@ -109,14 +109,13 @@ RUN wget https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jaco
 COPY install_javascript.sh /
 RUN /install_javascript.sh && rm /install_javascript.sh
 
-# Copy built ruby and ruzzy from builder
-COPY --from=base-ruby /usr/local/rvm /usr/local/rvm
-COPY --from=base-ruby /install/ruzzy /install/ruzzy
-COPY ruzzy /usr/bin/ruzzy
-ENV PATH="$PATH:/usr/local/rvm/rubies/ruby-3.3.1/bin"
-# RubyGems installation directory
-ENV GEM_HOME="$OUT/fuzz-gem"
-ENV GEM_PATH="/install/ruzzy"
+# Copy built ruby. It is up to the fuzzing harnesses
+# themselves to set GEM_HOME and GEM_PATH appropriately, as this depends
+# on how the harnesses are packaged.
+COPY --from=base-ruby /usr/local/bin/ruby /usr/local/bin/ruby
+COPY --from=base-ruby /usr/local/bin/gem /usr/local/bin/gem
+COPY --from=base-ruby /usr/local/lib/ruby /usr/local/lib/ruby
+COPY --from=base-ruby /usr/local/include/ruby-3.3.0 /usr/local/include/ruby-3.3.0
 
 # Do this last to make developing these files easier/faster due to caching.
 COPY bad_build_check \