diff --git a/Gemfile.dev b/Gemfile.dev
index 80f2b8920..4280e09b7 100644
--- a/Gemfile.dev
+++ b/Gemfile.dev
@@ -13,22 +13,39 @@ ruby_version = Gem::Version.new(RUBY_VERSION)
 if ruby_version >= Gem::Version.new("2.7.0")
   gem "debug", github: "ruby/debug", platform: :ruby
   gem "irb"
-  gem "ruby-lsp-rspec" if ruby_version >= Gem::Version.new("3.0.0") && RUBY_PLATFORM != "java"
 end
 
-if RUBY_VERSION >= "3.5"
+if ruby_version >= Gem::Version.new("2.5")
   gem "cgi"
 end
 
-# For RSpec
-gem "rspec", "~> 3.0"
+if ruby_version >= Gem::Version.new("3.4")
+  gem "drb"
+  gem "mutex_m"
+  gem "benchmark"
+  gem "base64"
+  gem "ostruct"
+end
+
+# RSpec
+gem "rspec"
 gem "rspec-retry"
+
+if ruby_version >= Gem::Version.new("3.0") && RUBY_PLATFORM != "java"
+  gem "ruby-lsp-rspec"
+end
+
+# Coverage
 gem "simplecov"
-gem "simplecov-cobertura", "~> 1.4"
-gem "rexml"
 
-if ruby_version >= Gem::Version.new("3.4")
-  gem "ostruct"
+# Do not change it without checking that `CI=true COVERAGE=true bundle exec rake`
+# passes in all projects
+if ruby_version >= Gem::Version.new("2.5")
+  gem "rexml", "3.4.1"
+  gem "simplecov-cobertura", "~> 3.0"
+else
+  gem "rexml", "3.2.5"
+  gem "simplecov-cobertura", "~> 1.4.0"
 end
 
 group :rubocop do
diff --git a/sentry-rails/Gemfile b/sentry-rails/Gemfile
index 6008cdf49..0bdcf787f 100644
--- a/sentry-rails/Gemfile
+++ b/sentry-rails/Gemfile
@@ -47,6 +47,7 @@ elsif rails_version >= Gem::Version.new("6.1.0")
     gem "sqlite3", "~> 1.6.9", platform: :ruby
   end
 else
+  gem "psych", "~> 3.0.0"
   gem "rspec-rails", "~> 4.0"
   gem "psych", "~> 3.0.0"