diff --git a/bundler/lib/bundler/errors.rb b/bundler/lib/bundler/errors.rb
index 3fa90c5eb855..6512325a007b 100644
--- a/bundler/lib/bundler/errors.rb
+++ b/bundler/lib/bundler/errors.rb
@@ -135,7 +135,8 @@ class GemRequireError < BundlerError
     attr_reader :orig_exception
 
     def initialize(orig_exception, msg)
-      full_message = msg + "\nGem Load Error is: #{orig_exception.message}\n"\
+      full_message = msg + "\nGem Load Error is:
+                      #{orig_exception.full_message(highlight: false)}\n"\
                       "Backtrace for gem load error is:\n"\
                       "#{orig_exception.backtrace.join("\n")}\n"\
                       "Bundler Error Backtrace:\n"
@@ -207,7 +208,9 @@ def initialize(underlying_error, message)
   class DirectoryRemovalError < BundlerError
     def initialize(orig_exception, msg)
       full_message = "#{msg}.\n" \
-                     "The underlying error was #{orig_exception.class}: #{orig_exception.message}, with backtrace:\n" \
+                     "The underlying error was #{orig_exception.class}:
+                     #{orig_exception.full_message(highlight: false)},
+                     with backtrace:\n" \
                      "  #{orig_exception.backtrace.join("\n  ")}\n\n" \
                      "Bundler Error Backtrace:"
       super(full_message)