diff --git a/bundler-audit.gemspec b/bundler-audit.gemspec
index 2905f3e0..c3472d60 100644
--- a/bundler-audit.gemspec
+++ b/bundler-audit.gemspec
@@ -21,7 +21,9 @@ Gem::Specification.new do |gem|
 
   glob = lambda { |patterns| gem.files & Dir[*patterns] }
 
-  gem.files = `git ls-files`.split($/)
+  gem.files = `git ls-files`.split($/).reject do |f|
+    f.match(%r{^(\.|gemspec.yml|Gemfile|spec/)})
+  end
   gem.files = glob[gemspec['files']] if gemspec['files']
 
   gem.executables = gemspec.fetch('executables') do