reference and update librdkafka sources (#474)

Author: mensfeld

ext/README.md

@@ -1,16 +1,17 @@
 # Ext
 
-This gem depends on the `librdkafka` C library. It is downloaded when
-this gem is installed.
+This gem depends on the `librdkafka` C library. It is downloaded, stored in
+`dist/` directory, and checked into source control.
 
 To update the `librdkafka` version follow the following steps:
 
 * Go to https://github.com/confluentinc/librdkafka/releases to get the new
   version number and asset checksum for `tar.gz`.
 * Change the version in `lib/rdkafka/version.rb`
 * Change the `sha256` in `lib/rdkafka/version.rb`
-* Run `bundle exec rake` in the `ext` directory to download and build
-  the new version
+* Run `bundle exec rake dist:download` in the `ext` directory to download the
+  new release and place it in the `dist/` for you
+* Run `bundle exec rake` in the `ext` directory to build the new version
 * Run `docker-compose pull` in the main gem directory to ensure the docker
   images used by the tests and run `docker-compose up`
 * Finally, run `bundle exec rspec` in the main gem directory to execute

ext/Rakefile

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require File.expand_path('../../lib/rdkafka/version', __FILE__)
+require "digest"
 require "fileutils"
 require "open-uri"
 
@@ -71,14 +72,50 @@ task :clean do
   FileUtils.rm_rf File.join(File.dirname(__FILE__), "tmp")
 end
 
+namespace :dist do
+  task :dir do
+    ENV["RDKAFKA_DIST_PATH"] ||= File.expand_path(File.join(File.dirname(__FILE__), '..', 'dist'))
+  end
+
+  task :file => "dist:dir" do
+    ENV["RDKAFKA_DIST_FILE"] ||= File.join(ENV["RDKAFKA_DIST_PATH"], "librdkafka_#{Rdkafka::LIBRDKAFKA_VERSION}.tar.gz")
+  end
+
+  task :clean => "dist:file" do
+    Dir.glob(File.join("#{ENV['RDKAFKA_DIST_PATH']}", "*")).each do |filename|
+      next if filename.include? ENV["RDKAFKA_DIST_FILE"]
+
+      FileUtils.rm_rf filename
+    end
+  end
+
+  task :download => "dist:file" do
+    version = Rdkafka::LIBRDKAFKA_VERSION
+    librdkafka_download = "https://codeload.github.com/confluentinc/librdkafka/tar.gz/v#{version}"
+
+    URI.open(librdkafka_download) do |file|
+      filename = ENV["RDKAFKA_DIST_FILE"]
+      data     = file.read
+
+      if Digest::SHA256.hexdigest(data) != Rdkafka::LIBRDKAFKA_SOURCE_SHA256
+        raise "SHA256 does not match downloaded file"
+      end
+
+      File.write(filename, data)
+    end
+  end
+
+  task :update => %w[dist:download dist:clean]
+end
+
 namespace :build do
   desc "Build librdkafka at the given git sha or tag"
   task :git, [:ref] do |task, args|
     ref = args[:ref]
     version = "git-#{ref}"
 
     recipe = MiniPortile.new("librdkafka", version)
-    recipe.files << "https://github.com/edenhill/librdkafka/archive/#{ref}.tar.gz"
+    recipe.files << "https://github.com/confluentinc/librdkafka/archive/#{ref}.tar.gz"
     recipe.configure_options = ["--host=#{recipe.host}","--enable-static", "--enable-zstd"]
     recipe.cook