SRCH-6124 dependencies cleanup (#1856)

* Remove unused gems
*Remove flickraw, sidekiq, and related gems, and their
initializers/configuration since they are not being used. Also remove
turbolinks and retry_block.
* Remove obsolete gems and unused CSS
* Remove `chunky_png` and `curb` gems as they are no longer used.
* Remove `turbolinks` and `turbolinks-source` gems as they are not used.
* Remove `sidekiq` and `sidekiq-failures` gems as they are not used.
* Remove the call to generate mobile CSS.
* Remove the deprecated `AffiliateCss` class.
* Add turbolinks back
* Add flickraw gem and initialize flickr config
* Remove unused mobile look and feel css template
* Fix stub request status code
* Update elapsed time expectations

Author: stevenbarragan

Gemfile

@@ -9,7 +9,6 @@ gem 'rack-contrib', '~> 2.5.0'
 gem 'rails-observers', '~> 0.1.5'
 gem 'responders', '~> 3.1.0'
 gem 'mysql2', '~> 0.5.5'
-gem 'curb', '~> 1.0.1'
 gem 'haml', '~> 6'
 gem 'will_paginate', '~> 3.3.1'
 gem 'nokogiri', '~> 1.16'
@@ -61,7 +60,6 @@ gem 'faraday_middleware', '~> 0.14.0'
 gem 'net-http-persistent', '~> 2.9.3'
 gem 'rash_alt', '~> 0.4.12', require: 'rash'
 gem 'geoip', '~> 1.6.4'
-gem 'htmlentities', '~> 4.3.4' # deprecated - only used in Google web search
 gem 'html_truncator', '~> 0.4.2'
 gem 'addressable', '~> 2.8.0'
 gem 'select2-rails', '~> 4.0.3'
@@ -89,7 +87,6 @@ gem 'activerecord-validate_unique_child_attribute', require: 'active_record/vali
 gem 'rack-cors', '~> 1.1.0', require: 'rack/cors'
 gem 'hashie', '~> 5.0.0'
 # retry_block is unsupported - consider replacing with retriable
-gem 'retry_block', '~> 1.2.0'
 gem 'colorize', '~> 0.8.1'
 gem 'http', '~> 5.0'
 gem 'robots_tag_parser', '~> 0.1.0'
@@ -118,11 +115,9 @@ gem 'exception_notification', '~> 4.5'
 # Temporary fix to remove warnings seen in Rails 7:
 # https://github.com/ruby/net-protocol/issues/10
 # This gem can be removed once we upgrade to Ruby 3.1.
-gem 'net-http'
 # Assets-related gems
 gem 'coffee-rails', '~> 5.0.0'
 gem 'uglifier', '~> 4.2.0'
-gem 'compass-blueprint', '~> 1.0.0'
 gem 'jquery-ui-rails', '~> 7.0.0'
 gem 'jquery-rails', '~> 4.4.0'
 gem 'twitter-typeahead-rails', '~> 0.11.1'
@@ -136,8 +131,6 @@ gem 'font-awesome-rails', '~> 4.7.0'
 # icons compiled into SVG/CSS+PNG using Grunticon. See
 # https://github.com/gsa/font-awesome-grunticon-rails
 # for instructions on how to add more icons to this set
-gem 'sidekiq', '~> 7.1.3'
-gem 'sidekiq-failures', '~> 1.0.0'
 gem 'font-awesome-grunticon-rails', git: 'https://github.com/gsa/font-awesome-grunticon-rails', ref: '8ad9734a65f7e2d2de934bebe4ee7b460734f96e'
 gem 'react-rails', '~> 3.0.0'
 # Locking to prevent a version mismatch between the gem and the NPM package version

Gemfile.lock

@@ -270,7 +270,6 @@ GEM
       launchy
     cgi (0.5.0)
     chronic (0.10.2)
-    chunky_png (1.4.0)
     cld3 (3.6.0)
     climate_control (0.2.0)
     coderay (1.1.3)
@@ -284,20 +283,6 @@ GEM
       execjs
     coffee-script-source (1.12.2)
     colorize (0.8.1)
-    compass (1.0.3)
-      chunky_png (~> 1.2)
-      compass-core (~> 1.0.2)
-      compass-import-once (~> 1.0.5)
-      rb-fsevent (>= 0.9.3)
-      rb-inotify (>= 0.9)
-      sass (>= 3.3.13, < 3.5)
-    compass-blueprint (1.0.0)
-      compass
-    compass-core (1.0.3)
-      multi_json (~> 1.0)
-      sass (>= 3.3.0, < 3.5)
-    compass-import-once (1.0.5)
-      sass (>= 3.2, < 3.5)
     concurrent-ruby (1.3.5)
     connection_pool (2.5.3)
     counter_culture (2.9.0)
@@ -349,7 +334,6 @@ GEM
     cucumber-wire (6.2.1)
       cucumber-core (~> 10.1, >= 10.1.0)
       cucumber-cucumber-expressions (~> 14.0, >= 14.0.0)
-    curb (1.0.9)
     database_cleaner (2.1.0)
       database_cleaner-active_record (>= 2, < 3)
     database_cleaner-active_record (2.2.1)
@@ -552,8 +536,6 @@ GEM
       ruby2_keywords (~> 0.0.1)
     mutex_m (0.2.0)
     mysql2 (0.5.6)
-    net-http (0.6.0)
-      uri
     net-http-persistent (2.9.4)
     net-imap (0.5.9)
       date
@@ -671,9 +653,6 @@ GEM
     rake (13.0.6)
     rash_alt (0.4.12)
       hashie (>= 3.4)
-    rb-fsevent (0.11.2)
-    rb-inotify (0.11.1)
-      ffi (~> 1.0)
     rdoc (6.14.2)
       erb
       psych (>= 4.0.0)
@@ -739,7 +718,6 @@ GEM
       rspec-expectations (>= 3.0.0)
       rspec-mocks (>= 3.0.0)
     retriable (3.1.2)
-    retry_block (1.2.0)
     rexml (3.4.1)
     robots_tag_parser (0.1.1)
     rspec-activemodel-mocks (1.2.1)
@@ -803,7 +781,6 @@ GEM
     rubyzip (2.4.1)
     rufus-scheduler (3.9.2)
       fugit (~> 1.1, >= 1.11.1)
-    sass (3.4.25)
     sass-rails (6.0.0)
       sassc-rails (~> 2.1, >= 2.1.1)
     sassc (2.4.0)
@@ -834,13 +811,6 @@ GEM
       shoulda-matchers (>= 2.1.0)
     shoulda-matchers (5.3.0)
       activesupport (>= 5.2.0)
-    sidekiq (7.1.6)
-      concurrent-ruby (< 2)
-      connection_pool (>= 2.3.0)
-      rack (>= 2.2.4)
-      redis-client (>= 0.14.0)
-    sidekiq-failures (1.0.4)
-      sidekiq (>= 4.0.0)
     signet (0.20.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
@@ -903,7 +873,6 @@ GEM
     unicode-display_width (3.1.4)
       unicode-emoji (~> 4.0, >= 4.0.4)
     unicode-emoji (4.0.4)
-    uri (1.0.3)
     validate_url (0.2.0)
       activemodel (>= 3.0.0)
     vcr (6.3.1)
@@ -967,13 +936,11 @@ DEPENDENCIES
   cld3 (~> 3.6.0)
   coffee-rails (~> 5.0.0)
   colorize (~> 0.8.1)
-  compass-blueprint (~> 1.0.0)
   counter_culture (~> 2.9.0)
   cssbundling-rails (~> 1.2)
   csv (~> 3.3)
   cucumber (~> 7.1)
   cucumber-rails (~> 2.4)
-  curb (~> 1.0.1)
   database_cleaner (~> 2.0)
   debug
   dotenv (~> 3.1, >= 3.1.2)
@@ -997,7 +964,6 @@ DEPENDENCIES
   haml (~> 6)
   hashie (~> 5.0.0)
   html_truncator (~> 0.4.2)
-  htmlentities (~> 4.3.4)
   http (~> 5.0)
   i18n-tasks (~> 0.9.19)
   iso8601 (~> 0.10.1)
@@ -1011,7 +977,6 @@ DEPENDENCIES
   medusa!
   mutex_m (~> 0.2.0)
   mysql2 (~> 0.5.5)
-  net-http
   net-http-persistent (~> 2.9.3)
   newrelic_rpm (~> 9.10)
   nokogiri (~> 1.16)
@@ -1044,7 +1009,6 @@ DEPENDENCIES
   resque-scheduler (~> 4.10.2)
   resque_spec (~> 0.18.0)
   retriable (~> 3.1)
-  retry_block (~> 1.2.0)
   robotex!
   robots_tag_parser (~> 0.1.0)
   rspec-activemodel-mocks (~> 1.1)
@@ -1063,8 +1027,6 @@ DEPENDENCIES
   shakapacker (~> 6.5.4)
   shoulda-kept-assign-to (~> 1.1)
   shoulda-matchers (~> 5.0)
-  sidekiq (~> 7.1.3)
-  sidekiq-failures (~> 1.0.0)
   simplecov (~> 0.17.0)
   truncator (~> 0.1.7)
   turbolinks (~> 5.2.1)

app/engines/search_engine.rb

@@ -29,7 +29,7 @@ def initialize(options = {})
   def execute_query
     http_params = params
     Rails.logger.debug "#{self.class.name} Url: #{api_endpoint}\nParams: #{http_params}"
-    retry_block(attempts: MAX_ATTEMPT_COUNT, catch: [Faraday::TimeoutError, Faraday::ConnectionFailed]) do |attempt|
+    Retriable.retriable(tries: MAX_ATTEMPT_COUNT, on: [Faraday::TimeoutError, Faraday::ConnectionFailed]) do |attempt|
       reset_timer
       @cached_response = api_connection.get(api_endpoint, http_params)
       process_cached_response(attempt)

app/models/affiliate.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require 'digest'
-require 'sass/css'
+
 
 class Affiliate < ApplicationRecord
   extend HumanAttributeName
@@ -175,7 +175,7 @@ class Affiliate < ApplicationRecord
   validates :secondary_header_links, length: { maximum: 3 }
 
   after_validation :update_error_keys
-  before_save :set_css_properties, :generate_look_and_feel_css, :set_json_fields, :set_search_labels
+  before_save :set_css_properties, :set_json_fields, :set_search_labels
   before_update :clear_existing_attachments
   after_commit :normalize_site_domains,             on: :create
   after_commit :remove_boosted_contents_from_index, on: :destroy
@@ -304,7 +304,6 @@ def indexing_locale
                                            managed_footer_links
                                            external_tracking_code
                                            submitted_external_tracking_code
-                                           mobile_look_and_feel_css
                                            logo_alt_text
                                            header_tagline
                                            header_tagline_url
@@ -713,10 +712,7 @@ def set_api_access_key
     self.api_access_key = Digest::SHA256.base64digest("#{name}:#{Time.current.to_i}:#{rand}").tr('+/', '-_')
   end
 
-  def generate_look_and_feel_css
-    renderer = AffiliateCss.new(build_css_hash)
-    self.mobile_look_and_feel_css = renderer.render_mobile_css
-  end
+
 
   def build_css_hash
     css_hash = {}

app/views/layouts/searches.html.haml

@@ -19,9 +19,6 @@
 
     = font_stylesheet_link_tag @search.affiliate
     = stylesheet_link_tag 'searches'
-    /[if IE]
-      = stylesheet_link_tag 'searches_mobile_ie'
-    ! <style>#{@search.affiliate.mobile_look_and_feel_css}</style>
     /[if lt IE 9]
       = stylesheet_link_tag 'searches_mobile_ie_lt9'
       = javascript_include_tag 'searches_ie_lt9'

config/initializers/sidekiq.rb

@@ -279,7 +279,7 @@
     mount Resque::Server.new, at: '/resque', constraints: AffiliateAdminConstraint
     get '/resque/(*all)', to: redirect(path: '/login')
 
-    mount Sidekiq::Web => '/sidekiq', constraints: AffiliateAdminConstraint
+
   end
 
   match '/admin/affiliates/:id/analytics' => 'admin/affiliates#analytics', :as => :affiliate_analytics_redirect, via: :get

config/routes.rb

@@ -3,28 +3,32 @@ module DocumentFetcher
 
   # Caveat fetcher: These requests cannot be recorded by VCR:
   # https://cm-jira.usa.gov/browse/SRCH-645
-  def self.fetch(url, connect_timeout: 2, read_timeout: 8)
-    easy = Curl::Easy.new(url) do |c|
-      c.connect_timeout = connect_timeout
-      c.follow_location = true
-      c.max_redirects = DEFAULT_MAX_REDIRECTS
-      c.timeout = read_timeout
-      c.useragent = DEFAULT_USER_AGENT
-      c.on_success { return handle_success_or_redirect easy }
-      c.on_redirect { return handle_success_or_redirect easy }
+  def self.fetch(url, connect_timeout: 2, read_timeout: 8, limit: DEFAULT_MAX_REDIRECTS)
+    raise ArgumentError, 'Too many redirects' if limit.zero?
+
+    uri = URI.parse(url)
+    http = Net::HTTP.new(uri.host, uri.port)
+    http.use_ssl = (uri.scheme == 'https')
+    http.open_timeout = connect_timeout
+    http.read_timeout = read_timeout
+    request = Net::HTTP::Get.new(uri.request_uri)
+    request['User-Agent'] = DEFAULT_USER_AGENT
+    response = http.request(request)
+
+    case response
+    when Net::HTTPSuccess
+      { status: response.code, body: response.body, last_effective_url: uri.to_s }
+    when Net::HTTPRedirection
+      fetch(response['location'], connect_timeout: connect_timeout, read_timeout: read_timeout, limit: limit - 1)
+    else
+      { error: "#{response.code} #{response.message}" }
     end
-    easy.perform
-    { error: "Unable to fetch #{url}" }
   rescue => e
     Rails.logger.error "#{self.name} fetch error url: #{url}", e
     { error: e.message }
   end
 
   private
 
-  def self.handle_success_or_redirect(easy)
-    { body: easy.body,
-      last_effective_url: easy.last_effective_url,
-      status: easy.status }
-  end
+
 end