SRCH-6222 SRCH-6227 Create OpenSearch engine and OpenSearch index (#1867)

stevenbarragan · web-flow · commit f228cb458f79 · 2025-09-24T10:29:22.000-07:00
* feat: Add OpenSearch search engine
* feat: Configure Open-Search client and index creation
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -41,6 +41,16 @@ executors:
           xpack.security.enabled: false
           ES_JAVA_OPTS: "-Xms256m -Xmx256m"
 
+      - image: opensearchproject/opensearch:3
+        environment:
+          cluster.name: os-cluster
+          discovery.type: single-node
+          bootstrap.memory_lock: true
+          http.port: 9300
+          plugins.security.disabled: "true"
+          OPENSEARCH_JAVA_OPTS: "-Xms256m -Xmx256m"
+          OPENSEARCH_INITIAL_ADMIN_PASSWORD: S3venDr4gon!
+
       - image: apache/tika:2.4.1
 
 commands:
diff --git a/.env.development b/.env.development
@@ -3,6 +3,10 @@ ES_HOSTS="http://localhost:9200"
 ES_CA_FINGERPRINT=FILL_THE_GENERATED_CA_FINGERPRINT
 ES_USER=elastic
 ES_PASSWORD=changeme
+OPENSEARCH_HOST=http://localhost:9300
+OPENSEARCH_INDEX=development-i14y-documents-searchgov
+OPENSEARCH_PASSWORD=S3venDr4gon!
+OPENSEARCH_USER=admin
 REDIS_SYSTEM_URL=redis://localhost:6379
 REDIS_PORT=6379
 I14Y_DOCKER_HOST=http://i14y:3200
diff --git a/.env.test b/.env.test
@@ -3,6 +3,10 @@ ES_HOSTS="http://localhost:9200"
 ES_CA_FINGERPRINT=FILL_THE_GENERATED_CA_FINGERPRINT
 ES_USER=elastic
 ES_PASSWORD=changeme
+OPENSEARCH_HOST=http://localhost:9300
+OPENSEARCH_INDEX=development-i14y-documents-searchgov
+OPENSEARCH_PASSWORD=S3venDr4gon!
+OPENSEARCH_USER=admin
 REDIS_SYSTEM_URL=redis://localhost:6379
 REDIS_PORT=6379
 I14Y_DOCKER_HOST=http://i14y:3200
diff --git a/Gemfile b/Gemfile
@@ -56,7 +56,7 @@ gem 'sass-rails', '~> 6.0'
 gem 'google_visualr',
     git: 'https://github.com/winston/google_visualr',
     ref: '17b97114a345baadd011e7b442b9a6c91a2b7ab5'
-gem 'faraday_middleware', '~> 0.14.0'
+gem 'faraday_middleware'
 gem 'net-http-persistent', '~> 2.9.3'
 gem 'rash_alt', '~> 0.4.12', require: 'rash'
 gem 'geoip', '~> 1.6.4'
@@ -77,6 +77,7 @@ gem 'elasticsearch', git: 'https://github.com/GSA/elasticsearch-ruby', branch: '
 gem 'elasticsearch-dsl', '~> 0.1.9'
 gem 'elasticsearch-xpack', '~> 7.4.0'
 gem "elasticsearch-persistence"
+gem 'opensearch-dsl'
 gem 'federal_register', '~> 0.6.3'
 gem 'redcarpet', '~> 3.6'
 gem 'google-api-client', '~> 0.53.0'
diff --git a/Gemfile.lock b/Gemfile.lock
@@ -564,6 +564,7 @@ GEM
       actionpack (>= 4.2)
       omniauth (~> 2.0)
     open_uri_redirections (0.2.1)
+    opensearch-dsl (0.2.1)
     os (1.1.4)
     ostruct (0.6.3)
     parallel (1.27.0)
@@ -811,10 +812,10 @@ GEM
       shoulda-matchers (>= 2.1.0)
     shoulda-matchers (5.3.0)
       activesupport (>= 5.2.0)
-    signet (0.20.0)
+    signet (0.21.0)
       addressable (~> 2.8)
       faraday (>= 0.17.5, < 3.a)
-      jwt (>= 1.5, < 3.0)
+      jwt (>= 1.5, < 4.0)
       multi_json (~> 1.10)
     simplecov (0.17.1)
       docile (~> 1.1)
@@ -951,7 +952,7 @@ DEPENDENCIES
   email_spec (~> 2.2)
   exception_notification (~> 4.5)
   faker (~> 1.8)
-  faraday_middleware (~> 0.14.0)
+  faraday_middleware
   federal_register (~> 0.6.3)
   feedjira (~> 3.2)
   flickraw (~> 0.9.9)
@@ -983,6 +984,7 @@ DEPENDENCIES
   omniauth (~> 2.1.0)
   omniauth-rails_csrf_protection (~> 1.0.1)
   omniauth_login_dot_gov!
+  opensearch-dsl
   pry-byebug (~> 3.5)
   pry-rails (~> 0.3.6)
   puma (~> 5.6)
diff --git a/app/controllers/api/v2/searches_controller.rb b/app/controllers/api/v2/searches_controller.rb
@@ -40,7 +40,10 @@ def docs
       private
 
       def selected_engine
-        if @search_options.site.search_elastic_engine?
+        case @search_options.site.search_engine
+        when "open_search"
+          OpenSearch::ApiEngine
+        when "search_elastic"
           ApiSearchElastic
         else
           ApiI14ySearch
diff --git a/app/controllers/searches_controller.rb b/app/controllers/searches_controller.rb
@@ -79,6 +79,8 @@ def advanced
   def pick_klass_vertical_template
     if get_commercial_results?
       [WebSearch, :web, :index]
+    elsif @affiliate.open_search_engine?
+      [OpenSearch::Engine, :SRCH, :i14y]
     elsif @affiliate.search_elastic_engine?
       [SearchElasticEngine, :SRCH, :i14y]
     elsif gets_i14y_results?
diff --git a/app/jobs/bulk_affiliate_search_engine_update_job.rb b/app/jobs/bulk_affiliate_search_engine_update_job.rb
@@ -3,7 +3,7 @@
 class BulkAffiliateSearchEngineUpdateJob < ApplicationJob
   queue_as :searchgov
 
-  VALID_SEARCH_ENGINES = %w[bing_v7 search_gov search_elastic].freeze
+  VALID_SEARCH_ENGINES = %w[bing_v7 search_gov search_elastic open_search].freeze
 
   def perform(requesting_user_email, file_name, s3_object_key)
     temp_file = download_from_s3(s3_object_key)
@@ -98,4 +98,4 @@ def log_parsing_failure(email, filename, results)
       Row Errors: #{results.error_details.count}.
     WARN
   end
-end
+end
diff --git a/app/models/open_search/api_engine.rb b/app/models/open_search/api_engine.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class OpenSearch::ApiEngine < OpenSearch::Engine
+  include ApiSearch
+
+  def as_json_result_hash(result)
+    super.merge(thumbnail_url: result.thumbnail_url)
+  end
+end
diff --git a/app/models/open_search/document_query.rb b/app/models/open_search/document_query.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class OpenSearch::DocumentQuery < SearchElastic::DocumentQuery
+  private
+
+  def parse_query(query)
+    site_params_parser = OpenSearch::QueryParser.new(query)
+    @site_filters = site_params_parser.site_filters
+    @included_sites = @site_filters[:included_sites]
+    @excluded_sites = @site_filters[:excluded_sites]
+    @query = site_params_parser.stripped_query
+  end
+end
diff --git a/app/models/open_search/document_repository.rb b/app/models/open_search/document_repository.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class OpenSearch::DocumentRepository < SearchElastic::DocumentRepository
+  client OPENSEARCH_CLIENT
+end
diff --git a/app/models/open_search/document_search.rb b/app/models/open_search/document_search.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class OpenSearch::DocumentSearch < SearchElastic::DocumentSearch
+  def initialize(options, affiliate:)
+    @doc_query = OpenSearch::DocumentQuery.new(options, affiliate:)
+    @indices = options[:indices]
+    @offset = options[:offset] || 0
+    @size = options[:size]
+  end
+
+  private
+
+  def execute_client_search
+    Rails.logger.debug "Query: *****\n#{doc_query.body.to_json}\n*****"
+
+    result = OPENSEARCH_CLIENT.search({
+      index: indices,
+      body: doc_query.body,
+      from: offset,
+      size: size,
+      rest_total_hits_as_int: true
+    })
+
+    OpenSearch::DocumentSearchResults.new(result, offset)
+  end
+end
diff --git a/app/models/open_search/document_search_results.rb b/app/models/open_search/document_search_results.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class OpenSearch::DocumentSearchResults < SearchElastic::DocumentSearchResults
+end
diff --git a/app/models/open_search/engine.rb b/app/models/open_search/engine.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class OpenSearch::Engine < SearchElasticEngine
+  def search
+    params = process_array_parameters(build_search_params).merge(indices: ENV.fetch('OPENSEARCH_INDEX'))
+    search_results = OpenSearch::DocumentSearch.new(params, affiliate: @affiliate).search
+    build_response(search_results)
+  end
+end
diff --git a/app/models/open_search/indexer.rb b/app/models/open_search/indexer.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class OpenSearch::Indexer
+  def self.create_index
+    index_name = ENV.fetch('OPENSEARCH_INDEX')
+
+    template_generator = OpenSearch::Template.new("*#{index_name}*")
+
+    OPENSEARCH_CLIENT.indices.put_template(
+      body: template_generator.body,
+      create: true,
+      name: index_name,
+      order: 0
+    )
+
+    repo = OpenSearch::DocumentRepository.new
+    repo.create_index!(index: index_name)
+  end
+end
diff --git a/app/models/open_search/query_parser.rb b/app/models/open_search/query_parser.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class OpenSearch::QueryParser < SearchElastic::QueryParser
+end
diff --git a/app/models/open_search/template.rb b/app/models/open_search/template.rb
@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+class OpenSearch::Template < SearchElastic::Template
+end
diff --git a/bin/docker b/bin/docker
@@ -11,6 +11,7 @@ setup() {
     docker build -f ${DOCKERFILE} -t ${IMAGE_NAME} .
     docker run -it --env-file ${ENV_FILE} --network host ${IMAGE_NAME} bin/rails usasearch:elasticsearch:create_indexes
     docker run -it --env-file ${ENV_FILE} --network host ${IMAGE_NAME} bin/rails db:setup
+    docker run -it --env-file ${ENV_FILE} --network host ${IMAGE_NAME} bin/rails css:build
 }
 
 build() {
diff --git a/config/initializers/constants.rb b/config/initializers/constants.rb
@@ -4,7 +4,7 @@
 DEFAULT_USER_AGENT                = ENV.fetch('DEFAULT_USER_AGENT', 'usasearch')
 DELIVER_FROM_EMAIL_ADDRESS        = 'no-reply@support.digitalgov.gov'
 NOTIFICATION_SENDER_EMAIL_ADDRESS = 'notification@support.digitalgov.gov'
-SEARCH_ENGINES                    = %w[BingV7 SearchGov SearchElastic].freeze
+SEARCH_ENGINES                    = %w[BingV7 SearchGov SearchElastic OpenSearch].freeze
 SUPPORT_EMAIL_ADDRESS             = ENV['SUPPORT_EMAIL_ADDRESS']
 BLOG_URL                          = ENV['BLOG_URL']
 PAGE_NOT_FOUND_URL                = ENV['PAGE_NOT_FOUND_URL']
diff --git a/config/initializers/opensearch_client.rb b/config/initializers/opensearch_client.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'elasticsearch'
+
+# NOTE: This is using the Elasticsearch::Client because the opensearch-ruby gem
+# has a dependency on a newer version of faraday than is allowed by the
+# omniauth_login_dot_gov gem.
+#
+# The elasticsearch-ruby gem is compatible with OpenSearch.
+OPENSEARCH_CLIENT = Elasticsearch::Client.new(
+  url: ENV.fetch('OPENSEARCH_HOST', 'http://localhost:9300'),
+  user: ENV['OPENSEARCH_USER'],
+  password: ENV['OPENSEARCH_PASSWORD'],
+  log: Rails.env.development?
+)
diff --git a/db/seeds.rb b/db/seeds.rb
@@ -9,6 +9,7 @@
   require_relative 'seeds/affiliate.rb'
   require_relative 'seeds/email_template.rb'
   require_relative 'seeds/search_module.rb'
+  require_relative 'seeds/drawer.rb'
   puts 'Creating custom index data'
   Dir[File.join(seed_dir, 'custom_index_data', '*.rb')].each {|file| require file }
 elsif Rails.env.test?
diff --git a/lib/tasks/opensearch.rake b/lib/tasks/opensearch.rake
@@ -0,0 +1,6 @@
+namespace :opensearch do
+  desc 'Create an index for opensearch engine'
+  task create_index: :environment do
+    OpenSearch::Indexer.create_index
+  end
+end
diff --git a/spec/models/open_search/document_search_results_spec.rb b/spec/models/open_search/document_search_results_spec.rb
@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe OpenSearch::DocumentSearchResults do
+  let(:document_search_results) { described_class.new(result) }
+
+  describe '#suggestion' do
+    subject(:suggestion) { document_search_results.suggestion }
+
+    context 'when no hits and suggestions are present' do
+      let(:result) do
+        { 'hits' => { 'total' => 0, 'hits' => [] },
+          'aggregations' => {},
+          'suggest' => suggestion_hash }
+      end
+      let(:suggestion_hash) do
+        { 'suggestion' =>
+                            [{ 'text' => 'blue',
+                               'options' => [{ 'text' => 'bulk',
+                                               'highlighted' => 'bulk' }] }] }
+      end
+
+      it { is_expected.to match(hash_including({ 'text' => 'bulk', 'highlighted' => 'bulk' })) }
+    end
+  end
+
+  describe '#results' do
+    subject(:results) { document_search_results.results }
+
+    context 'when hits are present' do
+      let(:result) do
+        { 'hits' => { 'total' => 1, 'hits' => [hits] },
+          'aggregations' => {},
+          'suggest' => [] }
+      end
+      let(:hits) do
+        { '_type' => '_doc',
+          '_source' => { 'path' => 'https://search.gov/about/',
+                         'created' => '2021-02-03T00:00:00.000-05:00',
+                         'language' => 'en',
+                         'title_en' => 'About Search.gov | Search.gov' },
+          'highlight' => { 'content_en' => ['Some highlighted content'] } }
+      end
+
+      it {
+        is_expected.to match(array_including({ 'path' => 'https://search.gov/about/',
+                                               'created' => '2021-02-03 05:00:00 UTC',
+                                               'language' => 'en',
+                                               'title' => 'About Search.gov | Search.gov',
+                                               'content' => 'Some highlighted content' }))
+      }
+    end
+  end
+
+  describe '#aggregations' do
+    subject(:aggregations) { document_search_results.aggregations }
+
+    context 'when aggregations are present' do
+      let(:result) do
+        { 'hits' => { 'total' => 1, 'hits' => [hits] },
+          'aggregations' => aggregations_hash,
+          'suggest' => [] }
+      end
+      let(:hits) do
+        { '_type' => '_doc',
+          '_source' => { 'path' => 'https://search.gov/about/',
+                         'created' => '2021-02-03T00:00:00.000-05:00',
+                         'language' => 'en',
+                         'title_en' => 'About Search.gov | Search.gov' },
+          'highlight' => { 'content_en' => ['Some highlighted content'] } }
+      end
+      let(:aggregations_hash) do
+        { 'content_type' => { 'doc_count_error_upper_bound' => 0,
+                              'sum_other_doc_count' => 0,
+                              'buckets' => [{ 'key' => 'article',
+                                              'doc_count' => 1 }] },
+          'tags' => { 'doc_count_error_upper_bound' => 0,
+                      'sum_other_doc_count' => 0,
+                      'buckets' => [] } }
+      end
+
+      it { is_expected.to match(array_including({ content_type: [{ agg_key: 'article', doc_count: 1 }] })) }
+
+      it { is_expected.not_to include(hash_including(:tags)) }
+    end
+  end
+end
diff --git a/spec/models/open_search/document_search_spec.rb b/spec/models/open_search/document_search_spec.rb
diff --git a/spec/models/open_search/engine_spec.rb b/spec/models/open_search/engine_spec.rb
