Adjust Content-Type for ES9

Signed-off-by: Hiroshi Hatake <hiroshi@chronosphere.io>

Author: cosmo0920

lib/fluent/plugin/out_elasticsearch.rb

@@ -73,6 +73,7 @@ def initialize(retry_stream)
     DEFAULT_RELOAD_AFTER = -1
     DEFAULT_TARGET_BULK_BYTES = -1
     DEFAULT_POLICY_ID = "logstash-policy"
+    ES9_CONTENT_TYPE = "application/vnd.elasticsearch+x-ndjson; compatible-with=9"
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -146,7 +147,7 @@ def initialize(retry_stream)
     config_param :sniffer_class_name, :string, :default => nil
     config_param :selector_class_name, :string, :default => nil
     config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
-    config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
+    config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson", :"#{ES9_CONTENT_TYPE}"], :default => :"application/json",
                  :deprecated => <<EOC
 elasticsearch gem v6.0.2 starts to use correct Content-Type. Please upgrade elasticserach gem and stop to use this option.
 see: https://github.com/elastic/elasticsearch-ruby/pull/514
@@ -350,10 +351,20 @@ class << self
           log.warn "Detected ES 7.x: `_doc` will be used as the document `_type`."
           @type_name = '_doc'.freeze
         end
-        if @last_seen_major_version >= 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
-          log.debug "Detected ES 8.x or above: This parameter has no effect."
+        if @last_seen_major_version == 8 && @type_name != DEFAULT_TYPE_NAME_ES_7x
+          log.debug "Detected ES 8.x: This parameter has no effect."
           @type_name = nil
         end
+        if @last_seen_major_version >= 9
+          if @type_name != DEFAULT_TYPE_NAME_ES_7x
+            log.debug "Detected ES 9.x or above: This parameter has no effect."
+            @type_name = nil
+          end
+          if @content_type != ES9_CONTENT_TYPE
+            log.trace "Detected ES 9.x or above: Content-Type will be adjusted."
+            @content_type = ES9_CONTENT_TYPE
+          end
+        end
       end
 
       if @validate_client_version && !dry_run?

test/plugin/test_out_elasticsearch.rb

@@ -90,7 +90,13 @@ def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_versio
   end
 
   def stub_elastic(url="http://localhost:9200/_bulk")
-    stub_request(:post, url).with do |req|
+    headers = if elasticsearch_miscellaneous_content_type?
+                { "Content-Type" => "application/vnd.elasticsearch+x-ndjson; compatible-with=9" }
+              else
+                {}
+              end
+
+    stub_request(:post, url).with({headers: headers}) do |req|
       @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
     end.to_return({:status => 200, :body => "", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
   end
@@ -4016,6 +4022,7 @@ def test_writes_to_default_index_with_compression
   data('Elasticsearch 6' => [6, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME],
        'Elasticsearch 7' => [7, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME_ES_7x],
        'Elasticsearch 8' => [8, nil],
+       'Elasticsearch 9' => [9, nil],
       )
   def test_writes_to_default_type(data)
     version, index_type = data