Merge pull request #11 from cosmo0920/validate-arbitrary-digits-timestamp

Validate arbitrary digits timestamp

Author: ecwws

README.md

@@ -33,6 +33,14 @@ elasticsearch.
 `subsecond_precision` controls the subsecond precision during the conversion.
 Default value is set to `3` (millisecond).
 
+Other `subsecond_precision` sample values are:
+
+* `6` (microsecond)
+* `9` (nanosecond)
+* `12` (picosecond)
+
+and more high precision is also supported.
+
 ## Usage
 
 ```

lib/fluent/plugin/filter_elasticsearch_timestamp_check.rb

@@ -2,6 +2,8 @@
 
 module Fluent::Plugin
   class ElasticsearchTimestampCheckFilter < Filter
+    attr_reader :timestamp_digits
+
     Fluent::Plugin.register_filter('elasticsearch_timestamp_check', self)
 
     config_param :subsecond_precision, :integer, default: 3
@@ -11,6 +13,15 @@ def configure(conf)
       require 'date'
       raise Fluent::ConfigError, "specify 1 or bigger number." if subsecond_precision < 1
       @strftime_format = "%Y-%m-%dT%H:%M:%S.%#{@subsecond_precision}N%z".freeze
+      @timestamp_digits = configure_digits
+    end
+
+    def configure_digits
+      subepoch_precision = 10 + @subsecond_precision
+      timestamp_digits = [10, 13]
+      timestamp_digits << subepoch_precision
+      timestamp_digits.uniq!
+      timestamp_digits
     end
 
     def start
@@ -29,10 +40,11 @@ def filter(tag, time, record)
           # all digit entry would be treated as epoch seconds or epoch millis
           if !!(timestamp =~ /\A[-+]?\d+\z/)
             num = timestamp.to_f
-            # epoch second or epoch millis should be either 10 or 13 digits
+            # By default, epoch second or epoch millis should be either 10 or 13 digits
             # other length should be considered invalid (until the next digit
             # rollover at 2286-11-20  17:46:40 Z
-            next unless [10, 13].include?(Math.log10(num).to_i + 1)
+            # For user-defined precision also should handle here.
+            next unless @timestamp_digits.include?(Math.log10(num).to_i + 1)
             record['@timestamp'] = record['fluent_converted_timestamp'] =
               Time.at(
                 num / (10 ** ((Math.log10(num).to_i + 1) - 10))

test/plugin/test_filter_elasticsearch_timestamp_check.rb

@@ -64,6 +64,30 @@ def test_timestamp_with_digit(data)
     formatted_time = Time.at(
       num / (10 ** ((Math.log10(num).to_i + 1) - 10))
     ).strftime('%Y-%m-%dT%H:%M:%S.%3N%z')
+    assert_equal([10, 13], d.instance.timestamp_digits)
+    assert_true(filtered.key?("@timestamp"))
+    assert_true(filtered.key?("fluent_converted_timestamp"))
+    assert_equal(formatted_time, filtered["fluent_converted_timestamp"])
+  end
+
+  data('@timestamp'       => '@timestamp',
+       'timestamp'        => 'timestamp',
+       'time'             => 'time',
+       'syslog_timestamp' => 'syslog_timestamp')
+  def test_timestamp_with_digit_and_micro_precision(data)
+    timekey = data
+    precision = 6
+    d = create_driver(%[subsecond_precision #{precision}])
+    timestamp = '1517878172013770'
+    d.run(default_tag: 'test') do
+      d.feed({'test' => 'notime'}.merge(timekey => timestamp))
+    end
+    filtered = d.filtered.map{|e| e.last}.first
+    num = timestamp.to_f
+    formatted_time = Time.at(
+      num / (10 ** ((Math.log10(num).to_i + 1) - 10))
+    ).strftime("%Y-%m-%dT%H:%M:%S.%#{precision}N%z")
+    assert_equal([10, 13, 10 + precision], d.instance.timestamp_digits)
     assert_true(filtered.key?("@timestamp"))
     assert_true(filtered.key?("fluent_converted_timestamp"))
     assert_equal(formatted_time, filtered["fluent_converted_timestamp"])
@@ -77,7 +101,7 @@ def test_timestamp_with_digit_and_nano_precision(data)
     timekey = data
     precision = 9
     d = create_driver(%[subsecond_precision #{precision}])
-    timestamp = '1505800348899'
+    timestamp = '1517878172013770000'
     d.run(default_tag: 'test') do
       d.feed({'test' => 'notime'}.merge(timekey => timestamp))
     end
@@ -86,6 +110,7 @@ def test_timestamp_with_digit_and_nano_precision(data)
     formatted_time = Time.at(
       num / (10 ** ((Math.log10(num).to_i + 1) - 10))
     ).strftime("%Y-%m-%dT%H:%M:%S.%#{precision}N%z")
+    assert_equal([10, 13, 10 + precision], d.instance.timestamp_digits)
     assert_true(filtered.key?("@timestamp"))
     assert_true(filtered.key?("fluent_converted_timestamp"))
     assert_equal(formatted_time, filtered["fluent_converted_timestamp"])