Merge pull request #34 from opus-codium/ingest-use-deduplication-window

Author: smortex

.github/workflows/ci.yml

@@ -33,6 +33,7 @@ jobs:
           - "3.1"
           - "3.2"
           - "3.3"
+          - "3.4"
     name: Ruby ${{ matrix.ruby }}
     steps:
       - uses: actions/checkout@v5
@@ -41,13 +42,5 @@ jobs:
         with:
           ruby-version: ${{ matrix.ruby }}
           bundler-cache: true
-      - name: Run tests without uploading code coverage
-        if: ${{ matrix.ruby != '3.0' }}
+      - name: Run tests
         run: bundle exec rake
-      - name: Run tests and upload coverage to Code Climate
-        if: ${{ matrix.ruby == '3.0' }}
-        uses: paambaati/[email protected]
-        env:
-          CC_TEST_REPORTER_ID: ${{ secrets.CODECLIMATE_TOKEN }}
-        with:
-          coverageCommand: bundle exec rake

.rubocop.yml

@@ -4,7 +4,7 @@ AllCops:
   TargetRubyVersion: 2.6
   NewCops: enable
 
-require:
+plugins:
   - rubocop-rake
   - rubocop-rspec
 

email-report-processor.gemspec

@@ -30,6 +30,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'activesupport'
+  spec.add_dependency 'datemath'
   spec.add_dependency 'mail'
   spec.add_dependency 'opensearch-ruby-cli'
   spec.add_dependency 'rexml'

exe/email-report-processor

@@ -6,6 +6,7 @@ require 'email_report_processor'
 require 'opensearch/cli'
 require 'faraday/net_http_persistent'
 
+require 'datemath'
 require 'openssl'
 require 'optparse'
 
@@ -93,6 +94,8 @@ end
 message_reader = EmailReportProcessor::MessageReader.new
 report_processor = EmailReportProcessor::ReportProcessor.new(client: cli.client, options: options)
 
+accepted_dates = Datemath::Parser.new(options[:deduplicate_since]).parse..Datemath::Parser.new(options[:deduplicate_until]).parse
+
 if ARGV.empty?
   mail = Mail.new($stdin.read)
   report_processor.send_report(message_reader.process_message(mail))
@@ -103,13 +106,17 @@ else
       m = EmailReportProcessor::Mbox.new(filename)
 
       while (mail = m.next_message)
+        next unless accepted_dates.cover?(mail.date)
+
         report_processor.send_report(message_reader.process_message(mail))
         p.step
       end
     elsif options[:maildir]
       m = EmailReportProcessor::MailDir.new(filename)
 
       while (mail = m.next_message)
+        next unless accepted_dates.cover?(mail.date)
+
         report_processor.send_report(message_reader.process_message(mail))
         p.step
       end