feat: backup and restore synthetics (#139)

* feat: backup and restore synthetic tests

* refactor: apply linter recommendations from rubocop

* refactor: rubocop pass

* chore: enable rubygems mfa

* refactor: respond_with200 to reduce a bit of visual noise

* fix: diffs generating false positives

* fix(deps): upgrade rb-fsevent

Author: jim80net

.rubocop.yml

@@ -1,78 +1,37 @@
-require: rubocop-rspec
+# The behavior of RuboCop can be controlled via the .rubocop.yml
+# configuration file. It makes it possible to enable/disable
+# certain cops (checks) and to alter their behavior if they accept
+# any parameters. The file can be placed either in your home
+# directory or in some project directory.
+#
+# RuboCop will start looking for the configuration file in the directory
+# where the inspected file is and continue its way up to the root directory.
+#
+# See https://docs.rubocop.org/rubocop/configuration
+require:
+  - rubocop-rspec
 
-Gemspec/DateAssignment: # (new in 1.10)
-  Enabled: true
-Layout/LineEndStringConcatenationIndentation: # (new in 1.18)
-  Enabled: true
-Layout/SpaceBeforeBrackets: # (new in 1.7)
-  Enabled: true
-Lint/AmbiguousAssignment: # (new in 1.7)
-  Enabled: true
-Lint/DeprecatedConstants: # (new in 1.8)
-  Enabled: true
-Lint/DuplicateBranch: # (new in 1.3)
-  Enabled: true
-Lint/DuplicateRegexpCharacterClassElement: # (new in 1.1)
-  Enabled: true
-Lint/EmptyBlock: # (new in 1.1)
-  Enabled: true
-Lint/EmptyClass: # (new in 1.3)
-  Enabled: true
-Lint/EmptyInPattern: # (new in 1.16)
-  Enabled: true
-Lint/LambdaWithoutLiteralBlock: # (new in 1.8)
-  Enabled: true
-Lint/NoReturnInBeginEndBlocks: # (new in 1.2)
-  Enabled: true
-Lint/NumberedParameterAssignment: # (new in 1.9)
-  Enabled: true
-Lint/OrAssignmentToConstant: # (new in 1.9)
-  Enabled: true
-Lint/RedundantDirGlobSort: # (new in 1.8)
-  Enabled: true
-Lint/SymbolConversion: # (new in 1.9)
-  Enabled: true
-Lint/ToEnumArguments: # (new in 1.1)
-  Enabled: true
-Lint/TripleQuotes: # (new in 1.9)
-  Enabled: true
-Lint/UnexpectedBlockArity: # (new in 1.5)
-  Enabled: true
-Lint/UnmodifiedReduceAccumulator: # (new in 1.1)
-  Enabled: true
-Naming/InclusiveLanguage: # (new in 1.18)
-  Enabled: true
-Style/ArgumentsForwarding: # (new in 1.1)
-  Enabled: true
-Style/CollectionCompact: # (new in 1.2)
-  Enabled: true
-Style/DocumentDynamicEvalDefinition: # (new in 1.1)
-  Enabled: true
-Style/EndlessMethod: # (new in 1.8)
-  Enabled: true
-Style/HashConversion: # (new in 1.10)
-  Enabled: true
-Style/HashExcept: # (new in 1.7)
-  Enabled: true
-Style/IfWithBooleanLiteralBranches: # (new in 1.9)
-  Enabled: true
-Style/InPatternThen: # (new in 1.16)
-  Enabled: true
-Style/MultilineInPatternThen: # (new in 1.16)
-  Enabled: true
-Style/NegatedIfElseCondition: # (new in 1.2)
-  Enabled: true
-Style/NilLambda: # (new in 1.3)
-  Enabled: true
-Style/QuotedSymbols: # (new in 1.16)
-  Enabled: true
-Style/RedundantArgument: # (new in 1.4)
-  Enabled: true
-Style/StringChars: # (new in 1.12)
-  Enabled: true
-Style/SwapValues: # (new in 1.1)
-  Enabled: true
-RSpec/IdenticalEqualityAssertion: # (new in 2.4)
-  Enabled: true
-RSpec/Rails/AvoidSetupHook: # (new in 2.4)
-  Enabled: true
+AllCops:
+  TargetRubyVersion: 2.7
+  NewCops: enable
+
+Layout/LineLength:
+  Enabled: false
+
+Metrics/BlockLength:
+  Enabled: false
+
+Metrics/ClassLength:
+  Enabled: false
+
+Metrics/MethodLength:
+  Enabled: false
+
+Naming/AccessorMethodName:
+  Enabled: false
+
+RSpec/MultipleMemoizedHelpers:
+  Enabled: false
+
+RSpec/ExampleLength:
+  Enabled: false

Gemfile.lock

@@ -62,7 +62,7 @@ GEM
       byebug (~> 11.0)
       pry (>= 0.13, < 0.15)
     rainbow (3.1.1)
-    rb-fsevent (0.11.1)
+    rb-fsevent (0.11.2)
     rb-inotify (0.10.1)
       ffi (~> 1.0)
     regexp_parser (2.5.0)

bin/datadog_backup

@@ -11,18 +11,17 @@ LOGGER.level = Logger::INFO
 
 require 'datadog_backup'
 
-
 def fatal(message)
   LOGGER.fatal(message)
   exit 1
 end
 
 def options_valid?(options)
   %w[backup diffs restore].include?(options[:action])
-  %w[DD_API_KEY DD_APP_KEY].all? { |key| ENV[key] }
+  %w[DD_API_KEY DD_APP_KEY].all? { |key| ENV.fetch(key, nil) }
 end
 
-def prereqs(defaults)
+def prereqs(defaults) # rubocop:disable Metrics/AbcSize
   ARGV << '--help' if ARGV.empty?
 
   result = defaults.dup
@@ -49,6 +48,9 @@ def prereqs(defaults)
     opts.on('--dashboards-only') do
       result[:resources] = [DatadogBackup::Dashboards]
     end
+    opts.on('--synthetics-only') do
+      result[:resources] = [DatadogBackup::Synthetics]
+    end
     opts.on(
       '--json',
       'format backups as JSON instead of YAML. Does not impact `diffs` nor `restore`, but do not mix formats in the same backup-dir.'
@@ -78,9 +80,9 @@ defaults = {
   action: nil,
   backup_dir: File.join(ENV.fetch('PWD'), 'backup'),
   diff_format: :color,
-  resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors],
+  resources: [DatadogBackup::Dashboards, DatadogBackup::Monitors, DatadogBackup::Synthetics],
   output_format: :yaml,
   force_restore: false
 }
 
-DatadogBackup::Cli.new(prereqs(defaults)).run!
+DatadogBackup::Cli.new(prereqs(defaults)).run!

datadog_backup.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.required_ruby_version = ['>= 2.7']
+  spec.required_ruby_version = '>= 2.7'
 
   spec.add_dependency 'amazing_print'
   spec.add_dependency 'concurrent-ruby'
@@ -28,11 +28,10 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'faraday'
   spec.add_dependency 'faraday-retry'
 
-
   spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'guard-rspec'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'guard-rspec'
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-rspec'

lib/datadog_backup.rb

@@ -8,12 +8,12 @@
 require_relative 'datadog_backup/core'
 require_relative 'datadog_backup/dashboards'
 require_relative 'datadog_backup/monitors'
+require_relative 'datadog_backup/synthetics'
 require_relative 'datadog_backup/thread_pool'
 require_relative 'datadog_backup/version'
 require_relative 'datadog_backup/deprecations'
 DatadogBackup::Deprecations.check
 
-
+# DatadogBackup is a gem for backing up and restoring Datadog monitors and dashboards.
 module DatadogBackup
 end
-

lib/datadog_backup/cli.rb

@@ -4,16 +4,17 @@
 require 'amazing_print'
 
 module DatadogBackup
+  # CLI is the command line interface for the datadog_backup gem.
   class Cli
     include ::DatadogBackup::Options
 
     def all_diff_futures
       LOGGER.info("Starting diffs on #{::DatadogBackup::ThreadPool::TPOOL.max_length} threads")
       any_resource_instance
         .all_file_ids_for_selected_resources
-        .map do |id|
-        Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, id) do |id|
-          [id, getdiff(id)]
+        .map do |file_id|
+        Concurrent::Promises.future_on(::DatadogBackup::ThreadPool::TPOOL, file_id) do |fid|
+          [fid, getdiff(fid)]
         end
       end
     end
@@ -32,32 +33,17 @@ def definitive_resource_instance(id)
       matching_resource_instance(any_resource_instance.class_from_id(id))
     end
 
-    def diffs
-      futures = all_diff_futures
-      ::DatadogBackup::ThreadPool.watcher.join
-
-      format_diff_output(
-        Concurrent::Promises
-          .zip(*futures)
-          .value!
-          .compact
-      )
-    end
-
     def getdiff(id)
       result = definitive_resource_instance(id).diff(id)
       case result
-      when ''
-        nil
-      when "\n"
-        nil
-      when '<div class="diff"></div>'
+      when '---' || '' || "\n" || '<div class="diff"></div>'
         nil
       else
         result
       end
     end
 
+    # rubocop:disable Style/StringConcatenation
     def format_diff_output(diff_output)
       case diff_format
       when nil, :color
@@ -69,58 +55,31 @@ def format_diff_output(diff_output)
           Diffy::CSS +
           '</style></head><body>' +
           diff_output.map do |id, diff|
-            "<br><br> ---<br><strong> id: #{id}</strong><br>" + diff
+            "<br><br> ---<br><strong> id: #{id}</strong><br>#{diff}"
           end.join('<br>') +
           '</body></html>'
       else
         raise 'Unexpected diff_format.'
       end
     end
+    # rubocop:enable Style/StringConcatenation
 
     def initialize(options)
       @options = options
     end
 
-    def matching_resource_instance(klass)
-      resource_instances.select { |resource_instance| resource_instance.instance_of?(klass) }.first
-    end
-
-    def resource_instances
-      @resource_instances ||= resources.map do |resource|
-        resource.new(@options)
-      end
-    end
-
     def restore
       futures = all_diff_futures
       watcher = ::DatadogBackup::ThreadPool.watcher
 
       futures.each do |future|
         id, diff = *future.value!
-        next unless diff
+        next if diff.nil? || diff.empty?
 
         if @options[:force_restore]
           definitive_resource_instance(id).restore(id)
         else
-          puts '--------------------------------------------------------------------------------'
-          puts format_diff_output([id, diff])
-          puts '(r)estore to Datadog, overwrite local changes and (d)ownload, (s)kip, or (q)uit?'
-          response = $stdin.gets.chomp
-          case response
-          when 'q'
-            exit
-          when 'r'
-            puts "Restoring #{id} to Datadog."
-            definitive_resource_instance(id).restore(id)
-          when 'd'
-            puts "Downloading #{id} from Datadog."
-            definitive_resource_instance(id).get_and_write_file(id)
-          when 's'
-            next
-          else
-            puts 'Invalid response, please try again.'
-            response = $stdin.gets.chomp
-          end
+          ask_to_restore(id, diff)
         end
       end
       watcher.join if watcher.status
@@ -131,5 +90,42 @@ def run!
     rescue SystemExit, Interrupt
       ::DatadogBackup::ThreadPool.shutdown
     end
+
+    private
+
+    def ask_to_restore(id, diff)
+      puts '--------------------------------------------------------------------------------'
+      puts format_diff_output([id, diff])
+      puts '(r)estore to Datadog, overwrite local changes and (d)ownload, (s)kip, or (q)uit?'
+      loop do
+        response = $stdin.gets.chomp
+        case response
+        when 'q'
+          exit
+        when 'r'
+          puts "Restoring #{id} to Datadog."
+          definitive_resource_instance(id).restore(id)
+          break
+        when 'd'
+          puts "Downloading #{id} from Datadog."
+          definitive_resource_instance(id).get_and_write_file(id)
+          break
+        when 's'
+          break
+        else
+          puts 'Invalid response, please try again.'
+        end
+      end
+    end
+
+    def matching_resource_instance(klass)
+      resource_instances.select { |resource_instance| resource_instance.instance_of?(klass) }.first
+    end
+
+    def resource_instances
+      @resource_instances ||= resources.map do |resource|
+        resource.new(@options)
+      end
+    end
   end
 end