qa: use clean expansion of LS tarball per fixture instance (#17082) (#17095)

* qa: use clean expansion of LS tarball per fixture instance

Because QA tests can _modify_ the Logstash installation (e.g. those that
invoke the plugin manager), it is important that the service wrapper
begins with a clean expansion of the logstash tarball.

* qa: enable safe reuse of ls_home in ls_to_ls tests

(cherry picked from commit d20eb4d)

Co-authored-by: Ry Biesemeyer <[email protected]>

Author: github-actions[bot]

build.gradle

@@ -418,7 +418,7 @@ def qaBuildPath = "${buildDir}/qa/integration"
 def qaVendorPath = "${qaBuildPath}/vendor"
 
 tasks.register("installIntegrationTestGems") {
-  dependsOn unpackTarDistribution
+  dependsOn assembleTarDistribution
   def gemfilePath = file("${projectDir}/qa/integration/Gemfile")
   inputs.files gemfilePath
   inputs.files file("${projectDir}/qa/integration/integration_tests.gemspec")

qa/integration/framework/settings.rb

@@ -16,6 +16,7 @@
 # under the License.
 
 require 'yaml'
+require 'delegate'
 
 # All settings for a test, global and per test
 class TestSettings
@@ -76,4 +77,23 @@ def feature_config_dir
     feature = feature_flag
     feature.empty? ? nil : File.join(FIXTURES_DIR, feature)
   end
+
+  def override(kv_map)
+    Override.new(self, kv_map)
+  end
+
+  class Override < SimpleDelegator# quacks like TestSettings
+    def initialize(test_settings, overrides)
+      super(test_settings)
+      @overrides = overrides
+    end
+
+    def is_set?(key)
+      @overrides.include?(key) || super
+    end
+
+    def get(key)
+      @overrides.fetch(key) { super }
+    end
+  end
 end

qa/integration/services/logstash_service.rb

@@ -53,22 +53,58 @@ def initialize(settings, api_port = 9600)
     if @settings.is_set?("ls_home_abs_path")
       @logstash_home = @settings.get("ls_home_abs_path")
     else
-      # use the LS which was just built in source repo
-      ls_version_file = YAML.load_file(LS_VERSION_FILE)
-      ls_file = "logstash-" + ls_version_file["logstash"]
-      # First try without the snapshot if it's there
-      @logstash_home = File.expand_path(File.join(LS_BUILD_DIR, ls_file), __FILE__)
-      @logstash_home += "-SNAPSHOT" unless Dir.exist?(@logstash_home)
-
-      puts "Using #{@logstash_home} as LS_HOME"
-      @logstash_bin = File.join("#{@logstash_home}", LS_BIN)
-      raise "Logstash binary not found in path #{@logstash_home}" unless File.file? @logstash_bin
+      @logstash_home = clean_expand_built_tarball
     end
 
+    puts "Using #{@logstash_home} as LS_HOME"
+    @logstash_bin = File.join("#{@logstash_home}", LS_BIN)
+    raise "Logstash binary not found in path #{@logstash_home}" unless File.file? @logstash_bin
+
     @default_settings_file = File.join(@logstash_home, LS_CONFIG_FILE)
     @monitoring_api = MonitoringAPI.new(api_port)
   end
 
+  ##
+  # @return [String] the path to a CLEAN expansion of the locally-built tarball
+  def clean_expand_built_tarball
+    build_dir = File.expand_path(LS_BUILD_DIR, __FILE__) # source of tarball
+    target_dir = File.join(build_dir, "qa-fixture")
+
+    # find the built tarball matching the current version, preferring non-SNAPSHOT
+    ls_version = YAML.load_file(LS_VERSION_FILE).fetch("logstash")
+    candidates = %W(
+          logstash-#{ls_version}.tar.gz
+          logstash-#{ls_version}-SNAPSHOT.tar.gz
+    )
+
+    candidates.each do |tarball_candidate|
+      tarball_candidate_path = File.join(build_dir, tarball_candidate)
+      if File.exist?(tarball_candidate_path)
+        expected_untar_directory = File.basename(tarball_candidate, ".tar.gz")
+        result_logstash_home = File.join(target_dir, expected_untar_directory)
+
+        if Dir.exist?(result_logstash_home)
+          puts "expunging(#{result_logstash_home})"
+          # FileUtils#rm_rf cannot be used here because it silently fails to remove the bundled jdk on MacOS
+          expunge_result = `rm -rf #{Shellwords.escape(result_logstash_home)} 2>&1`
+          fail("ERROR EXPUNGING: #{expunge_result}") unless $?.success?
+        end
+
+        puts "expanding(#{tarball_candidate_path})"
+        FileUtils.mkdir_p(target_dir) unless Dir.exist?(target_dir)
+        FileUtils.chdir(target_dir) do
+          expand_result = `tar -xzf #{Shellwords.escape(tarball_candidate_path)} 2>&1`
+          fail("ERROR EXPANDING: #{expand_result}") unless $?.success?
+        end
+
+        return result_logstash_home
+      end
+    end
+
+    fail("failed to find any matching build tarballs (looked for `#{candidates}` in `#{build_dir}`)")
+  end
+  private :clean_expand_built_tarball
+
   def alive?
     if @process.nil? || @process.exited?
       raise "Logstash process is not up because of an error, or it stopped"

qa/integration/specs/logstash_to_logstash_spec.rb

@@ -36,13 +36,6 @@
     @fixture.teardown
   }
 
-  def change_logstash_setting(logstash_service, name, value)
-    settings = {}.tap do |settings|
-      settings[name] = value
-    end
-    IO.write(logstash_service.application_settings_file, settings.to_yaml)
-  end
-
   def get_temp_path_dir
     tmp_path = Stud::Temporary.pathname
     tmp_data_path = File.join(tmp_path, "data")
@@ -51,10 +44,17 @@ def get_temp_path_dir
   end
 
   def run_logstash_instance(config_name, options = {})
-    api_port = 9600 + rand(1000)
-    logstash_service = LogstashService.new(@fixture.settings, api_port)
-    change_logstash_setting(logstash_service, "api.http.port", api_port)
-    logstash_service.spawn_logstash("-f", config_to_temp_file(@fixture.config(config_name, options)), "--path.data", get_temp_path_dir)
+    @next_api_port_offset = (@next_api_port_offset||0).next.modulo(1000) # cycle through 1000 possibles
+    api_port = 9600 + @next_api_port_offset
+
+    # to avoid LogstashService's clean-from-tarball default behaviour, we need
+    # to tell it where our LOGSTASH_HOME is in the existing service
+    existing_fixture_logstash_home = @fixture.get_service("logstash").logstash_home
+    logstash_service = LogstashService.new(@fixture.settings.override("ls_home_abs_path" => existing_fixture_logstash_home), api_port)
+
+    logstash_service.spawn_logstash("--path.config", config_to_temp_file(@fixture.config(config_name, options)),
+                                    "--path.data", get_temp_path_dir,
+                                    "--api.http.port", api_port.to_s)
     wait_for_logstash(logstash_service)
     logstash_service
   end