Reclassify timeouts on startup script retrieval to indicate the potential source issue from the startup script service

PiperOrigin-RevId: 852450925

Author: jacklacey11

perfkitbenchmarker/benchmark_status.py

@@ -73,6 +73,9 @@ class FailedSubstatus:
   # Failure when config values are invalid.
   INVALID_VALUE = 'INVALID_VALUE'
 
+  # VM under test is not ready, causing the benchmark to fail
+  VM_NOT_READY = 'VM_NOT_READY'
+
   # List of valid substatuses for use with --retries.
   # UNCATEGORIZED failures are not retryable. To make a specific UNCATEGORIZED
   # failure retryable, please raise an errors.Benchmarks.KnownIntermittentError.
@@ -87,6 +90,7 @@ class FailedSubstatus:
       UNSUPPORTED,
       COMMAND_TIMEOUT,
       RETRIES_EXCEEDED,
+      VM_NOT_READY,
   ]
 
 

perfkitbenchmarker/linux_packages/linux_boot.py

@@ -26,6 +26,10 @@
 FLAGS = flags.FLAGS
 
 
+class StartupScriptRetrievalError(vm_util.TimeoutExceededRetryError):
+  """Raised when GetStartupScriptOutput times out."""
+
+
 def PrepareBootScriptVM(aux_vm_ips: str, aux_vm_port: int) -> str:
   script_path = data.ResourcePath(
       os.path.join(DATA_DIR, BOOT_STARTUP_SCRIPT_TEMPLATE)
@@ -83,7 +87,12 @@ def CollectBootSamples(
   Returns:
     A list of sample.Sample objects.
   """
-  boot_output = GetStartupScriptOutput(vm, BOOT_SCRIPT_OUTPUT).split('\n')
+  try:
+    boot_output = GetStartupScriptOutput(vm, BOOT_SCRIPT_OUTPUT).split('\n')
+  except vm_util.TimeoutExceededRetryError as e:
+    raise StartupScriptRetrievalError(
+        'Timeout getting startup script output.'
+    ) from e
   boot_samples = ScrapeConsoleLogLines(
       boot_output, create_time, CONSOLE_FIRST_START_MATCHERS
   )
@@ -621,7 +630,12 @@ def CollectVmToVmSamples(
 ) -> List[sample.Sample]:
   """Collect samples related to vm-to-vm networking."""
   samples = []
-  vm_output = GetStartupScriptOutput(vm, BOOT_SCRIPT_OUTPUT)
+  try:
+    vm_output = GetStartupScriptOutput(vm, BOOT_SCRIPT_OUTPUT)
+  except vm_util.TimeoutExceededRetryError as e:
+    raise StartupScriptRetrievalError(
+        'Timeout getting startup script output.'
+    ) from e
   vm_internal_ip = vm.internal_ip
   vm_external_ip = vm.ip_address
 

perfkitbenchmarker/pkb.py

@@ -1294,6 +1294,12 @@ def RunBenchmark(
           spec.failed_substatus = (
               benchmark_status.FailedSubstatus.PROCESS_KILLED
           )
+        elif isinstance(
+            e, cluster_boot_benchmark.linux_boot.StartupScriptRetrievalError
+        ):
+          spec.failed_substatus = (
+              benchmark_status.FailedSubstatus.VM_NOT_READY
+          )
         elif _IsException(e, vm_util.TimeoutExceededRetryError):
           spec.failed_substatus = (
               benchmark_status.FailedSubstatus.COMMAND_TIMEOUT

tests/linux_packages/linux_boot_test.py

@@ -229,7 +229,7 @@ def testParseSystemDCriticalChain(self):
 
   def testCollectVmToVmSamples(self):
     """Test vm to vm networking result parsing."""
-    # Load startup script data, which ingress timestamps.
+    # Load startup script data, which contains ingress timestamps.
     with open(os.path.join(self.data_dir, 'boot.output')) as f:
       boot_output = f.read()
     # Load tcpdump output, which contains egress timestamps.
@@ -267,6 +267,20 @@ def testCollectVmToVmSamples(self):
         ],
     )
 
+  def testCollectBootSamplesTimeout(self):
+    """Test CollectBootSamples raises StartupScriptTimeoutError."""
+    self.enter_context(
+        mock.patch.object(
+            linux_boot,
+            'GetStartupScriptOutput',
+            side_effect=vm_util.TimeoutExceededRetryError,
+        )
+    )
+    with self.assertRaises(linux_boot.StartupScriptRetrievalError):
+      linux_boot.CollectBootSamples(
+          self.mock_vm, 0, ('', ''), datetime.datetime.now()
+      )
+
 
 if __name__ == '__main__':
   unittest.main()

tests/pkb_test.py

@@ -32,6 +32,7 @@
 from perfkitbenchmarker import sample
 from perfkitbenchmarker import stages
 from perfkitbenchmarker import test_util
+from perfkitbenchmarker.linux_packages import linux_boot
 from perfkitbenchmarker.providers.gcp import util as gcp_utils
 from tests import pkb_common_test_case
 
@@ -85,6 +86,21 @@ def testCreateProvisionFailedSample(self):
         self.spec, error_msg, stages.PROVISION, self.collector
     )
 
+  def testCreateScriptRetrievalFailedSample(self):
+    self.flags_mock.create_failed_run_samples = True
+    error_msg = 'error'
+    self.run_mock.side_effect = linux_boot.StartupScriptRetrievalError(
+        error_msg
+    )
+
+    self.assertRaises(
+        Exception, pkb.RunBenchmark, self.spec, self.collector, False
+    )
+    self.assertEqual(
+        self.spec.failed_substatus,
+        benchmark_status.FailedSubstatus.VM_NOT_READY,
+    )
+
   def testCreatePrepareFailedSample(self):
     self.flags_mock.create_failed_run_samples = True
     error_msg = 'error'