Merge pull request #70 from nasa/DAS-2481-hoss-noretry

DAS-2481 Add no reties for ServiceException response bad request 400

Author: vutrannasa

CHANGELOG.md

@@ -1,4 +1,11 @@
-## [v1.2.2] - 2026-03-23
+## [v1.2.4] - 2026-04-09
+
+### Changed
+
+- Ensure Harmony ServerExceptions with HTTP 500 are caught and retried,
+  while all other status codes are re-raised as NoRetryExceptions.
+
+## [v1.2.3] - 2026-03-23
 
 ### Changed
 

docker/service_version.txt

@@ -1 +1 @@
-1.2.3
+1.2.4

hoss/adapter.py

@@ -35,6 +35,7 @@
 from pystac import Asset, Item
 
 from hoss.dimension_utilities import is_index_subset
+from hoss.exceptions import StagingFailed
 from hoss.harmony_log_context import set_logger
 from hoss.subset import subset_granule
 from hoss.utilities import (
@@ -133,12 +134,10 @@ def process_item(self, item: Item, source: Source):
                     ),
                 )
                 mime, _ = get_file_mimetype(output_url)
-                url = stage(
+                url = self.hoss_stage(
                     output_url,
                     asset_name,
                     mime,
-                    location=self.message.stagingLocation,
-                    logger=self.logger,
                 )
 
             # Update the STAC record
@@ -189,3 +188,22 @@ def validate_message(self):
         for source in self.message.sources:
             if not hasattr(source, 'variables') or not source.variables:
                 self.logger.info('All variables will be retrieved.')
+
+    def hoss_stage(self, url: str, asset_name: str, output_mimetype: str) -> str:
+        """Stages the file to and S3 location and returns the url.
+        Throws a retriable exception when there is a failure.
+
+        """
+        try:
+            output_url = stage(
+                url,
+                asset_name,
+                output_mimetype,
+                location=self.message.stagingLocation,
+                logger=self.logger,
+            )
+
+            return output_url
+
+        except Exception as exception:
+            raise StagingFailed('Staging failed, ' + str(exception)) from exception

hoss/exceptions.py

@@ -325,3 +325,26 @@ def __init__(self, variable_names):
             f'Some variables requested are not supported and could not be processed: '
             f'"{variable_names}".',
         )
+
+
+class UrlAccessFailedWithNoRetries(CustomNoRetryError):
+    """This exception is raised when an HTTP request for a given URL fails due
+    to a server error other than response code 500.
+
+    """
+
+    def __init__(self, url, status_code):
+        super().__init__(
+            'UrlAccessFailedWithNoRetries',
+            f'{status_code} error retrieving: {url}',
+        )
+
+
+class StagingFailed(CustomError):
+    """This Exception is used as a catch any errors
+    during staging. This will be retriable exception.
+
+    """
+
+    def __init__(self, message):
+        super().__init__('StagingFailed', message)

hoss/utilities.py

@@ -5,6 +5,7 @@
 """
 
 import mimetypes
+import re
 from os import sep
 from os.path import splitext
 from shutil import move
@@ -21,7 +22,12 @@
 from harmony_service_lib.util import Config
 from harmony_service_lib.util import download as util_download
 
-from hoss.exceptions import CustomNoRetryError, UrlAccessFailed, UrlAccessForbidden
+from hoss.exceptions import (
+    CustomError,
+    UrlAccessFailed,
+    UrlAccessFailedWithNoRetries,
+    UrlAccessForbidden,
+)
 from hoss.harmony_log_context import get_logger
 
 
@@ -168,7 +174,7 @@ def download_url(
     except ForbiddenException as harmony_exception:
         raise UrlAccessForbidden(url, 403) from harmony_exception
     except ServerException as harmony_exception:
-        raise UrlAccessFailed(url, 500) from harmony_exception
+        handle_harmony_server_exception(url, harmony_exception)
     except Exception as harmony_exception:
         raise UrlAccessFailed(url, 'Unknown') from harmony_exception
 
@@ -204,16 +210,43 @@ def raise_from_hoss_exception(exception: Exception):
     """Convert a HOSS exception to an appropriate Harmony exception.
 
     Translates HOSS-specific exceptions into Harmony framework exceptions,
-    preserving the exception chain. CustomNoRetryError exceptions are converted
-    to NoRetryException, while all other exceptions become HarmonyException
-    instances.
+    preserving the exception chain. Ensures CustomError (download/staging)
+    are retried via HarmonyException, and all other errors fail permanently
+    with NoRetryException
 
     """
-    if issubclass(type(exception), CustomNoRetryError):
-        ExceptionClass = NoRetryException
-    else:
+    if issubclass(type(exception), CustomError):
         ExceptionClass = HarmonyException
+    else:
+        ExceptionClass = NoRetryException
 
     raise ExceptionClass(
         'Subsetter failed with error: ' + str(exception)
     ) from exception
+
+
+def handle_harmony_server_exception(url: str, harmony_exception: Exception):
+    """
+    Parses a harmony ServerException for the status code and raises the
+    appropriate UrlAccess exception based on whether it is a 500 or another code.
+
+    """
+
+    message = str(harmony_exception)
+
+    # Search for "status code: <number>" in the exception message
+    match = re.search(r'status code:\s*(\d+)', message)
+
+    if match:
+        status_code = int(match.group(1))
+
+        # Retry downloads on HTTP 500 reponse code (transient)
+        # Fail without retries for all other status codes
+        if status_code == 500:
+            raise UrlAccessFailed(url, 500) from harmony_exception
+
+        raise UrlAccessFailedWithNoRetries(url, status_code) from harmony_exception
+
+    # Fallback in case the exception message doesn't contain a status code
+    # Defaulting to 500
+    raise UrlAccessFailed(url, 500) from harmony_exception

tests/unit/test_adapter.py

@@ -1,4 +1,5 @@
 import json
+from os.path import join as path_join
 from typing import Dict, List, Optional
 from unittest import TestCase
 from unittest.mock import ANY, patch
@@ -8,6 +9,7 @@
 
 from hoss.adapter import HossAdapter
 from hoss.bbox_utilities import BBox
+from hoss.exceptions import CustomError, StagingFailed
 from tests.utilities import Granule, create_stac, spy_on
 
 
@@ -33,6 +35,7 @@ def setUpClass(cls):
         cls.africa_stac = create_stac(
             [Granule(cls.africa_granule_url, None, ['opendap', 'data'])]
         )
+        cls.staging_location = 's3://example-bucket'
 
     def setUp(self):
         self.config = config(validate=False)
@@ -559,3 +562,72 @@ def test_missing_variables(
             location='s3://example-bucket/',
             logger=hoss.logger,
         )
+
+    def test_hoss_stage(self, mock_stage, mock_subset_granule, mock_get_mimetype):
+        """Ensure hoss_stage successfully calls hoss_stage and returns the URL"""
+        mock_subset_granule.return_value = '/path/to/output.nc'
+        expected_output_basename = '/path/to/output.nc'
+        mock_get_mimetype.return_value = ('application/x-netcdf4', None)
+        expected_staged_url = path_join(self.staging_location, expected_output_basename)
+        mock_stage.return_value = expected_staged_url
+
+        collection_short_name = 'harmony_example_l2'
+
+        message = self.create_message(
+            'C1233860183-EEDTEST', collection_short_name, [], 'jlovell'
+        )
+
+        hoss = HossAdapter(message, config=self.config, catalog=self.africa_stac)
+
+        with patch.object(HossAdapter, 'process_item', self.process_item_spy):
+            hoss.invoke()
+
+        mock_subset_granule.assert_called_once_with(
+            self.africa_granule_url,
+            message.sources[0],
+            ANY,
+            hoss.message,
+            hoss.config,
+        )
+
+        mock_get_mimetype.assert_called_once_with('/path/to/output.nc')
+
+        result = hoss.hoss_stage(
+            expected_output_basename,
+            'africa.nc4',
+            'application/x-netcdf4',
+        )
+
+        assert result == expected_staged_url
+
+    def test_hoss_stage_raises_exception(
+        self, mock_stage, mock_subset_granule, mock_get_mimetype
+    ):
+        """Ensure hoss_stage raises an StagingFailed exception when
+        the Harmony stage function throws an exception.
+
+        """
+        mock_stage.side_effect = Exception("Connection timeout")
+        mock_subset_granule.return_value = '/path/to/output.nc'
+        mock_get_mimetype.return_value = ('application/x-netcdf4', None)
+
+        collection_short_name = 'harmony_example_l2'
+
+        message = self.create_message(
+            'C1233860183-EEDTEST', collection_short_name, [], 'jlovell'
+        )
+
+        hoss = HossAdapter(message, config=self.config, catalog=self.africa_stac)
+
+        with self.assertRaises(StagingFailed) as context_manager:
+            hoss.hoss_stage(
+                '/path/to/output.nc',
+                'africa.nc4',
+                'application/x-netcdf4',
+            )
+
+        self.assertIsInstance(context_manager.exception, CustomError)
+        self.assertEqual(
+            str(context_manager.exception),
+            'Staging failed, Connection timeout',
+        )

tests/unit/test_utilities.py

@@ -13,7 +13,9 @@
 from hoss.exceptions import (
     CustomError,
     CustomNoRetryError,
+    StagingFailed,
     UrlAccessFailed,
+    UrlAccessFailedWithNoRetries,
     UrlAccessForbidden,
 )
 from hoss.harmony_log_context import set_logger
@@ -37,6 +39,12 @@ class TestUtilities(TestCase):
     @classmethod
     def setUpClass(cls):
         cls.harmony_500_error = ServerException('I can\'t do that')
+        cls.harmony_500_status_code_error = ServerException(
+            'Unable to download due to status code: 500 and content'
+        )
+        cls.harmony_400_status_code_error = ServerException(
+            'Unable to download due to status code: 400 and content'
+        )
         cls.harmony_auth_error = ForbiddenException('You can\'t do that.')
         cls.config = config(validate=False)
         cls.logger = getLogger('test')
@@ -129,6 +137,28 @@ def test_download_url(self, mock_util_download):
             )
             mock_util_download.reset_mock()
 
+        with self.subTest('Harmony server exception 500 error should be retried.'):
+            mock_util_download.side_effect = [
+                self.harmony_500_status_code_error,
+                http_response,
+            ]
+
+            with self.assertRaises(UrlAccessFailed) as context:
+                download_url(test_url, output_directory, access_token, self.config)
+
+            self.assertIsInstance(context.exception, CustomError)
+            self.assertNotIsInstance(context.exception, CustomNoRetryError)
+
+            mock_util_download.assert_called_once_with(
+                test_url,
+                output_directory,
+                self.logger,
+                access_token=access_token,
+                data=None,
+                cfg=self.config,
+            )
+            mock_util_download.reset_mock()
+
         with self.subTest('A 403 error (forbidden) is not retried.'):
             mock_util_download.side_effect = [self.harmony_auth_error, http_response]
 
@@ -147,6 +177,28 @@ def test_download_url(self, mock_util_download):
             )
             mock_util_download.reset_mock()
 
+        with self.subTest('A 400 error (Bad Request) is not retried.'):
+            mock_util_download.side_effect = [
+                self.harmony_400_status_code_error,
+                http_response,
+            ]
+
+            with self.assertRaises(UrlAccessFailedWithNoRetries) as context:
+                download_url(test_url, output_directory, access_token, self.config)
+
+            self.assertIsInstance(context.exception, CustomNoRetryError)
+            self.assertNotIsInstance(context.exception, CustomError)
+
+            mock_util_download.assert_called_once_with(
+                test_url,
+                output_directory,
+                self.logger,
+                access_token=access_token,
+                data=None,
+                cfg=self.config,
+            )
+            mock_util_download.reset_mock()
+
         with self.subTest('Unknown/transient error is caught and is retryable.'):
             mock_util_download.side_effect = [
                 Exception('something broke'),
@@ -393,3 +445,16 @@ def test_raise_from_hoss_exception(self):
             with self.assertRaises(HarmonyException) as context:
                 raise_from_hoss_exception(failed_exception)
             self.assertNotIsInstance(context.exception, NoRetryException)
+
+        with self.subTest(
+            'UrlAccessFailedWithNoRetries (no-retry) raises NoRetryException.'
+        ):
+            failed_exception = UrlAccessFailedWithNoRetries(test_url, 400)
+            with self.assertRaises(NoRetryException):
+                raise_from_hoss_exception(failed_exception)
+
+        with self.subTest('StagingFailed (retryable) raises HarmonyException.'):
+            failed_exception = StagingFailed('Staging failed, Connection timeout')
+            with self.assertRaises(HarmonyException) as context:
+                raise_from_hoss_exception(failed_exception)
+            self.assertNotIsInstance(context.exception, NoRetryException)