fix(otlp-exporter-base): ensure retry on network errors during HTTP export

OpenTelemetry OTLP/HTTP specification states:

```
If the server disconnects without returning a response, the client SHOULD
retry and send the same request. The client SHOULD implement an exponential
backoff strategy between retries to avoid overwhelming the server.
...
If the client cannot connect to the server, the client SHOULD retry the
connection using an exponential backoff strategy between retries.
The interval between retries must have a random jitter.
```

The backoff infrastrucure was in place, it was just the glue code to
request APIs (fetch, http, xhr) that was reporting non-retryable state
in case of errors that might be temporary.

Author: Jakub Sokół

experimental/packages/otlp-exporter-base/src/is-export-retryable.ts

@@ -14,11 +14,63 @@
  * limitations under the License.
  */
 
-export function isExportRetryable(statusCode: number): boolean {
+export function isExportHTTPErrorRetryable(statusCode: number): boolean {
   const retryCodes = [429, 502, 503, 504];
   return retryCodes.includes(statusCode);
 }
 
+function getErrorCode(error: unknown): string | undefined {
+  if (!error || typeof error !== 'object') {
+    return undefined;
+  }
+
+  if ('code' in error && typeof error.code === 'string') {
+    return error.code;
+  }
+
+  const err = error as Error;
+  if (err.cause && typeof err.cause === 'object' && 'code' in err.cause) {
+    const code = (err.cause as { code: unknown }).code;
+    if (typeof code === 'string') {
+      return code;
+    }
+  }
+
+  return undefined;
+}
+
+export function isExportNetworkErrorRetryable(error: Error): boolean {
+  const RETRYABLE_ERROR_CODES = new Set([
+    'ECONNRESET',
+    'ECONNREFUSED',
+    'EPIPE',
+    'ETIMEDOUT',
+    'EAI_AGAIN',
+    'ENOTFOUND',
+    'ENETUNREACH',
+    'EHOSTUNREACH',
+    'UND_ERR_CONNECT_TIMEOUT',
+    'UND_ERR_HEADERS_TIMEOUT',
+    'UND_ERR_BODY_TIMEOUT',
+    'UND_ERR_SOCKET',
+  ]);
+
+  if (error.name === 'AbortError') {
+    return false;
+  }
+
+  const code = getErrorCode(error);
+  if (code && RETRYABLE_ERROR_CODES.has(code)) {
+    return true;
+  }
+
+  if (error instanceof TypeError && !error.cause) {
+    return true;
+  }
+
+  return false;
+}
+
 export function parseRetryAfterToMills(
   retryAfter?: string | undefined | null
 ): number | undefined {

experimental/packages/otlp-exporter-base/src/transport/fetch-transport.ts

@@ -18,7 +18,8 @@ import { IExporterTransport } from '../exporter-transport';
 import { ExportResponse } from '../export-response';
 import { diag } from '@opentelemetry/api';
 import {
-  isExportRetryable,
+  isExportHTTPErrorRetryable,
+  isExportNetworkErrorRetryable,
   parseRetryAfterToMills,
 } from '../is-export-retryable';
 import { HeadersFactory } from '../configuration/otlp-http-configuration';
@@ -53,7 +54,7 @@ class FetchTransport implements IExporterTransport {
       if (response.status >= 200 && response.status <= 299) {
         diag.debug('response success');
         return { status: 'success' };
-      } else if (isExportRetryable(response.status)) {
+      } else if (isExportHTTPErrorRetryable(response.status)) {
         const retryAfter = response.headers.get('Retry-After');
         const retryInMillis = parseRetryAfterToMills(retryAfter);
         return { status: 'retryable', retryInMillis };
@@ -63,10 +64,10 @@ class FetchTransport implements IExporterTransport {
         error: new Error('Fetch request failed with non-retryable status'),
       };
     } catch (error) {
-      if (error?.name === 'AbortError') {
+      if (isExportNetworkErrorRetryable(error)) {
         return {
-          status: 'failure',
-          error: new Error('Fetch request timed out', { cause: error }),
+          status: 'retryable',
+          retryInMillis: 0,
         };
       }
       return {

experimental/packages/otlp-exporter-base/src/transport/http-transport-utils.ts

@@ -19,7 +19,8 @@ import * as zlib from 'zlib';
 import { Readable } from 'stream';
 import { ExportResponse } from '../export-response';
 import {
-  isExportRetryable,
+  isExportHTTPErrorRetryable,
+  isExportNetworkErrorRetryable,
   parseRetryAfterToMills,
 } from '../is-export-retryable';
 import { OTLPExporterError } from '../types';
@@ -74,7 +75,7 @@ export function sendWithHttp(
           status: 'success',
           data: Buffer.concat(responseData),
         });
-      } else if (res.statusCode && isExportRetryable(res.statusCode)) {
+      } else if (res.statusCode && isExportHTTPErrorRetryable(res.statusCode)) {
         onDone({
           status: 'retryable',
           retryInMillis: parseRetryAfterToMills(res.headers['retry-after']),
@@ -96,16 +97,23 @@ export function sendWithHttp(
   req.setTimeout(timeoutMillis, () => {
     req.destroy();
     onDone({
-      status: 'failure',
-      error: new Error('Request Timeout'),
+      status: 'retryable',
+      retryInMillis: 0,
     });
   });
 
   req.on('error', (error: Error) => {
-    onDone({
-      status: 'failure',
-      error,
-    });
+    if (isExportNetworkErrorRetryable(error)) {
+      onDone({
+        status: 'retryable',
+        retryInMillis: 0,
+      });
+    } else {
+      onDone({
+        status: 'failure',
+        error,
+      });
+    }
   });
 
   compressAndSend(req, compression, data, (error: Error) => {

experimental/packages/otlp-exporter-base/src/transport/xhr-transport.ts

@@ -18,7 +18,7 @@ import { IExporterTransport } from '../exporter-transport';
 import { ExportResponse } from '../export-response';
 import { diag } from '@opentelemetry/api';
 import {
-  isExportRetryable,
+  isExportHTTPErrorRetryable,
   parseRetryAfterToMills,
 } from '../is-export-retryable';
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
@@ -49,8 +49,8 @@ class XhrTransport implements IExporterTransport {
 
       xhr.ontimeout = _ => {
         resolve({
-          status: 'failure',
-          error: new Error('XHR request timed out'),
+          status: 'retryable',
+          retryInMillis: 0,
         });
       };
 
@@ -60,7 +60,7 @@ class XhrTransport implements IExporterTransport {
           resolve({
             status: 'success',
           });
-        } else if (xhr.status && isExportRetryable(xhr.status)) {
+        } else if (xhr.status && isExportHTTPErrorRetryable(xhr.status)) {
           resolve({
             status: 'retryable',
             retryInMillis: parseRetryAfterToMills(
@@ -82,9 +82,10 @@ class XhrTransport implements IExporterTransport {
         });
       };
       xhr.onerror = () => {
+        // XHR onerror typically indicates network failures which are retryable
         resolve({
-          status: 'failure',
-          error: new Error('XHR request errored'),
+          status: 'retryable',
+          retryInMillis: 0,
         });
       };
 

experimental/packages/otlp-exporter-base/test/browser/fetch-transport.test.ts

@@ -122,7 +122,7 @@ describe('FetchTransport', function () {
       }, done /* catch any rejections */);
     });
 
-    it('returns failure when request times out', function (done) {
+    it('returns failure when request is aborted', function (done) {
       // arrange
       const abortError = new Error('aborted request');
       abortError.name = 'AbortError';
@@ -137,7 +137,7 @@ describe('FetchTransport', function () {
           assert.strictEqual(response.status, 'failure');
           assert.strictEqual(
             (response as ExportResponseFailure).error.message,
-            'Fetch request timed out'
+            'Fetch request errored'
           );
         } catch (e) {
           done(e);
@@ -147,7 +147,7 @@ describe('FetchTransport', function () {
       clock.tick(requestTimeout + 100);
     });
 
-    it('returns failure when no server exists', function (done) {
+    it('returns failure when fetch throws non-network error', function (done) {
       // arrange
       sinon.stub(globalThis, 'fetch').throws(new Error('fetch failed'));
       const clock = sinon.useFakeTimers();
@@ -169,5 +169,51 @@ describe('FetchTransport', function () {
       }, done /* catch any rejections */);
       clock.tick(requestTimeout + 100);
     });
+
+    it('returns retryable when browser fetch throws network error', function (done) {
+      // arrange
+      // Browser fetch throws TypeError for network errors
+      sinon.stub(globalThis, 'fetch').rejects(new TypeError('Failed to fetch'));
+      const transport = createFetchTransport(testTransportParameters);
+
+      //act
+      transport.send(testPayload, requestTimeout).then(response => {
+        // assert
+        try {
+          assert.strictEqual(response.status, 'retryable');
+          assert.strictEqual(
+            (response as ExportResponseRetryable).retryInMillis,
+            0
+          );
+        } catch (e) {
+          done(e);
+        }
+        done();
+      }, done /* catch any rejections */);
+    });
+
+    it('returns retryable when fetch throws network error with code', function (done) {
+      // arrange
+      const cause = new Error('network error') as NodeJS.ErrnoException;
+      cause.code = 'ECONNRESET';
+      const networkError = new TypeError('fetch failed', { cause });
+      sinon.stub(globalThis, 'fetch').rejects(networkError);
+      const transport = createFetchTransport(testTransportParameters);
+
+      //act
+      transport.send(testPayload, requestTimeout).then(response => {
+        // assert
+        try {
+          assert.strictEqual(response.status, 'retryable');
+          assert.strictEqual(
+            (response as ExportResponseRetryable).retryInMillis,
+            0
+          );
+        } catch (e) {
+          done(e);
+        }
+        done();
+      }, done /* catch any rejections */);
+    });
   });
 });

experimental/packages/otlp-exporter-base/test/browser/xhr-transport.test.ts

@@ -19,8 +19,8 @@ import * as assert from 'assert';
 import { createXhrTransport } from '../../src/transport/xhr-transport';
 import {
   ExportResponseRetryable,
-  ExportResponseFailure,
   ExportResponseSuccess,
+  ExportResponseFailure,
 } from '../../src';
 import { ensureHeadersContain } from '../testHelper';
 
@@ -58,6 +58,14 @@ function hasOnTimeout(request: unknown): request is { ontimeout: () => void } {
   return 'ontimeout' in request && typeof request['ontimeout'] === 'function';
 }
 
+function hasOnAbort(request: unknown): request is { onabort: () => void } {
+  if (request == null || typeof request != 'object') {
+    return false;
+  }
+
+  return 'onabort' in request && typeof request['onabort'] === 'function';
+}
+
 describe('XhrTransport', function () {
   afterEach(function () {
     sinon.restore();
@@ -151,7 +159,7 @@ describe('XhrTransport', function () {
       }, done /* catch any rejections */);
     });
 
-    it('returns failure when request times out', function (done) {
+    it('returns retryable when request times out', function (done) {
       // arrange
       // A fake server needed, otherwise the message will not be a timeout but a failure to connect.
       const server = sinon.useFakeServer();
@@ -170,10 +178,10 @@ describe('XhrTransport', function () {
       transport.send(testPayload, requestTimeout).then(response => {
         // assert
         try {
-          assert.strictEqual(response.status, 'failure');
+          assert.strictEqual(response.status, 'retryable');
           assert.strictEqual(
-            (response as ExportResponseFailure).error.message,
-            'XHR request timed out'
+            (response as ExportResponseRetryable).retryInMillis,
+            0
           );
         } catch (e) {
           done(e);
@@ -182,26 +190,56 @@ describe('XhrTransport', function () {
       }, done /* catch any rejections */);
     });
 
-    it('returns failure when no server exists', function (done) {
+    it('returns retryable when network error occurs', function (done) {
       // arrange
       const clock = sinon.useFakeTimers();
       const transport = createXhrTransport(testTransportParameters);
 
+      //act
+      transport.send(testPayload, requestTimeout).then(response => {
+        // assert
+        try {
+          assert.strictEqual(response.status, 'retryable');
+          assert.strictEqual(
+            (response as ExportResponseRetryable).retryInMillis,
+            0
+          );
+        } catch (e) {
+          done(e);
+        }
+        done();
+      }, done /* catch any rejections */);
+      clock.tick(requestTimeout + 100);
+    });
+
+    it('returns failure when request is aborted', function (done) {
+      // arrange
+      const server = sinon.useFakeServer();
+      const transport = createXhrTransport(testTransportParameters);
+
+      respondWhenRequestExists(server, () => {
+        // this executes after the act block
+        const request = server.requests[0];
+        // Manually trigger the onabort event
+        if (hasOnAbort(request)) {
+          request.onabort();
+        }
+      });
+
       //act
       transport.send(testPayload, requestTimeout).then(response => {
         // assert
         try {
           assert.strictEqual(response.status, 'failure');
           assert.strictEqual(
             (response as ExportResponseFailure).error.message,
-            'XHR request errored'
+            'XHR request aborted'
           );
         } catch (e) {
           done(e);
         }
         done();
       }, done /* catch any rejections */);
-      clock.tick(requestTimeout + 100);
     });
   });
 });