Skip to content

Commit 31740a9

Browse files
qtomlinsonqtomlinson
authored
Make retry configurable (#128)
- Increase timeout for test cases. - Add configuration for rest interval. - Add configuration for max retry count during fetch. - Implement retry logic for error code 429.
1 parent 549600b commit 31740a9

File tree

10 files changed

+40
-31
lines changed

10 files changed

+40
-31
lines changed

‎tools/integration/lib/fetch.js‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
const fetch = (...args) => import('node-fetch').then(({ default: fetch }) => fetch(...args))
55
// TODO: remove this once fetch is available in Node
66
const retry = require('async-retry')
7-
const RETRIES = 2
87

98
function buildPostOpts(json) {
109
return {
@@ -34,7 +33,6 @@ function verifyResponse(response) {
3433

3534
async function withRetry(retrier, opts = {}) {
3635
const defaultOpts = {
37-
retries: RETRIES,
3836
onRetry: (err, iAttempt) => {
3937
console.log(`Retry ${iAttempt} failed: ${err}`)
4038
}
@@ -45,15 +43,14 @@ async function withRetry(retrier, opts = {}) {
4543
async function fetchWithRetry(fetcher, retryOpts) {
4644
const response = await withRetry(async () => {
4745
const resp = await fetcher()
48-
// retry on 5xx
49-
if (resp?.status >= 500) verifyResponse(resp)
46+
if (resp?.status >= 500 || resp?.status === 429) verifyResponse(resp)
5047
return resp
5148
}, retryOpts)
5249
return verifyResponse(response)
5350
}
5451

55-
async function callFetchWithRetry(url, fetchOpts) {
56-
return fetchWithRetry(() => fetchResponse(url, fetchOpts))
52+
function createFetcherWithRetry({ maxRetries }) {
53+
return (url, fetchOpts) => fetchWithRetry(() => fetchResponse(url, fetchOpts), { retries: maxRetries })
5754
}
5855

59-
module.exports = { callFetch, buildPostOpts, callFetchWithRetry, fetchWithRetry }
56+
module.exports = { callFetch, buildPostOpts, createFetcherWithRetry, fetchWithRetry }

‎tools/integration/test/integration/e2e-test-service/attachmentTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,18 @@
11
// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
22
// SPDX-License-Identifier: MIT
33

4-
const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
5-
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
4+
const { createFetcherWithRetry } = require('../../../lib/fetch')
5+
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition, fetchRetry } = require('../testConfig')
66
const { strictEqual } = require('assert')
7+
const callFetch = createFetcherWithRetry(fetchRetry)
78

89
;(async function () {
910
const components = await getComponents()
1011
describe('Validation attachments between dev and prod', async function () {
1112
this.timeout(definition.timeout * 2)
1213

1314
//Rest a bit to avoid overloading the servers
14-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
15+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1516

1617
components.forEach(coordinates => {
1718
it(`should have the same attachement as prod for ${coordinates}`, () => fetchAndCompareAttachments(coordinates))

‎tools/integration/test/integration/e2e-test-service/curationTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,15 @@
22
// SPDX-License-Identifier: MIT
33

44
const { deepStrictEqual, strictEqual, ok } = require('assert')
5-
const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
6-
const { devApiBaseUrl, definition, curation } = require('../testConfig')
5+
const { createFetcherWithRetry, buildPostOpts } = require('../../../lib/fetch')
6+
const { devApiBaseUrl, definition, curation, fetchRetry } = require('../testConfig')
7+
const callFetch = createFetcherWithRetry(fetchRetry)
78

89
describe('Validate curation', function () {
910
this.timeout(definition.timeout)
1011

1112
//Rest a bit to avoid overloading the servers
12-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
13+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1314

1415
const coordinates = 'maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16'
1516
const title = curation.title || `test ${coordinates}`

‎tools/integration/test/integration/e2e-test-service/definitionTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,19 @@
33

44
const { omit, isEqual, pick } = require('lodash')
55
const { deepStrictEqual, strictEqual, ok } = require('assert')
6-
const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
7-
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
6+
const { createFetcherWithRetry, buildPostOpts } = require('../../../lib/fetch')
7+
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition, fetchRetry } = require('../testConfig')
88
const nock = require('nock')
99
const fs = require('fs')
10+
const callFetch = createFetcherWithRetry(fetchRetry)
1011

1112
;(async function () {
1213
const components = await getComponents()
1314
describe('Validate definitions', function () {
1415
this.timeout(definition.timeout)
1516

1617
//Rest a bit to avoid overloading the servers
17-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
18+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1819

1920
describe('Validation between dev and prod', function () {
2021
before(() => {

‎tools/integration/test/integration/e2e-test-service/noticeTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,19 @@
22
// SPDX-License-Identifier: MIT
33

44
const { deepStrictEqual } = require('assert')
5-
const { callFetchWithRetry: callFetch, buildPostOpts } = require('../../../lib/fetch')
6-
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition } = require('../testConfig')
5+
const { createFetcherWithRetry, buildPostOpts } = require('../../../lib/fetch')
6+
const { devApiBaseUrl, prodApiBaseUrl, getComponents, definition, fetchRetry } = require('../testConfig')
77
const nock = require('nock')
88
const fs = require('fs')
9+
const callFetch = createFetcherWithRetry(fetchRetry)
910

1011
;(async function () {
1112
const components = await getComponents()
1213
describe('Validate notice files between dev and prod', async function () {
1314
this.timeout(definition.timeout)
1415

1516
//Rest a bit to avoid overloading the servers
16-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout / 2)))
17+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1718

1819
before(() => {
1920
loadFixtures().forEach(([coordinatesString, notice]) => {

‎tools/integration/test/integration/e2e-test-service/originsTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,9 @@
22
// SPDX-License-Identifier: MIT
33

44
const assert = require('assert')
5-
const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
6-
const { devApiBaseUrl, getComponents, origins } = require('../testConfig')
5+
const { createFetcherWithRetry } = require('../../../lib/fetch')
6+
const { devApiBaseUrl, getComponents, origins, fetchRetry } = require('../testConfig')
7+
const callFetch = createFetcherWithRetry(fetchRetry)
78

89
const ORIGIN_EXCLUSION_LIST = ['go/golang', 'debsrc/debian', 'maven/mavengoogle']
910
const ORIGIN_REVISIONS_EXCLUSION_LIST = ['debsrc/debian']
@@ -18,7 +19,7 @@ const GRADLE_COMPONENT_ENDPOINT = 'gradleplugin/io.github.lognet.grpc-spring-boo
1819
describe('Validate Origins API Between Dev and Prod', function () {
1920
this.timeout(origins.timeout)
2021

21-
afterEach(() => new Promise(resolve => setTimeout(resolve, origins.timeout / 2)))
22+
afterEach(() => new Promise(resolve => setTimeout(resolve, origins.interval)))
2223

2324
components.filter(isComponentIncluded).forEach(component => {
2425
it(`Validates Origins API response for ${component}`, () => validateOriginResponses(component))

‎tools/integration/test/integration/e2e-test-service/statsTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
22
// SPDX-License-Identifier: MIT
33

4-
const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
5-
const { devApiBaseUrl, definition } = require('../testConfig')
4+
const { createFetcherWithRetry } = require('../../../lib/fetch')
5+
const { devApiBaseUrl, definition, fetchRetry } = require('../testConfig')
66
const { ok } = require('assert')
7+
const callFetch = createFetcherWithRetry(fetchRetry)
78

89
describe('Test for StatsService', function () {
910
this.timeout(definition.timeout * 10)
1011

1112
//Rest a bit to avoid overloading the servers
12-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
13+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1314

1415
it('should retrieve the list of supported stats', async function () {
1516
const url = `${devApiBaseUrl}/stats`

‎tools/integration/test/integration/e2e-test-service/statusTest.js‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,16 @@
11
// (c) Copyright 2024, SAP SE and ClearlyDefined contributors. Licensed under the MIT license.
22
// SPDX-License-Identifier: MIT
33

4-
const { callFetchWithRetry: callFetch } = require('../../../lib/fetch')
5-
const { devApiBaseUrl, definition } = require('../testConfig')
4+
const { createFetcherWithRetry } = require('../../../lib/fetch')
5+
const { devApiBaseUrl, definition, fetchRetry } = require('../testConfig')
66
const { ok } = require('assert')
7+
const callFetch = createFetcherWithRetry(fetchRetry)
78

89
describe('Test for StatusService', function () {
910
this.timeout(definition.timeout * 10)
1011

1112
//Rest a bit to avoid overloading the servers
12-
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.timeout)))
13+
afterEach(() => new Promise(resolve => setTimeout(resolve, definition.interval)))
1314

1415
it('should retrieve the list of supported status queries', async function () {
1516
const url = `${devApiBaseUrl}/status`

‎tools/integration/test/integration/testConfig.js‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,17 @@ module.exports = {
8686
timeout: 1000 * 60 * 60 * 4 // 4 hours for harvesting all the components
8787
},
8888
definition: {
89-
timeout: 1000 * 60 // for each component
89+
timeout: 1000 * 60 * 5, // for each component
90+
interval: 1000 * 3 // between requests
9091
},
9192
curation: {
9293
title: 'test maven/mavencentral/org.apache.httpcomponents/httpcore/4.4.16'
9394
},
9495
origins: {
95-
timeout: 1000 * 60 * 2
96+
timeout: 1000 * 60 * 5,
97+
interval: 1000 * 3 // between requests
98+
},
99+
fetchRetry: {
100+
maxRetries: 5
96101
}
97102
}

‎tools/integration/test/lib/fetchTest.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ describe('Test retry fetch', function () {
7575
async function test(fetcher) {
7676
let content = '',
7777
errorMessage = ''
78-
const retryOpts = { minTimeout: 10, retry: RETRIES }
78+
const retryOpts = { minTimeout: 10, retries: RETRIES }
7979
try {
8080
const resp = await fetchWithRetry(fetcher, retryOpts)
8181
content = await resp.text()

0 commit comments

Comments
 (0)