remove polling

Author: Rares Cheseli

packages/spacecat-shared-drs-client/README.md

@@ -1,7 +1,7 @@
 # Spacecat Shared - DRS Client
 
 A JavaScript client for the DRS (Data Retrieval Service) API, part of the SpaceCat Shared library.
-It allows you to submit data retrieval jobs to DRS, poll for their completion, look up previously
+It allows you to submit data retrieval jobs to DRS, check job status, look up previously
 retrieved URLs, and download results via presigned S3 URLs.
 
 ## Installation
@@ -82,16 +82,6 @@ if (status.status === 'COMPLETED') {
 }
 ```
 
-### Polling for Job Completion
-
-```js
-const result = await client.pollJobStatus('job-123', {
-  pollIntervalMs: 15000, // default: 15s
-  maxTimeoutMs: 600000,  // default: 10min
-});
-console.log(result.status); // 'COMPLETED' or 'FAILED'
-```
-
 ### Looking Up URLs
 
 Check the availability of previously retrieved URLs. Each URL in the response

packages/spacecat-shared-drs-client/src/index.d.ts

@@ -88,11 +88,6 @@ export interface JobStatusResponse {
   result_url_expires_in?: number;
 }
 
-export interface PollOptions {
-  pollIntervalMs?: number;
-  maxTimeoutMs?: number;
-}
-
 export type LookupUrlStatus = 'available' | 'scraping' | 'not_found';
 
 export interface LookupUrlResult {
@@ -126,7 +121,5 @@ export default class DrsClient {
 
   getJobStatus(jobId: string): Promise<JobStatusResponse>;
 
-  pollJobStatus(jobId: string, options?: PollOptions): Promise<JobStatusResponse>;
-
   lookupUrls(urls: string[]): Promise<LookupUrlsResponse>;
 }

packages/spacecat-shared-drs-client/src/index.js

@@ -17,9 +17,6 @@ import {
 const HTTP_BAD_REQUEST = 400;
 const HTTP_INTERNAL_SERVER_ERROR = 500;
 
-const DEFAULT_POLL_INTERVAL_MS = 15000;
-const DEFAULT_MAX_TIMEOUT_MS = 600000;
-
 export const VALID_DATASET_IDS = Object.freeze([
   'youtube_videos',
   'youtube_comments',
@@ -35,15 +32,9 @@ export const JOB_STATUSES = Object.freeze({
   FAILED: 'FAILED',
 });
 
-const TERMINAL_STATUSES = new Set([JOB_STATUSES.COMPLETED, JOB_STATUSES.FAILED]);
-
-const delay = (ms) => new Promise((resolve) => {
-  setTimeout(resolve, ms);
-});
-
 /**
  * Client for the Data Retrieval Service (DRS) API.
- * Provides methods to submit data retrieval jobs, poll for completion,
+ * Provides methods to submit data retrieval jobs, check job status,
  * and look up previously retrieved URLs.
  */
 export default class DrsClient {
@@ -198,48 +189,6 @@ export default class DrsClient {
     return this.#sendRequest('GET', `/jobs/${jobId}?include_result_url=true`);
   }
 
-  /**
-   * Polls a DRS job until it reaches a terminal status (COMPLETED or FAILED)
-   * or the timeout is exceeded.
-   * @param {string} jobId - The job identifier
-   * @param {Object} [options] - Polling options
-   * @param {number} [options.pollIntervalMs=15000] - Interval between polls in milliseconds
-   * @param {number} [options.maxTimeoutMs=600000] - Maximum time to poll before timing out
-   * @returns {Promise<Object>} Final job status response
-   */
-  async pollJobStatus(
-    jobId,
-    { pollIntervalMs = DEFAULT_POLL_INTERVAL_MS, maxTimeoutMs = DEFAULT_MAX_TIMEOUT_MS } = {},
-  ) {
-    if (!hasText(jobId)) {
-      throw this.#createError('Job ID is required', HTTP_BAD_REQUEST);
-    }
-
-    const startTime = Date.now();
-
-    // eslint-disable-next-line no-constant-condition
-    while (true) {
-      // eslint-disable-next-line no-await-in-loop
-      const status = await this.getJobStatus(jobId);
-
-      if (TERMINAL_STATUSES.has(status.status)) {
-        return status;
-      }
-
-      const elapsed = Date.now() - startTime;
-      if (elapsed + pollIntervalMs > maxTimeoutMs) {
-        throw this.#createError(
-          `Polling for job ${jobId} timed out after ${maxTimeoutMs}ms. Last status: ${status.status}`,
-          HTTP_INTERNAL_SERVER_ERROR,
-        );
-      }
-
-      this.log.info(`Job ${jobId} status: ${status.status}. Polling again in ${pollIntervalMs}ms...`);
-      // eslint-disable-next-line no-await-in-loop
-      await delay(pollIntervalMs);
-    }
-  }
-
   /**
    * Looks up previously retrieved URLs in the DRS data store.
    * Each URL in the response will have a status of 'available', 'scraping',

packages/spacecat-shared-drs-client/test/index.test.js

@@ -440,94 +440,6 @@ describe('DrsClient', () => {
     });
   });
 
-  describe('pollJobStatus', () => {
-    it('resolves immediately when job is already completed', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusCompletedResponse);
-
-      const result = await client.pollJobStatus('job-123', { pollIntervalMs: 10, maxTimeoutMs: 100 });
-      expect(result).to.deep.equal(jobStatusCompletedResponse);
-    });
-
-    it('resolves when job completes after multiple polls', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusQueuedResponse);
-
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusRunningResponse);
-
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusCompletedResponse);
-
-      const result = await client.pollJobStatus('job-123', { pollIntervalMs: 10, maxTimeoutMs: 5000 });
-      expect(result).to.deep.equal(jobStatusCompletedResponse);
-      expect(log.info.callCount).to.equal(2);
-    });
-
-    it('resolves when job fails', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusRunningResponse);
-
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusFailedResponse);
-
-      const result = await client.pollJobStatus('job-123', { pollIntervalMs: 10, maxTimeoutMs: 5000 });
-      expect(result.status).to.equal('FAILED');
-    });
-
-    it('throws error on timeout', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .times(10)
-        .reply(200, jobStatusRunningResponse);
-
-      await expect(client.pollJobStatus('job-123', { pollIntervalMs: 10, maxTimeoutMs: 25 }))
-        .to.be.rejectedWith('Polling for job job-123 timed out after 25ms. Last status: RUNNING');
-    });
-
-    it('propagates API error during polling', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusRunningResponse);
-
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(500, { message: 'Internal Server Error' });
-
-      await expect(client.pollJobStatus('job-123', { pollIntervalMs: 10, maxTimeoutMs: 5000 }))
-        .to.be.rejectedWith('DRS API request to /jobs/job-123?include_result_url=true failed with status: 500 - Internal Server Error');
-    });
-
-    it('throws error when job ID is missing', async () => {
-      await expect(client.pollJobStatus('')).to.be.rejectedWith('Job ID is required');
-    });
-
-    it('uses default poll options when none provided', async () => {
-      nock(API_BASE_URL)
-        .get('/jobs/job-123')
-        .query({ include_result_url: 'true' })
-        .reply(200, jobStatusCompletedResponse);
-
-      const result = await client.pollJobStatus('job-123');
-      expect(result).to.deep.equal(jobStatusCompletedResponse);
-    });
-  });
-
   describe('lookupUrls', () => {
     const lookupUrls = [
       'https://www.reddit.com/r/technology/comments/abc123/post_title/',