Updated the default value of CRAWLER_HARVESTS_QUEUE_VISIBILITY_TIMEOUT_SECONDS

Changed the default value of CRAWLER_HARVESTS_QUEUE_VISIBILITY_TIMEOUT_SECONDS to 0 to prevent any breaking changes in behavior.

For production crawlers, this value is explicitly set to 300 seconds (5 minutes) to ensure the desired functionality.

Author: qtomlinson

Dockerfile

@@ -48,6 +48,7 @@ ENV CRAWLER_NAME=cdcrawlerprod
 ENV CRAWLER_QUEUE_PREFIX=cdcrawlerprod
 ENV CRAWLER_QUEUE_PROVIDER=storageQueue
 ENV CRAWLER_STORE_PROVIDER=cdDispatch+cd(azblob)+azqueue
+ENV CRAWLER_HARVESTS_QUEUE_VISIBILITY_TIMEOUT_SECONDS=300
 ENV CRAWLER_WEBHOOK_URL=https://api.clearlydefined.io/webhook
 ENV CRAWLER_AZBLOB_CONTAINER_NAME=production
 

config/cdConfig.js

@@ -126,7 +126,7 @@ module.exports = {
       queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests',
       spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'),
       isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false,
-      visibilityTimeout: isNaN(azqueueVisibilityTimeoutSeconds) ? 5 * 60 : azqueueVisibilityTimeoutSeconds // 5 minutes default
+      visibilityTimeout: isNaN(azqueueVisibilityTimeoutSeconds) ? 0 : azqueueVisibilityTimeoutSeconds
     },
     'cd(azblob)': cd_azblob,
     'cd(file)': cd_file