Merge branch 'master' into dependabot/npm_and_yarn/micromatch-4.0.8

Author: qtomlinson

.github/workflows/build-and-deploy-dev.yml

@@ -0,0 +1,36 @@
+# This workflow will build a docker image, push it to ghcr.io, and deploy it to an Azure WebApp.
+name: Build and Deploy -- DEV
+
+on:
+  workflow_dispatch:
+  push:
+    branches: [master]
+
+jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
+  build-and-deploy:
+    name: Build and Deploy
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v3.2.0
+    secrets:
+      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_DEV }}
+      DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
+      PRODUCTION_DEPLOYERS: ${{ secrets.PRODUCTION_DEPLOYERS }}
+    with:
+      deploy-env: dev
+      application-type: worker
+      azure-app-base-name: cdcrawler
+      azure-app-name-postfix: -dev

.github/workflows/build-and-deploy-prod.yml

@@ -0,0 +1,37 @@
+# This workflow will build a docker image, push it to ghcr.io, and deploy it to an Azure WebApp.
+name: Build and Deploy -- PROD
+
+on:
+  workflow_dispatch:
+  release:
+    types: [published]
+
+jobs:
+  upload-package-lock-json:
+    name: Upload package-lock.json from this repo
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4.1.1
+
+      - name: Upload package-lock.json
+        uses: actions/upload-artifact@v4
+        with:
+          name: package-lock.json
+          path: package-lock.json
+
+  build-and-deploy-prod:
+    needs: upload-package-lock-json
+    uses: clearlydefined/operations/.github/workflows/app-build-and-deploy.yml@v3.2.0
+    secrets:
+      AZURE_CREDENTIALS: ${{ secrets.AZURE_CREDENTIALS }}
+      AZURE_WEBAPP_PUBLISH_PROFILE: ${{ secrets.AZURE_WEBAPP_PUBLISH_PROFILE_PROD }}
+      DEPLOY_TOKEN: ${{ secrets.DEPLOY_TOKEN }}
+      PRODUCTION_DEPLOYERS: ${{ secrets.PRODUCTION_DEPLOYERS }}
+      DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} 
+    with:
+      deploy-env: prod
+      application-type: worker
+      azure-app-base-name: cdcrawler
+      azure-app-name-postfix: -prod
+      docker-hub-username: ${{ vars.DOCKERHUB_USERNAME }}

DevDockerfile

@@ -4,8 +4,13 @@
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
 
+# Set environment variables from build arguments
 ARG BUILD_NUMBER=0
-ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
+ENV BUILD_NUMBER=$APP_VERSION
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 # Ruby and Python Dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests curl bzip2 build-essential libssl-dev libreadline-dev zlib1g-dev cmake python3 python3-dev python3-pip xz-utils libxml2-dev libxslt1-dev libpopt0 && \

Dockerfile

@@ -4,8 +4,11 @@
 FROM node:18-bullseye
 ENV APPDIR=/opt/service
 
-ARG BUILD_NUMBER=0
-ENV CRAWLER_BUILD_NUMBER=$BUILD_NUMBER
+# Set environment variables from build arguments
+ARG APP_VERSION="UNKNOWN"
+ENV APP_VERSION=$APP_VERSION
+ARG BUILD_SHA="UNKNOWN"
+ENV BUILD_SHA=$BUILD_SHA
 
 # Ruby and Python Dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends --no-install-suggests curl bzip2 build-essential libssl-dev libreadline-dev zlib1g-dev cmake python3 python3-dev python3-pip xz-utils libxml2-dev libxslt1-dev libpopt0 && \

config/cdConfig.js

@@ -5,7 +5,10 @@ const config = require('painless-config')
 
 const cd_azblob = {
   connection: config.get('CRAWLER_AZBLOB_CONNECTION_STRING'),
-  container: config.get('CRAWLER_AZBLOB_CONTAINER_NAME')
+  container: config.get('CRAWLER_AZBLOB_CONTAINER_NAME'),
+  account: config.get('CRAWLER_AZBLOB_ACCOUNT_NAME'),
+  spnAuth: config.get('CRAWLER_AZBLOB_SPN_AUTH'),
+  isSpnAuth: config.get('CRAWLER_AZBLOB_IS_SPN_AUTH') || false
 }
 
 const githubToken = config.get('CRAWLER_GITHUB_TOKEN')
@@ -95,10 +98,11 @@ module.exports = {
         // '--quiet'
       ],
       timeout: 1000,
-      processes: 2,
+      processes: Number(config.get('CRAWLER_SCANCODE_PARALLELISM') || process.env.CRAWLER_SCANCODE_PARALLELISM) || 2,
       format: '--json-pp'
     },
     source: {},
+    sourcearchive: {},
     top: { githubToken }
   },
   store: {
@@ -110,7 +114,10 @@ module.exports = {
     },
     azqueue: {
       connectionString: cd_azblob.connection,
-      queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests'
+      account: cd_azblob.account,
+      queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests',
+      spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'),
+      isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false
     },
     'cd(azblob)': cd_azblob,
     'cd(file)': cd_file
@@ -134,7 +141,12 @@ module.exports = {
       maxDequeueCount: 5,
       attenuation: {
         ttl: 3000
-      }
-    }
+      },
+      spnAuth: config.get('CRAWLER_QUEUE_AZURE_SPN_AUTH') || cd_azblob.spnAuth,
+      account: config.get('CRAWLER_QUEUE_AZURE_ACCOUNT_NAME') || cd_azblob.account,
+      isSpnAuth: config.get('CRAWLER_QUEUE_AZURE_IS_SPN_AUTH') || false
+    },
+    appVersion: config.get('APP_VERSION'),
+    buildsha: config.get('BUILD_SHA')
   }
 }

ghcrawler/app.js

@@ -22,9 +22,7 @@ function configureApp(service, logger) {
   app.use('/requests', require('./routes/requests')(service))
 
   // to keep AlwaysOn flooding logs with errors
-  app.get('/', (request, response) => {
-    response.helpers.send.noContent()
-  })
+  app.use('/', require('./routes/index')(config.get('BUILD_SHA'), config.get('APP_VERSION')))
 
   // Catch 404 and forward to error handler
   const requestHandler = (request, response, next) => {

ghcrawler/lib/crawler.js

@@ -638,7 +638,7 @@ class Crawler {
       metadata.errorMessage = request._error.message
       metadata.errorStack = request._error.stack
     }
-    metadata.version = 1
+    metadata.version = '1'
     metadata.meta = request.meta
     metadata.type = 'deadletter'
     metadata.url = request.url.replace('//', '//deadletter.')

ghcrawler/lib/traversalPolicy.js

@@ -143,6 +143,10 @@ class TraversalPolicy {
     return new TraversalPolicy('storageOnly', 'always', TraversalPolicy._resolveMapSpec(map))
   }
 
+  static reharvestAlways(map) {
+    return new TraversalPolicy('mutables', 'always', TraversalPolicy._resolveMapSpec(map))
+  }
+
   static clone(policy) {
     return new TraversalPolicy(policy.fetch, policy.freshness, policy.map)
   }