Merge pull request #563 from clearlydefined/master

Merge master into prod branch in preparation for the next release

Author: qtomlinson

.github/workflows/test.yml

@@ -0,0 +1,29 @@
+name: Run tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/[email protected]
+
+      - uses: actions/[email protected]
+        with:
+          node-version: 18
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run tests
+        run: npm test

.vscode/settings.json

@@ -6,8 +6,8 @@
   "editor.folding": false,
   "editor.tabSize": 2,
   "editor.detectIndentation": false,
-  "editor.formatOnSave": true,
+  "editor.formatOnSave": false,
   "editor.formatOnType": true,
   "editor.insertSpaces": true,
   "files.trimTrailingWhitespace": true
-}
+}

DevDockerfile

@@ -1,7 +1,7 @@
 # Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 # SPDX-License-Identifier: MIT
 
-FROM node:16
+FROM node:18-bullseye
 ENV APPDIR=/opt/service
 
 ARG BUILD_NUMBER=0
@@ -13,7 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   curl -L https://github.com/rbenv/ruby-build/archive/v20180822.tar.gz | tar -zxvf - -C /tmp/ && \
   cd /tmp/ruby-build-* && ./install.sh && cd / && \
   ruby-build -v 2.5.1 /usr/local && rm -rfv /tmp/ruby-build-* && \
-  gem install bundler --no-rdoc --no-ri
+  gem install bundler -v 2.3.26 --no-document
 
 # Scancode
 ARG SCANCODE_VERSION="30.1.0"
@@ -31,16 +31,19 @@ ENV SCANCODE_HOME=/usr/local/bin
 # the current RubyGem is 2.5.1. However, after upgrading RubyGem to 3.1.2, licensee:9.12.0 starts
 # to have hard time to find license in LICENSE file, like component npm/npmjs/-/caniuse-lite/1.0.30001344.
 # So we pin to the previous version of nokogiri and faraday.
-RUN gem install nokogiri:1.12.5 --no-rdoc --no-ri && \
-  gem install faraday:1.10.0 --no-rdoc --no-ri && \
-  gem install public_suffix:4.0.7 --no-rdoc --no-ri && \
-  gem install licensee:9.12.0 --no-rdoc --no-ri
+RUN gem install nokogiri:1.12.5 --no-document && \
+  gem install faraday:1.10.0 --no-document && \
+  gem install public_suffix:4.0.7 --no-document && \
+  gem install licensee:9.12.0 --no-document
 
 # REUSE
 RUN pip3 install setuptools
 RUN pip3 install reuse
 
+RUN git config --global --add safe.directory '*'
+
 COPY package*.json /tmp/
+COPY patches /tmp/patches
 RUN cd /tmp && npm install
 RUN mkdir -p "${APPDIR}" && cp -a /tmp/node_modules "${APPDIR}"
 

Dockerfile

@@ -5,7 +5,7 @@
 #COPY fossology_init.sh fossology_init.sh
 #RUN ./fossology_init.sh
 
-FROM node:16
+FROM node:18-bullseye
 ENV APPDIR=/opt/service
 #RUN apk update && apk upgrade && \
 #    apk add --no-cache bash git openssh
@@ -19,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends --no-install-su
   curl -L https://github.com/rbenv/ruby-build/archive/v20180822.tar.gz | tar -zxvf - -C /tmp/ && \
   cd /tmp/ruby-build-* && ./install.sh && cd / && \
   ruby-build -v 2.5.1 /usr/local && rm -rfv /tmp/ruby-build-* && \
-  gem install bundler --no-rdoc --no-ri
+  gem install bundler -v 2.3.26 --no-document
 
 # Scancode
 ARG SCANCODE_VERSION="30.1.0"
@@ -37,10 +37,10 @@ ENV SCANCODE_HOME=/usr/local/bin
 # the current RubyGem is 2.5.1. However, after upgrading RubyGem to 3.1.2, licensee:9.12.0 starts
 # to have hard time to find license in LICENSE file, like component npm/npmjs/-/caniuse-lite/1.0.30001344.
 # So we pin to the previous version of nokogiri and faraday.
-RUN gem install nokogiri:1.12.5 --no-rdoc --no-ri && \
-  gem install faraday:1.10.0 --no-rdoc --no-ri && \
-  gem install public_suffix:4.0.7 --no-rdoc --no-ri && \
-  gem install licensee:9.12.0 --no-rdoc --no-ri
+RUN gem install nokogiri:1.12.5 --no-document && \
+  gem install faraday:1.10.0 --no-document && \
+  gem install public_suffix:4.0.7 --no-document && \
+  gem install licensee:9.12.0 --no-document
 
 # REUSE
 RUN pip3 install setuptools
@@ -81,7 +81,10 @@ ENV CRAWLER_STORE_PROVIDER=cdDispatch+cd(azblob)+azqueue
 ENV CRAWLER_WEBHOOK_URL=https://api.clearlydefined.io/webhook
 ENV CRAWLER_AZBLOB_CONTAINER_NAME=production
 
+RUN git config --global --add safe.directory '*'
+
 COPY package*.json /tmp/
+COPY patches /tmp/patches
 RUN cd /tmp && npm install --production
 RUN mkdir -p "${APPDIR}" && cp -a /tmp/node_modules "${APPDIR}"
 

README.md

@@ -52,7 +52,7 @@ Here are a few example request objects.
 }
 ```
 
-The request `type` describes the crawling activity being requested. For example, "do `package` crawling". It is typically the same as the `type` in the url (see below). There are some more advanced scenarios where the two values are different but for starters, treat them as the same. The general form of a request URL is (note: it is a URL because of the underlying crawling infrastructure, the `cd` scheme is not particularly relevant)
+The request `type` describes the crawling activity being requested. For example, "do `package` crawling" (see [More on type](#more-on-type) for a description of valid type values). It is typically the same as the `type` in the url (see segments description below). There are some more advanced scenarios where the two values are different but for starters, treat them as the same. The general form of a request URL is (note: it is a URL because of the underlying crawling infrastructure, the `cd` scheme is not particularly relevant)
 
 ```
 cd:/type/provider/namespace/name/revision
@@ -80,6 +80,18 @@ Process the source, if any:
 
 The crawler's output is stored for use by the rest of the ClearlyDefined infrastructure -- it is not intended to be used directly by humans. Note that each tool's output is stored separately and the results of processing the component and the component source are also separated.
 
+### <a id="more-on-type"></a>More on `type`
+The `type` in the request object typically corresponds to an internal processor in CD.
+1. `component` is the most generic type.  Internally, it is converted to a `package` or `source` request by the component processor.
+2. `package` request is processed by the package processor and is further converted to a request with a specific type (`crate`, `deb`, `gem`, `go`, `maven`, `npm`, `nuget`, `composer`, `pod`, `pypi`).  For a `package` typed request, if the mentioned specific binary package type is known, the specific type (e.g. `npm`) can be used (instead of `package`) in the harvest request and skip the conversion step.  For example,
+```json
+{
+  "type": "npm",
+  "url": "cd:/npm/npmjs/-/redie/0.3.0"
+}
+```
+3. `source` requests are processed by the source processor, which subsequently dispatches a `clearlydefined` typed request for the supported source types and other requests (one for each scanning tool).  These are the more advanced scenarios where the request type and the coordinate type differ.
+
 # Configuration
 
 The crawler is quite configuable. Out of the box it is setup for demo-level use directly on your computer. In its full glory it can run with arbitrarily many distributed clients using an array of different queuing, caching and storage technologies.
@@ -121,7 +133,7 @@ If a CRAWLER_ID is specified, then each instance must have this setting globally
 ## Run Docker image from Docker Hub
 
 You can run the image as is from docker (this is w/o any port forwarding, which means the only way you can interact with the crawler locally is through the queue. See below for examples of how to run with ports exposed to do curl based testing).
-`docker run --env-file ../<env_name>.env.list clearlydefined/crawler`
+`docker run --platform linux/amd64 --env-file ../<env_name>.env.list clearlydefined/crawler`
 
 See `local.env.list`, `dev.env.list` and `prod.env.list` tempate files.
 
@@ -133,13 +145,13 @@ See `local.env.list`, `dev.env.list` and `prod.env.list` tempate files.
 
 ## Build and run Docker image locally
 
-`docker build -t cdcrawler:latest .`
+`docker build --platform linux/amd64 -t cdcrawler:latest .`
 
-`docker run --rm --env-file ../dev.env.list -p 5000:5000 -p 9229:9229 cdcrawler:latest`
+`docker run --platform linux/amd64 --rm --env-file ../dev.env.list -p 5000:5000 -p 9229:9229 cdcrawler:latest`
 
 With a debugger:
 
-`docker run --rm -d --env-file ../dev.env.list -p 9229:9229 -p 5000:5000 --entrypoint node cdcrawler:latest --inspect-brk=0.0.0.0:9229 index.js`
+`docker run --platform linux/amd64 --rm -d --env-file ../dev.env.list -p 9229:9229 -p 5000:5000 --entrypoint node cdcrawler:latest --inspect-brk=0.0.0.0:9229 index.js`
 
 At this point you can attach VS Code with the built in debugging profile (see .vscode/launch.json)
 

azure-pipelines.yml

@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license.
 // SPDX-License-Identifier: MIT
 const { DateTime } = require('luxon')
+const { spawn } = require('child_process')
+const { intersection } = require('lodash')
 
 const dateTimeFormats = [
   'EEE MMM d HH:mm:ss \'GMT\'ZZ yyyy'   //in pom properties
@@ -30,6 +32,12 @@ function trimAllParents(paths, parents) {
   return paths.map(path => trimParents(path, parents))
 }
 
+function isGitFile(file) {
+  if (!file) return false
+  const segments = file.split(/[\\/]/g)
+  return intersection(segments, ['.git']).length > 0
+}
+
 function extractDate(dateAndTime, formats = dateTimeFormats) {
   if (!dateAndTime) return dateAndTime
   let luxonResult = DateTime.fromISO(dateAndTime)
@@ -49,4 +57,29 @@ function extractDate(dateAndTime, formats = dateTimeFormats) {
   return (instant.isBefore(validStart) || instant.isAfter(validEnd)) ? null : luxonResult
 }
 
-module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, extractDate }
+function attachListeners(child, resolve, reject) {
+  let stdoutData = [], stderrData = []
+
+  child.stdout.on('data', chunk => stdoutData.push(chunk))
+  child.stderr.on('data', chunk => stderrData.push(chunk))
+
+  child
+  .on('error', (err) => reject(err))
+  .on('close', (code) => {
+    if (code === 0) resolve(stdoutData.join(''))
+    else {
+      const errorFromChild = new Error(stderrData.join(''))
+      errorFromChild.code = code
+      reject(errorFromChild)
+    }
+  })
+}
+
+function spawnPromisified(command, args, options) {
+  const childProcess = spawn(command, args, options)
+  return new Promise((resolve, reject) => {
+    attachListeners(childProcess, resolve, reject)
+  })
+}
+
+module.exports = { normalizePath, normalizePaths, trimParents, trimAllParents, isGitFile, extractDate, spawnPromisified }