Merge pull request #572 from clearlydefined/master

Merge `master` into `prod` for release `v1.0.2`

Author: elrayle

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "clearlydefined-crawler",
-  "version": "0.1.1",
+  "version": "1.0.2",
   "description": "A crawler that walks projects and packages looking for data of interest to the ClearlyDefined project.",
   "main": "./index.js",
   "scripts": {

package.json

@@ -7,7 +7,7 @@ const { clone } = require('lodash')
 const rimraf = require('rimraf')
 const FetchResult = require('../../lib/fetchResult')
 
-const providerDictionary = {
+const providerMap = {
   gitlab: 'https://gitlab.com',
   github: 'https://github.com'
 }
@@ -95,7 +95,7 @@ class GitCloner extends AbstractFetch {
 
   _buildUrl(spec) {
     const fullName = `${spec.namespace.replace(/\./g, '/')}/${spec.name}`
-    return `${providerDictionary[spec.provider]}/${fullName}.git`
+    return `${providerMap[spec.provider]}/${fullName}.git`
   }
 }
 

providers/fetch/gitCloner.js

@@ -9,6 +9,10 @@ const { parse: htmlParser } = require('node-html-parser')
 const { parse: spdxParser } = require('@clearlydefined/spdx')
 const FetchResult = require('../../lib/fetchResult')
 
+const providerMap = {
+  golang: 'https://proxy.golang.org'
+}
+
 class GoFetch extends AbstractFetch {
   constructor(options) {
     super(options)
@@ -69,7 +73,7 @@ class GoFetch extends AbstractFetch {
   }
 
   async _getLatestVersion(spec) {
-    const initial_url = `https://${spec.provider}/${spec.namespace}/${spec.name}/@v/list`
+    const initial_url = `${providerMap.golang}/${spec.namespace}/${spec.name}/@v/list`
     const replace_encoded_url = this._replace_encodings(initial_url)
     const url = replace_encoded_url.replace(/null\//g, '')
 
@@ -89,7 +93,7 @@ class GoFetch extends AbstractFetch {
   }
 
   _buildUrl(spec, extension = '.zip') {
-    let initial_url = `https://proxy.golang.org/${spec.namespace}/${spec.name}/@v/${spec.revision}${extension}`
+    let initial_url = `${providerMap.golang}/${spec.namespace}/${spec.name}/@v/${spec.revision}${extension}`
     return this._replace_encodings(this._remove_blank_fields(initial_url))
   }
 

providers/fetch/goFetch.js

@@ -9,16 +9,16 @@ const Request = require('../../../../ghcrawler').request
 const fs = require('fs')
 const { merge } = require('lodash')
 
-const stub = 'https://proxy.golang.org/'
+const goBaseURL = 'https://proxy.golang.org/'
 
 describe('Go utility functions', () => {
   it('builds URLs', () => {
     const fetch = GoFetch({})
-    expect(fetch._buildUrl(spec('go', 'golang', 'cloud.google.com', 'go', 'v0.56.0'))).to.equal(stub + 'cloud.google.com/go/@v/v0.56.0.zip')
-    expect(fetch._buildUrl(spec('go', 'golang', 'cloud.google.com', 'go', 'v0.56.0'), '.mod')).to.equal(stub + 'cloud.google.com/go/@v/v0.56.0.mod')
-    expect(fetch._buildUrl(spec('go', 'golang', '-', 'collectd.org', 'v0.5.0'))).to.equal(stub + 'collectd.org/@v/v0.5.0.zip')
-    expect(fetch._buildUrl(spec('go', 'golang', 'github.com%2fAzure%2fazure-event-hubs-go', 'v3', 'v3.2.0'))).to.equal(stub + 'github.com/Azure/azure-event-hubs-go/v3/@v/v3.2.0.zip')
-    expect(fetch._buildUrl(spec('go', 'golang', 'github.com%2FAzure%2Fazure-event-hubs-go', 'v3', 'v3.2.0'))).to.equal(stub + 'github.com/Azure/azure-event-hubs-go/v3/@v/v3.2.0.zip')
+    expect(fetch._buildUrl(spec('go', 'golang', 'cloud.google.com', 'go', 'v0.56.0'))).to.equal(goBaseURL + 'cloud.google.com/go/@v/v0.56.0.zip')
+    expect(fetch._buildUrl(spec('go', 'golang', 'cloud.google.com', 'go', 'v0.56.0'), '.mod')).to.equal(goBaseURL + 'cloud.google.com/go/@v/v0.56.0.mod')
+    expect(fetch._buildUrl(spec('go', 'golang', '-', 'collectd.org', 'v0.5.0'))).to.equal(goBaseURL + 'collectd.org/@v/v0.5.0.zip')
+    expect(fetch._buildUrl(spec('go', 'golang', 'github.com%2fAzure%2fazure-event-hubs-go', 'v3', 'v3.2.0'))).to.equal(goBaseURL + 'github.com/Azure/azure-event-hubs-go/v3/@v/v3.2.0.zip')
+    expect(fetch._buildUrl(spec('go', 'golang', 'github.com%2FAzure%2Fazure-event-hubs-go', 'v3', 'v3.2.0'))).to.equal(goBaseURL + 'github.com/Azure/azure-event-hubs-go/v3/@v/v3.2.0.zip')
   })
 })
 
@@ -47,6 +47,7 @@ describe('Go Proxy fetching', () => {
   beforeEach(() => {
     const requestPromiseStub = options => {
       if (options.url) {
+        expect(options.url).to.contain(goBaseURL)
         if (options.url.includes('error')) throw new Error('yikes')
         if (options.url.includes('code')) throw { statusCode: 500, message: 'Code' }
         if (options.url.includes('missing')) throw { statusCode: 404 }