Use hand-coded lexer requiring SPDX list packages

Prior to this, `npm prepublish` generated both lexer and parser
using Jison, hard-coding the versions of `spdx-license-ids` and
`spdx-exceptions` installed as `devDependencies` on the publisher's
machine into the package.  There were no `dependencies`.

This commit adds a hand-coded lexer, and uses Jison to generate only
the parser.  As a result, the SPDX list-data packages become regular
`dependencies`, `npm install` can install the latest versions,
and the lexer can `require()` them.  This will help ensure that
those relying on `spdx-expression-parse` get the latest SPDX lists,
without waiting for `spdx-expression-parse` to republish.

Author: kemitchell

README.md

@@ -38,7 +38,7 @@ The syntax comes from the [Software Package Data eXchange (SPDX)](https://spdx.o
 
 The bulk of the SPDX standard describes syntax and semantics of XML metadata files.  This package implements two lightweight, plain-text components of that larger standard:
 
-1.  The [license list](https://spdx.org/licenses), a mapping from specific string identifiers, like `Apache-2.0`, to standard form license texts and bolt-on license exceptions.  The [spdx-license-ids](https://www.npmjs.com/package/spdx-exceptions) and [spdx-exceptions](https://www.npmjs.com/package/spdx-license-ids) packages implement the license list.  They are development dependencies of this package.
+1.  The [license list](https://spdx.org/licenses), a mapping from specific string identifiers, like `Apache-2.0`, to standard form license texts and bolt-on license exceptions.  The [spdx-license-ids](https://www.npmjs.com/package/spdx-exceptions) and [spdx-exceptions](https://www.npmjs.com/package/spdx-license-ids) packages implement the license list.  They are dependencies of this package.
 
     Any license identifier from the license list is a valid license expression:
 

generate-parser.js

@@ -7,51 +7,7 @@ var options = {
 
 var words = ['AND', 'OR', 'WITH']
 
-var quote = function (argument) {
-  return '\'' + argument + '\''
-}
-
-var regexEscape = function (s) {
-  return s.replace(/[\^\\$*+?.()|{}[]\/]/g, '\\$&')
-}
-
-var handleLicensesAndExceptions = function () {
-  var ids = require('spdx-license-ids')
-  var exceptions = require('spdx-exceptions')
-
-  // Sort tokens longest-first (both license ids and exception strings)
-  var tokens = ids.concat(exceptions)
-  tokens.sort(function (a, b) { return b.length - a.length })
-  return tokens.map(function (t) {
-    var type = (ids.indexOf(t) >= 0) ? 'LICENSE' : 'EXCEPTION'
-    return [regexEscape(t), 'return ' + quote(type)]
-  })
-}
-
 var grammar = {
-  lex: {
-    macros: {},
-    rules: [
-      ['$', 'return ' + quote('EOS')],
-      ['\\s+', '/* skip whitespace */'],
-      ['\\+', 'return ' + quote('PLUS')],
-      ['\\(', 'return ' + quote('OPEN')],
-      ['\\)', 'return ' + quote('CLOSE')],
-      [':', 'return ' + quote('COLON')],
-      [
-        'DocumentRef-([0-9A-Za-z-+.]+)',
-        'return ' + quote('DOCUMENTREF')
-      ],
-      [
-        'LicenseRef-([0-9A-Za-z-+.]+)',
-        'return ' + quote('LICENSEREF')
-      ]
-    ]
-    .concat(words.map(function (word) {
-      return [word, 'return ' + quote(word)]
-    }))
-    .concat(handleLicensesAndExceptions())
-  },
   operators: [
     ['left', 'OR'],
     ['left', 'AND'],

index.js

@@ -1,5 +1,8 @@
-var parser = require('./parser').parser
+var Scanner = require('./scanner')
+var Parser = require('./parser').Parser
 
-module.exports = function (argument) {
-  return parser.parse(argument)
+module.exports = function (string) {
+  var parser = new Parser()
+  parser.lexer = new Scanner()
+  return parser.parse(string)
 }

package.json

@@ -6,14 +6,17 @@
   "files": [
     "AUTHORS",
     "index.js",
-    "parser.js"
+    "parser.js",
+    "scanner.js"
   ],
+  "dependencies": {
+    "spdx-exceptions": "^2.0.0",
+    "spdx-license-ids": "^2.0.1"
+  },
   "devDependencies": {
     "defence-cli": "^2.0.1",
     "jison": "^0.4.15",
     "replace-require-self": "^1.0.0",
-    "spdx-exceptions": "^2.0.0",
-    "spdx-license-ids": "^2.0.1",
     "standard": "^10.0.2"
   },
   "keywords": [

scanner.js

@@ -0,0 +1,111 @@
+var ids = require('spdx-license-ids')
+var exceptions = require('spdx-exceptions')
+
+module.exports = function () {
+  this.setInput = function (string) {
+    this.tokens = lex(string)
+  }
+  this.lex = function () {
+    var token = this.tokens.shift()
+    this.yylineno = 1
+    this.yytext = token.string
+    this.yyloc = {
+      first_line: 1,
+      last_line: 1,
+      first_column: token.start,
+      last_column: token.end
+    }
+    return token.type
+  }
+}
+
+var DOCUMENTREF = /^DocumentRef-([0-9A-Za-z-+.]+)$/
+var LICENSEREF = /^LicenseRef-([0-9A-Za-z-+.]+)$/
+var INVALID_CHARACTER = /[^ 0-9A-Za-z.+\-()]/
+var SINGLE_CHARACTER_TOKENS = ['(', ')', ':', '+']
+
+var includes = Array.prototype.includes
+  ? function (array, element) {
+    return array.includes(element)
+  }
+  : function (array, element) {
+    return array.indexOf(element) !== -1
+  }
+
+function lex (argument) {
+  if (INVALID_CHARACTER.test(argument)) {
+    throw new Error('Invalid character')
+  }
+  var tokens = []
+  var characterBuffer = ''
+  var startedBuffering = null
+  var length = argument.length
+  for (var offset = 0; offset < length; offset++) {
+    var character = argument[offset]
+    if (character === ' ') {
+      pushBuffered()
+    } else if (includes(SINGLE_CHARACTER_TOKENS, character)) {
+      pushBuffered()
+      tokens.push({
+        type: tokenTypeForString(character, offset),
+        string: character,
+        start: offset,
+        end: offset + 1
+      })
+    } else {
+      if (startedBuffering === null) {
+        startedBuffering = offset
+      }
+      characterBuffer += character
+    }
+  }
+  pushBuffered()
+  tokens.push({
+    type: 'EOS',
+    string: '',
+    start: argument.length,
+    end: argument.length
+  })
+  return tokens
+
+  function pushBuffered () {
+    if (characterBuffer) {
+      tokens.push({
+        type: tokenTypeForString(characterBuffer, startedBuffering),
+        string: characterBuffer,
+        start: startedBuffering,
+        end: startedBuffering + characterBuffer.length
+      })
+      characterBuffer = ''
+      startedBuffering = null
+    }
+  }
+}
+
+function tokenTypeForString (string, start) {
+  if (ids.indexOf(string) !== -1) {
+    return 'LICENSE'
+  } else if (string === 'AND') {
+    return string
+  } else if (string === 'OR') {
+    return string
+  } else if (string === 'WITH') {
+    return string
+  } else if (exceptions.indexOf(string) !== -1) {
+    return 'EXCEPTION'
+  } else if (LICENSEREF.test(string)) {
+    return 'LICENSEREF'
+  } else if (DOCUMENTREF.test(string)) {
+    return 'DOCUMENTREF'
+  } else if (string === '(') {
+    return 'OPEN'
+  } else if (string === ')') {
+    return 'CLOSE'
+  } else if (string === ':') {
+    return 'COLON'
+  } else if (string === '+') {
+    return 'PLUS'
+  } else {
+    throw new Error('Invalid input at offset ' + start)
+  }
+}