Refactor, add comments, and improve error messages

Author: kemitchell

scanner.js

@@ -2,9 +2,13 @@ var ids = require('spdx-license-ids')
 var exceptions = require('spdx-exceptions')
 
 module.exports = function () {
+  // Parse the input string all at once.
+  // Store the array of tokens.
   this.setInput = function (string) {
     this.tokens = lex(string)
   }
+
+  // Set the state for the Jison parser on each shift.
   this.lex = function () {
     var token = this.tokens.shift()
     this.yylineno = 1
@@ -24,18 +28,19 @@ var LICENSEREF = /^LicenseRef-([0-9A-Za-z-+.]+)$/
 var INVALID_CHARACTER = /[^ 0-9A-Za-z.+\-()]/
 var SINGLE_CHARACTER_TOKENS = ['(', ')', ':', '+']
 
+// Polyfill ECMAScript 2016 Array.prototype.includes.
 var includes = Array.prototype.includes
-  ? function (array, element) {
-    return array.includes(element)
-  }
-  : function (array, element) {
-    return array.indexOf(element) !== -1
-  }
+  ? function (array, element) { return array.includes(element) }
+  : function (array, element) { return array.indexOf(element) !== -1 }
 
 function lex (argument) {
-  if (INVALID_CHARACTER.test(argument)) {
-    throw new Error('Invalid character')
+  // If there are invalid characters, catch them ahead of time.
+  var match = INVALID_CHARACTER.exec(argument)
+  if (match) {
+    throw new Error('Invalid character at offset ' + match.index)
   }
+  // Iterate through the argument string, buffering
+  // non-space characters and building tokens.
   var tokens = []
   var characterBuffer = ''
   var startedBuffering = null
@@ -60,6 +65,7 @@ function lex (argument) {
     }
   }
   pushBuffered()
+  // Add an end-of-string token.
   tokens.push({
     type: 'EOS',
     string: '',
@@ -70,12 +76,14 @@ function lex (argument) {
 
   function pushBuffered () {
     if (characterBuffer) {
+      // Create a token for the buffered characters.
       tokens.push({
         type: tokenTypeForString(characterBuffer, startedBuffering),
         string: characterBuffer,
         start: startedBuffering,
         end: startedBuffering + characterBuffer.length
       })
+      // Reset the buffer.
       characterBuffer = ''
       startedBuffering = null
     }