Merge pull request #94 from electricimp/develop

Add path normalization to Github reader

Author: zandr

README.md

@@ -1,6 +1,6 @@
 <img src=docs/logo.png?2 width=180 alt=Builder><br />
 
-### Current version: 4.0.0 ###
+### Current version: 4.0.1 ###
 
 ![Build Status](https://cse-ci.electricimp.com/app/rest/builds/buildType:(id:Builder_BuildAndTest)/statusIcon)
 
@@ -181,6 +181,10 @@ builder.machine.directivesUseFile = <false|"PATH_TO_FILE">;
 
 const inputFile = "PATH_TO_YOUR_INPUT_FILE";
 
+// Set the directory of the input file as one of search dirs.
+// path.resolve() is used to get the absolute path (requires the path nodejs module)
+builder.machine.readers.file.inputFileDir = path.resolve(inputFile);
+
 const result = builder.machine.execute(`@include "${inputFile}"`);
 console.log(result);
 ```
@@ -849,7 +853,7 @@ If
 then *Builder* makes the following steps to find the include file:
 
 1. Only if the processed file is a local file, the final path to the include file is a concatenation of the path to the processed file and the path in the include. If the file is not found there, moves to the next step.
-1. The final path to the include file is a concatenation of the path to the file specified as the `<input_file>` parameter of the `pleasebuild` command and the path in the include. If the file is not found there, moves to the next step.
+1. The final path to the include file is a concatenation of the path to the file specified as the `<input_file>` parameter of the `pleasebuild` command and the path in the include. If the file is not found there, moves to the next step. **Note**: If you use *Builder* as a library and want this search step to work, make sure you specify the input file path (`builder.machine.readers.file.inputFileDir` variable) in your source code within the Builder initialization routine. See the [Library Installation section](#library-installation) to learn more.
 1. The final path to the include file is a concatenation of the path to the directory from where the `pleasebuild` command has been called and the path in the include. If the file is not found there, *Builder* reports an error.
 
 ##### Examples: #####
@@ -1058,6 +1062,8 @@ To reset the cache, use both the `--cache` and the `--clear-cache` options.
 
 If a resource should never be cached, it needs to be added to the `exclude-list.builder` file (see the example below). You can use wildcard characters to mask file names.
 
+**Note**: Non-identical paths, even if they point to absolutely the same location, are treated as different paths. Eg., if you include `/a/b/1.nut`, it becomes cached, and then you include `/a/b/../b/1.nut`, it will not be found in the cache.
+
 #### Wildcard Pattern Matching ####
 
 Pattern matching syntax is a similar to that of *.gitignore*. A string is a wildcard pattern if it contains '```?```' or '```*```' characters. Empty strings or strings that starts with '```#```' are ignored.
@@ -1172,6 +1178,8 @@ A typical `dependencies.json` file looks like this:
 ]
 ```
 
+**Note**: Non-identical paths, even if they point to absolutely the same location, are treated as different paths. Eg., if you have saved dependencies for `/a/b/1.nut` and then you include `/a/b/../b/1.nut` - the dependencies will not be applied to this file.
+
 ## Including JavaScript Libraries ##
 
 Builder can accept JavaScript libraries to add functionality to its global namespace. The library should export an object, the properties of which will be merged into the global namespace. For example, to include a function, *upper()*, to convert strings to uppercase, define your library file like so:

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "Builder",
-  "version": "4.0.0",
+  "version": "4.0.1",
   "description": "Builder Language Implementation",
   "main": "src/index.js",
   "bin": {

package.json

@@ -35,6 +35,12 @@ describe('GithubReader', () => {
 
     remote = reader.read('github.com/electricimp/Builder/spec/fixtures/sample-1/input.nut');
     expect(remote).toEqual(local);
+
+    remote = reader.read('github:electricimp/Builder/./spec/../spec/fixtures/sample-1/input.nut@master');
+    expect(remote).toEqual(local);
+
+    remote = reader.read('github.com:electricimp/Builder/./spec/../spec/fixtures/sample-1/input.nut');
+    expect(remote).toEqual(local);
   });
 
 });

spec/GithubReader.spec.js

@@ -52,7 +52,7 @@ describe('FileCache', () => {
     machine.useCache = true;
     machine.execute(`@include '${linkName}'`);
     expect(machine.fileCache._findFile(linkName) ? true : false).toEqual(true);
-    linkName = 'https://raw.githubusercontent.com/nobitlost/Builder/develop/spec/Builder.spec.js';
+    linkName = 'https://raw.githubusercontent.com/electricimp/Builder/develop/spec/Builder.spec.js';
     machine.execute(`@include '${linkName}'`);
     expect(machine.fileCache._findFile(linkName) ? true : false).toEqual(true);
   });
@@ -64,7 +64,7 @@ describe('FileCache', () => {
     machine.excludeList = __dirname + '/../fixtures/config/exclude-all.exclude';
     machine.execute(`@include '${linkName}'`);
     expect(machine.fileCache._findFile(linkName)).toEqual(false);
-    linkName = 'https://raw.githubusercontent.com/nobitlost/Builder/develop/spec/Builder.spec.js';
+    linkName = 'https://raw.githubusercontent.com/electricimp/Builder/develop/spec/Builder.spec.js';
     machine.execute(`@include '${linkName}'`);
     expect(machine.fileCache._findFile(linkName)).toEqual(false);
   });
@@ -78,10 +78,10 @@ describe('FileCache', () => {
       'github:electricimp/Builder/spec/fixtures/sample-11/OneLineSample.nut@v1.0.1',
       'github:electricimp/MessageManager/MessageManager.lib.nut',
       'github:electricimp/MessageManager/MessageManager.lib.js',
-      'https://raw.githubusercontent.com/nobitlost/Builder/v2.0.0/src/AstParser.js',
-      'https://raw.githubusercontent.com/nobitlost/Builder/v2.0.0/src/AstParser.js?nut=2',
-      'http://raw.githubusercontent.com/nobitlost/Builder/src/AstParser.js',
-      'http://raw.githubusercontent.com/nobitlost/Builder/src/AstParser.nut'
+      'https://raw.githubusercontent.com/electricimp/Builder/v2.0.0/src/AstParser.js',
+      'https://raw.githubusercontent.com/electricimp/Builder/v2.0.0/src/AstParser.js?nut=2',
+      'http://raw.githubusercontent.com/electricimp/Builder/src/AstParser.js',
+      'http://raw.githubusercontent.com/electricimp/Builder/src/AstParser.nut'
     ];
     const testFilesList = [
       'exclude-all.exclude',

spec/Machine/cache.spec.js

@@ -29,7 +29,7 @@ describe('Machine', () => {
     const rev1Content = "// included file a\n// included file b\n\n\n  // should be included\n\n    // l2 else\n\n\n  // should be included\n";
     const rev0GitBlobID = "9db26aa9017943a7812ab6751a699cd1c7247370";
     const rev0Content = "// included file a\n    // included file b\n\n\n      // should be included\n\n        // l2 else\n\n\n      // should be included\n";
-    const url = 'github:nobitlost/Builder/spec/fixtures/sample-1/input.nut.out';
+    const url = 'github:electricimp/Builder/spec/fixtures/sample-1/input.nut.out';
 
     // ensure that test dependencies JSON file does not exist
     if (fs.existsSync(dependenciesSaveFile)) {
@@ -64,7 +64,7 @@ describe('Machine', () => {
     const rev0GitBlobID = "9db26aa9017943a7812ab6751a699cd1c7247370";
     const rev0CommitID = "e2a5b434b34b5737b2ff52f51a92c5bbcc9f83bf";
     const rev0Content = "// included file a\n    // included file b\n\n\n      // should be included\n\n        // l2 else\n\n\n      // should be included\n";
-    const url = `github:nobitlost/Builder/spec/fixtures/sample-1/input.nut.out@${rev0CommitID}`;
+    const url = `github:electricimp/Builder/spec/fixtures/sample-1/input.nut.out@${rev0CommitID}`;
 
     // ensure that test dependencies JSON file does not exist
     if (fs.existsSync(dependenciesSaveFile)) {
@@ -85,7 +85,7 @@ describe('Machine', () => {
 
   it('Check ---save-dependecies/--use-dependencies options combination', () => {
     const rev1Content = "// included file a\n// included file b\n\n\n  // should be included\n\n    // l2 else\n\n\n  // should be included\n";
-    const url = `github:nobitlost/Builder/spec/fixtures/sample-1/input.nut.out`;
+    const url = `github:electricimp/Builder/spec/fixtures/sample-1/input.nut.out`;
 
     // ensure that test dependencies JSON file does not exist
     if (fs.existsSync(dependenciesUseFile)) {
@@ -119,7 +119,7 @@ describe('Machine', () => {
 
   it('Check case when dependencies JSON file is corrupted', () => {
     const rev1Content = "// included file a\n// included file b\n\n\n  // should be included\n\n    // l2 else\n\n\n  // should be included\n";
-    const url = `github:nobitlost/Builder/spec/fixtures/sample-1/input.nut.out`;
+    const url = `github:electricimp/Builder/spec/fixtures/sample-1/input.nut.out`;
 
     // ensure that test dependencies JSON file does not exist
     if (fs.existsSync(dependenciesSaveFile)) {
@@ -136,7 +136,7 @@ describe('Machine', () => {
     // corrupt the file
     fs.appendFileSync(dependenciesSaveFile, ']');
 
-    const fileCorruptedMessage = `The dependencies JSON file '${dependenciesSaveFile}' cannot be used: Unexpected token ] in JSON at position 127`;
+    const fileCorruptedMessage = `The dependencies JSON file '${dependenciesSaveFile}' cannot be used: Unexpected token ] in JSON at position 129`;
 
     // check exception error message
     try {

spec/Machine/use-dependencies.spec.js

@@ -66,7 +66,7 @@ class GithubReader extends AbstractReader {
 
     // process dependencies
     if (options && options.dependencies && options.dependencies.has(source)) {
-      this.gitBlobID = options.dependencies.get(source);
+      var gitBlobID = options.dependencies.get(source);
     }
 
     // spawn child process
@@ -77,7 +77,7 @@ class GithubReader extends AbstractReader {
         source,
         this.username,
         this.token,
-        this.gitBlobID,
+        gitBlobID,
       ],
       { timeout: this.timeout }
     );
@@ -198,10 +198,13 @@ class GithubReader extends AbstractReader {
 
     const octokit = new Octokit(octokitConfig);
 
+    const parsedUrl = this.parseUrl(source);
+    parsedUrl.path = upath.normalize(parsedUrl.path);
+
     if (gitBlobID !== 'undefined') {
       const args = {
-        owner: this.parseUrl(source).owner,
-        repo: this.parseUrl(source).repo,
+        owner: parsedUrl.owner,
+        repo: parsedUrl.repo,
         file_sha: gitBlobID,
       };
 
@@ -220,7 +223,7 @@ class GithubReader extends AbstractReader {
     }
 
     // @see https://developer.github.com/v3/repos/contents/#get-contents
-    octokit.repos.getContents(this.parseUrl(source))
+    octokit.repos.getContents(parsedUrl)
       .then((res) => {
         const ret = {
           data: Buffer.from(res.data.content, 'base64').toString(),