Merge pull request #544 from ethereum/solcjs-include-paths

[solcjs] `--include-path` option

Author: chriseth

README.md

@@ -27,10 +27,17 @@ solcjs --help
 
 To compile a contract that imports other contracts via relative paths:
 ```bash
-solcjs --bin --base-path . ./MainContract.sol
+solcjs --bin --include-path node_modules/ --base-path . MainContract.sol
 ```
-The option ``--base-path`` enables automatic loading of imports from the filesystem and
-takes a path as argument that contains the source files.
+Use the ``--base-path`` and ``--include-path`` options to describe the layout of your project.
+``--base-path`` represents the root of your own source tree while ``--include-path`` allows you to
+specify extra locations containing external code (e.g. libraries installed with a package manager).
+
+Note: ensure that all the files you specify on the command line are located inside the base path or
+one of the include paths.
+The compiler refers to files from outside of these directories using absolute paths.
+Having absolute paths in contract metadata will result in your bytecode being reproducible only
+when it's placed in these exact absolute locations.
 
 Note: this commandline interface is not compatible with `solc` provided by the Solidity compiler package and thus cannot be
 used in combination with an Ethereum client via the `eth.compile.solidity()` RPC method. Please refer to the

solcjs

@@ -37,7 +37,13 @@ program
   .option('--bin', 'Binary of the contracts in hex.')
   .option('--abi', 'ABI of the contracts.')
   .option('--standard-json', 'Turn on Standard JSON Input / Output mode.')
-  .option('--base-path <path>', 'Automatically resolve all imports inside the given path.')
+  .option('--base-path <path>', 'Root of the project source tree. ' +
+    'The import callback will attempt to interpret all import paths as relative to this directory.'
+  )
+  .option('--include-path <path...>', 'Extra source directories available to the import callback. ' +
+    'When using a package manager to install libraries, use this option to specify directories where packages are installed. ' +
+    'Can be used multiple times to provide multiple locations.'
+  )
   .option('-o, --output-dir <output-directory>', 'Output directory for the contracts.')
   .option('-p, --pretty-json', 'Pretty-print all JSON output.', false)
   .option('-v, --verbose', 'More detailed console output.', false);
@@ -54,16 +60,21 @@ function abort (msg) {
 }
 
 function readFileCallback(sourcePath) {
-  if (options.basePath)
-    sourcePath = options.basePath + '/' + sourcePath;
-  if (fs.existsSync(sourcePath)) {
-    try {
-      return { 'contents': fs.readFileSync(sourcePath).toString('utf8') }
-    } catch (e) {
-      return { error: 'Error reading ' + sourcePath + ': ' + e };
+  const prefixes = [options.basePath ? options.basePath : ""].concat(
+    options.includePath ? options.includePath : []
+  );
+  for (const prefix of prefixes) {
+    const prefixedSourcePath = (prefix ? prefix + '/' : "") + sourcePath;
+
+    if (fs.existsSync(prefixedSourcePath)) {
+      try {
+        return {'contents': fs.readFileSync(prefixedSourcePath).toString('utf8')}
+      } catch (e) {
+        return {error: 'Error reading ' + prefixedSourcePath + ': ' + e};
+      }
     }
-  } else
-    return { error: 'File not found at ' + sourcePath}
+  }
+  return {error: 'File not found inside the base path or any of the include paths.'}
 }
 
 function withUnixPathSeparators(filePath) {
@@ -74,8 +85,13 @@ function withUnixPathSeparators(filePath) {
   return filePath.replace(/\\/g, "/");
 }
 
-function stripBasePath(sourcePath) {
+function makeSourcePathRelativeIfPossible(sourcePath) {
   const absoluteBasePath = (options.basePath ? path.resolve(options.basePath) : path.resolve('.'));
+  const absoluteIncludePaths = (
+    options.includePath ?
+    options.includePath.map((prefix) => { return path.resolve(prefix); }) :
+    []
+  );
 
   // Compared to base path stripping logic in solc this is much simpler because path.resolve()
   // handles symlinks correctly (does not resolve them except in work dir) and strips .. segments
@@ -84,13 +100,16 @@ function stripBasePath(sourcePath) {
   // Windows and UNC paths are not handled in a special way (at least on Linux). Finally, it has
   // very little test coverage so there might be more differences that we are just not aware of.
   const absoluteSourcePath = path.resolve(sourcePath);
-  const relativeSourcePath = path.relative(absoluteBasePath, absoluteSourcePath);
 
-  if (relativeSourcePath.startsWith('../'))
-    // Path can't be made relative without stepping outside of base path so return absolute one.
-    return withUnixPathSeparators(absoluteSourcePath);
+  for (const absolutePrefix of [absoluteBasePath].concat(absoluteIncludePaths)) {
+    const relativeSourcePath = path.relative(absolutePrefix, absoluteSourcePath);
+
+    if (!relativeSourcePath.startsWith('../'))
+      return withUnixPathSeparators(relativeSourcePath);
+  }
 
-  return withUnixPathSeparators(relativeSourcePath);
+  // File is not located inside base path or include paths so use its absolute path.
+  return withUnixPathSeparators(absoluteSourcePath);
 }
 
 function toFormattedJson(input) {
@@ -148,11 +167,22 @@ if (!(options.bin || options.abi)) {
   abort('Invalid option selected, must specify either --bin or --abi');
 }
 
+if (!options.basePath && options.includePath && options.includePath.length > 0) {
+  abort('--include-path option requires a non-empty base path.');
+}
+
+if (options.includePath)
+  for (const includePath of options.includePath)
+    if (!includePath)
+      abort('Empty values are not allowed in --include-path.');
+
 var sources = {};
 
 for (var i = 0; i < files.length; i++) {
   try {
-    sources[stripBasePath(files[i])] = { content: fs.readFileSync(files[i]).toString() };
+    sources[makeSourcePathRelativeIfPossible(files[i])] = {
+      content: fs.readFileSync(files[i]).toString()
+    };
   } catch (e) {
     abort('Error reading ' + files[i] + ': ' + e);
   }

test/cli.js

@@ -131,6 +131,48 @@ tape('CLI', function (t) {
     spt.end();
   });
 
+  t.test('include paths', function (st) {
+    const spt = spawn(
+      st,
+      './solcjs --bin ' +
+        'test/resources/importCallback/base/contractB.sol ' +
+        'test/resources/importCallback/includeA/libY.sol ' +
+        './test/resources/importCallback/includeA//libY.sol ' +
+        path.resolve('test/resources/importCallback/includeA/libY.sol') + ' ' +
+        '--base-path test/resources/importCallback/base ' +
+        '--include-path test/resources/importCallback/includeA ' +
+        '--include-path ' + path.resolve('test/resources/importCallback/includeB/')
+    );
+    spt.stderr.empty();
+    spt.succeeds();
+    spt.end();
+  });
+
+  t.test('include paths without base path', function (st) {
+    const spt = spawn(
+      st,
+      './solcjs --bin ' +
+        'test/resources/importCallback/contractC.sol ' +
+        '--include-path test/resources/importCallback/includeA'
+    );
+    spt.stderr.match(/--include-path option requires a non-empty base path\./);
+    spt.fails();
+    spt.end();
+  });
+
+  t.test('empty include paths', function (st) {
+    const spt = spawn(
+      st,
+      './solcjs --bin ' +
+        'test/resources/importCallback/contractC.sol ' +
+        '--base-path test/resources/importCallback/base ' +
+        '--include-path='
+    );
+    spt.stderr.match(/Empty values are not allowed in --include-path\./);
+    spt.fails();
+    spt.end();
+  });
+
   t.test('standard json', function (st) {
     var input = {
       'language': 'Solidity',
@@ -187,4 +229,34 @@ tape('CLI', function (t) {
       spt.end();
     });
   });
+
+  t.test('standard json include paths', function (st) {
+    var input = {
+      'language': 'Solidity',
+      'sources': {
+        'contractB.sol': {
+          'content':
+            '// SPDX-License-Identifier: GPL-3.0\n' +
+            'pragma solidity >=0.0;\n' +
+            'import "./contractA.sol";\n'
+        }
+      }
+    };
+    var spt = spawn(
+      st,
+      './solcjs --standard-json ' +
+        '--base-path test/resources/importCallback/base ' +
+        '--include-path test/resources/importCallback/includeA ' +
+        '--include-path ' + path.resolve('test/resources/importCallback/includeB/')
+    );
+    spt.stdin.setEncoding('utf-8');
+    spt.stdin.write(JSON.stringify(input));
+    spt.stdin.end();
+    spt.stdin.on('finish', function () {
+      spt.stderr.empty();
+      spt.stdout.match(/{"sources":{"contractA.sol":{"id":0},"contractB.sol":{"id":1},"libX.sol":{"id":2},"libY.sol":{"id":3},"libZ.sol":{"id":4},"utils.sol":{"id":5}}}/);
+      spt.succeeds();
+      spt.end();
+    });
+  });
 });

test/resources/importCallback/base/contractA.sol

@@ -0,0 +1,8 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+import "libX.sol";
+import "libY.sol";
+import "libZ.sol";
+
+contract A {}

test/resources/importCallback/base/contractB.sol

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+import "./contractA.sol";
+
+contract B {}

test/resources/importCallback/contractC.sol

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+contract C {}

test/resources/importCallback/includeA/libX.sol

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+library X {}

test/resources/importCallback/includeA/libY.sol

@@ -0,0 +1,6 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+import "./utils.sol";
+
+library Y {}

test/resources/importCallback/includeA/utils.sol

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+library Utils {}

test/resources/importCallback/includeB/libZ.sol

@@ -0,0 +1,4 @@
+// SPDX-License-Identifier: GPL-3.0
+pragma solidity >=0.0;
+
+library Z {}