Merge pull request #640 from chriseth/globalPaths

Allow remappings to change depending on the context.

Author: chriseth

docs/layout-of-source-files.rst

@@ -6,6 +6,8 @@ Source files can contain an arbitrary number of contract definitions and include
 
 .. index:: source file, ! import
 
+.. _import:
+
 Importing other Source Files
 ============================
 
@@ -68,15 +70,20 @@ remappings so that e.g. ``github.com/ethereum/dapp-bin/library`` is remapped to
 ``/usr/local/dapp-bin/library`` and the compiler will read the files from there. If
 remapping keys are prefixes of each other, the longest is tried first. This
 allows for a "fallback-remapping" with e.g. ``""`` maps to
-``"/usr/local/include/solidity"``.
+``"/usr/local/include/solidity"``. Furthermore, these remappings can
+depend on the context, which allows you to configure packages to
+import e.g. different versions of a library of the same name.
 
 **solc**:
 
-For solc (the commandline compiler), these remappings are provided as ``key=value``
-arguments, where the ``=value`` part is optional (and defaults to key in that
+For solc (the commandline compiler), these remappings are provided as
+``context:prefix=target`` arguments, where both the ``context:`` and the
+``=target`` parts are optional (where target defaults to prefix in that
 case). All remapping values that are regular files are compiled (including
 their dependencies). This mechanism is completely backwards-compatible (as long
-as no filename contains a =) and thus not a breaking change.
+as no filename contains = or :) and thus not a breaking change. All imports
+in files in or below the directory ``context`` that import a file that
+starts with ``prefix`` are redirected by replacing ``prefix`` by ``target``.
 
 So as an example, if you clone
 ``github.com/ethereum/dapp-bin/`` locally to ``/usr/local/dapp-bin``, you can use
@@ -92,6 +99,19 @@ and then run the compiler as
 
   solc github.com/ethereum/dapp-bin/=/usr/local/dapp-bin/ source.sol
 
+As a more complex example, suppose you rely on some module that uses a
+very old version of dapp-bin. That old version of dapp-bin is checked
+out at ``/usr/local/dapp-bin_old``, then you can use 
+
+.. code-block:: bash
+
+  solc module1:github.com/ethereum/dapp-bin/=/usr/local/dapp-bin/ \
+       module2:github.com/ethereum/dapp-bin/=/usr/local/dapp-bin_old/ \
+       source.sol
+
+so that all imports in ``module2`` point to the old version but imports
+in ``module1`` get the new version.
+
 Note that solc only allows you to include files from certain directories:
 They have to be in the directory (or subdirectory) of one of the explicitly
 specified source files or in the directory (or subdirectory) of a remapping

docs/miscellaneous.rst

@@ -108,7 +108,7 @@ Using ``solc --help`` provides you with an explanation of all options. The compi
 If you only want to compile a single file, you run it as ``solc --bin sourceFile.sol`` and it will print the binary. Before you deploy your contract, activate the optimizer while compiling using ``solc --optimize --bin sourceFile.sol``. If you want to get some of the more advanced output variants of ``solc``, it is probably better to tell it to output everything to separate files using ``solc -o outputDirectory --bin --ast --asm sourceFile.sol``.
 
 The commandline compiler will automatically read imported files from the filesystem, but
-it is also possible to provide path redirects using ``prefix=path`` in the following way:
+it is also possible to provide path redirects using ``context:prefix=path`` in the following way:
 
 ::
 
@@ -121,6 +121,10 @@ always matches). ``solc`` will not read files from the filesystem that lie outsi
 the remapping targets and outside of the directories where explicitly specified source
 files reside, so things like ``import "/etc/passwd";`` only work if you add ``=/`` as a remapping.
 
+You can restrict remappings to only certain source files by prefixing a context.
+
+The section on :ref:`import` provides more details on remappings.
+
 If there are multiple matches due to remappings, the one with the longest common prefix is selected.
 
 If your contracts use :ref:`libraries <libraries>`, you will notice that the bytecode contains substrings of the form ``__LibraryName______``. You can use ``solc`` as a linker meaning that it will insert the library addresses for you at those points:

libsolidity/interface/CompilerStack.cpp

@@ -63,6 +63,24 @@ CompilerStack::CompilerStack(bool _addStandardSources, ReadFileCallback const& _
 		addSources(StandardSources, true); // add them as libraries
 }
 
+void CompilerStack::setRemappings(vector<string> const& _remappings)
+{
+	vector<Remapping> remappings;
+	for (auto const& remapping: _remappings)
+	{
+		auto eq = find(remapping.begin(), remapping.end(), '=');
+		if (eq == remapping.end())
+			continue; // ignore
+		auto colon = find(remapping.begin(), eq, ':');
+		Remapping r;
+		r.context = colon == eq ? string() : string(remapping.begin(), colon);
+		r.prefix = colon == eq ? string(remapping.begin(), eq) : string(colon + 1, eq);
+		r.target = string(eq + 1, remapping.end());
+		remappings.push_back(r);
+	}
+	swap(m_remappings, remappings);
+}
+
 void CompilerStack::reset(bool _keepSources, bool _addStandardSources)
 {
 	m_parseSuccessful = false;
@@ -384,37 +402,72 @@ tuple<int, int, int, int> CompilerStack::positionFromSourceLocation(SourceLocati
 	return make_tuple(++startLine, ++startColumn, ++endLine, ++endColumn);
 }
 
-StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _path)
+StringMap CompilerStack::loadMissingSources(SourceUnit const& _ast, std::string const& _sourcePath)
 {
 	StringMap newSources;
 	for (auto const& node: _ast.nodes())
 		if (ImportDirective const* import = dynamic_cast<ImportDirective*>(node.get()))
 		{
-			string path = absolutePath(import->path(), _path);
-			import->annotation().absolutePath = path;
-			if (m_sources.count(path) || newSources.count(path))
+			string importPath = absolutePath(import->path(), _sourcePath);
+			// The current value of `path` is the absolute path as seen from this source file.
+			// We first have to apply remappings before we can store the actual absolute path
+			// as seen globally.
+			importPath = applyRemapping(importPath, _sourcePath);
+			import->annotation().absolutePath = importPath;
+			if (m_sources.count(importPath) || newSources.count(importPath))
 				continue;
-			string contents;
-			string errorMessage;
-			if (!m_readFile)
-				errorMessage = "File not supplied initially.";
+
+			ReadFileResult result{false, string("File not supplied initially.")};
+			if (m_readFile)
+				result = m_readFile(importPath);
+
+			if (result.success)
+				newSources[importPath] = result.contentsOrErrorMesage;
 			else
-				tie(contents, errorMessage) = m_readFile(path);
-			if (!errorMessage.empty())
 			{
 				auto err = make_shared<Error>(Error::Type::ParserError);
 				*err <<
 					errinfo_sourceLocation(import->location()) <<
-					errinfo_comment("Source not found: " + errorMessage);
+					errinfo_comment("Source \"" + importPath + "\" not found: " + result.contentsOrErrorMesage);
 				m_errors.push_back(std::move(err));
 				continue;
 			}
-			else
-				newSources[path] = contents;
 		}
 	return newSources;
 }
 
+string CompilerStack::applyRemapping(string const& _path, string const& _context)
+{
+	// Try to find the longest prefix match in all remappings that are active in the current context.
+	auto isPrefixOf = [](string const& _a, string const& _b)
+	{
+		if (_a.length() > _b.length())
+			return false;
+		return std::equal(_a.begin(), _a.end(), _b.begin());
+	};
+
+	size_t longestPrefix = 0;
+	string longestPrefixTarget;
+	for (auto const& redir: m_remappings)
+	{
+		// Skip if we already have a closer match.
+		if (longestPrefix > 0 && redir.prefix.length() <= longestPrefix)
+			continue;
+		// Skip if redir.context is not a prefix of _context
+		if (!isPrefixOf(redir.context, _context))
+			continue;
+		// Skip if the prefix does not match.
+		if (!isPrefixOf(redir.prefix, _path))
+			continue;
+
+		longestPrefix = redir.prefix.length();
+		longestPrefixTarget = redir.target;
+	}
+	string path = longestPrefixTarget;
+	path.append(_path.begin() + longestPrefix, _path.end());
+	return path;
+}
+
 void CompilerStack::resolveImports()
 {
 	// topological sorting (depth first search) of the import graph, cutting potential cycles

libsolidity/interface/CompilerStack.h

@@ -75,15 +75,23 @@ enum class DocumentationType: uint8_t
 class CompilerStack: boost::noncopyable
 {
 public:
-	/// File reading callback, should return a pair of content and error message (exactly one nonempty)
-	/// for a given path.
-	using ReadFileCallback = std::function<std::pair<std::string, std::string>(std::string const&)>;
+	struct ReadFileResult
+	{
+		bool success;
+		std::string contentsOrErrorMesage;
+	};
+
+	/// File reading callback.
+	using ReadFileCallback = std::function<ReadFileResult(std::string const&)>;
 
 	/// Creates a new compiler stack.
 	/// @param _readFile callback to used to read files for import statements. Should return
 	/// @param _addStandardSources Adds standard sources if @a _addStandardSources.
 	explicit CompilerStack(bool _addStandardSources = true, ReadFileCallback const& _readFile = ReadFileCallback());
 
+	/// Sets path remappings in the format "context:prefix=target"
+	void setRemappings(std::vector<std::string> const& _remappings);
+
 	/// Resets the compiler to a state where the sources are not parsed or even removed.
 	void reset(bool _keepSources = false, bool _addStandardSources = true);
 
@@ -209,6 +217,7 @@ class CompilerStack: boost::noncopyable
 	/// @a m_readFile and stores the absolute paths of all imports in the AST annotations.
 	/// @returns the newly loaded sources.
 	StringMap loadMissingSources(SourceUnit const& _ast, std::string const& _path);
+	std::string applyRemapping(std::string const& _path, std::string const& _context);
 	void resolveImports();
 	/// Checks whether there are libraries with the same name, reports that as an error and
 	/// @returns false in this case.
@@ -226,7 +235,17 @@ class CompilerStack: boost::noncopyable
 	Contract const& contract(std::string const& _contractName = "") const;
 	Source const& source(std::string const& _sourceName = "") const;
 
+	struct Remapping
+	{
+		std::string context;
+		std::string prefix;
+		std::string target;
+	};
+
 	ReadFileCallback m_readFile;
+	/// list of path prefix remappings, e.g. mylibrary: github.com/ethereum = /usr/local/ethereum
+	/// "context:prefix=target"
+	std::vector<Remapping> m_remappings;
 	bool m_parseSuccessful;
 	std::map<std::string const, Source> m_sources;
 	std::shared_ptr<GlobalContext> m_globalContext;

solc/CommandLineInterface.cpp

@@ -318,36 +318,31 @@ void CommandLineInterface::readInputFilesAndConfigureRemappings()
 		}
 	}
 	else
-		for (string const& infile: m_args["input-file"].as<vector<string>>())
+		for (string path: m_args["input-file"].as<vector<string>>())
 		{
-			auto eq = find(infile.begin(), infile.end(), '=');
-			if (eq != infile.end())
-			{
-				string target(eq + 1, infile.end());
-				m_remappings.push_back(make_pair(string(infile.begin(), eq), target));
-				m_allowedDirectories.push_back(boost::filesystem::path(target).remove_filename());
-			}
+			auto eq = find(path.begin(), path.end(), '=');
+			if (eq != path.end())
+				path = string(eq + 1, path.end());
 			else
 			{
-				auto path = boost::filesystem::path(infile);
-				if (!boost::filesystem::exists(path))
+				auto infile = boost::filesystem::path(path);
+				if (!boost::filesystem::exists(infile))
 				{
 					cerr << "Skipping non existant input file \"" << infile << "\"" << endl;
 					continue;
 				}
 
-				if (!boost::filesystem::is_regular_file(path))
+				if (!boost::filesystem::is_regular_file(infile))
 				{
 					cerr << "\"" << infile << "\" is not a valid file. Skipping" << endl;
 					continue;
 				}
 
-				m_sourceCodes[path.string()] = dev::contentsString(path.string());
-				m_allowedDirectories.push_back(boost::filesystem::canonical(path).remove_filename());
+				m_sourceCodes[infile.string()] = dev::contentsString(infile.string());
+				path = boost::filesystem::canonical(infile).string();
 			}
+			m_allowedDirectories.push_back(boost::filesystem::path(path).remove_filename());
 		}
-	// Add empty remapping to try the path itself.
-	m_remappings.push_back(make_pair(string(), string()));
 }
 
 bool CommandLineInterface::parseLibraryOption(string const& _input)
@@ -534,67 +529,42 @@ bool CommandLineInterface::processInput()
 		return link();
 	}
 
-	function<pair<string,string>(string const&)> fileReader = [this](string const& _path)
+	CompilerStack::ReadFileCallback fileReader = [this](string const& _path)
 	{
-		// Try to find the longest prefix match in all remappings. At the end, there will bean
-		// empty remapping so that we also try the path itself, but any file should be either
-		// in (a subdirectory of) the directory of an explicit source or a remapping target.
-		int errorLevel = 0;
-		size_t longestPrefix = 0;
-		string bestMatchPath;
-		for (auto const& redir: m_remappings)
+		auto boostPath = boost::filesystem::path(_path);
+		if (!boost::filesystem::exists(boostPath))
+			return CompilerStack::ReadFileResult{false, "File not found."};
+		boostPath = boost::filesystem::canonical(boostPath);
+		bool isAllowed = false;
+		for (auto const& allowedDir: m_allowedDirectories)
 		{
-			auto const& virt = redir.first;
-			if (longestPrefix > 0 && virt.length() <= longestPrefix)
-				continue;
-			if (virt.length() > _path.length() || !std::equal(virt.begin(), virt.end(), _path.begin()))
-				continue;
-			string path = redir.second;
-			path.append(_path.begin() + virt.length(), _path.end());
-			auto boostPath = boost::filesystem::path(path);
-			if (!boost::filesystem::exists(boostPath))
+			// If dir is a prefix of boostPath, we are fine.
+			if (
+				std::distance(allowedDir.begin(), allowedDir.end()) <= std::distance(boostPath.begin(), boostPath.end()) &&
+				std::equal(allowedDir.begin(), allowedDir.end(), boostPath.begin())
+			)
 			{
-				errorLevel = max(errorLevel, 0);
-				continue;
-			}
-			boostPath = boost::filesystem::canonical(boostPath);
-			bool isAllowed = false;
-			for (auto const& dir: m_allowedDirectories)
-			{
-				// If dir is a prefix of boostPath, we are fine.
-				if (
-					std::distance(dir.begin(), dir.end()) <= std::distance(boostPath.begin(), boostPath.end()) &&
-					std::equal(dir.begin(), dir.end(), boostPath.begin())
-				)
-				{
-					isAllowed = true;
-					break;
-				}
-			}
-			if (!isAllowed)
-				errorLevel = max(errorLevel, 2);
-			else if (!boost::filesystem::is_regular_file(boostPath))
-				errorLevel = max(errorLevel, 1);
-			else
-			{
-				longestPrefix = virt.length();
-				bestMatchPath = path;
+				isAllowed = true;
+				break;
 			}
 		}
-		if (!bestMatchPath.empty())
-			return make_pair(m_sourceCodes[bestMatchPath] = dev::contentsString(bestMatchPath), string());
-		if (errorLevel == 0)
-			return make_pair(string(), string("File not found."));
-		else if (errorLevel == 1)
-			return make_pair(string(), string("Not a valid file."));
+		if (!isAllowed)
+			return CompilerStack::ReadFileResult{false, "File outside of allowed directories."};
+		else if (!boost::filesystem::is_regular_file(boostPath))
+			return CompilerStack::ReadFileResult{false, "Not a valid file."};
 		else
-			return make_pair(string(), string("File outside of allowed directories."));
+		{
+			auto contents = dev::contentsString(boostPath.string());
+			m_sourceCodes[boostPath.string()] = contents;
+			return CompilerStack::ReadFileResult{true, contents};
+		}
 	};
 
 	m_compiler.reset(new CompilerStack(m_args.count(g_argAddStandard) > 0, fileReader));
 	auto scannerFromSourceName = [&](string const& _sourceName) -> solidity::Scanner const& { return m_compiler->scanner(_sourceName); };
 	try
 	{
+		m_compiler->setRemappings(m_args["input-file"].as<vector<string>>());
 		for (auto const& sourceCode: m_sourceCodes)
 			m_compiler->addSource(sourceCode.first, sourceCode.second);
 		// TODO: Perhaps we should not compile unless requested

solc/CommandLineInterface.h

@@ -85,8 +85,6 @@ class CommandLineInterface
 	boost::program_options::variables_map m_args;
 	/// map of input files to source code strings
 	std::map<std::string, std::string> m_sourceCodes;
-	/// list of path prefix remappings, e.g. github.com/ethereum -> /usr/local/ethereum
-	std::vector<std::pair<std::string, std::string>> m_remappings;
 	/// list of allowed directories to read files from
 	std::vector<boost::filesystem::path> m_allowedDirectories;
 	/// map of library names to addresses