Merge branch 'develop' of github.com:tfire/solidity into fix/remove-namespace-ast-annotations

Author: tylerfxn

Changelog.md

@@ -1,12 +1,14 @@
 ### 0.8.13 (unreleased)
 
 Language Features:
+ * General: Allow annotating inline assembly as memory-safe to allow optimizations and stack limit evasion that rely on respecting Solidity's memory model.
 
 
 Compiler Features:
 * JSON-AST: Added selector field for errors and events.
 
 Bugfixes:
+ * Yul IR Code Generation: Optimize embedded creation code with correct settings. This fixes potential mismatches between the constructor code of a contract compiled in isolation and the bytecode in ``type(C).creationCode``, resp. the bytecode used for ``new C(...)``.
 
 
 ### 0.8.12 (2022-02-16)

cmake/EthUtils.cmake

@@ -48,3 +48,11 @@ function(detect_stray_source_files FILELIST DIRECTORY)
 		message(SEND_ERROR "The following source files are present but are not compiled: ${sources}")
 	endif()
 endfunction(detect_stray_source_files)
+
+# CreateExportedFunctionsForEMSDK(OUTPUT_VARIABLE Symbol1 Symbol2 ... SymbolN)
+function(CreateExportedFunctionsForEMSDK OUTPUT_VARIABLE)
+	list(TRANSFORM ARGN PREPEND "\"_")
+	list(TRANSFORM ARGN APPEND "\"")
+	list(JOIN ARGN "," ARGN)
+	set(${OUTPUT_VARIABLE} "[${ARGN}]" PARENT_SCOPE)
+endfunction()

docs/assembly.rst

@@ -228,6 +228,11 @@ of their block is reached.
 Conventions in Solidity
 -----------------------
 
+.. _assembly-typed-variables:
+
+Values of Typed Variables
+=========================
+
 In contrast to EVM assembly, Solidity has types which are narrower than 256 bits,
 e.g. ``uint24``. For efficiency, most arithmetic operations ignore the fact that
 types can be shorter than 256
@@ -237,6 +242,11 @@ This means that if you access such a variable
 from within inline assembly, you might have to manually clean the higher-order bits
 first.
 
+.. _assembly-memory-management:
+
+Memory Management
+=================
+
 Solidity manages memory in the following way. There is a "free memory pointer"
 at position ``0x40`` in memory. If you want to allocate memory, use the memory
 starting from where this pointer points at and update it.
@@ -268,3 +278,99 @@ first slot of the array and followed by the array elements.
     Statically-sized memory arrays do not have a length field, but it might be added later
     to allow better convertibility between statically- and dynamically-sized arrays, so
     do not rely on this.
+
+Memory Safety
+=============
+
+Without the use of inline assembly, the compiler can rely on memory to remain in a well-defined
+state at all times. This is especially relevant for :ref:`the new code generation pipeline via Yul IR <ir-breaking-changes>`:
+this code generation path can move local variables from stack to memory to avoid stack-too-deep errors and
+perform additional memory optimizations, if it can rely on certain assumptions about memory use.
+
+While we recommend to always respect Solidity's memory model, inline assembly allows you to use memory
+in an incompatible way. Therefore, moving stack variables to memory and additional memory optimizations are,
+by default, disabled in the presence of any inline assembly block that contains a memory operation or assigns
+to solidity variables in memory.
+
+However, you can specifically annotate an assembly block to indicate that it in fact respects Solidity's memory
+model as follows:
+
+.. code-block:: solidity
+
+    assembly ("memory-safe") {
+        ...
+    }
+
+In particular, a memory-safe assembly block may only access the following memory ranges:
+
+- Memory allocated by yourself using a mechanism like the ``allocate`` function described above.
+- Memory allocated by Solidity, e.g. memory within the bounds of a memory array you reference.
+- The scratch space between memory offset 0 and 64 mentioned above.
+- Temporary memory that is located *after* the value of the free memory pointer at the beginning of the assembly block,
+  i.e. memory that is "allocated" at the free memory pointer without updating the free memory pointer.
+
+Furthermore, if the assembly block assigns to Solidity variables in memory, you need to assure that accesses to
+the Solidity variables only access these memory ranges.
+
+Since this is mainly about the optimizer, these restrictions still need to be followed, even if the assembly block
+reverts or terminates. As an example, the following assembly snippet is not memory safe:
+
+.. code-block:: solidity
+
+    assembly {
+      returndatacopy(0, 0, returndatasize())
+      revert(0, returndatasize())
+    }
+
+But the following is:
+
+.. code-block:: solidity
+
+    assembly ("memory-safe") {
+      let p := mload(0x40)
+      returndatacopy(p, 0, returndatasize())
+      revert(p, returndatasize())
+    }
+
+Note that you do not need to update the free memory pointer if there is no following allocation,
+but you can only use memory starting from the current offset given by the free memory pointer.
+
+If the memory operations use a length of zero, it is also fine to just use any offset (not only if it falls into the scratch space):
+
+.. code-block:: solidity
+
+    assembly ("memory-safe") {
+      revert(0, 0)
+    }
+
+Note that not only memory operations in inline assembly itself can be memory-unsafe, but also assignments to
+solidity variables of reference type in memory. For example the following is not memory-safe:
+
+.. code-block:: solidity
+
+    bytes memory x;
+    assembly {
+      x := 0x40
+    }
+    x[0x20] = 0x42;
+
+Inline assembly that neither involves any operations that access memory nor assigns to any solidity variables
+in memory is automatically considered memory-safe and does not need to be annotated.
+
+.. warning::
+    It is your responsibility to make sure that the assembly actually satisfies the memory model. If you annotate
+    an assembly block as memory-safe, but violate one of the memory assumptions, this **will** lead to incorrect and
+    undefined behaviour that cannot easily be discovered by testing.
+
+In case you are developing a library that is meant to be compatible across multiple versions
+of solidity, you can use a special comment to annotate an assembly block as memory-safe:
+
+.. code-block:: solidity
+
+    /// @solidity memory-safe-assembly
+    assembly {
+        ...
+    }
+
+Note that we will disallow the annotation via comment in a future breaking release, so if you are not concerned with
+backwards-compatibility with older compiler versions, prefer using the dialect string.

docs/grammar/SolidityLexer.g4

@@ -251,6 +251,12 @@ mode AssemblyBlockMode;
 AssemblyDialect: '"evmasm"';
 AssemblyLBrace: '{' -> popMode, pushMode(YulMode);
 
+AssemblyFlagString: '"' DoubleQuotedStringCharacter+ '"';
+
+AssemblyBlockLParen: '(';
+AssemblyBlockRParen: ')';
+AssemblyBlockComma: ',';
+
 AssemblyBlockWS: [ \t\r\n\u000C]+ -> skip ;
 AssemblyBlockCOMMENT: '/*' .*? '*/' -> channel(HIDDEN) ;
 AssemblyBlockLINE_COMMENT: '//' ~[\r\n]* -> channel(HIDDEN) ;

docs/grammar/SolidityParser.g4

@@ -476,7 +476,13 @@ revertStatement: Revert expression callArgumentList Semicolon;
  * The contents of an inline assembly block use a separate scanner/lexer, i.e. the set of keywords and
  * allowed identifiers is different inside an inline assembly block.
  */
-assemblyStatement: Assembly AssemblyDialect? AssemblyLBrace yulStatement* YulRBrace;
+assemblyStatement: Assembly AssemblyDialect? assemblyFlags? AssemblyLBrace yulStatement* YulRBrace;
+
+/**
+ * Assembly flags.
+ * Comma-separated list of double-quoted strings as flags.
+ */
+assemblyFlags: AssemblyBlockLParen AssemblyFlagString (AssemblyBlockComma AssemblyFlagString)* AssemblyBlockRParen;
 
 //@doc:inline
 variableDeclarationList: variableDeclarations+=variableDeclaration (Comma variableDeclarations+=variableDeclaration)*;

docs/index.rst

@@ -5,10 +5,8 @@ Solidity is an object-oriented, high-level language for implementing smart
 contracts. Smart contracts are programs which govern the behaviour of accounts
 within the Ethereum state.
 
-Solidity is a `curly-bracket language <https://en.wikipedia.org/wiki/List_of_programming_languages_by_type#Curly-bracket_languages>`_.
-It is influenced by C++, Python and JavaScript, and is designed to target the Ethereum Virtual Machine (EVM).
-You can find more details about which languages Solidity has been inspired by in
-the :doc:`language influences <language-influences>` section.
+Solidity is a `curly-bracket language <https://en.wikipedia.org/wiki/List_of_programming_languages_by_type#Curly-bracket_languages>`_ designed to target the Ethereum Virtual Machine (EVM).
+It is influenced by C++, Python and JavaScript. You can find more details about which languages Solidity has been inspired by in the :doc:`language influences <language-influences>` section.
 
 Solidity is statically typed, supports inheritance, libraries and complex
 user-defined types among other features.
@@ -90,24 +88,25 @@ our `Gitter channel <https://gitter.im/ethereum/solidity/>`_.
 Translations
 ------------
 
-Community volunteers help translate this documentation into several languages.
-They have varying degrees of completeness and up-to-dateness. The English
+Community contributors help translate this documentation into several languages.
+Note that they have varying degrees of completeness and up-to-dateness. The English
 version stands as a reference.
 
+You can switch between languages by clicking on the flyout menu in the bottom-left corner
+and selecting the preferred language.
+
+* `French <https://docs.soliditylang.org/fr/latest/>`_
+* `Indonesian <https://github.com/solidity-docs/id-indonesian>`_
+* `Persian <https://github.com/solidity-docs/fa-persian>`_
+* `Japanese <https://github.com/solidity-docs/ja-japanese>`_
+* `Korean <https://github.com/solidity-docs/ko-korean>`_
+* `Chinese <https://github.com/solidity-docs/zh-cn-chinese/>`_
+
 .. note::
 
    We recently set up a new GitHub organization and translation workflow to help streamline the
    community efforts. Please refer to the `translation guide <https://github.com/solidity-docs/translation-guide>`_
-   for information on how to contribute to the community translations moving forward.
-
-* `French <https://solidity-fr.readthedocs.io>`_ (in progress)
-* `Italian <https://github.com/damianoazzolini/solidity>`_ (in progress)
-* `Japanese <https://solidity-jp.readthedocs.io>`_
-* `Korean <https://solidity-kr.readthedocs.io>`_ (in progress)
-* `Russian <https://github.com/ethereum/wiki/wiki/%5BRussian%5D-%D0%A0%D1%83%D0%BA%D0%BE%D0%B2%D0%BE%D0%B4%D1%81%D1%82%D0%B2%D0%BE-%D0%BF%D0%BE-Solidity>`_ (rather outdated)
-* `Simplified Chinese <https://learnblockchain.cn/docs/solidity/>`_ (in progress)
-* `Spanish <https://solidity-es.readthedocs.io>`_
-* `Turkish <https://github.com/denizozzgur/Solidity_TR/blob/master/README.md>`_ (partial)
+   for information on how to start a new language or contribute to the community translations.
 
 Contents
 ========

docs/ir-breaking-changes.rst

@@ -1,6 +1,8 @@
 
 .. index: ir breaking changes
 
+.. _ir-breaking-changes:
+
 *********************************
 Solidity IR-based Codegen Changes
 *********************************

docs/using-the-compiler.rst

@@ -410,12 +410,13 @@ Input Description
             "source1.sol": ["contract1"],
             "source2.sol": ["contract2", "contract3"]
           },
-          // Choose whether division and modulo operations should be replaced by
-          // multiplication with slack variables. Default is `true`.
-          // Using `false` here is recommended if you are using the CHC engine
+          // Choose how division and modulo operations should be encoded.
+          // When using `false` they are replaced by multiplication with slack
+          // variables. This is the default.
+          // Using `true` here is recommended if you are using the CHC engine
           // and not using Spacer as the Horn solver (using Eldarica, for example).
           // See the Formal Verification section for a more detailed explanation of this option.
-          "divModWithSlacks": true,
+          "divModNoSlacks": false,
           // Choose which model checker engine to use: all (default), bmc, chc, none.
           "engine": "chc",
           // Choose which types of invariants should be reported to the user: contract, reentrancy.

docs/yul.rst

@@ -1124,6 +1124,11 @@ regular strings in native encoding. For code,
 
 Above, ``Block`` refers to ``Block`` in the Yul code grammar explained in the previous chapter.
 
+.. note::
+
+    An object with a name that ends in ``_deployed`` is treated as deployed code by the Yul optimizer.
+    The only consequence of this is a different gas cost heuristic in the optimizer.
+
 .. note::
 
     Data objects or sub-objects whose names contain a ``.`` can be defined
@@ -1172,17 +1177,17 @@ An example Yul Object is shown below:
 
             // now return the runtime object (the currently
             // executing code is the constructor code)
-            size := datasize("runtime")
+            size := datasize("Contract1_deployed")
             offset := allocate(size)
             // This will turn into a memory->memory copy for Ewasm and
             // a codecopy for EVM
-            datacopy(offset, dataoffset("runtime"), size)
+            datacopy(offset, dataoffset("Contract1_deployed"), size)
             return(offset, size)
         }
 
         data "Table2" hex"4123"
 
-        object "runtime" {
+        object "Contract1_deployed" {
             code {
                 function allocate(size) -> ptr {
                     ptr := mload(0x40)
@@ -1204,7 +1209,7 @@ An example Yul Object is shown below:
                 // code here ...
             }
 
-            object "runtime" {
+            object "Contract2_deployed" {
                 code {
                     // code here ...
                 }

libevmasm/Assembly.cpp

@@ -397,26 +397,6 @@ AssemblyItem Assembly::newImmutableAssignment(string const& _identifier)
 	return AssemblyItem{AssignImmutable, h};
 }
 
-Assembly& Assembly::optimise(bool _enable, EVMVersion _evmVersion, bool _isCreation, size_t _runs)
-{
-	OptimiserSettings settings;
-	settings.isCreation = _isCreation;
-	settings.runInliner = true;
-	settings.runJumpdestRemover = true;
-	settings.runPeephole = true;
-	if (_enable)
-	{
-		settings.runDeduplicate = true;
-		settings.runCSE = true;
-		settings.runConstantOptimiser = true;
-	}
-	settings.evmVersion = _evmVersion;
-	settings.expectedExecutionsPerDeployment = _runs;
-	optimise(settings);
-	return *this;
-}
-
-
 Assembly& Assembly::optimise(OptimiserSettings const& _settings)
 {
 	optimiseInternal(_settings, {});
@@ -435,9 +415,8 @@ map<u256, u256> const& Assembly::optimiseInternal(
 	for (size_t subId = 0; subId < m_subs.size(); ++subId)
 	{
 		OptimiserSettings settings = _settings;
-		// Disable creation mode for sub-assemblies.
-		settings.isCreation = false;
-		map<u256, u256> const& subTagReplacements = m_subs[subId]->optimiseInternal(
+		Assembly& sub = *m_subs[subId];
+		map<u256, u256> const& subTagReplacements = sub.optimiseInternal(
 			settings,
 			JumpdestRemover::referencedTags(m_items, subId)
 		);
@@ -456,7 +435,7 @@ map<u256, u256> const& Assembly::optimiseInternal(
 				m_items,
 				_tagsReferencedFromOutside,
 				_settings.expectedExecutionsPerDeployment,
-				_settings.isCreation,
+				isCreation(),
 				_settings.evmVersion
 			}.optimise();
 
@@ -557,8 +536,8 @@ map<u256, u256> const& Assembly::optimiseInternal(
 
 	if (_settings.runConstantOptimiser)
 		ConstantOptimisationMethod::optimiseConstants(
-			_settings.isCreation,
-			_settings.isCreation ? 1 : _settings.expectedExecutionsPerDeployment,
+			isCreation(),
+			isCreation() ? 1 : _settings.expectedExecutionsPerDeployment,
 			_settings.evmVersion,
 			*this
 		);