Parse declaration attributes within #if...#endif blocks.

Introduce support for parsing declaration attributes that occur within
example:

    #if hasAttribute(frozen)
    @Frozen
    #endif
    public struct X { ... }

will apply to "frozen" attribute to the struct `X`, but only when the
compiler supports the "frozen" attribute.

Correctly determining whether a particular `#if` block contains
attributes to be associated with the following declaration vs.
starting a new declaration requires arbitrary lookahead. The parser
will ensure that at least one of the branches of the `#if` contains an
attribute, and that none of the branches contains something that does
not fit the attribute grammar, before committing to parsing the `#if`
clause as part of the declaration attributes. This lookahead does
occur at the top level (e.g., in the parsing of top-level declarations
and code), but should only need to scan past the first `#if` line to
the following token in the common case.

Unlike other `#if` when used to wrap statements or declarations, we
make no attempt to record the `#if` not taken anywhere in the AST.
This reflects a change in attitude in the design of the AST, because
we have found that trying to represent this information there (e.g.,
via `IfConfigDecl`) complicates clients while providing little value.
This information is best kept in the syntax tree, only.

Author: DougGregor

include/swift/Parse/Parser.h

@@ -100,6 +100,16 @@ class ConsumeTokenReceiver {
   virtual ~ConsumeTokenReceiver() = default;
 };
 
+/// The role of the elements in a given #if
+enum class IfConfigElementsRole {
+  // Parse normally.
+  Normal,
+  // Parse, but only for syntax. Throw away the results.
+  SyntaxOnly,
+  // Already skipped; no need to parse anything.
+  Skipped
+};
+
 /// The main class used for parsing a source file (.swift or .sil).
 ///
 /// Rather than instantiating a Parser yourself, use one of the parsing APIs
@@ -684,7 +694,8 @@ class Parser {
     while (Tok.isNot(K..., tok::eof, tok::r_brace, tok::pound_endif,
                      tok::pound_else, tok::pound_elseif,
                      tok::code_complete) &&
-           !isStartOfStmt() && !isStartOfSwiftDecl()) {
+           !isStartOfStmt() &&
+           !isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false)) {
       skipSingle();
     }
   }
@@ -945,7 +956,7 @@ class Parser {
   // Decl Parsing
 
   /// Returns true if parser is at the start of a Swift decl or decl-import.
-  bool isStartOfSwiftDecl();
+  bool isStartOfSwiftDecl(bool allowPoundIfAttributes = true);
 
   /// Returns true if the parser is at the start of a SIL decl.
   bool isStartOfSILDecl();
@@ -982,6 +993,7 @@ class Parser {
 
   ParserResult<Decl> parseDecl(ParseDeclOptions Flags,
                                bool IsAtStartOfLineOrPreviousHadSemi,
+                               bool IfConfigsAreDeclAttrs,
                                llvm::function_ref<void(Decl*)> Handler);
 
   std::pair<std::vector<Decl *>, Optional<Fingerprint>>
@@ -1004,12 +1016,29 @@ class Parser {
 
   ParserResult<TypeDecl> parseDeclAssociatedType(ParseDeclOptions Flags,
                                                  DeclAttributes &Attributes);
-  
+
+  /// Parse a #if ... #endif directive.
+  /// Delegate callback function to parse elements in the blocks and record
+  /// them however they wish. The parsing function will be provided with the
+  /// location of the clause token (`#if`, `#else`, etc.), the condition,
+  /// whether this is the active clause, and the role of the elements.
+  template<typename Result>
+  Result parseIfConfigRaw(
+      llvm::function_ref<void(SourceLoc clauseLoc, Expr *condition,
+                              bool isActive, IfConfigElementsRole role)>
+          parseElements,
+      llvm::function_ref<Result(SourceLoc endLoc, bool hadMissingEnd)> finish);
+
   /// Parse a #if ... #endif directive.
   /// Delegate callback function to parse elements in the blocks.
   ParserResult<IfConfigDecl> parseIfConfig(
     llvm::function_ref<void(SmallVectorImpl<ASTNode> &, bool)> parseElements);
 
+  /// Parse an #if ... #endif containing only attributes.
+  ParserStatus parseIfConfigDeclAttributes(
+    DeclAttributes &attributes, bool ifConfigsAreDeclAttrs,
+    PatternBindingInitializer *initContext);
+
   /// Parse a #error or #warning diagnostic.
   ParserResult<PoundDiagnosticDecl> parseDeclPoundDiagnostic();
 
@@ -1020,8 +1049,26 @@ class Parser {
   void setLocalDiscriminator(ValueDecl *D);
   void setLocalDiscriminatorToParamList(ParameterList *PL);
 
+  /// Skip an `#if` configuration block containing only attributes.
+  ///
+  /// \returns true if the skipping was successful, false otherwise.
+  bool skipIfConfigOfAttributes(bool &sawAnyAttributes);
+
+  /// Determine whether the `#if` at which the parser occurs only contains
+  /// attributes (in all branches), in which case it is treated as part of
+  /// an attribute list.
+  bool ifConfigContainsOnlyAttributes();
+
   /// Parse the optional attributes before a declaration.
-  ParserStatus parseDeclAttributeList(DeclAttributes &Attributes);
+  ParserStatus parseDeclAttributeList(DeclAttributes &Attributes,
+                                      bool IfConfigsAreDeclAttrs = false);
+
+  /// Parse the optional attributes before a declaration.
+  ///
+  /// This is the inner loop, which can be called recursively.
+  ParserStatus parseDeclAttributeList(DeclAttributes &Attributes,
+                                      bool IfConfigsAreDeclAttrs,
+                                      PatternBindingInitializer *initContext);
 
   /// Parse the optional modifiers before a declaration.
   bool parseDeclModifierList(DeclAttributes &Attributes, SourceLoc &StaticLoc,

lib/Parse/ParseDecl.cpp

@@ -3793,26 +3793,41 @@ ParserStatus Parser::parseTypeAttribute(TypeAttributes &Attributes,
   return makeParserSuccess();
 }
 
+ParserStatus Parser::parseDeclAttributeList(
+    DeclAttributes &Attributes, bool ifConfigsAreDeclAttrs,
+    PatternBindingInitializer *initContext) {
+  ParserStatus Status;
+  while (Tok.isAny(tok::at_sign, tok::pound_if)) {
+    if (Tok.is(tok::at_sign)) {
+      SyntaxParsingContext AttrCtx(SyntaxContext, SyntaxKind::Attribute);
+      SourceLoc AtLoc = consumeToken();
+      Status |= parseDeclAttribute(Attributes, AtLoc, initContext);
+    } else {
+      if (!ifConfigsAreDeclAttrs && !ifConfigContainsOnlyAttributes())
+        break;
+
+      Status |= parseIfConfigDeclAttributes(
+          Attributes, ifConfigsAreDeclAttrs, initContext);
+    }
+  }
+  return Status;
+}
+
 /// \verbatim
 ///   attribute-list:
 ///     /*empty*/
 ///     attribute-list-clause attribute-list
 ///   attribute-list-clause:
 ///     '@' attribute
 /// \endverbatim
-ParserStatus Parser::parseDeclAttributeList(DeclAttributes &Attributes) {
-  if (Tok.isNot(tok::at_sign))
+ParserStatus Parser::parseDeclAttributeList(
+    DeclAttributes &Attributes, bool IfConfigsAreDeclAttrs) {
+  if (Tok.isNot(tok::at_sign, tok::pound_if))
     return makeParserSuccess();
 
   PatternBindingInitializer *initContext = nullptr;
-  ParserStatus Status;
   SyntaxParsingContext AttrListCtx(SyntaxContext, SyntaxKind::AttributeList);
-  do {
-    SyntaxParsingContext AttrCtx(SyntaxContext, SyntaxKind::Attribute);
-    SourceLoc AtLoc = consumeToken();
-    Status |= parseDeclAttribute(Attributes, AtLoc, initContext);
-  } while (Tok.is(tok::at_sign));
-  return Status;
+  return parseDeclAttributeList(Attributes, IfConfigsAreDeclAttrs, initContext);
 }
 
 /// \verbatim
@@ -3902,7 +3917,8 @@ bool Parser::parseDeclModifierList(DeclAttributes &Attributes,
           BacktrackingScope Scope(*this);
 
           consumeToken(); // consume actor
-          isActorModifier = isStartOfSwiftDecl();
+          isActorModifier = isStartOfSwiftDecl(
+              /*allowPoundIfAttributes=*/false);
         }
 
         if (!isActorModifier)
@@ -4273,7 +4289,7 @@ static void skipAttribute(Parser &P) {
   }
 }
 
-bool Parser::isStartOfSwiftDecl() {
+bool Parser::isStartOfSwiftDecl(bool allowPoundIfAttributes) {
   if (Tok.is(tok::at_sign) && peekToken().is(tok::kw_rethrows)) {
     // @rethrows does not follow the general rule of @<identifier> so
     // it is needed to short circuit this else there will be an infinite
@@ -4307,7 +4323,7 @@ bool Parser::isStartOfSwiftDecl() {
   // check 'let' and 'var' right now.
   if (Tok.is(tok::kw_try))
     return peekToken().isAny(tok::kw_let, tok::kw_var);
-  
+
   // Skip an attribute, since it might be a type attribute.  This can't
   // happen at the top level of a scope, but we do use isStartOfSwiftDecl()
   // in positions like generic argument lists.
@@ -4318,10 +4334,20 @@ bool Parser::isStartOfSwiftDecl() {
 
     // If this attribute is the last element in the block,
     // consider it is a start of incomplete decl.
-    if (Tok.isAny(tok::r_brace, tok::eof, tok::pound_endif))
+    if (Tok.isAny(tok::r_brace, tok::eof) ||
+        (Tok.is(tok::pound_endif) && !allowPoundIfAttributes))
       return true;
 
-    return isStartOfSwiftDecl();
+    return isStartOfSwiftDecl(allowPoundIfAttributes);
+  }
+
+  // Skip a #if that contains only attributes in all branches. These will be
+  // parsed as attributes of a declaration, not as separate declarations.
+  if (Tok.is(tok::pound_if) && allowPoundIfAttributes) {
+    BacktrackingScope backtrack(*this);
+    bool sawAnyAttributes = false;
+    return skipIfConfigOfAttributes(sawAnyAttributes) && sawAnyAttributes &&
+        isStartOfSwiftDecl();
   }
 
   // If we have a decl modifying keyword, check if the next token is a valid
@@ -4343,13 +4369,13 @@ bool Parser::isStartOfSwiftDecl() {
           // If we found the start of a decl while trying to skip over the
           // paren, then we have something incomplete like 'private('. Return
           // true for better recovery.
-          if (isStartOfSwiftDecl())
+          if (isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false))
             return true;
 
           skipSingle();
         }
       }
-      return isStartOfSwiftDecl();
+      return isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false);
     }
   }
 
@@ -4376,7 +4402,7 @@ bool Parser::isStartOfSwiftDecl() {
     consumeToken(tok::l_paren);
     consumeToken(tok::identifier);
     consumeToken(tok::r_paren);
-    return isStartOfSwiftDecl();
+    return isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false);
   }
 
   if (Tok.isContextualKeyword("actor")) {
@@ -4388,7 +4414,7 @@ bool Parser::isStartOfSwiftDecl() {
     // it's an actor declaration, otherwise, it isn't.
     do {
       consumeToken();
-    } while (isStartOfSwiftDecl());
+    } while (isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false));
     return Tok.is(tok::identifier);
   }
 
@@ -4399,7 +4425,7 @@ bool Parser::isStartOfSwiftDecl() {
   // Otherwise, do a recursive parse.
   Parser::BacktrackingScope Backtrack(*this);
   consumeToken(tok::identifier);
-  return isStartOfSwiftDecl();
+  return isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false);
 }
 
 bool Parser::isStartOfSILDecl() {
@@ -4510,12 +4536,13 @@ setOriginalDeclarationForDifferentiableAttributes(DeclAttributes attrs,
 ParserResult<Decl>
 Parser::parseDecl(ParseDeclOptions Flags,
                   bool IsAtStartOfLineOrPreviousHadSemi,
+                  bool IfConfigsAreDeclAttrs,
                   llvm::function_ref<void(Decl*)> Handler) {
   ParserPosition BeginParserPosition;
   if (isCodeCompletionFirstPass())
     BeginParserPosition = getParserPosition();
 
-  if (Tok.is(tok::pound_if)) {
+  if (Tok.is(tok::pound_if) && !ifConfigContainsOnlyAttributes()) {
     auto IfConfigResult = parseIfConfig(
       [&](SmallVectorImpl<ASTNode> &Decls, bool IsActive) {
         ParserStatus Status;
@@ -4574,7 +4601,8 @@ Parser::parseDecl(ParseDeclOptions Flags,
   DeclAttributes Attributes;
   if (Tok.hasComment())
     Attributes.add(new (Context) RawDocCommentAttr(Tok.getCommentRange()));
-  ParserStatus AttrStatus = parseDeclAttributeList(Attributes);
+  ParserStatus AttrStatus = parseDeclAttributeList(
+      Attributes, IfConfigsAreDeclAttrs);
 
   // Parse modifiers.
   // Keep track of where and whether we see a contextual keyword on the decl.
@@ -5356,7 +5384,9 @@ ParserStatus Parser::parseDeclItem(bool &PreviousHadSemi,
   if (loadCurrentSyntaxNodeFromCache()) {
     return ParserStatus();
   }
-  Result = parseDecl(Options, IsAtStartOfLineOrPreviousHadSemi, handler);
+  Result = parseDecl(
+      Options, IsAtStartOfLineOrPreviousHadSemi,
+      /* IfConfigsAreDeclAttrs=*/false, handler);
   if (Result.isParseErrorOrHasCompletion())
     skipUntilDeclRBrace(tok::semi, tok::pound_endif);
   SourceLoc SemiLoc;
@@ -6200,7 +6230,8 @@ void Parser::skipSILUntilSwiftDecl() {
   // Tell the lexer we're about to start lexing SIL.
   Lexer::SILBodyRAII sbr(*L);
 
-  while (!Tok.is(tok::eof) && !isStartOfSwiftDecl()) {
+  while (!Tok.is(tok::eof) &&
+         !isStartOfSwiftDecl(/*allowPoundIfAttributes=*/false)) {
     // SIL pound dotted paths need to be skipped specially as they can contain
     // decl keywords like 'subscript'.
     if (consumeIf(tok::pound)) {