[flang] Don't perform macro replacement on exponents (#136176)

klausler · web-flow · commit aac53ad4d6ea · 2025-04-18T12:52:04.000-07:00
See new test. I inadvertently broke this behavior with a recent fix for
another problem, because the effects of the overloaded
TokenSequence::Put() member function on token merging were confusing.
Rename and document the various overloads.
diff --git a/flang/include/flang/Parser/token-sequence.h b/flang/include/flang/Parser/token-sequence.h
@@ -35,10 +35,10 @@ class Prescanner;
 class TokenSequence {
 public:
   TokenSequence() {}
-  TokenSequence(const TokenSequence &that) { Put(that); }
+  TokenSequence(const TokenSequence &that) { CopyAll(that); }
   TokenSequence(
       const TokenSequence &that, std::size_t at, std::size_t count = 1) {
-    Put(that, at, count);
+    AppendRange(that, at, count);
   }
   TokenSequence(TokenSequence &&that)
       : start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@@ -48,7 +48,7 @@ class TokenSequence {
 
   TokenSequence &operator=(const TokenSequence &that) {
     clear();
-    Put(that);
+    CopyAll(that);
     return *this;
   }
   TokenSequence &operator=(TokenSequence &&that);
@@ -100,14 +100,26 @@ class TokenSequence {
     start_.pop_back();
   }
 
-  void Put(const TokenSequence &);
-  void Put(const TokenSequence &, ProvenanceRange);
-  void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+  // These append characters with provenance and then close the token.
+  // When the last token of this sequence remains open beforehand,
+  // the new characters are appended to it.
   void Put(const char *, std::size_t, Provenance);
   void Put(const CharBlock &, Provenance);
   void Put(const std::string &, Provenance);
   void Put(llvm::raw_string_ostream &, Provenance);
 
+  // Appends a full copy of another sequence.  When the last token of this
+  // sequence remains open beforehand, it is closed before the new text
+  // is appended.
+  void CopyAll(const TokenSequence &);
+  // Copies a range of tokens from another sequence.  If the last token of
+  // this sequence remains open, the first token of the copied range will be
+  // appended to it.
+  void AppendRange(
+      const TokenSequence &, std::size_t at, std::size_t tokens = 1);
+  // Copies tokens (via Put above) with new provenance.
+  void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
+
   Provenance GetCharProvenance(std::size_t) const;
   Provenance GetTokenProvenance(
       std::size_t token, std::size_t offset = 0) const;
diff --git a/flang/lib/Parser/preprocessor.cpp b/flang/lib/Parser/preprocessor.cpp
@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
         continue;
       }
     }
-    result.Put(token, firstToken + j, 1);
+    result.AppendRange(token, firstToken + j, 1);
   }
   return result;
 }
@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
       }
     } else if (pasting && text.TokenAt(j).IsBlank()) {
     } else {
-      result.Put(text, j, 1);
+      result.AppendRange(text, j, 1);
       pasting = false;
     }
   }
@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
         CHECK(resultSize > 0 &&
             result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
         result.pop_back();
-        result.Put(Stringify(args[index], prescanner.allSources()));
+        result.CopyAll(Stringify(args[index], prescanner.allSources()));
       } else {
         const TokenSequence *arg{&args[index]};
         std::optional<TokenSequence> replaced;
@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
             }
           }
         }
-        result.Put(DEREF(arg));
+        result.CopyAll(DEREF(arg));
       }
     } else if (bytes == 11 && isVariadic_ &&
         token.ToString() == "__VA_ARGS__") {
@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
         if (k > argumentCount()) {
           result.Put(","s, commaProvenance);
         }
-        result.Put(args[k]);
+        result.CopyAll(args[k]);
       }
     } else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
         j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
           }
         }
       }
-      result.Put(replacement_, j);
+      result.AppendRange(replacement_, j);
     }
   }
   return TokenPasting(std::move(result));
@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
           inIfExpression](std::size_t after, const TokenSequence &replacement,
           std::size_t pFLMOffset) {
         if (after < input.SizeInTokens()) {
-          result.Put(replacement, 0, pFLMOffset);
+          result.AppendRange(replacement, 0, pFLMOffset);
           TokenSequence suffix;
-          suffix.Put(
+          suffix.AppendRange(
               replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
-          suffix.Put(input, after, input.SizeInTokens() - after);
+          suffix.AppendRange(input, after, input.SizeInTokens() - after);
           auto further{ReplaceMacros(
               suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
           if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
             // still not closed
             **partialFunctionLikeMacro += result.SizeInTokens();
           }
-          result.Put(further);
+          result.CopyAll(further);
           return true;
         } else {
           if (partialFunctionLikeMacro) {
@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
   for (; j < tokens; ++j) {
     CharBlock token{input.TokenAt(j)};
     if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     // Process identifier in replacement text.
@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       }
     }
     if (it == definitions_.end()) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     Definition *def{&it->second};
     if (def->isDisabled()) {
-      result.Put(input, j);
+      result.AppendRange(input, j);
       continue;
     }
     if (!def->isFunctionLike()) {
@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
         ProvenanceRange use{input.GetTokenProvenanceRange(j)};
         ProvenanceRange newRange{
             allSources_.AddMacroCall(from, use, replaced.ToString())};
-        result.Put(replaced, newRange);
+        result.CopyWithProvenance(replaced, newRange);
       }
     } else {
       // Possible function-like macro call.  Skip spaces and newlines to see
@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       if (!leftParen) {
         if (partialFunctionLikeMacro) {
           *partialFunctionLikeMacro = result.SizeInTokens();
-          result.Put(input, j, tokens - j);
+          result.AppendRange(input, j, tokens - j);
           return result;
         } else {
-          result.Put(input, j);
+          result.AppendRange(input, j);
           continue;
         }
       }
@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
       }
       if (k >= tokens && partialFunctionLikeMacro) {
         *partialFunctionLikeMacro = result.SizeInTokens();
-        result.Put(input, j, tokens - j);
+        result.AppendRange(input, j, tokens - j);
         return result;
       } else if (k >= tokens || argStart.size() < def->argumentCount() ||
           (argStart.size() > def->argumentCount() && !def->isVariadic())) {
-        result.Put(input, j);
+        result.AppendRange(input, j);
         continue;
       }
       std::vector<TokenSequence> args;
@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
         ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
         ProvenanceRange newRange{
             allSources_.AddMacroCall(from, use, replaced.ToString())};
-        result.Put(replaced, newRange);
+        result.CopyWithProvenance(replaced, newRange);
       }
       j = k; // advance to the terminal ')'
     }
diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp
@@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
         EmitCharAndAdvance(possible, *at_);
       }
       possible.CloseToken();
-      tokens.CloseToken();
-      tokens.Put(possible);
+      tokens.AppendRange(possible, 0); // appends to current token
       return true;
     }
     // Not an exponent; backtrack
@@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
       preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
       !preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
     // "_foo" is not defined, but "foo" is
-    tokens.Put(separate); // '_' "foo"
+    tokens.CopyAll(separate); // '_' "foo"
   } else {
-    tokens.Put(withUnderscore); // "_foo"
+    tokens.CopyAll(withUnderscore); // "_foo"
   }
   return true;
 }
@@ -1016,7 +1015,7 @@ void Prescanner::QuotedCharacterLiteral(
             ppTokens.Put(id, GetProvenance(idStart));
             if (auto replaced{
                     preprocessor_.MacroReplacement(ppTokens, *this)}) {
-              tokens.Put(*replaced);
+              tokens.CopyAll(*replaced);
               at_ = &idStart[idLen - 1];
               NextLine();
               continue; // try again on the next line
@@ -1836,7 +1835,7 @@ bool Prescanner::CompilerDirectiveContinuation(
   }
   if (ok) {
     tokens.pop_back(); // delete original '&'
-    tokens.Put(followingTokens, startAt, following - startAt);
+    tokens.AppendRange(followingTokens, startAt, following - startAt);
     tokens.RemoveRedundantBlanks();
   } else {
     nextLine_ = origNextLine;
@@ -1866,7 +1865,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
       }
       followingTokens.RemoveRedundantBlanks();
       tokens.pop_back(); // delete original '&'
-      tokens.Put(followingTokens);
+      tokens.CopyAll(followingTokens);
       return true;
     }
   }
diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
   return false;
 }
 
-void TokenSequence::Put(const TokenSequence &that) {
+void TokenSequence::CopyAll(const TokenSequence &that) {
   if (nextStart_ < char_.size()) {
     start_.push_back(nextStart_);
   }
@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
   provenances_.Put(that.provenances_);
 }
 
-void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
+void TokenSequence::CopyWithProvenance(
+    const TokenSequence &that, ProvenanceRange range) {
   std::size_t offset{0};
   std::size_t tokens{that.SizeInTokens()};
   for (std::size_t j{0}; j < tokens; ++j) {
@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
   CHECK(offset == range.size());
 }
 
-void TokenSequence::Put(
+void TokenSequence::AppendRange(
     const TokenSequence &that, std::size_t at, std::size_t tokens) {
   ProvenanceRange provenance;
   std::size_t offset{0};
@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
   TokenSequence result;
   for (std::size_t j{0}; j < tokens; ++j) {
     if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
-      result.Put(*this, j);
+      result.AppendRange(*this, j);
     }
   }
   swap(result);
@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
   for (std::size_t j{0}; j < tokens; ++j) {
     bool isBlank{TokenAt(j).IsBlank()};
     if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
-      result.Put(*this, j);
+      result.AppendRange(*this, j);
     }
     lastWasBlank = isBlank;
   }
@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
       } else {
         TokenSequence result;
         if (j > 0) {
-          result.Put(*this, 0, j - 1);
+          result.AppendRange(*this, 0, j - 1);
         }
         swap(result);
         return *this;
diff --git a/flang/test/Preprocessing/exponent-bug.F90 b/flang/test/Preprocessing/exponent-bug.F90
@@ -0,0 +1,5 @@
+! RUN: %flang -E %s 2>&1 | FileCheck %s
+! CHECK: 3.14159e00
+#define e00 e666
+print *, 3.14159e00
+end

Original file line number	Diff line number	Diff line change
`@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,`
`101`	`101`	`continue;`
`102`	`102`	`}`
`103`	`103`	`}`
`104`		`- result.Put(token, firstToken + j, 1);`
	`104`	`+ result.AppendRange(token, firstToken + j, 1);`
`105`	`105`	`}`
`106`	`106`	`return result;`
`107`	`107`	`}`
`@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {`
`170`	`170`	`}`
`171`	`171`	`} else if (pasting && text.TokenAt(j).IsBlank()) {`
`172`	`172`	`} else {`
`173`		`- result.Put(text, j, 1);`
	`173`	`+ result.AppendRange(text, j, 1);`
`174`	`174`	`pasting = false;`
`175`	`175`	`}`
`176`	`176`	`}`
`@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,`
`223`	`223`	`CHECK(resultSize > 0 &&`
`224`	`224`	`result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));`
`225`	`225`	`result.pop_back();`
`226`		`- result.Put(Stringify(args[index], prescanner.allSources()));`
	`226`	`+ result.CopyAll(Stringify(args[index], prescanner.allSources()));`
`227`	`227`	`} else {`
`228`	`228`	`const TokenSequence *arg{&args[index]};`
`229`	`229`	`std::optional<TokenSequence> replaced;`
`@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,`
`243`	`243`	`}`
`244`	`244`	`}`
`245`	`245`	`}`
`246`		`- result.Put(DEREF(arg));`
	`246`	`+ result.CopyAll(DEREF(arg));`
`247`	`247`	`}`
`248`	`248`	`} else if (bytes == 11 && isVariadic_ &&`
`249`	`249`	`token.ToString() == "__VA_ARGS__") {`
`@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,`
`254`	`254`	`if (k > argumentCount()) {`
`255`	`255`	`result.Put(","s, commaProvenance);`
`256`	`256`	`}`
`257`		`- result.Put(args[k]);`
	`257`	`+ result.CopyAll(args[k]);`
`258`	`258`	`}`
`259`	`259`	`} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&`
`260`	`260`	`j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&`
`@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,`
`274`	`274`	`}`
`275`	`275`	`}`
`276`	`276`	`}`
`277`		`- result.Put(replacement_, j);`
	`277`	`+ result.AppendRange(replacement_, j);`
`278`	`278`	`}`
`279`	`279`	`}`
`280`	`280`	`return TokenPasting(std::move(result));`
`@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`338`	`338`	`inIfExpression](std::size_t after, const TokenSequence &replacement,`
`339`	`339`	`std::size_t pFLMOffset) {`
`340`	`340`	`if (after < input.SizeInTokens()) {`
`341`		`- result.Put(replacement, 0, pFLMOffset);`
	`341`	`+ result.AppendRange(replacement, 0, pFLMOffset);`
`342`	`342`	`TokenSequence suffix;`
`343`		`- suffix.Put(`
	`343`	`+ suffix.AppendRange(`
`344`	`344`	`replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);`
`345`		`- suffix.Put(input, after, input.SizeInTokens() - after);`
	`345`	`+ suffix.AppendRange(input, after, input.SizeInTokens() - after);`
`346`	`346`	`auto further{ReplaceMacros(`
`347`	`347`	`suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};`
`348`	`348`	`if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {`
`349`	`349`	`// still not closed`
`350`	`350`	`**partialFunctionLikeMacro += result.SizeInTokens();`
`351`	`351`	`}`
`352`		`- result.Put(further);`
	`352`	`+ result.CopyAll(further);`
`353`	`353`	`return true;`
`354`	`354`	`} else {`
`355`	`355`	`if (partialFunctionLikeMacro) {`
`@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`362`	`362`	`for (; j < tokens; ++j) {`
`363`	`363`	`CharBlock token{input.TokenAt(j)};`
`364`	`364`	`if (token.IsBlank() \|\| !IsLegalIdentifierStart(token[0])) {`
`365`		`- result.Put(input, j);`
	`365`	`+ result.AppendRange(input, j);`
`366`	`366`	`continue;`
`367`	`367`	`}`
`368`	`368`	`// Process identifier in replacement text.`
`@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`388`	`388`	`}`
`389`	`389`	`}`
`390`	`390`	`if (it == definitions_.end()) {`
`391`		`- result.Put(input, j);`
	`391`	`+ result.AppendRange(input, j);`
`392`	`392`	`continue;`
`393`	`393`	`}`
`394`	`394`	`Definition *def{&it->second};`
`395`	`395`	`if (def->isDisabled()) {`
`396`		`- result.Put(input, j);`
	`396`	`+ result.AppendRange(input, j);`
`397`	`397`	`continue;`
`398`	`398`	`}`
`399`	`399`	`if (!def->isFunctionLike()) {`
`@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`444`	`444`	`ProvenanceRange use{input.GetTokenProvenanceRange(j)};`
`445`	`445`	`ProvenanceRange newRange{`
`446`	`446`	`allSources_.AddMacroCall(from, use, replaced.ToString())};`
`447`		`- result.Put(replaced, newRange);`
	`447`	`+ result.CopyWithProvenance(replaced, newRange);`
`448`	`448`	`}`
`449`	`449`	`} else {`
`450`	`450`	`// Possible function-like macro call. Skip spaces and newlines to see`
`@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`461`	`461`	`if (!leftParen) {`
`462`	`462`	`if (partialFunctionLikeMacro) {`
`463`	`463`	`*partialFunctionLikeMacro = result.SizeInTokens();`
`464`		`- result.Put(input, j, tokens - j);`
	`464`	`+ result.AppendRange(input, j, tokens - j);`
`465`	`465`	`return result;`
`466`	`466`	`} else {`
`467`		`- result.Put(input, j);`
	`467`	`+ result.AppendRange(input, j);`
`468`	`468`	`continue;`
`469`	`469`	`}`
`470`	`470`	`}`
`@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`491`	`491`	`}`
`492`	`492`	`if (k >= tokens && partialFunctionLikeMacro) {`
`493`	`493`	`*partialFunctionLikeMacro = result.SizeInTokens();`
`494`		`- result.Put(input, j, tokens - j);`
	`494`	`+ result.AppendRange(input, j, tokens - j);`
`495`	`495`	`return result;`
`496`	`496`	`} else if (k >= tokens \|\| argStart.size() < def->argumentCount() \|\|`
`497`	`497`	`(argStart.size() > def->argumentCount() && !def->isVariadic())) {`
`498`		`- result.Put(input, j);`
	`498`	`+ result.AppendRange(input, j);`
`499`	`499`	`continue;`
`500`	`500`	`}`
`501`	`501`	`std::vector<TokenSequence> args;`
`@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(`
`520`	`520`	`ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};`
`521`	`521`	`ProvenanceRange newRange{`
`522`	`522`	`allSources_.AddMacroCall(from, use, replaced.ToString())};`
`523`		`- result.Put(replaced, newRange);`
	`523`	`+ result.CopyWithProvenance(replaced, newRange);`
`524`	`524`	`}`
`525`	`525`	`j = k; // advance to the terminal ')'`
`526`	`526`	`}`
Original file line number	Diff line number	Diff line change
`@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {`
`96`	`96`	`return false;`
`97`	`97`	`}`
`98`	`98`
`99`		`-void TokenSequence::Put(const TokenSequence &that) {`
	`99`	`+void TokenSequence::CopyAll(const TokenSequence &that) {`
`100`	`100`	`if (nextStart_ < char_.size()) {`
`101`	`101`	`start_.push_back(nextStart_);`
`102`	`102`	`}`
`@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {`
`109`	`109`	`provenances_.Put(that.provenances_);`
`110`	`110`	`}`
`111`	`111`
`112`		`-void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {`
	`112`	`+void TokenSequence::CopyWithProvenance(`
	`113`	`+ const TokenSequence &that, ProvenanceRange range) {`
`113`	`114`	`std::size_t offset{0};`
`114`	`115`	`std::size_t tokens{that.SizeInTokens()};`
`115`	`116`	`for (std::size_t j{0}; j < tokens; ++j) {`
`@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {`
`120`	`121`	`CHECK(offset == range.size());`
`121`	`122`	`}`
`122`	`123`
`123`		`-void TokenSequence::Put(`
	`124`	`+void TokenSequence::AppendRange(`
`124`	`125`	`const TokenSequence &that, std::size_t at, std::size_t tokens) {`
`125`	`126`	`ProvenanceRange provenance;`
`126`	`127`	`std::size_t offset{0};`
`@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {`
`246`	`247`	`TokenSequence result;`
`247`	`248`	`for (std::size_t j{0}; j < tokens; ++j) {`
`248`	`249`	`if (!TokenAt(j).IsBlank() \|\| start_[j] < firstChar) {`
`249`		`- result.Put(*this, j);`
	`250`	`+ result.AppendRange(*this, j);`
`250`	`251`	`}`
`251`	`252`	`}`
`252`	`253`	`swap(result);`
`@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {`
`260`	`261`	`for (std::size_t j{0}; j < tokens; ++j) {`
`261`	`262`	`bool isBlank{TokenAt(j).IsBlank()};`
`262`	`263`	`if (!isBlank \|\| !lastWasBlank \|\| start_[j] < firstChar) {`
`263`		`- result.Put(*this, j);`
	`264`	`+ result.AppendRange(*this, j);`
`264`	`265`	`}`
`265`	`266`	`lastWasBlank = isBlank;`
`266`	`267`	`}`
`@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(`
`294`	`295`	`} else {`
`295`	`296`	`TokenSequence result;`
`296`	`297`	`if (j > 0) {`
`297`		`- result.Put(*this, 0, j - 1);`
	`298`	`+ result.AppendRange(*this, 0, j - 1);`
`298`	`299`	`}`
`299`	`300`	`swap(result);`
`300`	`301`	`return *this;`