Skip to content

Commit 1d58e59

Browse files
klauslerIanWood1
klausler
authored and
IanWood1
committed
[flang] Don't perform macro replacement on exponents (llvm#136176)
See new test. I inadvertently broke this behavior with a recent fix for another problem, because the effects of the overloaded TokenSequence::Put() member function on token merging were confusing. Rename and document the various overloads.
1 parent ac6095d commit 1d58e59

File tree

5 files changed

+55
-38
lines changed

5 files changed

+55
-38
lines changed

‎flang/include/flang/Parser/token-sequence.h‎

Lines changed: 18 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,10 @@ class Prescanner;
3535
class TokenSequence {
3636
public:
3737
TokenSequence() {}
38-
TokenSequence(const TokenSequence &that) { Put(that); }
38+
TokenSequence(const TokenSequence &that) { CopyAll(that); }
3939
TokenSequence(
4040
const TokenSequence &that, std::size_t at, std::size_t count = 1) {
41-
Put(that, at, count);
41+
AppendRange(that, at, count);
4242
}
4343
TokenSequence(TokenSequence &&that)
4444
: start_{std::move(that.start_)}, nextStart_{that.nextStart_},
@@ -48,7 +48,7 @@ class TokenSequence {
4848

4949
TokenSequence &operator=(const TokenSequence &that) {
5050
clear();
51-
Put(that);
51+
CopyAll(that);
5252
return *this;
5353
}
5454
TokenSequence &operator=(TokenSequence &&that);
@@ -100,14 +100,26 @@ class TokenSequence {
100100
start_.pop_back();
101101
}
102102

103-
void Put(const TokenSequence &);
104-
void Put(const TokenSequence &, ProvenanceRange);
105-
void Put(const TokenSequence &, std::size_t at, std::size_t tokens = 1);
103+
// These append characters with provenance and then close the token.
104+
// When the last token of this sequence remains open beforehand,
105+
// the new characters are appended to it.
106106
void Put(const char *, std::size_t, Provenance);
107107
void Put(const CharBlock &, Provenance);
108108
void Put(const std::string &, Provenance);
109109
void Put(llvm::raw_string_ostream &, Provenance);
110110

111+
// Appends a full copy of another sequence. When the last token of this
112+
// sequence remains open beforehand, it is closed before the new text
113+
// is appended.
114+
void CopyAll(const TokenSequence &);
115+
// Copies a range of tokens from another sequence. If the last token of
116+
// this sequence remains open, the first token of the copied range will be
117+
// appended to it.
118+
void AppendRange(
119+
const TokenSequence &, std::size_t at, std::size_t tokens = 1);
120+
// Copies tokens (via Put above) with new provenance.
121+
void CopyWithProvenance(const TokenSequence &, ProvenanceRange);
122+
111123
Provenance GetCharProvenance(std::size_t) const;
112124
Provenance GetTokenProvenance(
113125
std::size_t token, std::size_t offset = 0) const;

‎flang/lib/Parser/preprocessor.cpp‎

Lines changed: 19 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ TokenSequence Definition::Tokenize(const std::vector<std::string> &argNames,
101101
continue;
102102
}
103103
}
104-
result.Put(token, firstToken + j, 1);
104+
result.AppendRange(token, firstToken + j, 1);
105105
}
106106
return result;
107107
}
@@ -170,7 +170,7 @@ static TokenSequence TokenPasting(TokenSequence &&text) {
170170
}
171171
} else if (pasting && text.TokenAt(j).IsBlank()) {
172172
} else {
173-
result.Put(text, j, 1);
173+
result.AppendRange(text, j, 1);
174174
pasting = false;
175175
}
176176
}
@@ -223,7 +223,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
223223
CHECK(resultSize > 0 &&
224224
result.TokenAt(resultSize - 1) == replacement_.TokenAt(prev - 1));
225225
result.pop_back();
226-
result.Put(Stringify(args[index], prescanner.allSources()));
226+
result.CopyAll(Stringify(args[index], prescanner.allSources()));
227227
} else {
228228
const TokenSequence *arg{&args[index]};
229229
std::optional<TokenSequence> replaced;
@@ -243,7 +243,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
243243
}
244244
}
245245
}
246-
result.Put(DEREF(arg));
246+
result.CopyAll(DEREF(arg));
247247
}
248248
} else if (bytes == 11 && isVariadic_ &&
249249
token.ToString() == "__VA_ARGS__") {
@@ -254,7 +254,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
254254
if (k > argumentCount()) {
255255
result.Put(","s, commaProvenance);
256256
}
257-
result.Put(args[k]);
257+
result.CopyAll(args[k]);
258258
}
259259
} else if (bytes == 10 && isVariadic_ && token.ToString() == "__VA_OPT__" &&
260260
j + 2 < tokens && replacement_.TokenAt(j + 1).OnlyNonBlank() == '(' &&
@@ -274,7 +274,7 @@ TokenSequence Definition::Apply(const std::vector<TokenSequence> &args,
274274
}
275275
}
276276
}
277-
result.Put(replacement_, j);
277+
result.AppendRange(replacement_, j);
278278
}
279279
}
280280
return TokenPasting(std::move(result));
@@ -338,18 +338,18 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
338338
inIfExpression](std::size_t after, const TokenSequence &replacement,
339339
std::size_t pFLMOffset) {
340340
if (after < input.SizeInTokens()) {
341-
result.Put(replacement, 0, pFLMOffset);
341+
result.AppendRange(replacement, 0, pFLMOffset);
342342
TokenSequence suffix;
343-
suffix.Put(
343+
suffix.AppendRange(
344344
replacement, pFLMOffset, replacement.SizeInTokens() - pFLMOffset);
345-
suffix.Put(input, after, input.SizeInTokens() - after);
345+
suffix.AppendRange(input, after, input.SizeInTokens() - after);
346346
auto further{ReplaceMacros(
347347
suffix, prescanner, partialFunctionLikeMacro, inIfExpression)};
348348
if (partialFunctionLikeMacro && *partialFunctionLikeMacro) {
349349
// still not closed
350350
**partialFunctionLikeMacro += result.SizeInTokens();
351351
}
352-
result.Put(further);
352+
result.CopyAll(further);
353353
return true;
354354
} else {
355355
if (partialFunctionLikeMacro) {
@@ -362,7 +362,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
362362
for (; j < tokens; ++j) {
363363
CharBlock token{input.TokenAt(j)};
364364
if (token.IsBlank() || !IsLegalIdentifierStart(token[0])) {
365-
result.Put(input, j);
365+
result.AppendRange(input, j);
366366
continue;
367367
}
368368
// Process identifier in replacement text.
@@ -388,12 +388,12 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
388388
}
389389
}
390390
if (it == definitions_.end()) {
391-
result.Put(input, j);
391+
result.AppendRange(input, j);
392392
continue;
393393
}
394394
Definition *def{&it->second};
395395
if (def->isDisabled()) {
396-
result.Put(input, j);
396+
result.AppendRange(input, j);
397397
continue;
398398
}
399399
if (!def->isFunctionLike()) {
@@ -444,7 +444,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
444444
ProvenanceRange use{input.GetTokenProvenanceRange(j)};
445445
ProvenanceRange newRange{
446446
allSources_.AddMacroCall(from, use, replaced.ToString())};
447-
result.Put(replaced, newRange);
447+
result.CopyWithProvenance(replaced, newRange);
448448
}
449449
} else {
450450
// Possible function-like macro call. Skip spaces and newlines to see
@@ -461,10 +461,10 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
461461
if (!leftParen) {
462462
if (partialFunctionLikeMacro) {
463463
*partialFunctionLikeMacro = result.SizeInTokens();
464-
result.Put(input, j, tokens - j);
464+
result.AppendRange(input, j, tokens - j);
465465
return result;
466466
} else {
467-
result.Put(input, j);
467+
result.AppendRange(input, j);
468468
continue;
469469
}
470470
}
@@ -491,11 +491,11 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
491491
}
492492
if (k >= tokens && partialFunctionLikeMacro) {
493493
*partialFunctionLikeMacro = result.SizeInTokens();
494-
result.Put(input, j, tokens - j);
494+
result.AppendRange(input, j, tokens - j);
495495
return result;
496496
} else if (k >= tokens || argStart.size() < def->argumentCount() ||
497497
(argStart.size() > def->argumentCount() && !def->isVariadic())) {
498-
result.Put(input, j);
498+
result.AppendRange(input, j);
499499
continue;
500500
}
501501
std::vector<TokenSequence> args;
@@ -520,7 +520,7 @@ std::optional<TokenSequence> Preprocessor::MacroReplacement(
520520
ProvenanceRange use{input.GetIntervalProvenanceRange(j, k - j)};
521521
ProvenanceRange newRange{
522522
allSources_.AddMacroCall(from, use, replaced.ToString())};
523-
result.Put(replaced, newRange);
523+
result.CopyWithProvenance(replaced, newRange);
524524
}
525525
j = k; // advance to the terminal ')'
526526
}

‎flang/lib/Parser/prescan.cpp‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -903,8 +903,7 @@ bool Prescanner::HandleExponent(TokenSequence &tokens) {
903903
EmitCharAndAdvance(possible, *at_);
904904
}
905905
possible.CloseToken();
906-
tokens.CloseToken();
907-
tokens.Put(possible);
906+
tokens.AppendRange(possible, 0); // appends to current token
908907
return true;
909908
}
910909
// Not an exponent; backtrack
@@ -937,9 +936,9 @@ bool Prescanner::HandleKindSuffix(TokenSequence &tokens) {
937936
preprocessor_.IsNameDefined(separate.TokenAt(1)) &&
938937
!preprocessor_.IsNameDefined(withUnderscore.ToCharBlock())) {
939938
// "_foo" is not defined, but "foo" is
940-
tokens.Put(separate); // '_' "foo"
939+
tokens.CopyAll(separate); // '_' "foo"
941940
} else {
942-
tokens.Put(withUnderscore); // "_foo"
941+
tokens.CopyAll(withUnderscore); // "_foo"
943942
}
944943
return true;
945944
}
@@ -1016,7 +1015,7 @@ void Prescanner::QuotedCharacterLiteral(
10161015
ppTokens.Put(id, GetProvenance(idStart));
10171016
if (auto replaced{
10181017
preprocessor_.MacroReplacement(ppTokens, *this)}) {
1019-
tokens.Put(*replaced);
1018+
tokens.CopyAll(*replaced);
10201019
at_ = &idStart[idLen - 1];
10211020
NextLine();
10221021
continue; // try again on the next line
@@ -1836,7 +1835,7 @@ bool Prescanner::CompilerDirectiveContinuation(
18361835
}
18371836
if (ok) {
18381837
tokens.pop_back(); // delete original '&'
1839-
tokens.Put(followingTokens, startAt, following - startAt);
1838+
tokens.AppendRange(followingTokens, startAt, following - startAt);
18401839
tokens.RemoveRedundantBlanks();
18411840
} else {
18421841
nextLine_ = origNextLine;
@@ -1866,7 +1865,7 @@ bool Prescanner::SourceLineContinuation(TokenSequence &tokens) {
18661865
}
18671866
followingTokens.RemoveRedundantBlanks();
18681867
tokens.pop_back(); // delete original '&'
1869-
tokens.Put(followingTokens);
1868+
tokens.CopyAll(followingTokens);
18701869
return true;
18711870
}
18721871
}

‎flang/lib/Parser/token-sequence.cpp‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,7 @@ bool TokenSequence::IsAnythingLeft(std::size_t at) const {
9696
return false;
9797
}
9898

99-
void TokenSequence::Put(const TokenSequence &that) {
99+
void TokenSequence::CopyAll(const TokenSequence &that) {
100100
if (nextStart_ < char_.size()) {
101101
start_.push_back(nextStart_);
102102
}
@@ -109,7 +109,8 @@ void TokenSequence::Put(const TokenSequence &that) {
109109
provenances_.Put(that.provenances_);
110110
}
111111

112-
void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
112+
void TokenSequence::CopyWithProvenance(
113+
const TokenSequence &that, ProvenanceRange range) {
113114
std::size_t offset{0};
114115
std::size_t tokens{that.SizeInTokens()};
115116
for (std::size_t j{0}; j < tokens; ++j) {
@@ -120,7 +121,7 @@ void TokenSequence::Put(const TokenSequence &that, ProvenanceRange range) {
120121
CHECK(offset == range.size());
121122
}
122123

123-
void TokenSequence::Put(
124+
void TokenSequence::AppendRange(
124125
const TokenSequence &that, std::size_t at, std::size_t tokens) {
125126
ProvenanceRange provenance;
126127
std::size_t offset{0};
@@ -246,7 +247,7 @@ TokenSequence &TokenSequence::RemoveBlanks(std::size_t firstChar) {
246247
TokenSequence result;
247248
for (std::size_t j{0}; j < tokens; ++j) {
248249
if (!TokenAt(j).IsBlank() || start_[j] < firstChar) {
249-
result.Put(*this, j);
250+
result.AppendRange(*this, j);
250251
}
251252
}
252253
swap(result);
@@ -260,7 +261,7 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
260261
for (std::size_t j{0}; j < tokens; ++j) {
261262
bool isBlank{TokenAt(j).IsBlank()};
262263
if (!isBlank || !lastWasBlank || start_[j] < firstChar) {
263-
result.Put(*this, j);
264+
result.AppendRange(*this, j);
264265
}
265266
lastWasBlank = isBlank;
266267
}
@@ -294,7 +295,7 @@ TokenSequence &TokenSequence::ClipComment(
294295
} else {
295296
TokenSequence result;
296297
if (j > 0) {
297-
result.Put(*this, 0, j - 1);
298+
result.AppendRange(*this, 0, j - 1);
298299
}
299300
swap(result);
300301
return *this;

‎flang/test/Preprocessing/exponent-bug.F90‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
! RUN: %flang -E %s 2>&1 | FileCheck %s
2+
! CHECK: 3.14159e00
3+
#define e00 e666
4+
print *, 3.14159e00
5+
end

0 commit comments

Comments
 (0)