Skip to content

Commit 5c31f39

Browse files
abhina-sreeabhina-sree
committed
address comments
1 parent 304a0ed commit 5c31f39

File tree

14 files changed

+69
-43
lines changed

14 files changed

+69
-43
lines changed

‎clang/include/clang/Basic/TokenKinds.h‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,13 +111,18 @@ inline bool isLiteral(TokenKind K) {
111111
return isInLiteralRange;
112112
}
113113

114-
/// Return true if this is a utf literal kind.
114+
/// Return true if this is a UTF literal kind.
115115
inline bool isUTFLiteral(TokenKind K) {
116116
return K == tok::utf8_char_constant || K == tok::utf8_string_literal ||
117117
K == tok::utf16_char_constant || K == tok::utf16_string_literal ||
118118
K == tok::utf32_char_constant || K == tok::utf32_string_literal;
119119
}
120120

121+
/// Return true if this is a wide literal kind.
122+
inline bool isWideLiteral(TokenKind K) {
123+
return K == tok::wide_char_constant || K == tok::wide_string_literal;
124+
}
125+
121126
/// Return true if this is any of tok::annot_* kinds.
122127
bool isAnnotation(TokenKind K);
123128

‎clang/include/clang/Driver/Options.td‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2967,7 +2967,10 @@ def fexperimental_strict_floating_point : Flag<["-"], "fexperimental-strict-floa
29672967
def finput_charset_EQ : Joined<["-"], "finput-charset=">,
29682968
Visibility<[ClangOption, FlangOption, FC1Option]>, Group<f_Group>,
29692969
HelpText<"Specify the default character set for source files">;
2970-
def fexec_charset_EQ : Joined<["-"], "fexec-charset=">, Group<f_Group>;
2970+
def fexec_charset_EQ : Joined<["-"], "fexec-charset=">, Group<f_Group>,
2971+
HelpText<"Set the execution <charset> for string and character literals. "
2972+
"Supported character encodings include ISO-8859-1, UTF-8, IBM1047, "
2973+
"and possibly those supported by ICU or the host iconv library.">;
29712974
def finstrument_functions
29722975
: Flag<["-"], "finstrument-functions">,
29732976
Group<f_Group>,
@@ -7475,8 +7478,8 @@ def tune_cpu : Separate<["-"], "tune-cpu">,
74757478
MarshallingInfoString<TargetOpts<"TuneCPU">>;
74767479
def fexec_charset : Separate<["-"], "fexec-charset">, MetaVarName<"<charset>">,
74777480
HelpText<"Set the execution <charset> for string and character literals. "
7478-
"Supported character encodings include ISO8859-1, UTF-8, IBM-1047 "
7479-
"and those supported by the host icu or iconv library.">,
7481+
"Supported character encodings include ISO-8859-1, UTF-8, IBM1047, "
7482+
"and possibly those supported by ICU or the host iconv library.">,
74807483
MarshallingInfoString<LangOpts<"ExecEncoding">>;
74817484
def target_cpu : Separate<["-"], "target-cpu">,
74827485
HelpText<"Target a specific cpu type">,
@@ -9083,7 +9086,9 @@ def _SLASH_source_charset : CLCompileJoined<"source-charset:">,
90839086
HelpText<"Set source encoding, supports only UTF-8">,
90849087
Alias<finput_charset_EQ>;
90859088
def _SLASH_execution_charset : CLCompileJoined<"execution-charset:">,
9086-
HelpText<"Set runtime encoding, supports only UTF-8">,
9089+
HelpText<"Set the execution <charset> for string and character literals. "
9090+
"Supported character encodings include ISO-8859-1, UTF-8, IBM1047, "
9091+
"and possibly those supported by ICU or the host iconv library.">,
90879092
Alias<fexec_charset_EQ>;
90889093
def _SLASH_std : CLCompileJoined<"std:">,
90899094
HelpText<"Set language version (c++14,c++17,c++20,c++23preview,c++latest,c11,c17)">;

‎clang/include/clang/Lex/LiteralConverter.h‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,18 @@
1616
#include "llvm/ADT/StringRef.h"
1717
#include "llvm/Support/TextEncoding.h"
1818

19-
enum ConversionAction { NoConversion, ToSystemEncoding, ToExecEncoding };
19+
enum ConversionAction {
20+
CA_NoConversion,
21+
CA_ToSystemEncoding,
22+
CA_ToExecEncoding
23+
};
2024

2125
class LiteralConverter {
2226
llvm::StringRef InternalEncoding;
2327
llvm::StringRef SystemEncoding;
2428
llvm::StringRef ExecEncoding;
25-
llvm::TextEncodingConverter *ToSystemEncodingConverter;
26-
llvm::TextEncodingConverter *ToExecEncodingConverter;
29+
llvm::TextEncodingConverter *ToSystemEncodingConverter = nullptr;
30+
llvm::TextEncodingConverter *ToExecEncodingConverter = nullptr;
2731

2832
public:
2933
llvm::TextEncodingConverter *getConverter(ConversionAction Action);

‎clang/include/clang/Lex/LiteralSupport.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@ class StringLiteralParser {
251251
StringLiteralParser(
252252
ArrayRef<Token> StringToks, Preprocessor &PP,
253253
StringLiteralEvalMethod StringMethod = StringLiteralEvalMethod::Evaluated,
254-
ConversionAction Action = ToExecEncoding);
254+
ConversionAction Action = CA_ToExecEncoding);
255255
StringLiteralParser(ArrayRef<Token> StringToks, const SourceManager &sm,
256256
const LangOptions &features, const TargetInfo &target,
257257
DiagnosticsEngine *diags = nullptr)
@@ -260,7 +260,7 @@ class StringLiteralParser {
260260
Kind(tok::unknown), ResultPtr(ResultBuf.data()),
261261
EvalMethod(StringLiteralEvalMethod::Evaluated), hadError(false),
262262
Pascal(false) {
263-
init(StringToks, NoConversion);
263+
init(StringToks, CA_NoConversion);
264264
}
265265

266266
bool hadError;

‎clang/lib/Driver/ToolChains/Clang.cpp‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7366,18 +7366,20 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
73667366

73677367
// Set the default fexec-charset as the system charset.
73687368
CmdArgs.push_back("-fexec-charset");
7369-
CmdArgs.push_back(Args.MakeArgString(Triple.getDefaultTextEncoding()));
7369+
CmdArgs.push_back(Args.MakeArgString(Triple.getDefaultNarrowTextEncoding()));
73707370
if (Arg *execEncoding = Args.getLastArg(options::OPT_fexec_charset_EQ)) {
73717371
StringRef value = execEncoding->getValue();
7372-
llvm::ErrorOr<llvm::TextEncodingConverter> ErrorOrConverter =
7373-
llvm::TextEncodingConverter::create("UTF-8", value.data());
7374-
if (ErrorOrConverter) {
7375-
CmdArgs.push_back("-fexec-charset");
7376-
CmdArgs.push_back(Args.MakeArgString(value));
7377-
} else {
7378-
D.Diag(diag::err_drv_invalid_value)
7379-
<< execEncoding->getAsString(Args) << value;
7372+
bool KnownEncoding =
7373+
llvm::TextEncodingConverter::getKnownEncoding(value).has_value();
7374+
if (!KnownEncoding) {
7375+
llvm::ErrorOr<llvm::TextEncodingConverter> ErrorOrConverter =
7376+
llvm::TextEncodingConverter::create("UTF-8", value.data());
7377+
if (!ErrorOrConverter)
7378+
D.Diag(diag::err_drv_invalid_value)
7379+
<< execEncoding->getAsString(Args) << value;
73807380
}
7381+
CmdArgs.push_back("-fexec-charset");
7382+
CmdArgs.push_back(Args.MakeArgString(value));
73817383
}
73827384

73837385
RenderDiagnosticsOptions(D, Args, CmdArgs);

‎clang/lib/Frontend/InitPreprocessor.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1029,7 +1029,7 @@ static void InitializePredefinedMacros(const TargetInfo &TI,
10291029
Builder.defineMacro("__clang_literal_encoding__", LangOpts.ExecEncoding);
10301030
else
10311031
Builder.defineMacro("__clang_literal_encoding__",
1032-
TI.getTriple().getDefaultTextEncoding());
1032+
TI.getTriple().getDefaultNarrowTextEncoding());
10331033
if (TI.getTypeWidth(TI.getWCharType()) >= 32) {
10341034
// FIXME: 32-bit wchar_t signals UTF-32. This may change
10351035
// if -fwide-exec-charset= is ever supported.

‎clang/lib/Lex/LiteralConverter.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ using namespace llvm;
1313

1414
llvm::TextEncodingConverter *
1515
LiteralConverter::getConverter(ConversionAction Action) {
16-
if (Action == ToSystemEncoding)
16+
if (Action == CA_ToSystemEncoding)
1717
return ToSystemEncodingConverter;
18-
else if (Action == ToExecEncoding)
18+
else if (Action == CA_ToExecEncoding)
1919
return ToExecEncodingConverter;
2020
else
2121
return nullptr;
@@ -26,7 +26,7 @@ void LiteralConverter::setConvertersFromOptions(
2626
clang::DiagnosticsEngine &Diags) {
2727
using namespace llvm;
2828
InternalEncoding = "UTF-8";
29-
SystemEncoding = TInfo.getTriple().getDefaultTextEncoding();
29+
SystemEncoding = TInfo.getTriple().getDefaultNarrowTextEncoding();
3030
ExecEncoding =
3131
Opts.ExecEncoding.empty() ? InternalEncoding : Opts.ExecEncoding;
3232
// Create converter between internal and system encoding

‎clang/lib/Lex/LiteralSupport.cpp‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -147,7 +147,7 @@ static unsigned ProcessCharEscape(const char *ThisTokBegin,
147147
// that would have been \", which would not have been the end of string.
148148
unsigned ResultChar = *ThisTokBuf++;
149149
char Escape = ResultChar;
150-
bool Translate = true;
150+
bool Transcode = true;
151151
bool Invalid = false;
152152
switch (ResultChar) {
153153
// These map to themselves.
@@ -189,7 +189,7 @@ static unsigned ProcessCharEscape(const char *ThisTokBegin,
189189
ResultChar = 11;
190190
break;
191191
case 'x': { // Hex escape.
192-
Translate = false;
192+
Transcode = false;
193193
ResultChar = 0;
194194
if (ThisTokBuf != ThisTokEnd && *ThisTokBuf == '{') {
195195
Delimited = true;
@@ -253,7 +253,7 @@ static unsigned ProcessCharEscape(const char *ThisTokBegin,
253253
case '4': case '5': case '6': case '7': {
254254
// Octal escapes.
255255
--ThisTokBuf;
256-
Translate = false;
256+
Transcode = false;
257257
ResultChar = 0;
258258

259259
// Octal escapes are a series of octal digits with maximum length 3.
@@ -373,7 +373,7 @@ static unsigned ProcessCharEscape(const char *ThisTokBegin,
373373
HadError = true;
374374
}
375375

376-
if (Translate && Converter) {
376+
if (Transcode && Converter) {
377377
// Invalid escapes are written as '?' and then translated.
378378
char ByteChar = Invalid ? '?' : ResultChar;
379379
SmallString<8> ResultCharConv;
@@ -1834,8 +1834,8 @@ CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
18341834
}
18351835

18361836
llvm::TextEncodingConverter *Converter = nullptr;
1837-
if (!isUTFLiteral(Kind) && LiteralConv)
1838-
Converter = LiteralConv->getConverter(ToExecEncoding);
1837+
if (!isUTFLiteral(Kind) && !isWideLiteral(Kind) && LiteralConv)
1838+
Converter = LiteralConv->getConverter(CA_ToExecEncoding);
18391839

18401840
while (begin != end) {
18411841
// Is this a span of non-escape characters?
@@ -2142,7 +2142,7 @@ void StringLiteralParser::init(ArrayRef<Token> StringToks,
21422142
SourceLocation UDSuffixTokLoc;
21432143

21442144
llvm::TextEncodingConverter *Converter = nullptr;
2145-
if (!isUTFLiteral(Kind) && LiteralConv)
2145+
if (!isUTFLiteral(Kind) && !isWideLiteral(Kind) && LiteralConv)
21462146
Converter = LiteralConv->getConverter(Action);
21472147

21482148
for (unsigned i = 0, e = StringToks.size(); i != e; ++i) {

‎clang/test/Driver/cl-options.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@
251251
// source-charset-utf-16: invalid value 'utf-16' in '/source-charset:utf-16'
252252

253253
// /execution-charset: should warn on invalid charsets.
254-
// RUN: %clang_cl /execution-charset:invalid-charset -### -- %s 2>&1 | FileCheck -check-prefix=execution-charset-invalid %s
254+
// RUN: not %clang_cl /execution-charset:invalid-charset -### -- %s 2>&1 | FileCheck -check-prefix=execution-charset-invalid %s
255255
// execution-charset-invalid: invalid value 'invalid-charset' in '/execution-charset:invalid-charset'
256256
//
257257

‎clang/test/Driver/clang_f_opts.c‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -232,14 +232,20 @@
232232
// RUN: not %clang -### -S -finput-charset=iso-8859-1 -o /dev/null %s 2>&1 | FileCheck -check-prefix=CHECK-INVALID-INPUT-CHARSET %s
233233
// CHECK-INVALID-INPUT-CHARSET: error: invalid value 'iso-8859-1' in '-finput-charset=iso-8859-1'
234234

235-
// RUN: %clang -### -S -fexec-charset=invalid-charset -o /dev/null %s 2>&1 | FileCheck -check-prefix=CHECK-INVALID-INPUT-CHARSET %s
236-
// CHECK-INVALID-INPUT-CHARSET: error: invalid value 'invalid-charset' in '-fexec-charset=invalid-charset'
237-
238-
// Test that we support the following exec charsets.
239-
// RUN: %clang -### -S -fexec-charset=UTF-8 -o /dev/null %s 2>&1 | FileCheck --check-prefix=INVALID %s
240-
// RUN: %clang -### -S -fexec-charset=ISO8859-1 -o /dev/null %s 2>&1 | FileCheck --check-prefix=INVALID %s
241-
// RUN: %clang -### -S -fexec-charset=IBM-1047 -o /dev/null %s 2>&1 | FileCheck --check-prefix=INVALID %s
242-
// INVALID-NOT: error: invalid value
235+
// RUN: not %clang -### -S -fexec-charset=invalid-charset -o /dev/null %s 2>&1 | FileCheck -check-prefix=CHECK-INVALID-EXEC-CHARSET %s
236+
// CHECK-INVALID-EXEC-CHARSET: error: invalid value 'invalid-charset' in '-fexec-charset=invalid-charset'
237+
238+
// Test that we support the following exec charsets. The preferred MIME name is
239+
// `IBM1047`, but `IBM-1047` is the name used by z/OS USS utilities such as
240+
// `chtag`.
241+
// RUN: %clang -### -S -fexec-charset=UTF-8 -o /dev/null %s 2>&1 | FileCheck --check-prefix=CHECK-EXEC-CHARSET-UTF-8 %s
242+
// RUN: %clang -### -S -fexec-charset=ISO-8859-1 -o /dev/null %s 2>&1 | FileCheck --check-prefix=CHECK-EXEC-CHARSET-ISO-8859-1 %s
243+
// RUN: %clang -### -S -fexec-charset=IBM-1047 -o /dev/null %s 2>&1 | FileCheck --check-prefix=CHECK-EXEC-CHARSET-IBM-1047 %s
244+
// RUN: %clang -### -S -fexec-charset=IBM1047 -o /dev/null %s 2>&1 | FileCheck --check-prefix=CHECK-EXEC-CHARSET-IBM1047 %s
245+
// CHECK-EXEC-CHARSET-UTF-8: "-fexec-charset" "UTF-8"
246+
// CHECK-EXEC-CHARSET-ISO-8859-1: "-fexec-charset" "ISO-8859-1"
247+
// CHECK-EXEC-CHARSET-IBM-1047: "-fexec-charset" "IBM-1047"
248+
// CHECK-EXEC-CHARSET-IBM1047: "-fexec-charset" "IBM1047"
243249

244250
// Test that we don't error on these.
245251
// RUN: not %clang -### -S -Werror \

0 commit comments

Comments
 (0)