[lldb] improve the heuristics for checking if a terminal supports Unicode (llvm#171832)

This patch improves the way lldb checks if the terminal it's opened in
(if any) supports Unicode or not.

On POSIX systems, we check if `LANG` contains `UTF-8`.

On Windows, we always return `true` since we use the `WriteToConsoleW`
api.

This is a relanding of llvm#168603.

The tests failed because the bots support Unicode but the tests expect
ASCII. To avoid different outputs depending on the environment the tests
are running in, this patch always force ASCII in the tests.

(cherry picked from commit 59f107c)

Author: charles-zablit

lldb/include/lldb/Host/Terminal.h

@@ -68,6 +68,18 @@ class Terminal {
 
   llvm::Error SetHardwareFlowControl(bool enabled);
 
+  /// Returns whether or not the current terminal supports Unicode rendering.
+  ///
+  /// The value is cached after the first computation.
+  ///
+  /// On POSIX systems, we check if the LANG environment variable contains the
+  /// substring "UTF-8", case insensitive.
+  ///
+  /// On Windows, we always return true since we use the `WriteConsoleW` API
+  /// internally. Note that the default Windows codepage (437) does not support
+  /// all Unicode characters. This function does not check the codepage.
+  static bool SupportsUnicode();
+
 protected:
   struct Data;
 

lldb/include/lldb/Utility/DiagnosticsRendering.h

@@ -59,10 +59,27 @@ struct DiagnosticDetail {
 
 StructuredData::ObjectSP Serialize(llvm::ArrayRef<DiagnosticDetail> details);
 
+/// Renders an array of DiagnosticDetail instances.
+///
+/// \param[in] stream
+///     The stream to render the diagnostics to.
+/// \param offset_in_command
+///     An optional offset to the column position of the diagnostic in the
+///     source.
+/// \param show_inline
+///     Whether to show the diagnostics inline.
+/// \param details
+///     The array of DiagnosticsDetail to render.
+/// \param force_ascii
+///     Whether to force ascii rendering. If false, Unicode characters will be
+///     used if the output file supports them.
+///
+/// \see lldb_private::Terminal::SupportsUnicode
 void RenderDiagnosticDetails(Stream &stream,
                              std::optional<uint16_t> offset_in_command,
                              bool show_inline,
-                             llvm::ArrayRef<DiagnosticDetail> details);
+                             llvm::ArrayRef<DiagnosticDetail> details,
+                             bool force_ascii = false);
 
 class DiagnosticError
     : public llvm::ErrorInfo<DiagnosticError, CloneableECError> {

lldb/source/Host/common/Terminal.cpp

@@ -400,6 +400,23 @@ llvm::Error Terminal::SetHardwareFlowControl(bool enabled) {
 #endif // LLDB_ENABLE_TERMIOS
 }
 
+bool Terminal::SupportsUnicode() {
+#ifdef _WIN32
+  return true;
+#else
+  static std::optional<bool> g_result;
+  if (g_result)
+    return g_result.value();
+
+  const char *lang_var = std::getenv("LANG");
+  if (!lang_var)
+    return false;
+  g_result =
+      llvm::StringRef(lang_var).lower().find("utf-8") != std::string::npos;
+  return g_result.value();
+#endif
+}
+
 TerminalState::TerminalState(Terminal term, bool save_process_group)
     : m_tty(term) {
   Save(term, save_process_group);

lldb/source/Utility/DiagnosticsRendering.cpp

@@ -7,6 +7,8 @@
 //===----------------------------------------------------------------------===//
 
 #include "lldb/Utility/DiagnosticsRendering.h"
+#include "lldb/Host/Terminal.h"
+
 #include <cstdint>
 
 using namespace lldb_private;
@@ -85,7 +87,8 @@ static llvm::raw_ostream &PrintSeverity(Stream &stream,
 void RenderDiagnosticDetails(Stream &stream,
                              std::optional<uint16_t> offset_in_command,
                              bool show_inline,
-                             llvm::ArrayRef<DiagnosticDetail> details) {
+                             llvm::ArrayRef<DiagnosticDetail> details,
+                             bool force_ascii) {
   if (details.empty())
     return;
 
@@ -97,12 +100,8 @@ void RenderDiagnosticDetails(Stream &stream,
     return;
   }
 
-  // Since there is no other way to find this out, use the color
-  // attribute as a proxy for whether the terminal supports Unicode
-  // characters.  In the future it might make sense to move this into
-  // Host so it can be customized for a specific platform.
   llvm::StringRef cursor, underline, vbar, joint, hbar, spacer;
-  if (stream.AsRawOstream().colors_enabled()) {
+  if (Terminal::SupportsUnicode() && !force_ascii) {
     cursor = "˄";
     underline = "˜";
     vbar = "│";

lldb/test/Shell/Commands/command-dwim-print.test

@@ -1,16 +1,16 @@
 # RUN: echo quit | %lldb -o "dwim-print a" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
 #            (lldb) dwim-print a 
-# CHECK1:{{^                  \^}}
-# CHECK1: {{^                  error: use of undeclared identifier 'a'}}
+# CHECK1:{{^                  (\^|˄)}}
+# CHECK1: {{^                  (╰─ )?error: use of undeclared identifier 'a'}}
 # RUN: echo quit | %lldb -o "p a" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
 #            (lldb) p a 
-# CHECK2:{{^         \^}}
+# CHECK2:{{^         (\^|˄)}}
 # RUN: echo quit | %lldb -o "dwim-print -- a" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
 #            (lldb) dwim-print -- a 
-# CHECK3:{{^                     \^}}
+# CHECK3:{{^                     (\^|˄)}}
 # RUN: echo quit | %lldb -o "settings set show-inline-diagnostics false" \
 # RUN:   -o "dwim-print a" 2>&1 | FileCheck %s --check-prefix=CHECK4
 # CHECK4: error: <user expression 0>:1:1: use of undeclared identifier

lldb/test/Shell/Commands/command-expr-diagnostics.test

@@ -2,28 +2,28 @@
 # RUN: echo quit | %lldb -o "expression a+b" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
 #            (lldb) expression a+b
-# CHECK1:{{^                  \^ \^}}
-# CHECK1: {{^                  | error: use of undeclared identifier 'b'}}
-# CHECK1: {{^                  error: use of undeclared identifier 'a'}}
+# CHECK1:{{^                  (\^|˄) (\^|˄)}}
+# CHECK1: {{^                  (\||│) (╰─ )?error: use of undeclared identifier 'b'}}
+# CHECK1: {{^                  (╰─ )?error: use of undeclared identifier 'a'}}
 
 # RUN: echo quit | %lldb -o "expr a" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
 #            (lldb) expr a 
-# CHECK2:{{^            \^}}
+# CHECK2:{{^            (\^|˄)}}
 
 # RUN: echo quit | %lldb -o "expr -i 0 -o 0 -- a" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
 #            (lldb) expr -i 0 -o 0 -- a
-# CHECK3:{{^                         \^}}
-# CHECK3: {{^                         error: use of undeclared identifier 'a'}}
+# CHECK3:{{^                         (\^|˄)}}
+# CHECK3: {{^                         (╰─ )?error: use of undeclared identifier 'a'}}
 
 # RUN: echo "int main(){return 0;}">%t.c
 # RUN: %clang_host %t.c -o %t.exe
 # RUN: echo quit | %lldb %t.exe -o "b main" -o r -o \
 # RUN: "expr --top-level -- template<typename T> T FOO(T x) { return x/2;}" -o \
 # RUN: "expression -- FOO(\"\")" 2>&1 | FileCheck %s --check-prefix=CHECK4
 #            (lldb) expression -- FOO("")
-# CHECK4:{{^                     \^}}
+# CHECK4:{{^                     (\^|˄)}}
 # CHECK4: {{^                     note: in instantiation of function template}}
 # CHECK4: error: <user expression
 

lldb/test/Shell/Commands/command-options.test

@@ -1,16 +1,16 @@
 # RUN: echo quit | %lldb -O "log enable -x" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK1
 #            (lldb) log enable -x
-# CHECK1:{{^                  \^~}}
-# CHECK1: {{^                  error: unknown or ambiguous option}}
+# CHECK1:{{^                  (\^|˄)(~|˜)}}
+# CHECK1: {{^                  (╰─ )?error: unknown or ambiguous option}}
 
 # RUN: echo quit | %lldb -O "    log enable -xxxxxxx" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK2
 #            (lldb)     log enable -xxxxxxx
-# CHECK2:{{^                      \^~~~~~~~}}
-# CHECK2: {{^                      error: unknown or ambiguous option}}
+# CHECK2:{{^                      (\^|˄)(~|˜)+}}
+# CHECK2: {{^                      (╰─ )?error: unknown or ambiguous option}}
 # RUN: echo quit | %lldb -O "log enable dwarf all -f dwarf.log -x" \
 # RUN:   | FileCheck %s --strict-whitespace --check-prefix=CHECK3
 #            (lldb) log enable dwarf all -f dwarf.log -x
-# CHECK3:{{^                                         \^~}}
-# CHECK3: {{^                                         error: unknown or ambiguous option}}
+# CHECK3:{{^                                         (\^|˄)(~|˜)}}
+# CHECK3: {{^                                         (╰─ )?error: unknown or ambiguous option}}

lldb/unittests/Utility/DiagnosticsRenderingTest.cpp

@@ -10,7 +10,7 @@ class ErrorDisplayTest : public ::testing::Test {};
 
 std::string Render(std::vector<DiagnosticDetail> details) {
   StreamString stream;
-  RenderDiagnosticDetails(stream, 0, true, details);
+  RenderDiagnosticDetails(stream, 0, true, details, /*force_ascii=*/true);
   return stream.GetData();
 }
 } // namespace