[Diag] Refactor DiagnosticEngine to use YAML files - Reopen (swiftlang#32483)

Introduce localization support for diagnostics via file-per-language store in `YAML` format.
Add `LocalizationFormat` interface and its implementation `YAMLLocalizationProducer`.
Augment `DiagnosticEngine` to support localization when applicable.

Author: HassanElDesouky

include/swift/AST/CMakeLists.txt

@@ -0,0 +1,3 @@
+swift_install_in_component(DIRECTORY diagnostics
+                           DESTINATION "share/swift/"
+                           COMPONENT compiler)

include/swift/AST/DiagnosticEngine.h

@@ -18,9 +18,10 @@
 #ifndef SWIFT_BASIC_DIAGNOSTICENGINE_H
 #define SWIFT_BASIC_DIAGNOSTICENGINE_H
 
-#include "swift/AST/TypeLoc.h"
 #include "swift/AST/DeclNameLoc.h"
 #include "swift/AST/DiagnosticConsumer.h"
+#include "swift/AST/LocalizationFormat.h"
+#include "swift/AST/TypeLoc.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Allocator.h"
 #include "llvm/Support/VersionTuple.h"
@@ -629,7 +630,7 @@ namespace swift {
     DiagnosticState(DiagnosticState &&) = default;
     DiagnosticState &operator=(DiagnosticState &&) = default;
   };
-    
+
   /// Class responsible for formatting diagnostics and presenting them
   /// to the user.
   class DiagnosticEngine {
@@ -663,6 +664,10 @@ namespace swift {
     /// but rather stored until all transactions complete.
     llvm::StringSet<llvm::BumpPtrAllocator &> TransactionStrings;
 
+    /// Diagnostic producer to handle the logic behind retrieving a localized
+    /// diagnostic message.
+    std::unique_ptr<diag::LocalizationProducer> localization;
+
     /// The number of open diagnostic transactions. Diagnostics are only
     /// emitted once all transactions have closed.
     unsigned TransactionCount = 0;
@@ -734,6 +739,12 @@ namespace swift {
       return diagnosticDocumentationPath;
     }
 
+    void setLocalization(std::string locale, std::string path) {
+      if (!locale.empty() && !path.empty())
+        localization =
+            std::make_unique<diag::YAMLLocalizationProducer>(locale, path);
+    }
+
     void ignoreDiagnostic(DiagID id) {
       state.setDiagnosticBehavior(id, DiagnosticState::Behavior::Ignore);
     }
@@ -955,8 +966,7 @@ namespace swift {
     void emitTentativeDiagnostics();
 
   public:
-    static const char *diagnosticStringFor(const DiagID id,
-                                           bool printDiagnosticName);
+    const char *diagnosticStringFor(const DiagID id, bool printDiagnosticName);
 
     /// If there is no clear .dia file for a diagnostic, put it in the one
     /// corresponding to the SourceLoc given here.

include/swift/AST/LocalizationFormat.h

@@ -0,0 +1,69 @@
+//===--- LocalizationFormat.h - YAML format for Diagnostic Messages ---*-
+// C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the format for localized diagnostic messages.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef SWIFT_LOCALIZATIONFORMAT_H
+#define SWIFT_LOCALIZATIONFORMAT_H
+
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <string>
+#include <type_traits>
+
+namespace swift {
+enum class DiagID : uint32_t;
+
+namespace diag {
+class LocalizationProducer {
+public:
+  /// If the  message isn't available/localized in the current `yaml` file,
+  /// return the fallback default message.
+  virtual llvm::StringRef getMessageOr(swift::DiagID id,
+                                       llvm::StringRef defaultMessage) const {
+    return defaultMessage;
+  }
+
+  virtual ~LocalizationProducer() {}
+};
+
+class YAMLLocalizationProducer final : public LocalizationProducer {
+public:
+  std::vector<std::string> diagnostics;
+  explicit YAMLLocalizationProducer(std::string locale, std::string path);
+  llvm::StringRef getMessageOr(swift::DiagID id,
+                               llvm::StringRef defaultMessage) const override;
+};
+
+class LocalizationInput : public llvm::yaml::Input {
+  using Input::Input;
+
+  /// Read diagnostics in the YAML file iteratively
+  template <typename T, typename Context>
+  friend typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
+                                 void>::type
+  readYAML(llvm::yaml::IO &io, T &Seq, bool, Context &Ctx);
+
+  template <typename T>
+  friend typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
+                                 LocalizationInput &>::type
+  operator>>(LocalizationInput &yin, T &diagnostics);
+};
+
+} // namespace diag
+} // namespace swift
+
+#endif

include/swift/AST/diagnostics/en.yaml

@@ -0,0 +1,19 @@
+#===--- en.yaml - Localized diagnostic messages for English ---*- YAML -*-===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===----------------------------------------------------------------------===#
+#
+# This file defines the diagnostic messages for the English language.
+# Each diagnostic is described in the following format:
+# - id: "<diagnostic-id>"
+#   msg: "<diagnostic-message>"
+# 
+#===----------------------------------------------------------------------===#
+

include/swift/AST/diagnostics/fr.yaml

@@ -0,0 +1,19 @@
+#===--- fr.yaml - Localized diagnostic messages for French ---*- YAML -*-===#
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+#===----------------------------------------------------------------------===#
+#
+# This file defines the diagnostic messages for the French language.
+# Each diagnostic is described in the following format:
+# - id: "<diagnostic-id>"
+#   msg: "<diagnostic-message>"
+# 
+#===----------------------------------------------------------------------===#
+

lib/AST/CMakeLists.txt

@@ -56,6 +56,7 @@ add_swift_host_library(swiftAST STATIC
   IndexSubset.cpp
   InlinableText.cpp
   LayoutConstraint.cpp
+  LocalizationFormat.cpp
   Module.cpp
   ModuleDependencies.cpp
   ModuleLoader.cpp

lib/AST/DiagnosticEngine.cpp

@@ -20,6 +20,7 @@
 #include "swift/AST/ASTPrinter.h"
 #include "swift/AST/Decl.h"
 #include "swift/AST/DiagnosticSuppression.h"
+#include "swift/AST/LocalizationFormat.h"
 #include "swift/AST/Module.h"
 #include "swift/AST/Pattern.h"
 #include "swift/AST/PrintOptions.h"
@@ -1011,10 +1012,17 @@ void DiagnosticEngine::emitDiagnostic(const Diagnostic &diagnostic) {
 
 const char *DiagnosticEngine::diagnosticStringFor(const DiagID id,
                                                   bool printDiagnosticName) {
+  // TODO: Print diagnostic names from `localization`.
   if (printDiagnosticName) {
     return debugDiagnosticStrings[(unsigned)id];
   }
-  return diagnosticStrings[(unsigned)id];
+  auto defaultMessage = diagnosticStrings[(unsigned)id];
+  if (localization) {
+    auto localizedMessage =
+        localization.get()->getMessageOr(id, defaultMessage);
+    return localizedMessage.data();
+  }
+  return defaultMessage;
 }
 
 const char *InFlightDiagnostic::fixItStringFor(const FixItID id) {

lib/AST/LocalizationFormat.cpp

@@ -0,0 +1,129 @@
+//===--- LocalizationFormat.cpp - YAML format for Diagnostic Messages ---*-
+// C++ -*-===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2020 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements the format for localized diagnostic messages.
+//
+//===----------------------------------------------------------------------===//
+
+#include "swift/AST/LocalizationFormat.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/YAMLParser.h"
+#include "llvm/Support/YAMLTraits.h"
+#include <string>
+#include <type_traits>
+
+namespace {
+enum LocalDiagID : uint32_t {
+#define DIAG(KIND, ID, Options, Text, Signature) ID,
+#include "swift/AST/DiagnosticsAll.def"
+  NumDiags
+};
+
+struct DiagnosticNode {
+  uint32_t id;
+  std::string msg;
+};
+} // namespace
+
+namespace llvm {
+namespace yaml {
+
+template <> struct ScalarEnumerationTraits<LocalDiagID> {
+  static void enumeration(IO &io, LocalDiagID &value) {
+#define DIAG(KIND, ID, Options, Text, Signature)                               \
+  io.enumCase(value, #ID, LocalDiagID::ID);
+#include "swift/AST/DiagnosticsAll.def"
+  }
+};
+
+template <> struct MappingTraits<DiagnosticNode> {
+  static void mapping(IO &io, DiagnosticNode &node) {
+    LocalDiagID diagID;
+    io.mapRequired("id", diagID);
+    io.mapRequired("msg", node.msg);
+    node.id = static_cast<uint32_t>(diagID);
+  }
+};
+
+} // namespace yaml
+} // namespace llvm
+
+namespace swift {
+namespace diag {
+
+YAMLLocalizationProducer::YAMLLocalizationProducer(std::string locale,
+                                                   std::string path) {
+  llvm::SmallString<128> DiagnosticsFilePath(path);
+  llvm::sys::path::append(DiagnosticsFilePath, locale);
+  llvm::sys::path::replace_extension(DiagnosticsFilePath, ".yaml");
+  auto FileBufOrErr = llvm::MemoryBuffer::getFileOrSTDIN(DiagnosticsFilePath);
+  // Absence of localizations shouldn't crash the compiler.
+  if (!FileBufOrErr)
+    return;
+  llvm::MemoryBuffer *document = FileBufOrErr->get();
+  diag::LocalizationInput yin(document->getBuffer());
+  yin >> diagnostics;
+}
+
+llvm::StringRef
+YAMLLocalizationProducer::getMessageOr(swift::DiagID id,
+                                       llvm::StringRef defaultMessage) const {
+  if (diagnostics.empty())
+    return defaultMessage;
+  const std::string &diagnosticMessage = diagnostics[(unsigned)id];
+  if (diagnosticMessage.empty())
+    return defaultMessage;
+  return diagnosticMessage;
+}
+
+template <typename T, typename Context>
+typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value, void>::type
+readYAML(llvm::yaml::IO &io, T &Seq, bool, Context &Ctx) {
+  unsigned count = io.beginSequence();
+  if (count)
+    Seq.resize(LocalDiagID::NumDiags);
+  for (unsigned i = 0; i < count; ++i) {
+    void *SaveInfo;
+    if (io.preflightElement(i, SaveInfo)) {
+      DiagnosticNode current;
+      yamlize(io, current, true, Ctx);
+      io.postflightElement(SaveInfo);
+      // YAML file isn't guaranteed to have diagnostics in order of their
+      // declaration in `.def` files, to accommodate that we need to leave
+      // holes in diagnostic array for diagnostics which haven't yet been
+      // localized and for the ones that have `DiagnosticNode::id`
+      // indicates their position.
+      Seq[static_cast<unsigned>(current.id)] = std::move(current.msg);
+    }
+  }
+  io.endSequence();
+}
+
+template <typename T>
+typename std::enable_if<llvm::yaml::has_SequenceTraits<T>::value,
+                        LocalizationInput &>::type
+operator>>(LocalizationInput &yin, T &diagnostics) {
+  llvm::yaml::EmptyContext Ctx;
+  if (yin.setCurrentDocument()) {
+    // If YAML file's format doesn't match the current format in
+    // DiagnosticMessageFormat, will throw an error.
+    readYAML(yin, diagnostics, true, Ctx);
+  }
+  return yin;
+}
+
+} // namespace diag
+} // namespace swift