From 7c18f69fdd54fe44e852883a871ec9df6d2512ac Mon Sep 17 00:00:00 2001 From: Paul Kirth Date: Sat, 31 May 2025 13:30:32 -0700 Subject: [PATCH] [llvm] Add a tool to check mustache compliance against the public spec This is a cli tool to that tests the conformance of LLVM's mustache implementation against the public Mustache spec, hosted at https://github.com/mustache/spec. This is a revised version of the patches in #111487. Co-authored-by: Peter Chou --- llvm/CMakeLists.txt | 1 + llvm/docs/CommandGuide/index.rst | 1 + .../CommandGuide/llvm-test-mustache-spec.rst | 37 +++ .../llvm-test-mustache-spec/CMakeLists.txt | 5 + .../llvm-test-mustache-spec.cpp | 268 ++++++++++++++++++ 5 files changed, 312 insertions(+) create mode 100644 llvm/docs/CommandGuide/llvm-test-mustache-spec.rst create mode 100644 llvm/utils/llvm-test-mustache-spec/CMakeLists.txt create mode 100644 llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 206f009b45f59..cfb67472aa71e 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -1313,6 +1313,7 @@ if( LLVM_INCLUDE_UTILS ) add_subdirectory(utils/yaml-bench) add_subdirectory(utils/split-file) add_subdirectory(utils/mlgo-utils) + add_subdirectory(utils/llvm-test-mustache-spec) if( LLVM_INCLUDE_TESTS ) set(LLVM_SUBPROJECT_TITLE "Third-Party/Google Test") add_subdirectory(${LLVM_THIRD_PARTY_DIR}/unittest ${CMAKE_CURRENT_BINARY_DIR}/third-party/unittest) diff --git a/llvm/docs/CommandGuide/index.rst b/llvm/docs/CommandGuide/index.rst index 643951eca2a26..88fc1fd326b76 100644 --- a/llvm/docs/CommandGuide/index.rst +++ b/llvm/docs/CommandGuide/index.rst @@ -87,6 +87,7 @@ Developer Tools llvm-exegesis llvm-ifs llvm-locstats + llvm-test-mustache-spec llvm-pdbutil llvm-profgen llvm-tli-checker diff --git a/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst b/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst new file mode 100644 index 0000000000000..8cd5a349e7e49 --- /dev/null +++ b/llvm/docs/CommandGuide/llvm-test-mustache-spec.rst @@ -0,0 +1,37 @@ +llvm-test-mustache-spec - LLVM tool to test Mustache library compliance +======================================================================= + +.. program:: llvm-test-mustache-spec + +SYNOPSIS +-------- + +:program:`llvm-test-mustache-spec` [*inputs...*] + +Description +----------- + +``llvm-test-mustache-spec`` tests the mustache spec conformance of the LLVM +mustache library. The spec can be found here: https://github.com/mustache/spec + +To test against the spec, simply download the spec and pass the test JSON files +to the driver. Each spec file should have a list of tests for compliance with +the spec. These are loaded as test cases, and rendered with our Mustache +implementation, which is then compared against the expected output from the +spec. + +The current implementation only supports non-optional parts of the spec, so +we do not expect any of the dynamic-names, inheritance, or lambda tests to +pass. Additionally, Triple Mustache is not supported. Unsupported tests are +marked as XFail and are removed from the XFail list as they are fixed. + +The tool prints the number of test failures and successes in each of the test +files to standard output. + +EXAMPLE +------- + +.. code-block:: console + + $ llvm-test-mustache-spec path/to/specs/\*.json + diff --git a/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt b/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt new file mode 100644 index 0000000000000..dc1aa73371ffc --- /dev/null +++ b/llvm/utils/llvm-test-mustache-spec/CMakeLists.txt @@ -0,0 +1,5 @@ +add_llvm_utility(llvm-test-mustache-spec + llvm-test-mustache-spec.cpp +) + +target_link_libraries(llvm-test-mustache-spec PRIVATE LLVMSupport) diff --git a/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp b/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp new file mode 100644 index 0000000000000..28ed1b876672d --- /dev/null +++ b/llvm/utils/llvm-test-mustache-spec/llvm-test-mustache-spec.cpp @@ -0,0 +1,268 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Simple drivers to test the mustache spec found at: +// https://github.com/mustache/spec +// +// It is used to verify that the current implementation conforms to the spec. +// Simply download the spec and pass the test JSON files to the driver. Each +// spec file should have a list of tests for compliance with the spec. These +// are loaded as test cases, and rendered with our Mustache implementation, +// which is then compared against the expected output from the spec. +// +// The current implementation only supports non-optional parts of the spec, so +// we do not expect any of the dynamic-names, inheritance, or lambda tests to +// pass. Additionally, Triple Mustache is not supported. Unsupported tests are +// marked as XFail and are removed from the XFail list as they are fixed. +// +// Usage: +// llvm-test-mustache-spec path/to/test/file.json path/to/test/file2.json ... +//===----------------------------------------------------------------------===// + +#include "llvm/ADT/StringSet.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/Debug.h" +#include "llvm/Support/Error.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Mustache.h" +#include "llvm/Support/Path.h" +#include + +using namespace llvm; +using namespace llvm::json; +using namespace llvm::mustache; + +#define DEBUG_TYPE "llvm-test-mustache-spec" + +static cl::OptionCategory Cat("llvm-test-mustache-spec Options"); + +static cl::list + InputFiles(cl::Positional, cl::desc(""), cl::OneOrMore); + +static cl::opt ReportErrors("report-errors", + cl::desc("Report errors in spec tests"), + cl::cat(Cat)); + +static ExitOnError ExitOnErr; + +static int NumXFail = 0; +static int NumSuccess = 0; + +static const StringMap> XFailTestNames = {{ + {"delimiters.json", + { + "Pair Behavior", + "Special Characters", + "Sections", + "Inverted Sections", + "Partial Inheritence", + "Post-Partial Behavior", + "Standalone Tag", + "Indented Standalone Tag", + "Standalone Line Endings", + "Standalone Without Previous Line", + "Standalone Without Newline", + }}, + {"~dynamic-names.json", + { + "Basic Behavior - Partial", + "Basic Behavior - Name Resolution", + "Context", + "Dotted Names", + "Dotted Names - Failed Lookup", + "Dotted names - Context Stacking", + "Dotted names - Context Stacking Under Repetition", + "Dotted names - Context Stacking Failed Lookup", + "Recursion", + "Surrounding Whitespace", + "Inline Indentation", + "Standalone Line Endings", + "Standalone Without Previous Line", + "Standalone Without Newline", + "Standalone Indentation", + "Padding Whitespace", + }}, + {"~inheritance.json", + { + "Default", + "Variable", + "Triple Mustache", + "Sections", + "Negative Sections", + "Mustache Injection", + "Inherit", + "Overridden content", + "Data does not override block default", + "Two overridden parents", + "Override parent with newlines", + "Inherit indentation", + "Only one override", + "Parent template", + "Recursion", + "Multi-level inheritance, no sub child", + "Text inside parent", + "Text inside parent", + "Block scope", + "Standalone parent", + "Standalone block", + "Block reindentation", + "Intrinsic indentation", + "Nested block reindentation", + + }}, + {"~lambdas.json", + { + "Interpolation", + "Interpolation - Expansion", + "Interpolation - Alternate Delimiters", + "Interpolation - Multiple Calls", + "Escaping", + "Section", + "Section - Expansion", + "Section - Alternate Delimiters", + "Section - Multiple Calls", + + }}, + {"interpolation.json", + { + "Triple Mustache", + "Triple Mustache Integer Interpolation", + "Triple Mustache Decimal Interpolation", + "Triple Mustache Null Interpolation", + "Triple Mustache Context Miss Interpolation", + "Dotted Names - Triple Mustache Interpolation", + "Implicit Iterators - Triple Mustache", + "Triple Mustache - Surrounding Whitespace", + "Triple Mustache - Standalone", + "Triple Mustache With Padding", + }}, + {"partials.json", {"Standalone Indentation"}}, + {"sections.json", {"Implicit Iterator - Triple mustache"}}, +}}; + +struct TestData { + static Expected createTestData(json::Object *TestCase, + StringRef InputFile) { + // If any of the needed elements are missing, we cannot continue. + // NOTE: partials are optional in the test schema. + if (!TestCase || !TestCase->getString("template") || + !TestCase->getString("expected") || !TestCase->getString("name") || + !TestCase->get("data")) + return createStringError( + llvm::inconvertibleErrorCode(), + "invalid JSON schema in test file: " + InputFile + "\n"); + + return TestData{TestCase->getString("template").value(), + TestCase->getString("expected").value(), + TestCase->getString("name").value(), TestCase->get("data"), + TestCase->get("partials")}; + } + + TestData() = default; + + StringRef TemplateStr; + StringRef ExpectedStr; + StringRef Name; + Value *Data; + Value *Partials; +}; + +static void reportTestFailure(const TestData &TD, StringRef ActualStr, + bool IsXFail) { + LLVM_DEBUG(dbgs() << "Template: " << TD.TemplateStr << "\n"); + if (TD.Partials) { + LLVM_DEBUG(dbgs() << "Partial: "); + LLVM_DEBUG(TD.Partials->print(dbgs())); + LLVM_DEBUG(dbgs() << "\n"); + } + LLVM_DEBUG(dbgs() << "JSON Data: "); + LLVM_DEBUG(TD.Data->print(dbgs())); + LLVM_DEBUG(dbgs() << "\n"); + outs() << formatv("Test {}: {}\n", (IsXFail ? "XFailed" : "Failed"), TD.Name); + if (ReportErrors) { + outs() << " Expected: \'" << TD.ExpectedStr << "\'\n" + << " Actual: \'" << ActualStr << "\'\n" + << " ====================\n"; + } +} + +static void registerPartials(Value *Partials, Template &T) { + if (!Partials) + return; + for (const auto &[Partial, Str] : *Partials->getAsObject()) + T.registerPartial(Partial.str(), Str.getAsString()->str()); +} + +static json::Value readJsonFromFile(StringRef &InputFile) { + std::unique_ptr Buffer = + ExitOnErr(errorOrToExpected(MemoryBuffer::getFile(InputFile))); + return ExitOnErr(parse(Buffer->getBuffer())); +} + +static bool isTestXFail(StringRef FileName, StringRef TestName) { + auto P = llvm::sys::path::filename(FileName); + auto It = XFailTestNames.find(P); + return It != XFailTestNames.end() && It->second.contains(TestName); +} + +static bool evaluateTest(StringRef &InputFile, TestData &TestData, + std::string &ActualStr) { + bool IsXFail = isTestXFail(InputFile, TestData.Name); + bool Matches = TestData.ExpectedStr == ActualStr; + if ((Matches && IsXFail) || (!Matches && !IsXFail)) { + reportTestFailure(TestData, ActualStr, IsXFail); + return false; + } + IsXFail ? NumXFail++ : NumSuccess++; + return true; +} + +static void runTest(StringRef InputFile) { + NumXFail = 0; + NumSuccess = 0; + outs() << "Running Tests: " << InputFile << "\n"; + json::Value Json = readJsonFromFile(InputFile); + + json::Object *Obj = Json.getAsObject(); + Array *TestArray = Obj->getArray("tests"); + // Even though we parsed the JSON, it can have a bad format, so check it. + if (!TestArray) + ExitOnErr(createStringError( + llvm::inconvertibleErrorCode(), + "invalid JSON schema in test file: " + InputFile + "\n")); + + const size_t Total = TestArray->size(); + + for (Value V : *TestArray) { + auto TestData = + ExitOnErr(TestData::createTestData(V.getAsObject(), InputFile)); + Template T(TestData.TemplateStr); + registerPartials(TestData.Partials, T); + + std::string ActualStr; + raw_string_ostream OS(ActualStr); + T.render(*TestData.Data, OS); + evaluateTest(InputFile, TestData, ActualStr); + } + + const int NumFailed = Total - NumSuccess - NumXFail; + outs() << formatv("===Results===\n" + " Suceeded: {}\n" + " Expectedly Failed: {}\n" + " Failed: {}\n" + " Total: {}\n", + NumSuccess, NumXFail, NumFailed, Total); +} + +int main(int argc, char **argv) { + ExitOnErr.setBanner(std::string(argv[0]) + " error: "); + cl::ParseCommandLineOptions(argc, argv); + for (const auto &FileName : InputFiles) + runTest(FileName); + return 0; +}