diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e00d42e59..69e87175ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,7 +47,10 @@ jobs: - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 - id: build - run: win_build_vs.bat build /CI /S /A + run: | + PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.17763.0\x86 + PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2017\Enterprise\MSBuild\15.0\Bin\Roslyn + win_build_vs.bat build /CI /S /A shell: cmd windows_VS2019: name: "windows-vs2019" @@ -59,7 +62,10 @@ jobs: - name: Setup PATH uses: microsoft/setup-msbuild@v1.0.2 - id: build - run: win_build_vs.bat build /2019 /64 /CI + run: | + PATH %PATH%;C:\Program Files (x86)\Windows Kits\10\bin\10.0.19041.0\x64 + PATH %PATH%;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\Roslyn + win_build_vs.bat build /2019 /64 /CI shell: cmd ubuntu_16_04: name: "ubuntu-16.04" diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp index 22442f9b33..f2624022e4 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.cpp +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.cpp @@ -34,6 +34,7 @@ #include "wel/MetadataWalker.h" #include "wel/XMLString.h" #include "wel/UnicodeConversion.h" +#include "wel/JSONUtils.h" #include "io/BufferStream.h" #include "core/ProcessContext.h" @@ -134,8 +135,16 @@ core::Property ConsumeWindowsEventLog::OutputFormat( core::PropertyBuilder::createProperty("Output Format")-> isRequired(true)-> withDefaultValue(Both)-> - withAllowableValues({XML, Plaintext, Both})-> - withDescription("Set the output format type. In case \'Both\' is selected the processor generates two flow files for every event captured")-> + withAllowableValues({XML, Plaintext, Both, JSON})-> + withDescription("Set the output format type. In case \'Both\' is selected the processor generates two flow files for every event captured in format XML and Plaintext")-> + build()); + +core::Property ConsumeWindowsEventLog::JSONFormat( + core::PropertyBuilder::createProperty("JSON Format")-> + isRequired(true)-> + withDefaultValue(JSONSimple)-> + withAllowableValues({JSONSimple, JSONFlattened, JSONRaw})-> + withDescription("Set the json format type. Only applicable if Output Format is set to 'JSON'")-> build()); core::Property ConsumeWindowsEventLog::BatchCommitSize( @@ -162,7 +171,8 @@ core::Property ConsumeWindowsEventLog::ProcessOldEvents( core::Relationship ConsumeWindowsEventLog::Success("success", "Relationship for successfully consumed events."); ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name, utils::Identifier uuid) - : core::Processor(name, uuid), logger_(logging::LoggerFactory::getLogger()), apply_identifier_function_(false), batch_commit_size_(0U) { + : core::Processor(name, uuid), + logger_(logging::LoggerFactory::getLogger()) { char buff[MAX_COMPUTERNAME_LENGTH + 1]; DWORD size = sizeof(buff); if (GetComputerName(buff, &size)) { @@ -170,9 +180,6 @@ ConsumeWindowsEventLog::ConsumeWindowsEventLog(const std::string& name, utils::I } else { LogWindowsError(); } - - writeXML_ = false; - writePlainText_ = false; } void ConsumeWindowsEventLog::notifyStop() { @@ -199,7 +206,7 @@ void ConsumeWindowsEventLog::initialize() { //! Set the supported properties setSupportedProperties({ Channel, Query, MaxBufferSize, InactiveDurationToReconnect, IdentifierMatcher, IdentifierFunction, ResolveAsAttributes, - EventHeaderDelimiter, EventHeader, OutputFormat, BatchCommitSize, BookmarkRootDirectory, ProcessOldEvents + EventHeaderDelimiter, EventHeader, OutputFormat, JSONFormat, BatchCommitSize, BookmarkRootDirectory, ProcessOldEvents }); //! Set the supported relationships @@ -252,11 +259,31 @@ void ConsumeWindowsEventLog::onSchedule(const std::shared_ptrgetProperty(OutputFormat.getName(), mode); - writeXML_ = (mode == Both || mode == XML); - - writePlainText_ = (mode == Both || mode == Plaintext); + output_ = {}; + if (mode == XML) { + output_.xml = true; + } else if (mode == Plaintext) { + output_.plaintext = true; + } else if (mode == Both) { + output_.xml = true; + output_.plaintext = true; + } else if (mode == JSON) { + std::string json_format; + context->getProperty(JSONFormat.getName(), json_format); + if (json_format == JSONRaw) { + output_.json.type = JSONType::Raw; + } else if (json_format == JSONSimple) { + output_.json.type = JSONType::Simple; + } else if (json_format == JSONFlattened) { + output_.json.type = JSONType::Flattened; + } + } else { + // in the future this might be considered an error, but for now due to backwards + // compatibility we just fall through and execute the processor outputing nothing + // throw Exception(PROCESS_SCHEDULE_EXCEPTION, "Unrecognized output format: " + mode); + } - if (writeXML_ && !hMsobjsDll_) { + if ((output_.xml || output_.json) && !hMsobjsDll_) { char systemDir[MAX_PATH]; if (GetSystemDirectory(systemDir, sizeof(systemDir))) { hMsobjsDll_ = LoadLibrary((systemDir + std::string("\\msobjs.dll")).c_str()); @@ -564,7 +591,7 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e logger_->log_debug("Finish doc traversing, performing writing..."); - if (writePlainText_) { + if (output_.plaintext) { logger_->log_trace("Writing event in plain text"); auto handler = getEventLogHandler(providerName); @@ -583,30 +610,47 @@ bool ConsumeWindowsEventLog::createEventRender(EVT_HANDLE hEvent, EventRender& e // set the delimiter log_header.setDelimiter(header_delimiter_); // render the header. - eventRender.rendered_text_ = log_header.getEventHeader([&walker](wel::METADATA metadata) { return walker.getMetadata(metadata); }); - eventRender.rendered_text_ += "Message" + header_delimiter_ + " "; - eventRender.rendered_text_ += message; + eventRender.plaintext = log_header.getEventHeader([&walker](wel::METADATA metadata) { return walker.getMetadata(metadata); }); + eventRender.plaintext += "Message" + header_delimiter_ + " "; + eventRender.plaintext += message; } logger_->log_trace("Finish writing in plain text"); } - if (writeXML_) { - logger_->log_trace("Writing event in XML"); + if (output_.xml || output_.json) { substituteXMLPercentageItems(doc); logger_->log_trace("Finish substituting %% in XML"); if (resolve_as_attributes_) { - eventRender.matched_fields_ = walker.getFieldValues(); + eventRender.matched_fields = walker.getFieldValues(); } + } + + if (output_.xml) { + logger_->log_trace("Writing event in XML"); wel::XmlString writer; doc.print(writer, "", pugi::format_raw); // no indentation or formatting xml = writer.xml_; - eventRender.text_ = std::move(xml); + eventRender.xml = std::move(xml); logger_->log_trace("Finish writing in XML"); } + if (output_.json.type == JSONType::Raw) { + logger_->log_trace("Writing event in raw JSON"); + eventRender.json = wel::jsonToString(wel::toRawJSON(doc)); + logger_->log_trace("Finish writing in raw JSON"); + } else if (output_.json.type == JSONType::Simple) { + logger_->log_trace("Writing event in simple JSON"); + eventRender.json = wel::jsonToString(wel::toSimpleJSON(doc)); + logger_->log_trace("Finish writing in simple JSON"); + } else if (output_.json.type == JSONType::Flattened) { + logger_->log_trace("Writing event in flattened JSON"); + eventRender.json = wel::jsonToString(wel::toFlattenedJSON(doc)); + logger_->log_trace("Finish writing in flattened JSON"); + } + return true; } @@ -658,39 +702,45 @@ void ConsumeWindowsEventLog::putEventRenderFlowFileToSession(const EventRender& const std::string& str_; }; - if (writeXML_) { - auto flowFile = session.create(); - logger_->log_trace("Writing rendered XML to a flow file"); - + auto commitFlowFile = [&] (const std::shared_ptr& flowFile, const std::string& content, const std::string& mimeType) { { - WriteCallback wc{ eventRender.text_ }; + WriteCallback wc{ content }; session.write(flowFile, &wc); } - for (const auto &fieldMapping : eventRender.matched_fields_) { - if (!fieldMapping.second.empty()) { - session.putAttribute(flowFile, fieldMapping.first, fieldMapping.second); - } - } - session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "application/xml"); + session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, mimeType); session.putAttribute(flowFile, "Timezone name", timezone_name_); session.putAttribute(flowFile, "Timezone offset", timezone_offset_); session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); session.transfer(flowFile, Success); - } + }; - if (writePlainText_) { + if (output_.xml) { auto flowFile = session.create(); - logger_->log_trace("Writing rendered plain text to a flow file"); + logger_->log_trace("Writing rendered XML to a flow file"); - { - WriteCallback wc{ eventRender.rendered_text_ }; - session.write(flowFile, &wc); + for (const auto &fieldMapping : eventRender.matched_fields) { + if (!fieldMapping.second.empty()) { + session.putAttribute(flowFile, fieldMapping.first, fieldMapping.second); + } } - session.putAttribute(flowFile, core::SpecialFlowAttribute::MIME_TYPE, "text/plain"); - session.putAttribute(flowFile, "Timezone name", timezone_name_); - session.putAttribute(flowFile, "Timezone offset", timezone_offset_); - session.getProvenanceReporter()->receive(flowFile, provenanceUri_, getUUIDStr(), "Consume windows event logs", 0); - session.transfer(flowFile, Success); + + commitFlowFile(flowFile, eventRender.xml, "application/xml"); + } + + if (output_.plaintext) { + logger_->log_trace("Writing rendered plain text to a flow file"); + commitFlowFile(session.create(), eventRender.plaintext, "text/plain"); + } + + if (output_.json.type == JSONType::Raw) { + logger_->log_trace("Writing rendered raw JSON to a flow file"); + commitFlowFile(session.create(), eventRender.json, "application/json"); + } else if (output_.json.type == JSONType::Simple) { + logger_->log_trace("Writing rendered simple JSON to a flow file"); + commitFlowFile(session.create(), eventRender.json, "application/json"); + } else if (output_.json.type == JSONType::Flattened) { + logger_->log_trace("Writing rendered flattened JSON to a flow file"); + commitFlowFile(session.create(), eventRender.json, "application/json"); } } diff --git a/extensions/windows-event-log/ConsumeWindowsEventLog.h b/extensions/windows-event-log/ConsumeWindowsEventLog.h index db4f13b7a7..d0820bf33d 100644 --- a/extensions/windows-event-log/ConsumeWindowsEventLog.h +++ b/extensions/windows-event-log/ConsumeWindowsEventLog.h @@ -43,9 +43,10 @@ namespace minifi { namespace processors { struct EventRender { - std::map matched_fields_; - std::string text_; - std::string rendered_text_; + std::map matched_fields; + std::string xml; + std::string plaintext; + std::string json; }; class Bookmark; @@ -77,6 +78,7 @@ class ConsumeWindowsEventLog : public core::Processor { static core::Property EventHeaderDelimiter; static core::Property EventHeader; static core::Property OutputFormat; + static core::Property JSONFormat; static core::Property BatchCommitSize; static core::Property BookmarkRootDirectory; static core::Property ProcessOldEvents; @@ -107,9 +109,13 @@ class ConsumeWindowsEventLog : public core::Processor { bool createEventRender(EVT_HANDLE eventHandle, EventRender& eventRender); void substituteXMLPercentageItems(pugi::xml_document& doc); - static constexpr const char * const XML = "XML"; - static constexpr const char * const Both = "Both"; - static constexpr const char * const Plaintext = "Plaintext"; + static constexpr const char* XML = "XML"; + static constexpr const char* Both = "Both"; + static constexpr const char* Plaintext = "Plaintext"; + static constexpr const char* JSON = "JSON"; + static constexpr const char* JSONRaw = "Raw"; + static constexpr const char* JSONSimple = "Simple"; + static constexpr const char* JSONFlattened = "Flattened"; private: struct TimeDiff { @@ -132,18 +138,30 @@ class ConsumeWindowsEventLog : public core::Processor { std::wstring wstrChannel_; std::wstring wstrQuery_; std::string regex_; - bool resolve_as_attributes_; - bool apply_identifier_function_; + bool resolve_as_attributes_{false}; + bool apply_identifier_function_{false}; std::string provenanceUri_; std::string computerName_; uint64_t maxBufferSize_{}; DWORD lastActivityTimestamp_{}; std::mutex cache_mutex_; std::map providers_; - uint64_t batch_commit_size_; + uint64_t batch_commit_size_{}; + + enum class JSONType {None, Raw, Simple, Flattened}; + + struct OutputFormat { + bool xml{false}; + bool plaintext{false}; + struct JSON { + JSONType type{JSONType::None}; + + explicit operator bool() const noexcept { + return type != JSONType::None; + } + } json; + } output_; - bool writeXML_; - bool writePlainText_; std::unique_ptr bookmark_; std::mutex on_trigger_mutex_; std::unordered_map xmlPercentageItemsResolutions_; diff --git a/extensions/windows-event-log/tests/CMakeLists.txt b/extensions/windows-event-log/tests/CMakeLists.txt index 4d144ce364..1f8c9f3538 100644 --- a/extensions/windows-event-log/tests/CMakeLists.txt +++ b/extensions/windows-event-log/tests/CMakeLists.txt @@ -17,7 +17,15 @@ # under the License. # -file(GLOB WEL_INTEGRATION_TESTS "*.cpp") +set(WEL_INTEGRATION_TESTS "BookmarkTests.cpp" "ConsumeWindowsEventLogTests.cpp" "MetadataWalkerTests.cpp") +if (TEST_CUSTOM_WEL_PROVIDER) + execute_process(COMMAND + "${CMAKE_CURRENT_LIST_DIR}/custom-provider/generate-and-register.bat" + "${CMAKE_CURRENT_LIST_DIR}/custom-provider" + ) + list(APPEND WEL_INTEGRATION_TESTS "CWELCustomProviderTests.cpp") +endif() + SET(WEL_TEST_COUNT 0) FOREACH(testfile ${WEL_INTEGRATION_TESTS}) get_filename_component(testfilename "${testfile}" NAME_WE) diff --git a/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp new file mode 100644 index 0000000000..57eb4bfcc6 --- /dev/null +++ b/extensions/windows-event-log/tests/CWELCustomProviderTests.cpp @@ -0,0 +1,176 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#undef NDEBUG + +#include "ConsumeWindowsEventLog.h" + +#include "core/ConfigurableComponent.h" +#include "processors/LogAttribute.h" +#include "processors/PutFile.h" +#include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" +#include "rapidjson/document.h" +#include "wel/UniqueEvtHandle.h" +#include "IntegrationTestUtils.h" + +#include "CWELTestUtils.h" + +// generated from the manifest file "custom-provider/unit-test-provider.man" +// using the command "mc -um unit-test-provider.man" +#include "custom-provider/unit-test-provider.h" + +namespace { + +struct CustomEventData { + std::wstring first; + std::wstring second; + std::wstring third; + int binary_length; + const unsigned char* binary_data; +}; + +const std::string CUSTOM_PROVIDER_NAME = "minifi_unit_test_provider"; +const std::string CUSTOM_CHANNEL = CUSTOM_PROVIDER_NAME + "/Log"; + +bool dispatchCustomEvent(const CustomEventData& event) { + static auto provider_initialized = EventRegisterminifi_unit_test_provider(); + REQUIRE(provider_initialized == ERROR_SUCCESS); + + auto result = EventWriteCustomEvent( + event.first.c_str(), + event.second.c_str(), + event.third.c_str(), + event.binary_length, + event.binary_data + ); + return result == ERROR_SUCCESS; +} + +using org::apache::nifi::minifi::wel::unique_evt_handle; + +bool advanceBookmark(const unique_evt_handle& hBookmark, const std::string& channel, const std::string& query, bool advance_to_last = false) { + const auto hEventResults = unique_evt_handle{ EvtQuery(0, std::wstring{channel.begin(), channel.end()}.c_str(), std::wstring{query.begin(), query.end()}.c_str(), EvtQueryChannelPath) }; + if (!hEventResults) { + return false; + } + + if (advance_to_last) { + if (!EvtSeek(hEventResults.get(), 0, 0, 0, EvtSeekRelativeToLast)) { + return false; + } + } else { + if (!EvtSeek(hEventResults.get(), 1, hBookmark.get(), 0, EvtSeekRelativeToBookmark)) { + return false; + } + } + + const unique_evt_handle hEvent = [&hEventResults] { + DWORD dwReturned{}; + EVT_HANDLE hEvent{ nullptr }; + EvtNext(hEventResults.get(), 1, &hEvent, INFINITE, 0, &dwReturned); + return unique_evt_handle{ hEvent }; + }(); + + if (!hEvent) { + return false; + } + + REQUIRE(EvtUpdateBookmark(hBookmark.get(), hEvent.get())); + + return true; +} + +class CustomProviderController : public OutputFormatTestController { + public: + CustomProviderController(std::string format, std::string json_format) : OutputFormatTestController(CUSTOM_CHANNEL, "*", std::move(format), std::move(json_format)) { + bookmark_.reset(EvtCreateBookmark(0)); + advanceBookmark(bookmark_, channel_, query_, true); + REQUIRE(bookmark_); + } + + protected: + void dispatchBookmarkEvent() override { + auto binary = reinterpret_cast("\x0c\x10"); + REQUIRE(dispatchCustomEvent({L"Bookmark", L"Second", L"Third", 2, binary})); + REQUIRE(checkNewEventAvailable()); + } + void dispatchCollectedEvent() override { + auto binary = reinterpret_cast("\x09\x01"); + REQUIRE(dispatchCustomEvent({L"Actual event", L"Second", L"Third", 2, binary})); + REQUIRE(checkNewEventAvailable()); + } + + private: + bool checkNewEventAvailable() { + return org::apache::nifi::minifi::utils::verifyEventHappenedInPollTime(std::chrono::seconds{5}, [&] { + return advanceBookmark(bookmark_, channel_, query_); + }); + } + unique_evt_handle bookmark_; +}; + +const std::string EVENT_DATA_JSON = R"( + [{ + "Type": "Data", + "Content": "Actual event", + "Name": "param1" + }, { + "Type": "Data", + "Content": "Second", + "Name": "param2" + }, { + "Type": "Data", + "Content": "Third", + "Name": "Channel" + }, { + "Type": "Binary", + "Content": "0901", + "Name": "" + }] +)"; + +} // namespace + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly custom provider", "[onTrigger]") { + std::string event = CustomProviderController{"JSON", "Simple"}.run(); + verifyJSON(event, R"( + { + "System": { + "Provider": { + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(" + }, + "Channel": ")" + CUSTOM_CHANNEL + R"(" + }, + "EventData": )" + EVENT_DATA_JSON + R"( + } + )"); +} + +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly custom provider", "[onTrigger]") { + std::string event = CustomProviderController{"JSON", "Flattened"}.run(); + verifyJSON(event, R"( + { + "Name": ")" + CUSTOM_PROVIDER_NAME + R"(", + "Channel": ")" + CUSTOM_CHANNEL /* Channel is not overwritten by data named "Channel" */ + R"(", + "EventData": )" + EVENT_DATA_JSON /* EventData is not discarded */ + R"(, + "param1": "Actual event", + "param2": "Second" + } + )"); +} diff --git a/extensions/windows-event-log/tests/CWELTestUtils.h b/extensions/windows-event-log/tests/CWELTestUtils.h new file mode 100644 index 0000000000..3820ca58d3 --- /dev/null +++ b/extensions/windows-event-log/tests/CWELTestUtils.h @@ -0,0 +1,117 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "ConsumeWindowsEventLog.h" +#include "processors/PutFile.h" +#include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" +#include "utils/OptionalUtils.h" + +core::Relationship Success{"success", "Everything is fine"}; + +using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; +using PutFile = org::apache::nifi::minifi::processors::PutFile; + +class OutputFormatTestController : public TestController { + public: + OutputFormatTestController(std::string channel, std::string query, std::string output_format, utils::optional json_format = {}) + : channel_(std::move(channel)), + query_(std::move(query)), + output_format_(std::move(output_format)), + json_format_(std::move(json_format)) {} + + std::string run() { + LogTestController::getInstance().setDebug(); + LogTestController::getInstance().setDebug(); + std::shared_ptr test_plan = createPlan(); + + auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), channel_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), query_); + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), output_format_); + if (json_format_) { + test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::JSONFormat.getName(), json_format_.value()); + } + + auto dir = utils::createTempDir(this); + + auto put_file = test_plan->addProcessor("PutFile", "putFile", Success, true); + test_plan->setProperty(put_file, PutFile::Directory.getName(), dir); + + { + dispatchBookmarkEvent(); + + runSession(test_plan); + } + + test_plan->reset(); + LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); + + + { + dispatchCollectedEvent(); + + runSession(test_plan); + + auto files = utils::file::list_dir_all(dir, LogTestController::getInstance().getLogger(), false); + REQUIRE(files.size() == 1); + + std::ifstream file{utils::file::concat_path(files[0].first, files[0].second)}; + return {std::istreambuf_iterator{file}, {}}; + } + } + + protected: + virtual void dispatchBookmarkEvent() = 0; + virtual void dispatchCollectedEvent() = 0; + + std::string channel_; + std::string query_; + std::string output_format_; + utils::optional json_format_; +}; + +// carries out a loose match on objects, i.e. it doesn't matter if the +// actual object has extra fields than expected +void matchJSON(const rapidjson::Value& json, const rapidjson::Value& expected) { + if (expected.IsObject()) { + REQUIRE(json.IsObject()); + for (const auto& expected_member : expected.GetObject()) { + REQUIRE(json.HasMember(expected_member.name)); + matchJSON(json[expected_member.name], expected_member.value); + } + } else if (expected.IsArray()) { + REQUIRE(json.IsArray()); + REQUIRE(json.Size() == expected.Size()); + for (size_t idx{0}; idx < expected.Size(); ++idx) { + matchJSON(json[idx], expected[idx]); + } + } else { + REQUIRE(json == expected); + } +} + +void verifyJSON(const std::string& json_str, const std::string& expected_str) { + rapidjson::Document json, expected; + REQUIRE_FALSE(json.Parse(json_str.c_str()).HasParseError()); + REQUIRE_FALSE(expected.Parse(expected_str.c_str()).HasParseError()); + + matchJSON(json, expected); +} diff --git a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp index 02aae26507..1976f404c8 100644 --- a/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp +++ b/extensions/windows-event-log/tests/ConsumeWindowsEventLogTests.cpp @@ -19,17 +19,22 @@ #include "core/ConfigurableComponent.h" #include "processors/LogAttribute.h" +#include "processors/PutFile.h" #include "TestBase.h" +#include "utils/TestUtils.h" +#include "utils/file/FileUtils.h" +#include "rapidjson/document.h" + +#include "CWELTestUtils.h" using ConsumeWindowsEventLog = org::apache::nifi::minifi::processors::ConsumeWindowsEventLog; using LogAttribute = org::apache::nifi::minifi::processors::LogAttribute; +using PutFile = org::apache::nifi::minifi::processors::PutFile; using ConfigurableComponent = org::apache::nifi::minifi::core::ConfigurableComponent; using IdGenerator = org::apache::nifi::minifi::utils::IdGenerator; namespace { -core::Relationship Success{"success", "Everything is fine"}; - const std::string APPLICATION_CHANNEL = "Application"; constexpr DWORD CWEL_TESTS_OPCODE = 14985; // random opcode hopefully won't clash with something important @@ -41,6 +46,19 @@ void reportEvent(const std::string& channel, const char* message, WORD log_level ReportEventA(event_source, log_level, 0, CWEL_TESTS_OPCODE, nullptr, 1, 0, &message, nullptr); } +class SimpleFormatTestController : public OutputFormatTestController { + public: + using OutputFormatTestController::OutputFormatTestController; + + protected: + void dispatchBookmarkEvent() { + reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); + } + void OutputFormatTestController::dispatchCollectedEvent() { + reportEvent(APPLICATION_CHANNEL, "Event one"); + } +}; + } // namespace TEST_CASE("ConsumeWindowsEventLog constructor works", "[create]") { @@ -305,48 +323,75 @@ TEST_CASE("ConsumeWindowsEventLog output format can be set", "[create][output_fo // TEST_CASE("ConsumeWindowsEventLog prints events in plain text correctly", "[onTrigger]") TEST_CASE("ConsumeWindowsEventLog prints events in XML correctly", "[onTrigger]") { - TestController test_controller; - LogTestController::getInstance().setDebug(); - LogTestController::getInstance().setDebug(); - std::shared_ptr test_plan = test_controller.createPlan(); - - auto cwel_processor = test_plan->addProcessor("ConsumeWindowsEventLog", "cwel"); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Channel.getName(), APPLICATION_CHANNEL); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::Query.getName(), QUERY); - test_plan->setProperty(cwel_processor, ConsumeWindowsEventLog::OutputFormat.getName(), "XML"); - - auto logger_processor = test_plan->addProcessor("LogAttribute", "logger", Success, true); - test_plan->setProperty(logger_processor, LogAttribute::FlowFilesToLog.getName(), "0"); - test_plan->setProperty(logger_processor, LogAttribute::LogPayload.getName(), "true"); - test_plan->setProperty(logger_processor, LogAttribute::MaxPayloadLineLength.getName(), "1024"); - - { - reportEvent(APPLICATION_CHANNEL, "Event zero: this is in the past"); - - test_controller.runSession(test_plan); - } - - test_plan->reset(); - LogTestController::getInstance().resetStream(LogTestController::getInstance().log_output); + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, QUERY, "XML"}.run(); + + REQUIRE(event.find(R"()") != std::string::npos); + REQUIRE(event.find(R"(14985)") != std::string::npos); + REQUIRE(event.find(R"(4)") != std::string::npos); + REQUIRE(event.find(R"(0)") != std::string::npos); + REQUIRE(event.find(R"(0x80000000000000)") != std::string::npos); + // the ID of the event goes here (a number) + REQUIRE(event.find(R"()") != std::string::npos); + REQUIRE(event.find(R"(Application)") != std::string::npos); + // the computer name goes here + REQUIRE(event.find(R"(Event one)") != std::string::npos); +} - { - reportEvent(APPLICATION_CHANNEL, "Event one"); +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Simple correctly", "[onTrigger]") { + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON", "Simple"}.run(); + verifyJSON(event, R"json( + { + "System": { + "Provider": { + "Name": "Application" + }, + "Channel": "Application" + }, + "EventData": [{ + "Type": "Data", + "Content": "Event one", + "Name": "" + }] + } + )json"); +} - test_controller.runSession(test_plan); +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Flattened correctly", "[onTrigger]") { + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON", "Flattened"}.run(); + verifyJSON(event, R"json( + { + "Name": "Application", + "Channel": "Application", + "EventData": [{ + "Type": "Data", + "Content": "Event one", + "Name": "" + }] + } + )json"); +} - REQUIRE(LogTestController::getInstance().contains(R"()")); - REQUIRE(LogTestController::getInstance().contains(R"(14985)")); - REQUIRE(LogTestController::getInstance().contains(R"(4)")); - REQUIRE(LogTestController::getInstance().contains(R"(0)")); - REQUIRE(LogTestController::getInstance().contains(R"(0x80000000000000)")); - // the ID of the event goes here (a number) - REQUIRE(LogTestController::getInstance().contains(R"()")); - REQUIRE(LogTestController::getInstance().contains(R"(Application)")); - // the computer name goes here - REQUIRE(LogTestController::getInstance().contains(R"(Event one)")); - } +TEST_CASE("ConsumeWindowsEventLog prints events in JSON::Raw correctly", "[onTrigger]") { + std::string event = SimpleFormatTestController{APPLICATION_CHANNEL, "*", "JSON", "Raw"}.run(); + verifyJSON(event, R"json( + [ + { + "name": "Event", + "children": [ + {"name": "System"}, + { + "name": "EventData", + "children": [{ + "name": "Data", + "text": "Event one" + }] + } + ] + } + ] + )json"); } namespace { diff --git a/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat new file mode 100644 index 0000000000..c74090b0da --- /dev/null +++ b/extensions/windows-event-log/tests/custom-provider/generate-and-register.bat @@ -0,0 +1,62 @@ +@echo off &setlocal enabledelayedexpansion +rem Licensed to the Apache Software Foundation (ASF) under one or more +rem contributor license agreements. See the NOTICE file distributed with +rem this work for additional information regarding copyright ownership. +rem The ASF licenses this file to You under the Apache License, Version 2.0 +rem (the "License"); you may not use this file except in compliance with +rem the License. You may obtain a copy of the License at +rem +rem http://www.apache.org/licenses/LICENSE-2.0 +rem +rem Unless required by applicable law or agreed to in writing, software +rem distributed under the License is distributed on an "AS IS" BASIS, +rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +rem See the License for the specific language governing permissions and +rem limitations under the License. + +cd %1 + +( + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^ + echo ^