diff --git a/src/openvic-dataloader/AbstractSyntaxTree.hpp b/src/openvic-dataloader/AbstractSyntaxTree.hpp index eea99f8..79a6100 100644 --- a/src/openvic-dataloader/AbstractSyntaxTree.hpp +++ b/src/openvic-dataloader/AbstractSyntaxTree.hpp @@ -60,7 +60,7 @@ namespace ovdl { BasicAbstractSyntaxTree() = default; explicit BasicAbstractSyntaxTree(file_type&& file) - : AbstractSyntaxTree(file.size() * file.visit_buffer([](auto&& buffer) -> size_t { return sizeof(typename std::decay_t::char_type); })), + : AbstractSyntaxTree(file.size() * sizeof(typename std::decay_t::char_type)), _file { std::move(file) } {} template diff --git a/src/openvic-dataloader/DiagnosticLogger.hpp b/src/openvic-dataloader/DiagnosticLogger.hpp index 39ce32a..9bd9761 100644 --- a/src/openvic-dataloader/DiagnosticLogger.hpp +++ b/src/openvic-dataloader/DiagnosticLogger.hpp @@ -335,23 +335,22 @@ namespace ovdl { [[nodiscard]] Writer& annotation(AnnotationKind kind, BasicNodeLocation loc, format_str fmt, Args&&... args) { std::basic_string output; - _file.visit_buffer([&](auto&& buffer) { - using char_type = typename std::decay_t::encoding::char_type; + lexy::buffer const& buffer = _file.buffer(); + using char_type = typename std::decay_t::encoding::char_type; - BasicNodeLocation converted_loc = loc; + BasicNodeLocation converted_loc = loc; - auto begin_loc = lexy::get_input_location(buffer, converted_loc.begin()); + auto begin_loc = lexy::get_input_location(buffer, converted_loc.begin()); - auto stream = _logger.make_callback_stream(output); - auto iter = _logger.make_ostream_iterator(stream); + auto stream = _logger.make_callback_stream(output); + auto iter = _logger.make_ostream_iterator(stream); - lexy_ext::diagnostic_writer _impl { buffer, { lexy::visualize_fancy } }; - _impl.write_empty_annotation(iter); - _impl.write_annotation(iter, kind, begin_loc, converted_loc.end(), - [&](auto out, lexy::visualization_options) { - return lexy::_detail::write_str(out, fmt::format(fmt, std::forward(args)...).c_str()); - }); - }); + lexy_ext::diagnostic_writer _impl { buffer, { lexy::visualize_fancy } }; + _impl.write_empty_annotation(iter); + _impl.write_annotation(iter, kind, begin_loc, converted_loc.end(), + [&](auto out, lexy::visualization_options) { + return lexy::_detail::write_str(out, fmt::format(fmt, std::forward(args)...).c_str()); + }); error::Annotation* annotation; auto message = _logger.intern(output); @@ -443,11 +442,9 @@ namespace ovdl { template T, typename... Args> void log_with_error(T* error, DiagnosticKind kind, format_str fmt, Args&&... args) { - file().visit_buffer( - [&](auto&& buffer) { - lexy_ext::diagnostic_writer impl { buffer }; - log_with_impl(impl, error, kind, fmt, std::forward(args)...); - }); + lexy::buffer const& buffer = file().buffer(); + lexy_ext::diagnostic_writer impl { buffer }; + log_with_impl(impl, error, kind, fmt, std::forward(args)...); } template T, typename... Args> @@ -483,10 +480,9 @@ namespace ovdl { Writer result(*this, file(), semantic); - file().visit_buffer([&](auto&& buffer) { - lexy_ext::diagnostic_writer impl { buffer }; - log_with_impl(impl, semantic, kind, fmt, std::forward(args)...); - }); + lexy::buffer const& buffer = file().buffer(); + lexy_ext::diagnostic_writer impl { buffer }; + log_with_impl(impl, semantic, kind, fmt, std::forward(args)...); if (kind == DiagnosticKind::error) { _errored = true; diff --git a/src/openvic-dataloader/File.cpp b/src/openvic-dataloader/File.cpp index 9e721a8..cd5a3f1 100644 --- a/src/openvic-dataloader/File.cpp +++ b/src/openvic-dataloader/File.cpp @@ -15,9 +15,9 @@ const char* File::path() const noexcept { } bool File::is_valid() const noexcept { - return _buffer.index() != 0 && !_buffer.valueless_by_exception() && visit_buffer([](auto&& buffer) { return buffer.data() != nullptr; }); + return _buffer.data() != nullptr; } std::size_t File::size() const noexcept { - return _buffer.index() != 0 && !_buffer.valueless_by_exception() ? _buffer_size : 0; + return _buffer.size(); } \ No newline at end of file diff --git a/src/openvic-dataloader/File.hpp b/src/openvic-dataloader/File.hpp index fa01d33..d0982ad 100644 --- a/src/openvic-dataloader/File.hpp +++ b/src/openvic-dataloader/File.hpp @@ -3,11 +3,11 @@ #include #include // IWYU pragma: keep #include -#include #include #include +#include #include #include @@ -33,87 +33,17 @@ namespace ovdl { std::size_t size() const noexcept; - template - constexpr bool is_buffer() const { - return buffer_ids::type_id>() + 1 == _buffer.index(); + lexy::buffer& buffer() { + return _buffer; } - template - lexy::buffer* try_get_buffer_as() { - return std::get_if>(&_buffer); + lexy::buffer const& buffer() const { + return _buffer; } - template - const lexy::buffer* try_get_buffer_as() const { - return std::get_if>(&_buffer); - } - - template - lexy::buffer& get_buffer_as() { - assert((is_buffer())); - return *std::get_if>(&_buffer); - } - - template - const lexy::buffer& get_buffer_as() const { - assert((is_buffer())); - return *std::get_if>(&_buffer); - } - -#define SWITCH_LIST \ - X(1) \ - X(2) \ - X(3) \ - X(4) \ - X(5) \ - X(6) - -#define X(NUM) \ - case NUM: \ - return visitor(std::get(_buffer)); - - template - decltype(auto) visit_buffer(Visitor&& visitor) { - switch (_buffer.index()) { - SWITCH_LIST - case 0: return visitor(lexy::buffer<> {}); - default: ovdl::detail::unreachable(); - } - } - - template - Return visit_buffer(Visitor&& visitor) { - switch (_buffer.index()) { - SWITCH_LIST - case 0: return visitor(lexy::buffer<> {}); - default: ovdl::detail::unreachable(); - } - } - - template - decltype(auto) visit_buffer(Visitor&& visitor) const { - switch (_buffer.index()) { - SWITCH_LIST - case 0: return visitor(lexy::buffer<> {}); - default: ovdl::detail::unreachable(); - } - } - - template - Return visit_buffer(Visitor&& visitor) const { - switch (_buffer.index()) { - SWITCH_LIST - case 0: return visitor(lexy::buffer<> {}); - default: ovdl::detail::unreachable(); - } - } -#undef X -#undef SWITCH_LIST - protected: const char* _path = ""; - std::size_t _buffer_size = 0; - detail::type_prepend_t _buffer; + lexy::buffer _buffer; }; template @@ -122,17 +52,13 @@ namespace ovdl { BasicFile() = default; - template - explicit BasicFile(const char* path, lexy::buffer&& buffer) + explicit BasicFile(const char* path, lexy::buffer&& buffer) : File(path) { - _buffer_size = buffer.size(); _buffer = static_cast&&>(buffer); } - template - explicit BasicFile(lexy::buffer&& buffer) + explicit BasicFile(lexy::buffer&& buffer) : File("") { - _buffer_size = buffer.size(); _buffer = static_cast&&>(buffer); } diff --git a/src/openvic-dataloader/ParseState.hpp b/src/openvic-dataloader/ParseState.hpp index 4288788..fe567ee 100644 --- a/src/openvic-dataloader/ParseState.hpp +++ b/src/openvic-dataloader/ParseState.hpp @@ -86,12 +86,10 @@ namespace ovdl { _logger { this->file() }, BasicParseState(encoding) {} - template - FileParseState(lexy::buffer&& buffer, detail::Encoding encoding) + FileParseState(lexy::buffer&& buffer, detail::Encoding encoding) : FileParseState(file_type { std::move(buffer) }, encoding) {} - template - FileParseState(const char* path, lexy::buffer&& buffer, detail::Encoding encoding) + FileParseState(const char* path, lexy::buffer&& buffer, detail::Encoding encoding) : FileParseState(file_type { path, std::move(buffer) }, encoding) {} FileParseState(const FileParseState&) = delete; diff --git a/src/openvic-dataloader/csv/Parser.cpp b/src/openvic-dataloader/csv/Parser.cpp index ef8e1ce..11327ed 100644 --- a/src/openvic-dataloader/csv/Parser.cpp +++ b/src/openvic-dataloader/csv/Parser.cpp @@ -39,12 +39,11 @@ struct Parser::ParseHandler final : detail::BasicFileParseHandler switch (parse_state().encoding()) { using enum detail::Encoding; case Ascii: - return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); case Utf8: case Windows1251: case Windows1252: case Gbk: - return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); + return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); OVDL_DEFAULT_CASE_UNREACHABLE(Unknown); } }(); @@ -211,24 +210,15 @@ const FilePosition Parser::get_error_position(const error::Error* error) const { return {}; } -// TODO: Remove reinterpret_cast -// WARNING: This almost certainly breaks on utf16 and utf32 encodings, luckily we don't parse in that format -// This is purely to silence the node_location errors because char8_t is useless -#define REINTERPRET_IT(IT) reinterpret_cast::encoding::char_type*>((IT)) - - return _parse_handler->parse_state().file().visit_buffer( - [&](auto&& buffer) -> FilePosition { - auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.begin())); - FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; - if (err_location.begin() < err_location.end()) { - auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.end()), loc_begin.anchor()); - result.end_line = loc_end.line_nr(); - result.end_column = loc_end.column_nr(); - } - return result; - }); - -#undef REINTERPRET_IT + lexy::buffer const& buffer = _parse_handler->buffer(); + auto loc_begin = lexy::get_input_location(buffer, err_location.begin()); + FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; + if (err_location.begin() < err_location.end()) { + auto loc_end = lexy::get_input_location(buffer, err_location.end(), loc_begin.anchor()); + result.end_line = loc_end.line_nr(); + result.end_column = loc_end.column_nr(); + } + return result; } void Parser::print_errors_to(std::basic_ostream& stream) const { diff --git a/src/openvic-dataloader/detail/InternalConcepts.hpp b/src/openvic-dataloader/detail/InternalConcepts.hpp index c7ffbb3..9a21846 100644 --- a/src/openvic-dataloader/detail/InternalConcepts.hpp +++ b/src/openvic-dataloader/detail/InternalConcepts.hpp @@ -89,7 +89,7 @@ namespace ovdl::detail { T t, const T ct, typename T::ast_type::file_type&& file, - lexy::buffer&& buffer, + lexy::buffer&& buffer, ovdl::detail::Encoding encoding, const char* path // ) { @@ -109,7 +109,7 @@ namespace ovdl::detail { T t, const T ct, typename T::file_type&& file, - lexy::buffer&& buffer, + lexy::buffer&& buffer, ovdl::detail::Encoding encoding, const char* path // ) { diff --git a/src/openvic-dataloader/detail/ParseHandler.hpp b/src/openvic-dataloader/detail/ParseHandler.hpp index c8fa412..8bdb6c0 100644 --- a/src/openvic-dataloader/detail/ParseHandler.hpp +++ b/src/openvic-dataloader/detail/ParseHandler.hpp @@ -83,17 +83,17 @@ namespace ovdl::detail { constexpr virtual buffer_error load_buffer_impl(lexy::buffer&& buffer, const char* path = "", std::optional fallback = std::nullopt) = 0; virtual const char* path_impl() const = 0; - template + template static constexpr auto generate_state = [](State* state, const char* path, auto&& buffer, Encoding encoding) { if (path[0] != '\0') { *state = { path, - lexy::buffer(std::move(buffer)), + lexy::buffer(std::move(buffer)), encoding }; return; } - *state = { lexy::buffer(std::move(buffer)), encoding }; + *state = { lexy::buffer(std::move(buffer)), encoding }; }; template @@ -141,12 +141,9 @@ namespace ovdl::detail { auto [encoding, is_alone] = encoding_detect::Detector { .default_fallback = fallback.value() }.detect_assess(buffer); switch (encoding) { using enum Encoding; - case Ascii: { - generate_state(state, path, std::move(buffer), encoding); - break; - } + case Ascii: case Utf8: { - generate_state(state, path, std::move(buffer), encoding); + generate_state(state, path, std::move(buffer), encoding); break; } case Unknown: { @@ -211,9 +208,12 @@ namespace ovdl::detail { return _parse_state; } - template - constexpr const auto& buffer() const { - return _parse_state.file().template get_buffer_as(); + constexpr lexy::buffer& buffer() { + return _parse_state.file().buffer(); + } + + constexpr lexy::buffer const& buffer() const { + return _parse_state.file().buffer(); } protected: @@ -248,9 +248,12 @@ namespace ovdl::detail { return _parse_state; } - template - constexpr const auto& buffer() const { - return _parse_state.ast().file().template get_buffer_as(); + constexpr lexy::buffer& buffer() { + return _parse_state.ast().file().buffer(); + } + + constexpr lexy::buffer const& buffer() const { + return _parse_state.ast().file().buffer(); } protected: diff --git a/src/openvic-dataloader/v2script/Parser.cpp b/src/openvic-dataloader/v2script/Parser.cpp index 6b569d3..27784a1 100644 --- a/src/openvic-dataloader/v2script/Parser.cpp +++ b/src/openvic-dataloader/v2script/Parser.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include #include @@ -62,12 +61,11 @@ struct Parser::ParseHandler final : detail::BasicStateParseHandler(buffer(), parse_state(), parse_state().logger().error_callback()); case Utf8: case Windows1251: case Windows1252: case Gbk: - return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); + return lexy::parse(buffer(), parse_state(), parse_state().logger().error_callback()); OVDL_DEFAULT_CASE_UNREACHABLE(Unknown); } }(); @@ -290,11 +288,6 @@ std::string Parser::make_list_string() const { return _parse_handler->parse_state().ast().make_list_visualizer(); } -// TODO: Remove reinterpret_cast -// WARNING: This almost certainly breaks on utf16 and utf32 encodings, luckily we don't parse in that format -// This is purely to silence the node_location errors because char8_t is useless -#define REINTERPRET_IT(IT) reinterpret_cast::encoding::char_type*>((IT)) - const FilePosition Parser::get_position(const ast::Node* node) const { if (!node || !node->is_linked_in_tree()) { return {}; @@ -308,17 +301,15 @@ const FilePosition Parser::get_position(const ast::Node* node) const { return FilePosition {}; } - return _parse_handler->parse_state().ast().file().visit_buffer( - [&](auto&& buffer) -> FilePosition { - auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(node_location.begin())); - FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; - if (node_location.begin() < node_location.end()) { - auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(node_location.end()), loc_begin.anchor()); - result.end_line = loc_end.line_nr(); - result.end_column = loc_end.column_nr(); - } - return result; - }); + lexy::buffer const& buffer = _parse_handler->buffer(); + auto loc_begin = lexy::get_input_location(buffer, node_location.begin()); + FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; + if (node_location.begin() < node_location.end()) { + auto loc_end = lexy::get_input_location(buffer, node_location.end(), loc_begin.anchor()); + result.end_line = loc_end.line_nr(); + result.end_column = loc_end.column_nr(); + } + return result; } Parser::error_range Parser::get_errors() const { @@ -339,21 +330,17 @@ const FilePosition Parser::get_error_position(const error::Error* error) const { return FilePosition {}; } - return _parse_handler->parse_state().ast().file().visit_buffer( - [&](auto&& buffer) -> FilePosition { - auto loc_begin = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.begin())); - FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; - if (err_location.begin() < err_location.end()) { - auto loc_end = lexy::get_input_location(buffer, REINTERPRET_IT(err_location.end()), loc_begin.anchor()); - result.end_line = loc_end.line_nr(); - result.end_column = loc_end.column_nr(); - } - return result; - }); + lexy::buffer const& buffer = _parse_handler->buffer(); + auto loc_begin = lexy::get_input_location(buffer, err_location.begin()); + FilePosition result { loc_begin.line_nr(), loc_begin.line_nr(), loc_begin.column_nr(), loc_begin.column_nr() }; + if (err_location.begin() < err_location.end()) { + auto loc_end = lexy::get_input_location(buffer, err_location.end(), loc_begin.anchor()); + result.end_line = loc_end.line_nr(); + result.end_column = loc_end.column_nr(); + } + return result; } -#undef REINTERPRET_IT - void Parser::print_errors_to(std::basic_ostream& stream) const { auto errors = get_errors(); if (errors.empty()) {