Merge pull request #818 from davidgiven/convert

Add a fe-convert (plus all the necessary backend work).

Author: davidgiven

.github/workflows/ccpp.yml

@@ -20,7 +20,7 @@ jobs:
         path: 'fluxengine-testdata'
     - name: apt
       run: |
-        sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.0-gtk3-dev libfmt-dev libprotobuf-dev wx-common
+        sudo apt install libudev-dev libsqlite3-dev protobuf-compiler libwxgtk3.0-gtk3-dev libfmt-dev libprotobuf-dev
     - name: make
       run: CXXFLAGS="-Wp,-D_GLIBCXX_ASSERTIONS" make -j`nproc` -C fluxengine
 

Makefile

@@ -25,8 +25,10 @@ ifeq ($(BUILDTYPE),windows)
 	EXT = .exe
 else
 	CC = gcc
-	CXX = g++ -std=c++17
-	CFLAGS = -g -O3
+	CXX = g++ -std=c++20
+	CFLAGS = -g -O3 \
+		-Wno-deprecated-enum-float-conversion \
+		-Wno-deprecated-enum-enum-conversion
 	LDFLAGS =
 	AR = ar
 	PKG_CONFIG = pkg-config
@@ -37,7 +39,7 @@ else
 endif
 
 HOSTCC = gcc
-HOSTCXX = g++ -std=c++17
+HOSTCXX = g++ -std=c++20
 HOSTCFLAGS = -g -O3
 HOSTLDFLAGS =
 

README.md

@@ -137,6 +137,7 @@ choices because they can store multiple types of file system.
 | [`rx50`](doc/disk-rx50.md) | Digital RX50: 400kB 5.25" 80-track 10-sector SSDD | 🦖 | 🦖 |  |
 | [`smaky6`](doc/disk-smaky6.md) | Smaky 6: 308kB 5.25" 77-track 16-sector SSDD, hard sectored | 🦖 |  | SMAKY6  |
 | [`tartu`](doc/disk-tartu.md) | Tartu: The Palivere and variations | 🦄 | 🦖 | CPMFS  |
+| [`ti99`](doc/disk-ti99.md) | TI-99: 90kB 35-track SSSD | 🦖 |  |  |
 | [`tids990`](doc/disk-tids990.md) | Texas Instruments DS990: 1126kB 8" DSSD | 🦖 | 🦖 |  |
 | [`tiki`](doc/disk-tiki.md) | Tiki 100: CP/M |  |  | CPMFS  |
 | [`victor9k`](doc/disk-victor9k.md) | Victor 9000 / Sirius One: 1224kB 5.25" DSDD GCR | 🦖 | 🦖 |  |
@@ -258,6 +259,10 @@ package, written by Robert Leslie et al, taken from
 https://www.mars.org/home/rob/proj/hfs. It is GPL 2.0 licensed. Please see the
 contents of the directory for the full text.
 
+As an exception, `dep/lexy` contains a partial copy of the lexy package, written
+by foonathen@github, taken from https://github.com/foonathan/lexy. It is BSL 1.0
+licensed. Please see the contents of the directory for the full text.
+
 __Important:__ Because of all these exceptions, if you distribute the
 FluxEngine package as a whole, you must comply with the terms of _all_ of the
 licensing terms. This means that __effectively the FluxEngine package is

dep/lexy/LICENSE

@@ -0,0 +1,23 @@
+Boost Software License - Version 1.0 - August 17th, 2003
+
+Permission is hereby granted, free of charge, to any person or organization
+obtaining a copy of the software and accompanying documentation covered by
+this license (the "Software") to use, reproduce, display, distribute,
+execute, and transmit the Software, and to prepare derivative works of the
+Software, and to permit third-parties to whom the Software is furnished to
+do so, all subject to the following:
+
+The copyright notices in the Software and this entire statement, including
+the above license grant, this restriction and the following disclaimer,
+must be included in all copies of the Software, in whole or in part, and
+all derivative works of the Software, unless such copies or derivative
+works are solely in the form of machine-executable object code generated by
+a source language processor.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.

dep/lexy/README.adoc

@@ -0,0 +1,175 @@
+= lexy
+
+ifdef::env-github[]
+image:https://img.shields.io/endpoint?url=https%3A%2F%2Fwww.jonathanmueller.dev%2Fproject%2Flexy%2Findex.json[Project Status,link=https://www.jonathanmueller.dev/project/]
+image:https://github.com/foonathan/lexy/workflows/Main%20CI/badge.svg[Build Status]
+image:https://img.shields.io/badge/try_it_online-blue[Playground,link=https://lexy.foonathan.net/playground]
+endif::[]
+
+lexy is a parser combinator library for {cpp}17 and onwards.
+It allows you to write a parser by specifying it in a convenient {cpp} DSL,
+which gives you all the flexibility and control of a handwritten parser without any of the manual work.
+
+ifdef::env-github[]
+*Documentation*: https://lexy.foonathan.net/[lexy.foonathan.net]
+endif::[]
+
+.IPv4 address parser
+--
+ifndef::env-github[]
+[.godbolt-example]
+.+++<a href="https://godbolt.org/z/scvajjE17", title="Try it online">{{< svg "icons/play.svg" >}}</a>+++
+endif::[]
+[source,cpp]
+----
+namespace dsl = lexy::dsl;
+
+// Parse an IPv4 address into a `std::uint32_t`.
+struct ipv4_address
+{
+    // What is being matched.
+    static constexpr auto rule = []{
+        // Match a sequence of (decimal) digits and convert it into a std::uint8_t.
+        auto octet = dsl::integer<std::uint8_t>;
+
+        // Match four of them separated by periods.
+        return dsl::times<4>(octet, dsl::sep(dsl::period)) + dsl::eof;
+    }();
+
+    // How the matched output is being stored.
+    static constexpr auto value
+        = lexy::callback<std::uint32_t>([](std::uint8_t a, std::uint8_t b, std::uint8_t c, std::uint8_t d) {
+            return (a << 24) | (b << 16) | (c << 8) | d;
+        });
+};
+----
+--
+
+== Features
+
+Full control::
+  * *Describe the parser, not some abstract grammar*:
+    Unlike parser generators that use some table driven magic for parsing, lexy's grammar is just syntax sugar for a hand-written recursive descent parser.
+    The parsing algorithm does exactly what you've instructed it to do -- no more ambiguities or weird shift/reduce errors!
+  * *No implicit backtracking or lookahead*:
+    It will only backtrack when you say it should, and only lookahead when and how far you want it.
+    Don't worry about rules that have side-effects, they won't be executed unnecessarily thanks to the user-specified lookahead conditions.
+    https://lexy.foonathan.net/playground?example=peek[Try it online].
+  * *Escape hatch for manual parsing*:
+    Sometimes you want to parse something that can't be expressed easily with lexy's facilities.
+    Don't worry, you can integrate a hand-written parser into the grammar at any point.
+    https://lexy.foonathan.net/playground/?example=scan[Try it online].
+  * *Tracing*:
+    Figure out why the grammar isn't working the way you want it to.
+    https://lexy.foonathan.net/playground/?example=trace&mode=trace[Try it online].
+
+Easily integrated::
+  * *A pure {cpp} DSL*:
+    No need to use an external grammar file; embed the grammar directly in your {cpp} project using operator overloading and functions.
+  * *Bring your own data structures*:
+    You can directly store results into your own types and have full control over all heap allocations.
+  * *Fully `constexpr` parsing*:
+    You want to parse a string literal at compile-time? You can do so.
+  * *Minimal standard library dependencies*:
+    The core parsing library only depends on fundamental headers such as `<type_traits>` or `<cstddef>`; no big includes like `<vector>` or `<algorithm>`.
+  * *Header-only core library* (by necessity, not by choice -- it's `constexpr` after all).
+
+ifdef::env-github[Designed for text::]
+ifndef::env-github[Designed for text (e.g. {{< github-example json >}}, {{< github-example xml >}}, {{< github-example email >}}) ::]
+  * *Unicode support*: parse UTF-8, UTF-16, or UTF-32, and access the Unicode character database to query char classes or perform case folding.
+    https://lexy.foonathan.net/playground?example=identifier-unicode[Try it online].
+  * *Convenience*:
+    Built-in rules for parsing nested structures, quotes and escape sequences.
+    https://lexy.foonathan.net/playground?example=parenthesized[Try it online].
+  * *Automatic whitespace skipping*:
+    No need to manually handle whitespace or comments.
+    https://lexy.foonathan.net/playground/?example=whitespace_comment[Try it online].
+
+ifdef::env-github[Designed for programming languages::]
+ifndef::env-github[Designed for programming languages (e.g. {{< github-example calculator >}}, {{< github-example shell >}})::]
+  * *Keyword and identifier parsing*:
+    Reserve a set of keywords that won't be matched as regular identifiers.
+    https://lexy.foonathan.net/playground/?example=reserved_identifier[Try it online].
+  * *Operator parsing*:
+    Parse unary/binary operators with different precedences and associativity, including chained comparisons `a < b < c`.
+    https://lexy.foonathan.net/playground/?example=expr[Try it online].
+  * *Automatic error recovery*:
+    Log an error, recover, and continue parsing!
+    https://lexy.foonathan.net/playground/?example=recover[Try it online].
+
+ifdef::env-github[Designed for binary input::]
+ifndef::env-github[Designed for binary input (e.g. {{< github-example protobuf >}})::]
+  * *Bytes*: Rules for parsing `N` bytes or Nbit big/little endian integer.
+  * *Bits*: Rules for parsing individual bit patterns.
+  * *Blobs*: Rules for parsing TLV formats.
+
+== FAQ
+
+Why should I use lexy over XYZ?::
+  lexy is closest to other PEG parsers.
+  However, they usually do more implicit backtracking, which can hurt performance and you need to be very careful with rules that have side-effects.
+  This is not the case for lexy, where backtracking is controlled using branch conditions.
+  lexy also gives you a lot of control over error reporting, supports error recovery, special support for operator precedence parsing, and other advanced features.
+
+  http://boost-spirit.com/home/[Boost.Spirit]:::
+    The main difference: it is not a Boost library.
+    In addition, Boost.Spirit is quite old and doesn't support e.g. non-common ranges as input. 
+    Boost.Spirit also eagerly creates attributes from the rules, which can lead to nested tuples/variants while lexy uses callbacks which enables zero-copy parsing directly into your own data structure.
+    However, lexy's grammar is more verbose and designed to parser bigger grammars instead of the small one-off rules that Boost.Spirit is good at.
+  https://github.com/taocpp/PEGTL[PEGTL]:::
+    PEGTL is very similar and was a big inspiration.
+    The biggest difference is that lexy uses an operator based DSL instead of inheriting from templated classes as PEGTL does;
+    depending on your preference this can be an advantage or disadvantage.
+  Hand-written Parsers:::
+    Writing a handwritten parser is more manual work and error prone.
+    lexy automates that away without having to sacrifice control.
+    You can use it to quickly prototype a parser and then slowly replace more and more with a handwritten parser over time;
+    mixing a hand-written parser and a lexy grammar works seamlessly.
+
+How bad are the compilation times?::
+They're not as bad as you might expect (in debug mode, that is).
++
+The example JSON parser compiles in about 2s on my machine.
+If we remove all the lexy specific parts and just benchmark the time it takes for the compiler to process the datastructure (and stdlib includes),
+that takes about 700ms.
+If we validate JSON only instead of parsing it, so remove the data structures and keep only the lexy specific parts, we're looking at about 840ms.
++
+Keep in mind, that you can fully isolate lexy in a single translation unit that only needs to be touched when you change the parser.
+You can also split a lexy grammar into multiple translation units using the `dsl::subgrammar` rule.
+
+How bad are the {cpp} error messages if you mess something up?::
+  They're certainly worse than the error message lexy gives you.
+  The big problem here is that the first line gives you the error, followed by dozens of template instantiations, which end at your `lexy::parse` call.
+  Besides providing an external tool to filter those error messages, there is nothing I can do about that.
+
+How fast is it?::
+  Benchmarks are available in the `benchmarks/` directory.
+  A sample result of the JSON validator benchmark which compares the example JSON parser with various other implementations is available https://lexy.foonathan.net/benchmark_json/[here].
+
+Why is it called lexy?::
+  I previously had a tokenizer library called foonathan/lex.
+  I've tried adding a parser to it, but found that the line between pure tokenization and parsing has become increasingly blurred.
+  lexy is a re-imagination on of the parser I've added to foonathan/lex, and I've simply kept a similar name.
+
+ifdef::env-github[]
+== Documentation
+
+The documentation, including tutorials, reference documentation, and an interactive playground can be found at https://lexy.foonathan.net/[lexy.foonathan.net].
+
+A minimal `CMakeLists.txt` that uses lexy can look like this:
+
+.`CMakeLists.txt`
+```cmake
+project(lexy-example)
+
+include(FetchContent)
+FetchContent_Declare(lexy URL https://lexy.foonathan.net/download/lexy-src.zip)
+FetchContent_MakeAvailable(lexy)
+
+add_executable(lexy_example)
+target_sources(lexy_example PRIVATE main.cpp)
+target_link_libraries(lexy_example PRIVATE foonathan::lexy)
+```
+
+endif::[]
+

dep/lexy/UPSTREAM.md

@@ -0,0 +1,2 @@
+This is a heavily truncated copy of https://github.com/foonathan/lexy, commit
+20926cf.

dep/lexy/build.py

@@ -0,0 +1,11 @@
+from build.c import cxxlibrary
+from glob import glob
+
+cxxlibrary(
+    name="lexy",
+    srcs=[],
+    hdrs={
+        h: f"./include/{h}"
+        for h in glob("**/*.hpp", root_dir="dep/lexy/include", recursive=True)
+    },
+)

dep/lexy/include/lexy/_detail/any_ref.hpp

@@ -0,0 +1,68 @@
+// Copyright (C) 2020-2025 Jonathan Müller and lexy contributors
+// SPDX-License-Identifier: BSL-1.0
+
+#ifndef LEXY_DETAIL_ANY_REF_HPP_INCLUDED
+#define LEXY_DETAIL_ANY_REF_HPP_INCLUDED
+
+#include <lexy/_detail/config.hpp>
+
+// Essentially a void*, but we can cast it in a constexpr context.
+// The cost is an extra layer of indirection.
+
+namespace lexy::_detail
+{
+template <typename T>
+class any_holder;
+
+// Store a pointer to this instead of a void*.
+class any_base
+{
+public:
+    any_base(const any_base&)            = delete;
+    any_base& operator=(const any_base&) = delete;
+
+    template <typename T>
+    constexpr T& get() noexcept
+    {
+        return static_cast<any_holder<T>*>(this)->get();
+    }
+    template <typename T>
+    constexpr const T& get() const noexcept
+    {
+        return static_cast<const any_holder<T>*>(this)->get();
+    }
+
+private:
+    constexpr any_base() = default;
+    ~any_base()          = default;
+
+    template <typename T>
+    friend class any_holder;
+};
+
+using any_ref  = any_base*;
+using any_cref = const any_base*;
+
+// Need to store the object in here.
+template <typename T>
+class any_holder : public any_base
+{
+public:
+    constexpr explicit any_holder(T&& obj) : _obj(LEXY_MOV(obj)) {}
+
+    constexpr T& get() noexcept
+    {
+        return _obj;
+    }
+    constexpr const T& get() const noexcept
+    {
+        return _obj;
+    }
+
+private:
+    T _obj;
+};
+} // namespace lexy::_detail
+
+#endif // LEXY_DETAIL_ANY_REF_HPP_INCLUDED
+

dep/lexy/include/lexy/_detail/assert.hpp

@@ -0,0 +1,51 @@
+// Copyright (C) 2020-2025 Jonathan Müller and lexy contributors
+// SPDX-License-Identifier: BSL-1.0
+
+#ifndef LEXY_DETAIL_ASSERT_HPP_INCLUDED
+#define LEXY_DETAIL_ASSERT_HPP_INCLUDED
+
+#include <lexy/_detail/config.hpp>
+
+#ifndef LEXY_ENABLE_ASSERT
+
+// By default, enable assertions if NDEBUG is not defined.
+
+#    if NDEBUG
+#        define LEXY_ENABLE_ASSERT 0
+#    else
+#        define LEXY_ENABLE_ASSERT 1
+#    endif
+
+#endif
+
+#if LEXY_ENABLE_ASSERT
+
+// We want assertions: use assert() if that's available, otherwise abort.
+// We don't use assert() directly as that's not constexpr.
+
+#    if NDEBUG
+
+#        include <cstdlib>
+#        define LEXY_PRECONDITION(Expr) ((Expr) ? void(0) : std::abort())
+#        define LEXY_ASSERT(Expr, Msg) ((Expr) ? void(0) : std::abort())
+
+#    else
+
+#        include <cassert>
+
+#        define LEXY_PRECONDITION(Expr) ((Expr) ? void(0) : assert(Expr))
+#        define LEXY_ASSERT(Expr, Msg) ((Expr) ? void(0) : assert((Expr) && (Msg)))
+
+#    endif
+
+#else
+
+// We don't want assertions.
+
+#    define LEXY_PRECONDITION(Expr) static_cast<void>(sizeof(Expr))
+#    define LEXY_ASSERT(Expr, Msg) static_cast<void>(sizeof(Expr))
+
+#endif
+
+#endif // LEXY_DETAIL_ASSERT_HPP_INCLUDED
+