Introduce input::SourceListing (#313)

* Introduce `input::SourceListing`

To hold a source file along with its logical lines.

* Include `vector`

* Include `utility`

to access `to_underlying`

Author: GabrielDosReis

include/ipr/input

@@ -49,8 +49,36 @@ namespace ipr::input {
     struct PhysicalLine {
         Morsel morsel { };
         std::uint32_t number { };
+
+        bool empty() const { return morsel.length == 0; }
+    };
+
+    // A logical line is either a simple phyiscal line or a composite of multiple
+    // physical lines spliced together.
+    enum class LineSort : std::uint8_t {
+        Simple = 0x01,          // A simple, non-continuating physical line.
+        Composite = 0x02,       // Result of spliced multiple physical lines.
+    };
+
+    struct SimpleLine {
+        PhysicalLine line;
+    };
+
+    struct CompositeLine {
+        std::vector<PhysicalLine> lines;
+    };
+
+    struct LineIndex {
+        LineIndex(LineSort, std::uint32_t);
+        LineSort sort() const { return static_cast<LineSort>(srt); }
+        std::uint32_t index() const { return idx; }
+    private:
+        std::uint32_t srt : 2;
+        std::uint32_t idx : 30;
     };
 
+    static_assert(sizeof(LineIndex) == 4);
+
     // Input source file mapped to memory as sequence of raw bytes.
     // UTF-8 is assumed as the encoding of the text.
     struct SourceFile {
@@ -112,4 +140,21 @@ namespace ipr::input {
     {
         return iterator{nullptr};
     }
+
+    // A depot of lines read from an input source file.
+    struct LineDepot {
+        std::vector<SimpleLine> simples;
+        std::vector<CompositeLine> composites;
+        std::vector<LineIndex> indices;
+    };
+
+    // An input source listing is a source file with its lines read into logical lines.
+    struct SourceListing : SourceFile {
+        explicit SourceListing(const SystemPath&);
+        const SimpleLine& simple_line(LineIndex) const;
+        const CompositeLine& composite_line(LineIndex) const;
+        const std::vector<LineIndex>& logical_lines() const { return depot.indices; }
+    private:
+        LineDepot depot;
+    };
 }

src/input.cxx

@@ -15,9 +15,21 @@
 
 #include <assert.h>
 #include <iostream>
+#include <utility>
 #include "ipr/input"
 
 namespace ipr::input {
+    static constexpr std::uint32_t index_watermark { 1u << 31 }; 
+
+
+    LineIndex::LineIndex(LineSort s, std::uint32_t i)
+        : srt{(assert(s == LineSort::Simple || s == LineSort::Composite), std::to_underlying(s))}, 
+          idx{(assert(i < index_watermark), i)}
+    {
+    }
+
+
+
 #ifdef _WIN32
     // Helper type for automatically closing a handle on scope exit.
     struct SystemHandle {
@@ -109,6 +121,17 @@ namespace ipr::input {
     constexpr char8_t carriage_return = 0x0D;    // '\r';
     constexpr char8_t line_feed = 0x0A;          // '\n';
 
+    static inline bool white_space(char8_t c)
+    {
+        switch (c)
+        {
+        case u' ': case u8'\t': case u8'\v': case u8'\f':
+            return true;
+        default:
+            return false;
+        }
+    }
+
     void SourceFile::LineRange::next_line() noexcept
     {
         const auto offset = static_cast<std::uint64_t>(ptr - src->view.data());
@@ -156,4 +179,68 @@ namespace ipr::input {
 
         return *this;
     }
+
+    namespace {
+        LineDepot read_lines(const SourceFile& src)
+        {
+            LineDepot depot { };
+            const auto file_start = src.contents().data();
+
+            CompositeLine composite { };
+            for (auto line: src.lines())
+            {
+                if (line.empty())
+                    continue;
+                // Trim any trailing whitespace character when determining logical line continuation.
+                const auto line_start = file_start + line.morsel.offset;
+                auto cursor = line_start + line.morsel.length;
+                while (--cursor > line_start and white_space(*cursor))
+                    ;
+                if (cursor <= line_start)
+                    continue;               // skip entirely blank lines.
+                if (*cursor == u8'\\')
+                {
+                    line.morsel.length = cursor - line_start;
+                    composite.lines.push_back(line);
+                    continue;
+                }
+                else if (not composite.lines.empty())
+                {
+                    composite.lines.push_back(line);
+                    auto idx = depot.composites.size();
+                    depot.composites.push_back(composite);
+                    depot.indices.emplace_back(LineSort::Composite, idx);
+                    composite.lines.clear();
+                }
+                else
+                {
+                    auto idx = depot.simples.size();
+                    depot.indices.emplace_back(LineSort::Simple, idx);
+                    depot.simples.emplace_back(line);
+                }
+            }
+
+            return depot;
+        }
+    }
+
+    SourceListing::SourceListing(const SystemPath& path) 
+        : SourceFile{path}, depot{read_lines(*this)}
+    { }
+
+    const SimpleLine& SourceListing::simple_line(LineIndex line) const
+    {
+        assert(idx.sort() == LineSort::Simple);
+        auto n = line.index();
+        assert(n < depot.simples.size());
+        return depot.simples[n];
+    }
+
+    const CompositeLine& SourceListing::composite_line(LineIndex line) const
+    {
+        assert(idx.sort() == LineSort::Composite);
+        auto n = line.index();
+        assert(n < depot.composites.size());
+        return depot.composites[n];
+    }
 }

tests/unit-tests/lines.cxx

@@ -7,12 +7,15 @@
 #  define WIDEN(S) S
 #endif
 
+#define  DUP(S) \
+    S ## S
+
 #include <iostream>
 #include "ipr/input"
 
 TEST_CASE("echo input file") {
     ipr::input::SystemPath path = WIDEN(__FILE__);
-    ipr::input::SourceFile file{path};
+    ipr::input::SourceListing file{path};
     std::cout << "file.size: " << file.contents().size() << std::endl;
     std::uint32_t last_line_number = 0;
     for (auto line : file.lines())
@@ -22,5 +25,5 @@ TEST_CASE("echo input file") {
                 << ", length: " << line.morsel.length << "}\n";
         last_line_number = line.number;
     }
-    CHECK(last_line_number == 26); // Adjust this number based on the actual number of lines in the file
+    CHECK(last_line_number == 29); // Adjust this number based on the actual number of lines in the file
 }