io/BufferedReader: new class, replacing fopen()/fgets()

Author: MaxKellermann

src/IniFile.cxx

@@ -4,15 +4,15 @@
 #include "IniFile.hxx"
 #include "lib/fmt/RuntimeError.hxx"
 #include "lib/fmt/SystemError.hxx"
+#include "io/BufferedReader.hxx"
+#include "io/FileReader.hxx"
 #include "util/CharUtil.hxx"
 #include "util/ScopeExit.hxx"
 #include "util/StringSplit.hxx"
 #include "util/StringStrip.hxx"
 
 #include <string_view>
 
-#include <stdio.h>
-
 static constexpr bool
 IsValidSectionNameChar(char ch) noexcept
 {
@@ -124,13 +124,12 @@ IniParser::ParseLine(std::string_view line)
 }
 
 static IniFile
-ReadIniFile(const char *path, FILE *file)
+ReadIniFile(const char *path, BufferedReader &reader)
 {
 	IniParser parser;
 
 	unsigned no = 1;
-	char buffer[4096];
-	while (auto line = fgets(buffer, sizeof(buffer), file)) {
+	while (const char *line = reader.ReadLine()) {
 		try {
 			parser.ParseLine(line);
 		} catch (...) {
@@ -147,11 +146,8 @@ ReadIniFile(const char *path, FILE *file)
 IniFile
 ReadIniFile(const char *path)
 {
-	FILE *file = fopen(path, "r");
-	if (file == nullptr)
-		throw FmtErrno("Failed to open {:?}", path);
-
-	AtScopeExit(file) { fclose(file); };
+	FileReader file{path};
+	BufferedReader reader{file};
 
-	return ReadIniFile(path, file);
+	return ReadIniFile(path, reader);
 }

src/ReadConfig.cxx

@@ -3,6 +3,8 @@
 
 #include "ReadConfig.hxx"
 #include "lib/fmt/RuntimeError.hxx"
+#include "io/BufferedReader.hxx"
+#include "io/FileReader.hxx"
 #include "util/Compiler.h"
 #include "util/ScopeExit.hxx"
 #include "util/StringStrip.hxx"
@@ -318,25 +320,14 @@ parse_ignore_list_line(std::string_view input)
 static IgnoreList*
 load_ignore_list(const std::string& path, Config::IgnoreListMap& ignore_lists)
 {
-
-	FILE *file = fopen(path.c_str(), "r");
-	if (file == nullptr) {
-		throw FmtRuntimeError("Cannot load ignore file: cannot open {:?} for reading", path);
-	}
-
-	AtScopeExit(file) { fclose(file); };
+	FileReader file{path.c_str()};
+	BufferedReader reader{file};
 
 	IgnoreList ignore_list;
 
-	char line_buf[4096];
-	size_t line_num = 0;
-	while (fgets(line_buf, sizeof(line_buf), file)) {
-		std::string_view line(line_buf);
-		if (line.back() == '\n') {
-			line.remove_suffix(1);
-		}
+	while (const char *_line = reader.ReadLine()) {
+		std::string_view line{_line};
 
-		line_num++;
 		if (line.empty()) {
 			continue;
 		}
@@ -363,7 +354,7 @@ load_ignore_list(const std::string& path, Config::IgnoreListMap& ignore_lists)
 			ignore_list.entries.emplace_back(std::move(entry));
 		} catch (const std::runtime_error& error) {
 			throw FmtRuntimeError("Error loading ignore list {:?}: Error parsing line {}: {}",
-					      path, line_num, error.what());
+					      path, reader.GetLineNumber(), error.what());
 		}
 	}
 

src/io/BufferedReader.cxx

@@ -0,0 +1,104 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// author: Max Kellermann <[email protected]>
+
+#include "BufferedReader.hxx"
+#include "Reader.hxx"
+#include "util/TextFile.hxx"
+
+#include <algorithm> // for std::copy_n()
+#include <cassert>
+#include <cstdint>
+#include <stdexcept>
+
+bool
+BufferedReader::Fill(bool need_more)
+{
+	if (eof)
+		return !need_more;
+
+	auto w = buffer.Write();
+	if (w.empty()) {
+		if (buffer.GetCapacity() >= MAX_SIZE)
+			return !need_more;
+
+		buffer.Grow(buffer.GetCapacity() * 2);
+		w = buffer.Write();
+		assert(!w.empty());
+	}
+
+	std::size_t nbytes = reader.Read(w);
+	if (nbytes == 0) {
+		eof = true;
+		return !need_more;
+	}
+
+	buffer.Append(nbytes);
+	return true;
+}
+
+void *
+BufferedReader::ReadFull(std::size_t size)
+{
+	while (true) {
+		auto r = Read();
+		if (r.size() >= size)
+			return r.data();
+
+		if (!Fill(true))
+			throw std::runtime_error("Premature end of file");
+	}
+}
+
+std::size_t
+BufferedReader::ReadFromBuffer(std::span<std::byte> dest) noexcept
+{
+	const auto src = Read();
+	std::size_t nbytes = std::min(src.size(), dest.size());
+	std::copy_n(src.data(), nbytes, dest.data());
+	Consume(nbytes);
+	return nbytes;
+}
+
+void
+BufferedReader::ReadFull(std::span<std::byte> dest)
+{
+	while (true) {
+		std::size_t nbytes = ReadFromBuffer(dest);
+		dest = dest.subspan(nbytes);
+		if (dest.empty())
+			break;
+
+		if (!Fill(true))
+			throw std::runtime_error("Premature end of file");
+	}
+}
+
+char *
+BufferedReader::ReadLine()
+{
+	do {
+		char *line = ReadBufferedLine(buffer);
+		if (line != nullptr) {
+			++line_number;
+			return line;
+		}
+	} while (Fill(true));
+
+	if (!eof || buffer.empty())
+		return nullptr;
+
+	auto w = buffer.Write();
+	if (w.empty()) {
+		buffer.Grow(buffer.GetCapacity() + 1);
+		w = buffer.Write();
+		assert(!w.empty());
+	}
+
+	/* terminate the last line */
+	w[0] = {};
+
+	char *line = reinterpret_cast<char *>(buffer.Read().data());
+	buffer.Clear();
+	++line_number;
+	return line;
+}

src/io/BufferedReader.hxx

@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// author: Max Kellermann <[email protected]>
+
+#pragma once
+
+#include "util/DynamicFifoBuffer.hxx"
+
+#include <cstddef>
+#include <span>
+
+class Reader;
+
+class BufferedReader {
+	static constexpr std::size_t MAX_SIZE = 512 * 1024;
+
+	Reader &reader;
+
+	DynamicFifoBuffer<std::byte> buffer;
+
+	bool eof = false;
+
+	unsigned line_number = 0;
+
+public:
+	explicit BufferedReader(Reader &_reader) noexcept
+		:reader(_reader), buffer(16384) {}
+
+	/**
+	 * Reset the internal state.  Should be called after rewinding
+	 * the underlying #Reader.
+	 */
+	void Reset() noexcept {
+		buffer.Clear();
+		eof = false;
+		line_number = 0;
+	}
+
+	bool Fill(bool need_more);
+
+	[[gnu::pure]]
+	std::span<std::byte> Read() const noexcept {
+		return std::as_writable_bytes(buffer.Read());
+	}
+
+	/**
+	 * Read a buffer of exactly the given size (without consuming
+	 * it).  Throws std::runtime_error if not enough data is
+	 * available.
+	 */
+	void *ReadFull(std::size_t size);
+
+	void Consume(std::size_t n) noexcept {
+		buffer.Consume(n);
+	}
+
+	/**
+	 * Read (and consume) data from the input buffer into the
+	 * given buffer.  Does not attempt to refill the buffer.
+	 */
+	std::size_t ReadFromBuffer(std::span<std::byte> dest) noexcept;
+
+	/**
+	 * Read data into the given buffer and consume it from our
+	 * buffer.  Throw an exception if the request cannot be
+	 * forfilled.
+	 */
+	void ReadFull(std::span<std::byte> dest);
+
+	template<typename T>
+	void ReadFullT(T &dest) {
+		ReadFull(std::as_writable_bytes(std::span{&dest, 1}));
+	}
+
+	template<typename T>
+	T ReadFullT() {
+		T dest;
+		ReadFullT<T>(dest);
+		return dest;
+	}
+
+	char *ReadLine();
+
+	unsigned GetLineNumber() const noexcept {
+		return line_number;
+	}
+};

src/io/FileAt.hxx

@@ -0,0 +1,16 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// Copyright CM4all GmbH
+// author: Max Kellermann <[email protected]>
+
+#pragma once
+
+#include "FileDescriptor.hxx"
+
+/**
+ * Reference to a file by an anchor directory (which can be an
+ * `O_PATH` descriptor) and a path name relative to it.
+ */
+struct FileAt {
+	FileDescriptor directory;
+	const char *name;
+};

src/io/FileReader.cxx

@@ -0,0 +1,52 @@
+// SPDX-License-Identifier: BSD-2-Clause
+// author: Max Kellermann <[email protected]>
+
+#include "FileReader.hxx"
+#include "lib/fmt/SystemError.hxx"
+#include "io/Open.hxx"
+
+#include <cassert>
+
+#ifdef _WIN32
+
+FileReader::FileReader(const char *path)
+	:handle(CreateFile(path, GENERIC_READ, FILE_SHARE_READ,
+			   nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL,
+			   nullptr))
+{
+	if (handle == INVALID_HANDLE_VALUE)
+		throw FmtLastError("Failed to open {}", path);
+}
+
+std::size_t
+FileReader::Read(std::span<std::byte> dest)
+{
+	assert(IsDefined());
+
+	DWORD nbytes;
+	if (!ReadFile(handle, dest.data(), dest.size(), &nbytes, nullptr))
+		throw MakeLastError("Failed to read from file");
+
+	return nbytes;
+}
+
+#else
+
+FileReader::FileReader(const char *path)
+	:fd(OpenReadOnly(path))
+{
+}
+
+std::size_t
+FileReader::Read(std::span<std::byte> dest)
+{
+	assert(IsDefined());
+
+	ssize_t nbytes = fd.Read(dest);
+	if (nbytes < 0)
+		throw MakeErrno("Failed to read from file");
+
+	return nbytes;
+}
+
+#endif