Handle invalid glibc FILE buffer

Author: vitaut

include/fmt/format-inl.h

@@ -1549,7 +1549,8 @@ template <typename F> class glibc_file : public file_base<F> {
   bool needs_flush() const {
     if ((this->file_->_flags & line_buffered) == 0) return false;
     char* end = this->file_->_IO_write_end;
-    return memchr(end, '\n', to_unsigned(this->file_->_IO_write_ptr - end));
+    auto size = max_of<ptrdiff_t>(this->file_->_IO_write_ptr - end, 0);
+    return memchr(end, '\n', static_cast<size_t>(size));
   }
 
   void flush() { fflush_unlocked(this->file_); }

test/format-test.cc

@@ -2549,6 +2549,20 @@ TEST(format_test, writer) {
   EXPECT_EQ(s.str(), "foo");
 }
 
+#if FMT_USE_FCNTL && !defined(_WIN32)
+TEST(format_test, invalid_glibc_buffer) {
+  auto pipe = fmt::pipe();
+  auto write_end = pipe.write_end.fdopen("w");
+  auto file = write_end.get();
+
+  // This results in _IO_write_ptr < _IO_write_end.
+  fprintf(file, "111\n");
+  setvbuf(file, nullptr, _IOLBF, 0);
+
+  fmt::print(file, "------\n");
+}
+#endif  // FMT_USE_FCNTL
+
 #if FMT_USE_BITINT
 FMT_PRAGMA_CLANG(diagnostic ignored "-Wbit-int-extension")