[flang][runtime] Formatted input optimizations

Make some minor tweaks (inlining, caching) to the formatting input
path to improve integer input in a SPEC code.  (None of the I/O
library has been tuned yet for performance, and there are some
easy optimizations for common cases.)  Input integer values are
now calculated with native C/C++ 128-bit integers.

A benchmark that only reads about 5M lines of three integer values each
speeds up from over 8 seconds to under 3 in my environment with these
changeds.

If this works out, the code here can be used to optimize the formatted
input paths for real and character data, too.

Fixes #134026.

Author: klausler

flang-rt/include/flang-rt/runtime/connection.h

@@ -45,19 +45,36 @@ struct ConnectionAttributes {
 };
 
 struct ConnectionState : public ConnectionAttributes {
-  RT_API_ATTRS bool
-  IsAtEOF() const; // true when read has hit EOF or endfile record
-  RT_API_ATTRS bool
-  IsAfterEndfile() const; // true after ENDFILE until repositioned
+  RT_API_ATTRS bool IsAtEOF() const {
+    // true when read has hit EOF or endfile record
+    return endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber;
+  }
+  RT_API_ATTRS bool IsAfterEndfile() const {
+    // true after ENDFILE until repositioned
+    return endfileRecordNumber && currentRecordNumber > *endfileRecordNumber;
+  }
 
   // All positions and measurements are always in units of bytes,
   // not characters.  Multi-byte character encodings are possible in
   // both internal I/O (when the character kind of the variable is 2 or 4)
   // and external formatted I/O (when the encoding is UTF-8).
-  RT_API_ATTRS std::size_t RemainingSpaceInRecord() const;
-  RT_API_ATTRS bool NeedAdvance(std::size_t) const;
-  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t);
-  RT_API_ATTRS void HandleRelativePosition(std::int64_t);
+  RT_API_ATTRS std::size_t RemainingSpaceInRecord() const {
+    auto recl{recordLength.value_or(openRecl.value_or(
+        executionEnvironment.listDirectedOutputLineLengthLimit))};
+    return positionInRecord >= recl ? 0 : recl - positionInRecord;
+  }
+  RT_API_ATTRS bool NeedAdvance(std::size_t width) const {
+    return positionInRecord > 0 && width > RemainingSpaceInRecord();
+  }
+  RT_API_ATTRS void HandleAbsolutePosition(std::int64_t n) {
+    positionInRecord = (n < 0 ? 0 : n) + leftTabLimit.value_or(0);
+  }
+  RT_API_ATTRS void HandleRelativePosition(std::int64_t n) {
+    auto least{leftTabLimit.value_or(0)};
+    auto newPos{positionInRecord + n};
+    positionInRecord = newPos < least ? least : newPos;
+    ;
+  }
 
   RT_API_ATTRS void BeginRecord() {
     positionInRecord = 0;

flang-rt/include/flang-rt/runtime/io-stmt.h

@@ -130,20 +130,95 @@ class IoStatementState {
   }
 
   // Vacant after the end of the current record
-  RT_API_ATTRS Fortran::common::optional<char32_t> GetCurrentChar(
+  RT_API_ATTRS Fortran::common::optional<char32_t> GetCurrentCharSlow(
       std::size_t &byteCount);
 
+  // For faster formatted input editing, this structure can be built by
+  // GetUpcomingFastAsciiField() and used to save significant time in
+  // GetCurrentChar, NextInField() and other input utilities when the input
+  // is buffered, does not require UTF-8 conversion, and comprises only
+  // single byte characters.
+  class FastAsciiField {
+  public:
+    RT_API_ATTRS FastAsciiField(ConnectionState &connection)
+        : connection_{connection} {}
+    RT_API_ATTRS FastAsciiField(
+        ConnectionState &connection, const char *start, std::size_t bytes)
+        : connection_{connection}, at_{start}, limit_{start + bytes} {
+      CheckForAsterisk();
+    }
+    RT_API_ATTRS ConnectionState &connection() { return connection_; }
+    RT_API_ATTRS std::size_t got() const { return got_; }
+
+    RT_API_ATTRS bool IsActive() const { return at_ != nullptr; }
+
+    RT_API_ATTRS Fortran::common::optional<char32_t> Next() const {
+      if (at_ && at_ < limit_) {
+        return *at_;
+      } else {
+        return std::nullopt;
+      }
+    }
+    RT_API_ATTRS void NextRecord(IoStatementState &io) {
+      if (at_) {
+        if (std::size_t bytes{io.GetNextInputBytes(at_)}) {
+          limit_ = at_ + bytes;
+          CheckForAsterisk();
+        } else {
+          at_ = limit_ = nullptr;
+        }
+      }
+    }
+    RT_API_ATTRS void Advance(int gotten, std::size_t bytes) {
+      if (at_ && at_ < limit_) {
+        ++at_;
+        got_ += gotten;
+      }
+      connection_.HandleRelativePosition(bytes);
+    }
+    RT_API_ATTRS void SkipRestOfRecord() { at_ = limit_; }
+    RT_API_ATTRS bool MightHaveAsterisk() const { return !at_ || hasAsterisk_; }
+
+  private:
+    RT_API_ATTRS void CheckForAsterisk() {
+      hasAsterisk_ =
+          at_ && at_ < limit_ && std::memchr(at_, '*', limit_ - at_) != nullptr;
+    }
+
+    ConnectionState &connection_;
+    const char *at_{nullptr};
+    const char *limit_{nullptr};
+    std::size_t got_{0}; // for READ(..., SIZE=)
+    bool hasAsterisk_{false};
+  };
+
+  RT_API_ATTRS FastAsciiField GetUpcomingFastAsciiField();
+
+  RT_API_ATTRS Fortran::common::optional<char32_t> GetCurrentChar(
+      std::size_t &byteCount, FastAsciiField *field = nullptr) {
+    if (field) {
+      if (auto ch{field->Next()}) {
+        byteCount = ch ? 1 : 0;
+        return ch;
+      } else if (field->IsActive()) {
+        return std::nullopt;
+      }
+    }
+    return GetCurrentCharSlow(byteCount);
+  }
+
   // The result of CueUpInput() and the "remaining" arguments to SkipSpaces()
   // and NextInField() are always in units of bytes, not characters; the
   // distinction matters for internal input from CHARACTER(KIND=2 and 4).
 
   // For fixed-width fields, return the number of remaining bytes.
   // Skip over leading blanks.
-  RT_API_ATTRS Fortran::common::optional<int> CueUpInput(const DataEdit &edit) {
+  RT_API_ATTRS Fortran::common::optional<int> CueUpInput(
+      const DataEdit &edit, FastAsciiField *fastField = nullptr) {
     Fortran::common::optional<int> remaining;
     if (edit.IsListDirected()) {
       std::size_t byteCount{0};
-      GetNextNonBlank(byteCount);
+      GetNextNonBlank(byteCount, fastField);
     } else {
       if (edit.width.value_or(0) > 0) {
         remaining = *edit.width;
@@ -152,16 +227,17 @@ class IoStatementState {
           *remaining *= bytesPerChar;
         }
       }
-      SkipSpaces(remaining);
+      SkipSpaces(remaining, fastField);
     }
     return remaining;
   }
 
   RT_API_ATTRS Fortran::common::optional<char32_t> SkipSpaces(
-      Fortran::common::optional<int> &remaining) {
+      Fortran::common::optional<int> &remaining,
+      FastAsciiField *fastField = nullptr) {
     while (!remaining || *remaining > 0) {
       std::size_t byteCount{0};
-      if (auto ch{GetCurrentChar(byteCount)}) {
+      if (auto ch{GetCurrentChar(byteCount, fastField)}) {
         if (*ch != ' ' && *ch != '\t') {
           return ch;
         }
@@ -172,7 +248,11 @@ class IoStatementState {
           GotChar(byteCount);
           *remaining -= byteCount;
         }
-        HandleRelativePosition(byteCount);
+        if (fastField) {
+          fastField->Advance(0, byteCount);
+        } else {
+          HandleRelativePosition(byteCount);
+        }
       } else {
         break;
       }
@@ -183,25 +263,35 @@ class IoStatementState {
   // Acquires the next input character, respecting any applicable field width
   // or separator character.
   RT_API_ATTRS Fortran::common::optional<char32_t> NextInField(
-      Fortran::common::optional<int> &remaining, const DataEdit &);
+      Fortran::common::optional<int> &remaining, const DataEdit &,
+      FastAsciiField *field = nullptr);
 
   // Detect and signal any end-of-record condition after input.
   // Returns true if at EOR and remaining input should be padded with blanks.
-  RT_API_ATTRS bool CheckForEndOfRecord(std::size_t afterReading);
+  RT_API_ATTRS bool CheckForEndOfRecord(
+      std::size_t afterReading, const ConnectionState &);
 
   // Skips spaces, advances records, and ignores NAMELIST comments
   RT_API_ATTRS Fortran::common::optional<char32_t> GetNextNonBlank(
-      std::size_t &byteCount) {
-    auto ch{GetCurrentChar(byteCount)};
+      std::size_t &byteCount, FastAsciiField *fastField = nullptr) {
+    auto ch{GetCurrentChar(byteCount, fastField)};
     bool inNamelist{mutableModes().inNamelist};
     while (!ch || *ch == ' ' || *ch == '\t' || *ch == '\n' ||
         (inNamelist && *ch == '!')) {
       if (ch && (*ch == ' ' || *ch == '\t' || *ch == '\n')) {
-        HandleRelativePosition(byteCount);
-      } else if (!AdvanceRecord()) {
+        if (fastField) {
+          fastField->Advance(0, byteCount);
+        } else {
+          HandleRelativePosition(byteCount);
+        }
+      } else if (AdvanceRecord()) {
+        if (fastField) {
+          fastField->NextRecord(*this);
+        }
+      } else {
         return Fortran::common::nullopt;
       }
-      ch = GetCurrentChar(byteCount);
+      ch = GetCurrentChar(byteCount, fastField);
     }
     return ch;
   }

flang-rt/lib/runtime/connection.cpp

@@ -9,37 +9,10 @@
 #include "flang-rt/runtime/connection.h"
 #include "flang-rt/runtime/environment.h"
 #include "flang-rt/runtime/io-stmt.h"
-#include <algorithm>
 
 namespace Fortran::runtime::io {
 RT_OFFLOAD_API_GROUP_BEGIN
 
-RT_API_ATTRS std::size_t ConnectionState::RemainingSpaceInRecord() const {
-  auto recl{recordLength.value_or(openRecl.value_or(
-      executionEnvironment.listDirectedOutputLineLengthLimit))};
-  return positionInRecord >= recl ? 0 : recl - positionInRecord;
-}
-
-RT_API_ATTRS bool ConnectionState::NeedAdvance(std::size_t width) const {
-  return positionInRecord > 0 && width > RemainingSpaceInRecord();
-}
-
-RT_API_ATTRS bool ConnectionState::IsAtEOF() const {
-  return endfileRecordNumber && currentRecordNumber >= *endfileRecordNumber;
-}
-
-RT_API_ATTRS bool ConnectionState::IsAfterEndfile() const {
-  return endfileRecordNumber && currentRecordNumber > *endfileRecordNumber;
-}
-
-RT_API_ATTRS void ConnectionState::HandleAbsolutePosition(std::int64_t n) {
-  positionInRecord = std::max(n, std::int64_t{0}) + leftTabLimit.value_or(0);
-}
-
-RT_API_ATTRS void ConnectionState::HandleRelativePosition(std::int64_t n) {
-  positionInRecord = std::max(leftTabLimit.value_or(0), positionInRecord + n);
-}
-
 SavedPosition::SavedPosition(IoStatementState &io) : io_{io} {
   ConnectionState &conn{io_.GetConnectionState()};
   saved_ = conn;

flang-rt/lib/runtime/edit-input.cpp

@@ -169,17 +169,18 @@ static inline RT_API_ATTRS char32_t GetRadixPointChar(const DataEdit &edit) {
 // Prepares input from a field, and returns the sign, if any, else '\0'.
 static RT_API_ATTRS char ScanNumericPrefix(IoStatementState &io,
     const DataEdit &edit, Fortran::common::optional<char32_t> &next,
-    Fortran::common::optional<int> &remaining) {
-  remaining = io.CueUpInput(edit);
-  next = io.NextInField(remaining, edit);
+    Fortran::common::optional<int> &remaining,
+    IoStatementState::FastAsciiField *fastField = nullptr) {
+  remaining = io.CueUpInput(edit, fastField);
+  next = io.NextInField(remaining, edit, fastField);
   char sign{'\0'};
   if (next) {
     if (*next == '-' || *next == '+') {
       sign = *next;
       if (!edit.IsListDirected()) {
-        io.SkipSpaces(remaining);
+        io.SkipSpaces(remaining, fastField);
       }
-      next = io.NextInField(remaining, edit);
+      next = io.NextInField(remaining, edit, fastField);
     }
   }
   return sign;
@@ -213,17 +214,18 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
   }
   Fortran::common::optional<int> remaining;
   Fortran::common::optional<char32_t> next;
-  char sign{ScanNumericPrefix(io, edit, next, remaining)};
+  auto fastField{io.GetUpcomingFastAsciiField()};
+  char sign{ScanNumericPrefix(io, edit, next, remaining, &fastField)};
   if (sign == '-' && !isSigned) {
     io.GetIoErrorHandler().SignalError("Negative sign in UNSIGNED input field");
     return false;
   }
-  common::UnsignedInt128 value{0};
+  common::uint128_t value{0};
   bool any{!!sign};
   bool overflow{false};
   const char32_t comma{GetSeparatorChar(edit)};
-  static constexpr auto maxu128{~common::UnsignedInt128{0}};
-  for (; next; next = io.NextInField(remaining, edit)) {
+  static constexpr auto maxu128{~common::uint128_t{0}};
+  for (; next; next = io.NextInField(remaining, edit, &fastField)) {
     char32_t ch{*next};
     if (ch == ' ' || ch == '\t') {
       if (edit.modes.editingFlags & blankZero) {
@@ -243,7 +245,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
         // input, like a few other Fortran compilers do.
         // TODO: also process exponents?  Some compilers do, but they obviously
         // can't just be ignored.
-        while ((next = io.NextInField(remaining, edit))) {
+        while ((next = io.NextInField(remaining, edit, &fastField))) {
           if (*next < '0' || *next > '9') {
             break;
           }
@@ -271,7 +273,7 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
     return false;
   }
   if (isSigned) {
-    auto maxForKind{common::UnsignedInt128{1} << ((8 * kind) - 1)};
+    auto maxForKind{common::uint128_t{1} << ((8 * kind) - 1)};
     overflow |= value >= maxForKind && (value > maxForKind || sign != '-');
   } else {
     auto maxForKind{maxu128 >> (((16 - kind) * 8) + (isSigned ? 1 : 0))};
@@ -287,7 +289,16 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
   }
   if (any || !io.GetIoErrorHandler().InError()) {
     // The value is stored in the lower order bits on big endian platform.
-    // When memcpy, shift the value to the higher order bit.
+    // For memcpy, shift the value to the highest order bits.
+#if USING_NATIVE_INT128_T
+    auto shft{static_cast<int>(sizeof value - kind)};
+    if (!isHostLittleEndian && shft >= 0) {
+      auto l{value << shft};
+      std::memcpy(n, &l, kind);
+    } else {
+      std::memcpy(n, &value, kind); // a blank field means zero
+    }
+#else
     auto shft{static_cast<int>(sizeof(value.low())) - kind};
     // For kind==8 (i.e. shft==0), the value is stored in low_ in big endian.
     if (!isHostLittleEndian && shft >= 0) {
@@ -296,6 +307,8 @@ RT_API_ATTRS bool EditIntegerInput(IoStatementState &io, const DataEdit &edit,
     } else {
       std::memcpy(n, &value, kind); // a blank field means zero
     }
+#endif
+    io.GotChar(fastField.got());
     return true;
   } else {
     return false;
@@ -1070,7 +1083,7 @@ RT_API_ATTRS bool EditCharacterInput(IoStatementState &io, const DataEdit &edit,
       readyBytes = io.GetNextInputBytes(input);
       if (readyBytes == 0 ||
           (readyBytes < remainingChars && edit.modes.nonAdvancing)) {
-        if (io.CheckForEndOfRecord(readyBytes)) {
+        if (io.CheckForEndOfRecord(readyBytes, connection)) {
           if (readyBytes == 0) {
             // PAD='YES' and no more data
             Fortran::runtime::fill_n(x, lengthChars, ' ');

flang-rt/lib/runtime/io-api.cpp

@@ -1057,14 +1057,15 @@ bool IODEF(InputDescriptor)(Cookie cookie, const Descriptor &descriptor) {
 }
 
 bool IODEF(InputInteger)(Cookie cookie, std::int64_t &n, int kind) {
-  if (!cookie->CheckFormattedStmtType<Direction::Input>("InputInteger")) {
-    return false;
+  IoStatementState &io{*cookie};
+  if (io.BeginReadingRecord()) {
+    if (auto edit{io.GetNextDataEdit()}) {
+      return edit->descriptor == DataEdit::ListDirectedNullValue ||
+          EditIntegerInput(io, *edit, reinterpret_cast<void *>(&n), kind,
+              /*isSigned=*/true);
+    }
   }
-  StaticDescriptor<0> staticDescriptor;
-  Descriptor &descriptor{staticDescriptor.descriptor()};
-  descriptor.Establish(
-      TypeCategory::Integer, kind, reinterpret_cast<void *>(&n), 0);
-  return descr::DescriptorIO<Direction::Input>(*cookie, descriptor);
+  return false;
 }
 
 bool IODEF(InputReal32)(Cookie cookie, float &x) {