cherry pick several runtime and intrinsic fixes (#1006)

* [flang] Fix regression from recent runtime input fix

A recent runtime I/O change[1] was meant to improve the handling of
input from external files missing a terminal newline on their last
records; the change was "triggered" by the wrong circumstances and
causing reads that should have pulled more data into the buffer to be
treated as EOFs.  So fix that, and also don't retain input data
in the buffer once an input record has been finished unless it's
known that list-directed or NAMELIST input of a repeated input item
may need to backspace a non-positionable external unit to return
to the beginning of the repeated item.

[1] 6578893

Differential Revision: https://reviews.llvm.org/D108164

* [flang] Add missing call to BeginReadingRecord()

NAMELIST input needs to start with a call to BeginReadingRecord().
Internal unit input doesn't care (so unit tests were passing), but
external unit input does need the call and will assert without it.

Differential Revision: https://reviews.llvm.org/D108051

* [flang] Fix the vector version of EOSHIFT with a BOUNDARY argument

When the vector version of EOSHIFT was called, the BOUNDARY argument was being
ignored.  I fixed that and added a test that would not pass without this fix.

Differential Revision: https://reviews.llvm.org/D108249

Co-authored-by: peter klausler <[email protected]>
Co-authored-by: Peter Steinfeld <[email protected]>

Author: 3 people

flang/runtime/connection.h

@@ -57,6 +57,12 @@ struct ConnectionState : public ConnectionAttributes {
   // or an end-of-file READ condition on a sequential access file
   std::optional<std::int64_t> endfileRecordNumber;
 
+  // Set when processing repeated items during list-directed & NAMELIST input
+  // in order to keep a span of records in frame on a non-positionable file,
+  // so that backspacing to the beginning of the repeated item doesn't require
+  // repositioning the external storage medium when that's impossible.
+  std::optional<std::int64_t> resumptionRecordNumber;
+
   // Mutable modes set at OPEN() that can be overridden in READ/WRITE & FORMAT
   MutableModes modes; // BLANK=, DECIMAL=, SIGN=, ROUND=, PAD=, DELIM=, kP
 };

flang/runtime/io-stmt.cpp

@@ -653,7 +653,11 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
     comma = ';';
   }
   if (remaining_ > 0 && !realPart_) { // "r*c" repetition in progress
-    while (connection.currentRecordNumber > repeatRecordNumber_) {
+    RUNTIME_CHECK(
+        io.GetIoErrorHandler(), connection.resumptionRecordNumber.has_value());
+    while (connection.currentRecordNumber >
+        connection.resumptionRecordNumber.value_or(
+            connection.currentRecordNumber)) {
       io.BackspaceRecord();
     }
     connection.HandleAbsolutePosition(repeatPositionInRecord_);
@@ -666,6 +670,9 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
       }
     }
     remaining_ -= edit.repeat;
+    if (remaining_ <= 0) {
+      connection.resumptionRecordNumber.reset();
+    }
     return edit;
   }
   // Skip separators, handle a "r*c" repeat count; see 13.10.2 in Fortran 2018
@@ -726,7 +733,11 @@ ListDirectedStatementState<Direction::Input>::GetNextDataEdit(
       }
       edit.repeat = std::min<int>(r, maxRepeat);
       remaining_ = r - edit.repeat;
-      repeatRecordNumber_ = connection.currentRecordNumber;
+      if (remaining_ > 0) {
+        connection.resumptionRecordNumber = connection.currentRecordNumber;
+      } else {
+        connection.resumptionRecordNumber.reset();
+      }
       repeatPositionInRecord_ = connection.positionInRecord;
     } else { // not a repetition count, just an integer value; rewind
       connection.positionInRecord = start;

flang/runtime/io-stmt.h

@@ -229,7 +229,6 @@ class ListDirectedStatementState<Direction::Input>
 
 private:
   int remaining_{0}; // for "r*" repetition
-  std::int64_t repeatRecordNumber_;
   std::int64_t repeatPositionInRecord_;
   bool eatComma_{false}; // consume comma after previously read item
   bool hitSlash_{false}; // once '/' is seen, nullify further items

flang/runtime/namelist.cpp

@@ -263,6 +263,7 @@ bool IONAME(InputNamelist)(Cookie cookie, const NamelistGroup &group) {
   auto *listInput{io.get_if<ListDirectedStatementState<Direction::Input>>()};
   RUNTIME_CHECK(handler, listInput != nullptr);
   // Check the group header
+  io.BeginReadingRecord();
   std::optional<char32_t> next{io.GetNextNonBlank()};
   if (!next || *next != '&') {
     handler.SignalError(

flang/runtime/transformational.cpp

@@ -284,6 +284,8 @@ void RTNAME(EoshiftVector)(Descriptor &result, const Descriptor &source,
     SubscriptValue sourceAt{lb + j - 1 + shift};
     if (sourceAt >= lb && sourceAt < lb + extent) {
       CopyElement(result, &j, source, &sourceAt, terminator);
+    } else if (boundary) {
+      CopyElement(result, &j, *boundary, 0, terminator);
     }
   }
 }

flang/runtime/unit.cpp

@@ -350,13 +350,10 @@ bool ExternalFileUnit::SetSequentialVariableFormattedRecordLength() {
       if (*recordLength > 0 && record[*recordLength - 1] == '\r') {
         --*recordLength;
       }
-    } else {
-      recordLength = bytes; // final record w/o \n
+      return true;
     }
-    return true;
-  } else {
-    return false;
   }
+  return false;
 }
 
 void ExternalFileUnit::SetLeftTabLimit() {
@@ -421,7 +418,10 @@ void ExternalFileUnit::FinishReadingRecord(IoErrorHandler &handler) {
             Frame()[recordOffsetInFrame_ + *recordLength] == '\n') {
           ++recordOffsetInFrame_;
         }
-        recordOffsetInFrame_ += *recordLength;
+        if (!resumptionRecordNumber || mayPosition()) {
+          frameOffsetInFile_ += recordOffsetInFrame_ + *recordLength;
+          recordOffsetInFrame_ = 0;
+        }
         recordLength.reset();
       }
     }
@@ -612,10 +612,17 @@ void ExternalFileUnit::BeginSequentialVariableFormattedInputRecord(
   }
   std::size_t length{0};
   do {
-    std::size_t need{recordOffsetInFrame_ + length + 1};
-    length = ReadFrame(frameOffsetInFile_, need, handler);
+    std::size_t need{length + 1};
+    length =
+        ReadFrame(frameOffsetInFile_, recordOffsetInFrame_ + need, handler) -
+        recordOffsetInFrame_;
     if (length < need) {
-      handler.SignalEnd();
+      if (length > 0) {
+        // final record w/o \n
+        recordLength = length;
+      } else {
+        handler.SignalEnd();
+      }
       break;
     }
   } while (!SetSequentialVariableFormattedRecordLength());

flang/unittests/RuntimeGTest/Transformational.cpp

@@ -110,6 +110,26 @@ TEST(Transformational, Shifts) {
         *result.ZeroBasedIndexedElement<std::int32_t>(j), eoshiftExpect1[j]);
   }
   result.Destroy();
+
+  // VECTOR EOSHIFT
+  StaticDescriptor<0> boundaryDescriptor;
+  Descriptor vectorBoundary{boundaryDescriptor.descriptor()};
+  std::int32_t boundaryValue{343};
+  vectorBoundary.Establish(TypeCategory::Integer, 4,
+      const_cast<void *>(reinterpret_cast<const void *>(&boundaryValue)), 0);
+  RTNAME(EoshiftVector)
+  (vectorResult, *vector, 2, &vectorBoundary, __FILE__, __LINE__);
+  EXPECT_EQ(vectorResult.type(), array->type());
+  EXPECT_EQ(vectorResult.rank(), 1);
+  EXPECT_EQ(vectorResult.GetDimension(0).LowerBound(), 1);
+  EXPECT_EQ(vectorResult.GetDimension(0).Extent(), 6);
+  EXPECT_EQ(vectorResult.type(), (TypeCode{TypeCategory::Integer, 4}));
+  static std::int32_t eoshiftVectorExpect[6]{3, 4, 5, 6, 343, 343};
+  for (int j{0}; j < 6; ++j) {
+    EXPECT_EQ(*vectorResult.ZeroBasedIndexedElement<std::int32_t>(j),
+        eoshiftVectorExpect[j]);
+  }
+  vectorResult.Destroy();
 }
 
 TEST(Transformational, Pack) {