Fix the case where reading a line has spaces in it (#43)

* Fix the case where reading a line has spaces in it

**NOTE**: There doesn't seem to be any way of doing it with easier std libs other than bufio Buffer or Scanner, but this is meant to be general.
therefore, it copies part of fmt.Scanln without tokenizing on space.

* Add some comments.

Co-authored-by: Sheridan C Rawlins <[email protected]>

Author: scr-oath

io/rune-reader.go

@@ -0,0 +1,75 @@
+package io
+
+import (
+	"io"
+	"strings"
+	"unicode/utf8"
+)
+
+//UnbufferedRuneReader doesn't attempt to buffer the underlying reader, but does buffer enough to decode utf8 runes.
+type UnbufferedRuneReader struct {
+	reader io.Reader
+	buf    [utf8.UTFMax]byte // used only inside ReadRune
+	single [1]byte           // to read one byte
+}
+
+//ReadLine reads a line from any reader.
+func ReadLine(reader io.Reader) (string, error) {
+	rr := ToRuneReader(reader)
+	var sb strings.Builder
+	var r rune
+	var err error
+	for r, _, err = rr.ReadRune(); err == nil && r != '\n'; r, _, err = rr.ReadRune() {
+		sb.WriteRune(r)
+	}
+	if err == io.EOF && sb.Len() > 0 {
+		err = nil
+	}
+	return sb.String(), err
+}
+
+//ToRuneReader Converts reader to an io.RuneReader
+func ToRuneReader(reader io.Reader) io.RuneReader {
+	if ret, ok := reader.(io.RuneReader); ok {
+		return ret
+	}
+	return &UnbufferedRuneReader{
+		reader: reader,
+	}
+}
+
+func (u *UnbufferedRuneReader) readByte() (b byte, err error) {
+	n, err := io.ReadFull(u.reader, u.single[:])
+	if n != 1 {
+		return 0, err
+	}
+	return u.single[0], err
+}
+
+//ReadRune reads a single rune, and returns the rune, its byte-length, and possibly an error.
+// see the code for fmt.Scanln - which is not public, but which does, but tokenizing on space, which is not desirable.
+// The implementation in fmt.Scanln also implements io.RuneScanner, which is not needed here as newlines are discarded.
+func (u *UnbufferedRuneReader) ReadRune() (r rune, size int, err error) {
+	u.buf[0], err = u.readByte()
+	if err != nil {
+		return
+	}
+	if u.buf[0] < utf8.RuneSelf { // fast check for common ASCII case
+		r = rune(u.buf[0])
+		size = 1
+		return
+	}
+	var n int
+	for n = 1; !utf8.FullRune(u.buf[:n]); n++ {
+		u.buf[n], err = u.readByte()
+		if err != nil {
+			if err == io.EOF {
+				err = nil
+				break
+			}
+			return
+		}
+	}
+	r, size = utf8.DecodeRune(u.buf[:n])
+	return
+}

io/wrappers.go

@@ -227,8 +227,7 @@ func IOReaderRead(L *lua.LState) int {
 			L.Push(lua.LString(data))
 			return 1
 		case 'l':
-			var line lua.LString
-			_, err := fmt.Fscanln(reader, &line)
+			line, err := ReadLine(reader)
 			if err == io.EOF {
 				L.Push(lua.LNil)
 				return 1
@@ -237,7 +236,7 @@ func IOReaderRead(L *lua.LState) int {
 				L.RaiseError("%v", err)
 				return 0
 			}
-			L.Push(line)
+			L.Push(lua.LString(line))
 			return 1
 		}
 	}

strings/test/test_api.lua

@@ -82,4 +82,36 @@ function TestFields(t)
     assert(fields[2] == "b", string.format("%s ~= 'b'", fields[2]))
     assert(fields[3] == "c", string.format("%s ~= 'c'", fields[3]))
     assert(fields[4] == "d", string.format("%s ~= 'd'", fields[3]))
+end
+
+function assert_equal(expected, actual)
+    assert(expected == actual, string.format([[expected "%s": got "%s"]], expected, actual))
+end
+
+function TestReadLine(t)
+    t:Run("single line", function(t)
+        local reader = strings.new_reader("single line\n")
+        local line = reader:read("*l")
+        assert_equal("single line", line)
+        line = reader:read("*l")
+        assert(not line, line)
+    end)
+
+    t:Run("single line no newline", function(t)
+        local reader = strings.new_reader("single line")
+        local line = reader:read("*l")
+        assert_equal("single line", line)
+        line = reader:read("*l")
+        assert(not line, line)
+    end)
+
+    t:Run("two lines with spaces in first", function(t)
+        local reader = strings.new_reader("foo bar\nbaz\n")
+        local line = reader:read("*l")
+        assert_equal("foo bar", line)
+        line = reader:read("*l")
+        assert_equal("baz", line)
+        line = reader:read("*l")
+        assert(not line, line)
+    end)
 end