Fix: FetchResults truncates rows returned by the server (#18)

Apache Hive thrift always sends HasMoreResults==false

This change updates the Next() method to continuing fetching pages of
results until an empty page is received.

Also updates the changelog and version number

Signed-off-by: Jesse Whitehouse <[email protected]>

Author: Jesse

CHANGELOG.md

@@ -2,6 +2,11 @@
 
 ## 0.1.x (Unreleased)
 
+## 0.1.4 (2022-07-30)
+
+- Fix: Could not fetch rowsets greater than the value of `maxRows` (#18)
+- Updated default user agent
+- Updated README and CONTRIBUTING
 
 ## 0.1.3 (2022-06-16)
 

README.md

@@ -42,7 +42,7 @@ The DSN format is:
 databricks://:[your token]@[Workspace hostname][Endpoint HTTP Path]
 ```
 
-You can set HTTP Timeout value by appending a `timeout` query parameter (in milliseconds) and you can set max rows to retrieve by setting the `maxRows` query parameter:
+You can set HTTP Timeout value by appending a `timeout` query parameter (in milliseconds) and you can set max rows to retrieve per network request by setting the `maxRows` query parameter:
 
 ```
 databricks://:[your token]@[Workspace hostname][Endpoint HTTP Path]?timeout=1000&maxRows=1000

connection_test.go

@@ -1,6 +1,7 @@
 package dbsql
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"os"
@@ -61,6 +62,38 @@ func TestSelect(t *testing.T) {
 	}
 }
 
+func TestExhaustCursor(t *testing.T) {
+	// GIVEN: Session MaxRows < expected result length
+	// WHEN: Caller fetches all results
+	// THEN: The expected result length is pulled by successive calls to
+
+	// MaxRows is defined in databricks.go as 10_000
+
+	db := open(t)
+	defer db.Close()
+
+	// Pull more results than the known MaxRows value
+	ctx := context.Background()
+	rows, _ := db.QueryContext(ctx, "SELECT id FROM RANGE(100000)")
+
+	rowIds := make([]int, 0)
+
+	// Fetch all results from the `rows` iterator
+	var rowId int
+	for rows.Next() {
+		rows.Scan(&rowId)
+		rowIds = append(rowIds, rowId)
+	}
+
+	expectedLength := 100_000
+	actualLength := len(rowIds)
+
+	if expectedLength != actualLength {
+		t.Errorf("Result length mismatch. Expected %d, Actual: %d", expectedLength, actualLength)
+
+	}
+}
+
 func open(t *testing.T) *sql.DB {
 	dsn := os.Getenv("DATABRICKS_DSN")
 	if dsn == "" {

hive/result_set.go

@@ -22,19 +22,26 @@ type ResultSet struct {
 
 // Next ...
 func (rs *ResultSet) Next(dest []driver.Value) error {
-	if rs.idx >= rs.length {
-		if !rs.more {
-			return io.EOF
-		}
 
+	var thisPageExhausted bool = rs.idx >= length(rs.result)
+
+	if thisPageExhausted {
+
+		// Fetch another page
 		resp, err := rs.fetchfn()
+
 		if err != nil {
 			return err
 		}
+
+		// Replace previous page of results with new page of results
 		rs.result = resp.Results
-		rs.more = resp.GetHasMoreRows()
-		rs.idx = 0
+
+		// Replace previous page length with new page length
 		rs.length = length(rs.result)
+
+		// Reset index to the start of the new page
+		rs.idx = 0
 	}
 
 	if rs.length == 0 {
@@ -43,11 +50,14 @@ func (rs *ResultSet) Next(dest []driver.Value) error {
 
 	for i := range dest {
 		val, err := value(rs.result.Columns[i], rs.schema.Columns[i], rs.idx)
+
 		if err != nil {
 			return err
 		}
+
 		dest[i] = val
 	}
+
 	rs.idx++
 	return nil
 }