Skip to content

Commit 53e47b3

Browse files
ethanyzhangethanyzhang
committed
save: add --no-analyze option
1 parent 2c8f1c2 commit 53e47b3

File tree

6 files changed

+51
-37
lines changed

6 files changed

+51
-37
lines changed

‎cmd/save.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ func init() {
3434
saveCmd.Flags().StringVarP(&save.OutputPath, "output-path", "o", wd, "Output directory path")
3535
saveCmd.Flags().StringVarP(&save.Catalog, "catalog", "", "", "Catalog name")
3636
saveCmd.Flags().StringVarP(&save.Schema, "schema", "", "", "Schema name")
37+
saveCmd.Flags().BoolVarP(&save.NoAnalyze, "no-analyze", "n", false, "Do not run additional queries to analyze table when stats were missing.")
3738
saveCmd.Flags().StringArrayVarP(&save.Session, "session", "", nil,
3839
"Session property (property can be used multiple times; format is\nkey=value; use 'SHOW SESSION' in Presto CLI to see available properties)")
3940
saveCmd.Flags().StringVarP(&save.InputFilePath, "file", "f", "", "CSV file to read catalog,schema,table.")

‎cmd/save/main.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020

2121
var (
2222
PrestoFlags utils.PrestoFlags
23+
NoAnalyze bool
2324
OutputPath string
2425
Schema string
2526
Catalog string
@@ -126,7 +127,7 @@ func saveTable(ctx context.Context, client *presto.Client, catalog, schema, tabl
126127
return
127128
}
128129
ts := &TableSummary{Catalog: catalog, Schema: schema, Name: table}
129-
ts.QueryTableSummary(ctx, client)
130+
ts.QueryTableSummary(ctx, client, !NoAnalyze)
130131
filePath := filepath.Join(OutputPath,
131132
fmt.Sprintf("%s_%s_%s.json", catalog, schema, table))
132133
if err := ts.SaveToFile(filePath); err != nil {

‎cmd/save/table_summary.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ func handleQueryError(err error, abortOnError bool) (retry bool) {
9292
return
9393
}
9494

95-
func (s *TableSummary) QueryTableSummary(ctx context.Context, client *presto.Client) {
95+
func (s *TableSummary) QueryTableSummary(ctx context.Context, client *presto.Client, analyze bool) {
9696
fullyQualifiedTableName := fmt.Sprintf("%s.%s.%s", s.Catalog, s.Schema, s.Name)
9797

9898
defer func() {
@@ -118,7 +118,7 @@ func (s *TableSummary) QueryTableSummary(ctx context.Context, client *presto.Cli
118118
}
119119

120120
// Zero rows, no need to do anything more.
121-
if *s.RowCount == 0 {
121+
if *s.RowCount == 0 || !analyze {
122122
return
123123
}
124124

‎cmd/save/table_summary_test.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,17 @@ package save
33
import (
44
"context"
55
"fmt"
6-
"github.com/stretchr/testify/assert"
76
"pbench/presto"
87
"testing"
98
)
109

1110
func TestTableSummary_QueryTableSummary(t *testing.T) {
12-
//t.SkipNow()
1311
client, _ := presto.NewClient("http://localhost:8080", false)
12+
if _, _, err := client.GetClusterInfo(context.Background()); err != nil {
13+
t.Skip("local cluster is not ready")
14+
}
1415
client.Catalog("tpch").Schema("sf1")
1516
ts := &TableSummary{Catalog: "tpch", Schema: "sf1", Name: "lineitem"}
16-
assert.Nil(t, ts.QueryTableSummary(context.Background(), client))
17+
ts.QueryTableSummary(context.Background(), client, true)
1718
fmt.Println(ts)
1819
}

‎presto/unmarshal_test.go‎

Lines changed: 37 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ import (
99
)
1010

1111
func TestBuiltinRows(t *testing.T) {
12-
t.SkipNow()
1312
pRows, pCols := getRowsFromPresto(t)
1413
bRows, bCols := getBuiltinRows(t)
1514
assert.Equal(t, pRows, bRows)
@@ -18,6 +17,9 @@ func TestBuiltinRows(t *testing.T) {
1817

1918
func TestPrestoUnmarshalScalar(t *testing.T) {
2019
client, _ := NewClient("http://localhost:8080", false)
20+
if _, _, err := client.GetClusterInfo(context.Background()); err != nil {
21+
t.Skip("local cluster is not ready")
22+
}
2123
client.Catalog("tpch").Schema("sf1")
2224
ctx := context.Background()
2325
var ddl string
@@ -42,8 +44,36 @@ func TestPrestoUnmarshal(t *testing.T) {
4244
err = UnmarshalQueryData(rows, columnHeaders, columnsStats)
4345
assert.ErrorIs(t, err, UnmarshalError) // not a pointer
4446

47+
newFloat64 := func(f float64) *float64 {
48+
return &f
49+
}
50+
newString := func(v string) *string {
51+
return &v
52+
}
53+
zero := newFloat64(0)
54+
expectedStats := []ColumnStats{
55+
{ColumnName: "orderkey", DistinctValuesCount: newFloat64(1500254), NullsFraction: zero, LowValue: newString("1"), HighValue: newString("6000000")},
56+
{ColumnName: "partkey", DistinctValuesCount: newFloat64(200044), NullsFraction: zero, LowValue: newString("1"), HighValue: newString("200000")},
57+
{ColumnName: "suppkey", DistinctValuesCount: newFloat64(10000.0), NullsFraction: zero, LowValue: newString("1"), HighValue: newString("10000")},
58+
{ColumnName: "linenumber", DistinctValuesCount: newFloat64(7.0), NullsFraction: zero, LowValue: newString("1"), HighValue: newString("7")},
59+
{ColumnName: "quantity", DistinctValuesCount: newFloat64(50.0), NullsFraction: zero, LowValue: newString("1.0"), HighValue: newString("50.0")},
60+
{ColumnName: "extendedprice", DistinctValuesCount: newFloat64(933985.0), NullsFraction: zero, LowValue: newString("901.0"), HighValue: newString("104949.5")},
61+
{ColumnName: "discount", DistinctValuesCount: newFloat64(11.0), NullsFraction: zero, LowValue: newString("0.0"), HighValue: newString("0.1")},
62+
{ColumnName: "tax", DistinctValuesCount: newFloat64(9.0), NullsFraction: zero, LowValue: newString("0.0"), HighValue: newString("0.08")},
63+
{ColumnName: "returnflag", DataSize: newFloat64(6001215.0), DistinctValuesCount: newFloat64(3.0), NullsFraction: zero},
64+
{ColumnName: "linestatus", DataSize: newFloat64(6001215.0), DistinctValuesCount: newFloat64(2.0), NullsFraction: zero},
65+
{ColumnName: "shipdate", DistinctValuesCount: newFloat64(2526.0), NullsFraction: zero, LowValue: newString("1992-01-02"), HighValue: newString("1998-12-01")},
66+
{ColumnName: "commitdate", DistinctValuesCount: newFloat64(2466.0), NullsFraction: zero, LowValue: newString("1992-01-31"), HighValue: newString("1998-10-31")},
67+
{ColumnName: "receiptdate", DistinctValuesCount: newFloat64(2554.0), NullsFraction: zero, LowValue: newString("1992-01-04"), HighValue: newString("1998-12-31")},
68+
{ColumnName: "shipinstruct", DataSize: newFloat64(7.2006409e7), DistinctValuesCount: newFloat64(4.0), NullsFraction: zero},
69+
{ColumnName: "shipmode", DataSize: newFloat64(2.5717034e7), DistinctValuesCount: newFloat64(7.0), NullsFraction: zero},
70+
{ColumnName: "comment", DataSize: newFloat64(1.58997209e8), DistinctValuesCount: newFloat64(4580252.0), NullsFraction: zero},
71+
{RowCount: newFloat64(6001215.0)},
72+
}
73+
4574
err = UnmarshalQueryData(rows, columnHeaders, &columnsStats[0])
46-
assert.ErrorIs(t, err, UnmarshalError) // not an array/slice pointer
75+
assert.Nil(t, err)
76+
assert.Equal(t, expectedStats[0], columnsStats[0])
4777

4878
// UnmarshalQueryData into a []json.RawMessage
4979
var decodedRows []json.RawMessage
@@ -71,36 +101,14 @@ func TestPrestoUnmarshal(t *testing.T) {
71101
err = UnmarshalQueryData(rows, columnHeaders, &columnsStats)
72102
assert.Nil(t, err)
73103
assert.Equal(t, len(rows), len(columnsStats))
74-
newFloat64 := func(f float64) *float64 {
75-
return &f
76-
}
77-
newString := func(v string) *string {
78-
return &v
79-
}
80-
zero := newFloat64(0)
81-
assert.Equal(t, []ColumnStats{
82-
{"orderkey", nil, newFloat64(1500254), zero, nil, newString("1"), newString("6000000"), nil, nil, nil},
83-
{"partkey", nil, newFloat64(200044), zero, nil, newString("1"), newString("200000"), nil, nil, nil},
84-
{"suppkey", nil, newFloat64(10000.0), zero, nil, newString("1"), newString("10000"), nil, nil, nil},
85-
{"linenumber", nil, newFloat64(7.0), zero, nil, newString("1"), newString("7"), nil, nil, nil},
86-
{"quantity", nil, newFloat64(50.0), zero, nil, newString("1.0"), newString("50.0"), nil, nil, nil},
87-
{"extendedprice", nil, newFloat64(933985.0), zero, nil, newString("901.0"), newString("104949.5"), nil, nil, nil},
88-
{"discount", nil, newFloat64(11.0), zero, nil, newString("0.0"), newString("0.1"), nil, nil, nil},
89-
{"tax", nil, newFloat64(9.0), zero, nil, newString("0.0"), newString("0.08"), nil, nil, nil},
90-
{"returnflag", newFloat64(6001215.0), newFloat64(3.0), zero, nil, nil, nil, nil, nil, nil},
91-
{"linestatus", newFloat64(6001215.0), newFloat64(2.0), zero, nil, nil, nil, nil, nil, nil},
92-
{"shipdate", nil, newFloat64(2526.0), zero, nil, newString("1992-01-02"), newString("1998-12-01"), nil, nil, nil},
93-
{"commitdate", nil, newFloat64(2466.0), zero, nil, newString("1992-01-31"), newString("1998-10-31"), nil, nil, nil},
94-
{"receiptdate", nil, newFloat64(2554.0), zero, nil, newString("1992-01-04"), newString("1998-12-31"), nil, nil, nil},
95-
{"shipinstruct", newFloat64(7.2006409e7), newFloat64(4.0), zero, nil, nil, nil, nil, nil, nil},
96-
{"shipmode", newFloat64(2.5717034e7), newFloat64(7.0), zero, nil, nil, nil, nil, nil, nil},
97-
{"comment", newFloat64(1.58997209e8), newFloat64(4580252.0), zero, nil, nil, nil, nil, nil, nil},
98-
{"", nil, nil, nil, newFloat64(6001215.0), nil, nil, nil, nil, nil},
99-
}, columnsStats)
104+
assert.Equal(t, expectedStats, columnsStats)
100105
}
101106

102107
func getRowsFromPresto(t *testing.T) ([]json.RawMessage, []Column) {
103108
client, _ := NewClient("http://localhost:8080", false)
109+
if _, _, err := client.GetClusterInfo(context.Background()); err != nil {
110+
t.Skip("local cluster is not ready")
111+
}
104112
client.Catalog("tpch").Schema("sf1")
105113
ctx := context.Background()
106114

@@ -114,9 +122,8 @@ func getRowsFromPresto(t *testing.T) ([]json.RawMessage, []Column) {
114122
})
115123
if assert.Nil(t, err) {
116124
return rows, clientResult.Columns
117-
} else {
118-
return rows, nil
119125
}
126+
return rows, nil
120127
}
121128

122129
func getBuiltinRows(t *testing.T) ([]json.RawMessage, []Column) {

‎presto/unmarshaller.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,11 @@ func UnmarshalQueryData(data []json.RawMessage, columns []Column, v any) error {
9797
if vPtr.Kind() != reflect.Pointer {
9898
return fmt.Errorf("%w must be a pointer, but it is %T", UnmarshalError, v)
9999
} else if vPtr.IsNil() {
100-
vPtr.Set(reflect.New(vPtr.Elem().Type()))
100+
if vPtr.CanAddr() {
101+
vPtr.Set(reflect.New(vPtr.Type().Elem()))
102+
} else {
103+
return fmt.Errorf("%w non-addressable value", UnmarshalError)
104+
}
101105
}
102106

103107
vArrayOrStruct := vPtr.Elem()

0 commit comments

Comments
 (0)