Skip to content

Commit 1eb027c

Browse files
committed
feat: Add std.parseCsv and std.manifestCsv
1 parent fed90cd commit 1eb027c

17 files changed

+201
-7
lines changed

builtins.go

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"crypto/sha256"
2424
"crypto/sha512"
2525
"encoding/base64"
26+
"encoding/csv"
2627
"encoding/hex"
2728
"encoding/json"
2829
"fmt"
@@ -1512,6 +1513,170 @@ func builtinParseYAML(i *interpreter, str value) (value, error) {
15121513
return jsonToValue(i, elems[0])
15131514
}
15141515

1516+
func builtinParseCSVWithHeader(i *interpreter, arguments []value) (value, error) {
1517+
strv := arguments[0]
1518+
dv := arguments[1]
1519+
1520+
sval, err := i.getString(strv)
1521+
if err != nil {
1522+
return nil, err
1523+
}
1524+
s := sval.getGoString()
1525+
1526+
d := ',' // default delimiter
1527+
if dv.getType() != nullType {
1528+
dval, err := i.getString(dv)
1529+
if err != nil {
1530+
return nil, err
1531+
}
1532+
ds := dval.getGoString()
1533+
if len(ds) != 1 {
1534+
return nil, i.Error(fmt.Sprintf("Delimiter %s is invalid", ds))
1535+
}
1536+
d = rune(ds[0]) // conversion to rune
1537+
}
1538+
1539+
json := make([]interface{}, 0)
1540+
var keys []string
1541+
1542+
reader := csv.NewReader(strings.NewReader(s))
1543+
reader.Comma = d
1544+
1545+
for row := 0; ; row++ {
1546+
record, err := reader.Read()
1547+
if err == io.EOF {
1548+
break
1549+
}
1550+
if err != nil {
1551+
return nil, i.Error(fmt.Sprintf("failed to parse CSV: %s", err.Error()))
1552+
}
1553+
1554+
if row == 0 { // consider first row as header
1555+
// detect and handle duplicate headers
1556+
keyCount := map[string]int{}
1557+
for _, k := range record {
1558+
keyCount[k]++
1559+
if c := keyCount[k]; c > 1 {
1560+
keys = append(keys, fmt.Sprintf("%s__%d", k, c-1))
1561+
} else {
1562+
keys = append(keys, k)
1563+
}
1564+
}
1565+
} else {
1566+
j := make(map[string]interface{})
1567+
for i, k := range keys {
1568+
j[k] = record[i]
1569+
}
1570+
json = append(json, j)
1571+
}
1572+
}
1573+
return jsonToValue(i, json)
1574+
}
1575+
1576+
func builtinManifestCsv(i *interpreter, arguments []value) (value, error) {
1577+
arrv := arguments[0]
1578+
hv := arguments[1]
1579+
1580+
arr, err := i.getArray(arrv)
1581+
if err != nil {
1582+
return nil, err
1583+
}
1584+
1585+
var headers []string
1586+
if hv.getType() == nullType {
1587+
if len(arr.elements) == 0 { // no elements to select headers
1588+
return makeValueString(""), nil
1589+
}
1590+
1591+
// default to all headers
1592+
obj, err := i.evaluateObject(arr.elements[0])
1593+
if err != nil {
1594+
return nil, err
1595+
}
1596+
1597+
simpleObj := obj.uncached.(*simpleObject)
1598+
for fieldName := range simpleObj.fields {
1599+
headers = append(headers, fieldName)
1600+
}
1601+
} else {
1602+
// headers are provided
1603+
ha, err := i.getArray(hv)
1604+
if err != nil {
1605+
return nil, err
1606+
}
1607+
1608+
for _, elem := range ha.elements {
1609+
header, err := i.evaluateString(elem)
1610+
if err != nil {
1611+
return nil, err
1612+
}
1613+
headers = append(headers, header.getGoString())
1614+
}
1615+
}
1616+
1617+
var buf bytes.Buffer
1618+
w := csv.NewWriter(&buf)
1619+
1620+
// Write headers
1621+
w.Write(headers)
1622+
1623+
// Write rest of the rows
1624+
for _, elem := range arr.elements {
1625+
obj, err := i.evaluateObject(elem)
1626+
if err != nil {
1627+
return nil, err
1628+
}
1629+
1630+
record := make([]string, len(headers))
1631+
for c, h := range headers {
1632+
val, err := obj.index(i, h)
1633+
if err != nil { // no corresponding column
1634+
// skip to next column
1635+
continue
1636+
}
1637+
1638+
s, err := stringFromValue(i, val)
1639+
if err != nil {
1640+
return nil, err
1641+
}
1642+
record[c] = s
1643+
}
1644+
w.Write(record)
1645+
}
1646+
1647+
w.Flush()
1648+
1649+
return makeValueString(buf.String()), nil
1650+
}
1651+
1652+
func stringFromValue(i *interpreter, v value) (string, error) {
1653+
switch v.getType() {
1654+
case stringType:
1655+
s, err := i.getString(v)
1656+
if err != nil {
1657+
return "", err
1658+
}
1659+
return s.getGoString(), nil
1660+
case numberType:
1661+
n, err := i.getNumber(v)
1662+
if err != nil {
1663+
return "", err
1664+
}
1665+
return fmt.Sprint(n.value), nil
1666+
case booleanType:
1667+
b, err := i.getBoolean(v)
1668+
if err != nil {
1669+
return "", err
1670+
}
1671+
return fmt.Sprint(b.value), nil
1672+
case nullType:
1673+
return "", nil
1674+
default:
1675+
// for functionType, objectType and arrayType
1676+
return "", i.Error("invalid string conversion")
1677+
}
1678+
}
1679+
15151680
func jsonEncode(v interface{}) (string, error) {
15161681
buf := new(bytes.Buffer)
15171682
enc := json.NewEncoder(buf)
@@ -2520,6 +2685,8 @@ var funcBuiltins = buildBuiltinMap([]builtin{
25202685
&unaryBuiltin{name: "parseInt", function: builtinParseInt, params: ast.Identifiers{"str"}},
25212686
&unaryBuiltin{name: "parseJson", function: builtinParseJSON, params: ast.Identifiers{"str"}},
25222687
&unaryBuiltin{name: "parseYaml", function: builtinParseYAML, params: ast.Identifiers{"str"}},
2688+
&generalBuiltin{name: "parseCsvWithHeader", function: builtinParseCSVWithHeader, params: []generalBuiltinParameter{{name: "str"}, {name: "delimiter", defaultValue: &nullValue}}},
2689+
&generalBuiltin{name: "manifestCsv", function: builtinManifestCsv, params: []generalBuiltinParameter{{name: "json"}, {name: "headers", defaultValue: &nullValue}}},
25232690
&generalBuiltin{name: "manifestJsonEx", function: builtinManifestJSONEx, params: []generalBuiltinParameter{{name: "value"}, {name: "indent"},
25242691
{name: "newline", defaultValue: &valueFlatString{value: []rune("\n")}},
25252692
{name: "key_val_sep", defaultValue: &valueFlatString{value: []rune(": ")}}}},

linter/internal/types/stdlib.go

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,14 @@ func prepareStdlib(g *typeGraph) {
106106

107107
// Parsing
108108

109-
"parseInt": g.newSimpleFuncType(numberType, "str"),
110-
"parseOctal": g.newSimpleFuncType(numberType, "str"),
111-
"parseHex": g.newSimpleFuncType(numberType, "str"),
112-
"parseJson": g.newSimpleFuncType(jsonType, "str"),
113-
"parseYaml": g.newSimpleFuncType(jsonType, "str"),
114-
"encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"),
115-
"decodeUTF8": g.newSimpleFuncType(stringType, "arr"),
109+
"parseInt": g.newSimpleFuncType(numberType, "str"),
110+
"parseOctal": g.newSimpleFuncType(numberType, "str"),
111+
"parseHex": g.newSimpleFuncType(numberType, "str"),
112+
"parseJson": g.newSimpleFuncType(jsonType, "str"),
113+
"parseYaml": g.newSimpleFuncType(jsonType, "str"),
114+
"parseCsvWithHeader": g.newFuncType(jsonType, []ast.Parameter{required("str"), optional("delimiter")}),
115+
"encodeUTF8": g.newSimpleFuncType(numberArrayType, "str"),
116+
"decodeUTF8": g.newSimpleFuncType(stringType, "arr"),
116117

117118
// Manifestation
118119

@@ -124,6 +125,7 @@ func prepareStdlib(g *typeGraph) {
124125
"manifestJsonMinified": g.newSimpleFuncType(stringType, "value"),
125126
"manifestYamlDoc": g.newFuncType(stringType, []ast.Parameter{required("value"), optional("indent_array_in_object"), optional("quote_keys")}),
126127
"manifestYamlStream": g.newSimpleFuncType(stringType, "value"),
128+
"manifestCsv": g.newFuncType(stringType, []ast.Parameter{required("json"), optional("headers")}),
127129
"manifestXmlJsonml": g.newSimpleFuncType(stringType, "value"),
128130

129131
// Arrays

testdata/builtinManifestCsv.golden

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"head1,head2\nval1,val2\n,1\nval3,\n"

testdata/builtinManifestCsv.jsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
std.manifestCsv([{ "head1": "val1", "head2": "val2", "head3": "foo" }, { "head2": 1, "head3": "bar" }, { "head1": "val3" }], ["head1", "head2"])

testdata/builtinManifestCsv.linter.golden

Whitespace-only changes.

testdata/builtinManifestCsv2.golden

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"head1\nval1\nval2\n"

testdata/builtinManifestCsv2.jsonnet

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
std.manifestCsv([{ "head1": "val1" }, { "head1": "val2" }])

testdata/builtinManifestCsv2.linter.golden

Whitespace-only changes.
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
[
2+
{
3+
"head1": "val1",
4+
"head2": "val2"
5+
}
6+
]
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
std.parseCsvWithHeader("head1,head2\nval1,val2")

0 commit comments

Comments
 (0)