@@ -23,6 +23,7 @@ import (
23
23
"crypto/sha256"
24
24
"crypto/sha512"
25
25
"encoding/base64"
26
+ "encoding/csv"
26
27
"encoding/hex"
27
28
"encoding/json"
28
29
"fmt"
@@ -1512,6 +1513,170 @@ func builtinParseYAML(i *interpreter, str value) (value, error) {
1512
1513
return jsonToValue (i , elems [0 ])
1513
1514
}
1514
1515
1516
+ func builtinParseCSVWithHeader (i * interpreter , arguments []value ) (value , error ) {
1517
+ strv := arguments [0 ]
1518
+ dv := arguments [1 ]
1519
+
1520
+ sval , err := i .getString (strv )
1521
+ if err != nil {
1522
+ return nil , err
1523
+ }
1524
+ s := sval .getGoString ()
1525
+
1526
+ d := ',' // default delimiter
1527
+ if dv .getType () != nullType {
1528
+ dval , err := i .getString (dv )
1529
+ if err != nil {
1530
+ return nil , err
1531
+ }
1532
+ ds := dval .getGoString ()
1533
+ if len (ds ) != 1 {
1534
+ return nil , i .Error (fmt .Sprintf ("Delimiter %s is invalid" , ds ))
1535
+ }
1536
+ d = rune (ds [0 ]) // conversion to rune
1537
+ }
1538
+
1539
+ json := make ([]interface {}, 0 )
1540
+ var keys []string
1541
+
1542
+ reader := csv .NewReader (strings .NewReader (s ))
1543
+ reader .Comma = d
1544
+
1545
+ for row := 0 ; ; row ++ {
1546
+ record , err := reader .Read ()
1547
+ if err == io .EOF {
1548
+ break
1549
+ }
1550
+ if err != nil {
1551
+ return nil , i .Error (fmt .Sprintf ("failed to parse CSV: %s" , err .Error ()))
1552
+ }
1553
+
1554
+ if row == 0 { // consider first row as header
1555
+ // detect and handle duplicate headers
1556
+ keyCount := map [string ]int {}
1557
+ for _ , k := range record {
1558
+ keyCount [k ]++
1559
+ if c := keyCount [k ]; c > 1 {
1560
+ keys = append (keys , fmt .Sprintf ("%s__%d" , k , c - 1 ))
1561
+ } else {
1562
+ keys = append (keys , k )
1563
+ }
1564
+ }
1565
+ } else {
1566
+ j := make (map [string ]interface {})
1567
+ for i , k := range keys {
1568
+ j [k ] = record [i ]
1569
+ }
1570
+ json = append (json , j )
1571
+ }
1572
+ }
1573
+ return jsonToValue (i , json )
1574
+ }
1575
+
1576
+ func builtinManifestCsv (i * interpreter , arguments []value ) (value , error ) {
1577
+ arrv := arguments [0 ]
1578
+ hv := arguments [1 ]
1579
+
1580
+ arr , err := i .getArray (arrv )
1581
+ if err != nil {
1582
+ return nil , err
1583
+ }
1584
+
1585
+ var headers []string
1586
+ if hv .getType () == nullType {
1587
+ if len (arr .elements ) == 0 { // no elements to select headers
1588
+ return makeValueString ("" ), nil
1589
+ }
1590
+
1591
+ // default to all headers
1592
+ obj , err := i .evaluateObject (arr .elements [0 ])
1593
+ if err != nil {
1594
+ return nil , err
1595
+ }
1596
+
1597
+ simpleObj := obj .uncached .(* simpleObject )
1598
+ for fieldName := range simpleObj .fields {
1599
+ headers = append (headers , fieldName )
1600
+ }
1601
+ } else {
1602
+ // headers are provided
1603
+ ha , err := i .getArray (hv )
1604
+ if err != nil {
1605
+ return nil , err
1606
+ }
1607
+
1608
+ for _ , elem := range ha .elements {
1609
+ header , err := i .evaluateString (elem )
1610
+ if err != nil {
1611
+ return nil , err
1612
+ }
1613
+ headers = append (headers , header .getGoString ())
1614
+ }
1615
+ }
1616
+
1617
+ var buf bytes.Buffer
1618
+ w := csv .NewWriter (& buf )
1619
+
1620
+ // Write headers
1621
+ w .Write (headers )
1622
+
1623
+ // Write rest of the rows
1624
+ for _ , elem := range arr .elements {
1625
+ obj , err := i .evaluateObject (elem )
1626
+ if err != nil {
1627
+ return nil , err
1628
+ }
1629
+
1630
+ record := make ([]string , len (headers ))
1631
+ for c , h := range headers {
1632
+ val , err := obj .index (i , h )
1633
+ if err != nil { // no corresponding column
1634
+ // skip to next column
1635
+ continue
1636
+ }
1637
+
1638
+ s , err := stringFromValue (i , val )
1639
+ if err != nil {
1640
+ return nil , err
1641
+ }
1642
+ record [c ] = s
1643
+ }
1644
+ w .Write (record )
1645
+ }
1646
+
1647
+ w .Flush ()
1648
+
1649
+ return makeValueString (buf .String ()), nil
1650
+ }
1651
+
1652
+ func stringFromValue (i * interpreter , v value ) (string , error ) {
1653
+ switch v .getType () {
1654
+ case stringType :
1655
+ s , err := i .getString (v )
1656
+ if err != nil {
1657
+ return "" , err
1658
+ }
1659
+ return s .getGoString (), nil
1660
+ case numberType :
1661
+ n , err := i .getNumber (v )
1662
+ if err != nil {
1663
+ return "" , err
1664
+ }
1665
+ return fmt .Sprint (n .value ), nil
1666
+ case booleanType :
1667
+ b , err := i .getBoolean (v )
1668
+ if err != nil {
1669
+ return "" , err
1670
+ }
1671
+ return fmt .Sprint (b .value ), nil
1672
+ case nullType :
1673
+ return "" , nil
1674
+ default :
1675
+ // for functionType, objectType and arrayType
1676
+ return "" , i .Error ("invalid string conversion" )
1677
+ }
1678
+ }
1679
+
1515
1680
func jsonEncode (v interface {}) (string , error ) {
1516
1681
buf := new (bytes.Buffer )
1517
1682
enc := json .NewEncoder (buf )
@@ -2520,6 +2685,8 @@ var funcBuiltins = buildBuiltinMap([]builtin{
2520
2685
& unaryBuiltin {name : "parseInt" , function : builtinParseInt , params : ast.Identifiers {"str" }},
2521
2686
& unaryBuiltin {name : "parseJson" , function : builtinParseJSON , params : ast.Identifiers {"str" }},
2522
2687
& unaryBuiltin {name : "parseYaml" , function : builtinParseYAML , params : ast.Identifiers {"str" }},
2688
+ & generalBuiltin {name : "parseCsvWithHeader" , function : builtinParseCSVWithHeader , params : []generalBuiltinParameter {{name : "str" }, {name : "delimiter" , defaultValue : & nullValue }}},
2689
+ & generalBuiltin {name : "manifestCsv" , function : builtinManifestCsv , params : []generalBuiltinParameter {{name : "json" }, {name : "headers" , defaultValue : & nullValue }}},
2523
2690
& generalBuiltin {name : "manifestJsonEx" , function : builtinManifestJSONEx , params : []generalBuiltinParameter {{name : "value" }, {name : "indent" },
2524
2691
{name : "newline" , defaultValue : & valueFlatString {value : []rune ("\n " )}},
2525
2692
{name : "key_val_sep" , defaultValue : & valueFlatString {value : []rune (": " )}}}},
0 commit comments