fix: upgrade to custom JSON handlers

rsdoiel · rsdoiel · commit 5eea8bef5387 · 2023-09-19T13:15:36.000-07:00
diff --git a/.gitignore b/.gitignore
@@ -37,3 +37,5 @@ bin/*
 # Project ignores
 *.json
 testout
+temp.*
+temp?.*
diff --git a/CITATION.cff b/CITATION.cff
@@ -16,7 +16,6 @@ maintainers:
     orcid: "https://orcid.org/0000-0003-0900-6903"
 
 repository-code: "https://github.com/caltechlibrary/datatools"
-version: 1.2.3
+version: 1.2.4
 license-url: "https://data.caltech.edu/license"
 keywords: [ "csv", "excel", "sql", "json", "xlsx", "golang", "bash" ]
-date-released: 2022-06-30
diff --git a/about.md b/about.md
@@ -11,16 +11,16 @@ authors:
     orcid: "https://orcid.org/0000-0003-0900-6903"
 
 repository-code: "https://github.com/caltechlibrary/datatools"
-version: 1.2.3
+version: 1.2.4
 license-url: "https://data.caltech.edu/license"
 keywords: [ "csv", "excel", "sql", "json", "xlsx", "golang", "bash" ]
-date-released: 2022-06-30
+
 ---
 
 About this software
 ===================
 
-## datatools 1.2.3
+## datatools 1.2.4
 
 ### Authors
 
diff --git a/cmd/csv2json/csv2json.go b/cmd/csv2json/csv2json.go
@@ -22,7 +22,6 @@ package main
 
 import (
 	"encoding/csv"
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -137,6 +136,7 @@ Convert data1.csv to JSON blobs, one line per blob
 	trimLeadingSpace bool
 	fieldsPerRecord  int
 	reuseRecord      bool
+	pretty           bool
 )
 
 func fmtTxt(src string, appName string, version string) string {
@@ -167,6 +167,7 @@ func main() {
 	flag.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in fields for CSV input")
 	flag.BoolVar(&reuseRecord, "reuse-record", false, "reuse the backing array")
 	flag.IntVar(&fieldsPerRecord, "fields-per-record", 0, "Set the number of fields expected in the CSV read, -1 to turn off")
+	flag.BoolVar(&pretty, "pretty", false, "pretty print the JSON output")
 
 	// Parse environment and options
 	flag.Parse()
@@ -260,7 +261,12 @@ func main() {
 				object[fmt.Sprintf("col_%d", col)] = val
 			}
 		}
-		src, err := json.Marshal(object)
+		var src []byte
+		if pretty {
+			src, err = datatools.JSONMarshalIndent(object, "", "    ")
+		} else {
+			src, err = datatools.JSONMarshal(object)
+		}
 		if err != nil {
 			if !quiet {
 				fmt.Fprintf(eout, "error row %d, %s\n", rowNo, err)
diff --git a/cmd/finddir/finddir.go b/cmd/finddir/finddir.go
@@ -116,8 +116,6 @@ Find all subdirectories starting with "img".
 	generateMarkdown bool
 	generateManPage  bool
 	quiet            bool
-	newLine          bool
-	eol              string
 
 	// Application Options
 	showModificationTime bool
@@ -197,8 +195,6 @@ func main() {
 	flag.StringVar(&outputFName, "o", "", "output filename")
 	flag.StringVar(&outputFName, "output", "", "output filename")
 	flag.BoolVar(&quiet, "quiet", false, "suppress error messages")
-	flag.BoolVar(&newLine, "nl", false, "if true add a trailing newline")
-	flag.BoolVar(&newLine, "newline", false, "if true add a trailing newline")
 
 	// Application Specific Options
 	flag.BoolVar(&showModificationTime, "m", false, "display file modification time before the path")
@@ -250,9 +246,6 @@ func main() {
 		fmt.Fprintf(out, "%s %s\n", appName, datatools.Version)
 		os.Exit(0)
 	}
-	if newLine {
-		eol = "\n"
-	}
 
 	if findPrefix == false && findSuffix == false && findContains == false {
 		findAll = true
@@ -283,5 +276,4 @@ func main() {
 			os.Exit(1)
 		}
 	}
-	fmt.Fprintf(out, "%s", eol)
 }
diff --git a/cmd/findfile/findfile.go b/cmd/findfile/findfile.go
@@ -116,8 +116,6 @@ Search the current directory and subdirectories for Markdown files with extensio
 	generateMarkdown bool
 	generateManPage  bool
 	quiet            bool
-	newLine          bool
-	eol              string
 
 	// App Specific Options
 	showModificationTime bool
@@ -197,8 +195,6 @@ func main() {
 	flag.StringVar(&outputFName, "o", "", "output filename")
 	flag.StringVar(&outputFName, "output", "", "output filename")
 	flag.BoolVar(&quiet, "quiet", false, "suppress error messages")
-	flag.BoolVar(&newLine, "nl", false, "if true add a trailing newline")
-	flag.BoolVar(&newLine, "newline", false, "if true add a trailing newline")
 
 	// App Specific Options
 	flag.BoolVar(&showModificationTime, "m", false, "display file modification time before the path")
@@ -249,9 +245,6 @@ func main() {
 		fmt.Fprintf(out, "%s %s\n", appName, datatools.Version)
 		os.Exit(0)
 	}
-	if newLine {
-		eol = "\n"
-	}
 
 	if findPrefix == false && findSuffix == false && findContains == false {
 		findAll = true
@@ -282,5 +275,4 @@ func main() {
 			os.Exit(1)
 		}
 	}
-	fmt.Fprintf(out, "%s", eol)
 }
diff --git a/cmd/jsoncols/jsoncols.go b/cmd/jsoncols/jsoncols.go
@@ -282,9 +282,9 @@ func main() {
 				row = append(row, result.(json.Number).String())
 			default:
 				if prettyPrint {
-					src, err = json.MarshalIndent(result, "", "    ")
+					src, err = datatools.JSONMarshalIndent(result, "", "    ")
 				} else {
-					src, err = json.Marshal(result)
+					src, err = datatools.JSONMarshal(result)
 				}
 				if err != nil {
 					fmt.Fprintln(eout, err)
@@ -338,9 +338,9 @@ func main() {
 				fmt.Fprintf(out, "%s", result.(json.Number).String())
 			default:
 				if prettyPrint {
-					src, err = json.MarshalIndent(result, "", "    ")
+					src, err = datatools.JSONMarshalIndent(result, "", "    ")
 				} else {
-					src, err = json.Marshal(result)
+					src, err = datatools.JSONMarshal(result)
 				}
 				if err != nil {
 					fmt.Fprintln(eout, err)
diff --git a/cmd/jsonrange/jsonrange.go b/cmd/jsonrange/jsonrange.go
@@ -7,7 +7,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io/ioutil"
@@ -287,7 +286,7 @@ func mapVals(data map[string]interface{}, limit int) ([]string, error) {
 		if i == limit {
 			return result, nil
 		}
-		outSrc, err := json.Marshal(val)
+		outSrc, err := datatools.JSONMarshal(val)
 		if err != nil {
 			return nil, err
 		}
@@ -303,7 +302,7 @@ func arrayVals(data []interface{}, limit int) ([]string, error) {
 		if i == limit {
 			return result, nil
 		}
-		outSrc, err := json.Marshal(val)
+		outSrc, err := datatools.JSONMarshal(val)
 		if err != nil {
 			return nil, err
 		}
diff --git a/cmd/string/string.go b/cmd/string/string.go
@@ -331,7 +331,7 @@ func fnSplit(in io.Reader, out io.Writer, eout io.Writer, args []string) int {
 	for _, arg := range args {
 		parts := strings.Split(arg, delimiter)
 		// Now assemble our JSON array and display it
-		src, err := json.Marshal(parts)
+		src, err := datatools.JSONMarshal(parts)
 		exitOnError(eout, err, quiet)
 		fmt.Fprintf(out, "%s%s", src, eol)
 	}
@@ -362,7 +362,7 @@ func fnSplitN(in io.Reader, out io.Writer, eout io.Writer, args []string) int {
 		parts := strings.SplitN(arg, delimiter, cnt)
 
 		// Now assemble our JSON array and display it
-		src, err := json.Marshal(parts)
+		src, err := datatools.JSONMarshal(parts)
 		exitOnError(eout, err, quiet)
 		fmt.Fprintf(out, "%s%s", src, eol)
 	}
diff --git a/cmd/toml2json/toml2json.go b/cmd/toml2json/toml2json.go
@@ -20,7 +20,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -126,9 +125,9 @@ func toml2JSON(in io.Reader, out io.Writer, printPrint bool) error {
 		return err
 	}
 	if prettyPrint == true {
-		src, err = json.MarshalIndent(m, "", "    ")
+		src, err = datatools.JSONMarshalIndent(m, "", "    ")
 	} else {
-		src, err = json.Marshal(m)
+		src, err = datatools.JSONMarshal(m)
 	}
 	fmt.Fprintf(out, "%s", src)
 	return nil
diff --git a/cmd/xlsx2json/xlsx2json.go b/cmd/xlsx2json/xlsx2json.go
@@ -20,7 +20,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
@@ -174,7 +173,7 @@ func xlsx2JSON(out io.Writer, workBookName, sheetName string) error {
 			}
 			results = append(results, cells)
 		}
-		src, err := json.Marshal(results)
+		src, err := datatools.JSONMarshal(results)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/yaml2json/yaml2json.go b/cmd/yaml2json/yaml2json.go
@@ -139,7 +139,7 @@ func yaml2JSON(in io.Reader, out io.Writer, printPrint bool) error {
 		if err != nil {
 			return err
 		}
-		src, err = json.MarshalIndent(m, "", "    ")
+		src, err = datatools.JSONMarshalIndent(m, "", "    ")
 		if err != nil {
 			return err
 		}
diff --git a/codemeta.json b/codemeta.json
@@ -4,12 +4,11 @@
     "license": "https://data.caltech.edu/license",
     "codeRepository": "https://github.com/caltechlibrary/datatools",
     "dateCreated": "2017-02-06",
-    "dateModified": "2023-06-30",
-    "datePublished": "2022-06-30",
+	"dateRelease": "2023-09-19",
     "downloadUrl": "https://github.com/caltechlibrary/datatools/releases/",
     "issueTracker": "https://github.com/caltechlibrary/datatools/issues",
     "name": "datatools",
-    "version": "1.2.3",
+    "version": "1.2.4",
     "description": "A set of command line tools for working with CSV, Excel Workbooks, JSON and structured text documents.",
     "applicationCategory": "computer programming",
     "developmentStatus": "active",
diff --git a/codemeta/codemeta.go b/codemeta/codemeta.go
@@ -1,7 +1,6 @@
 package codemeta
 
 import (
-	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -55,11 +54,11 @@ type Codemeta struct {
 }
 
 func (person *PersonOrOrganization) ToJSON() ([]byte, error) {
-	return json.MarshalIndent(person, "", "\t")
+	return JSONMarshalIndent(person, "", "\t")
 }
 
 func (cm *Codemeta) ToJSON() ([]byte, error) {
-	return json.MarshalIndent(cm, "", "\t")
+	return JSONMarshalIndent(cm, "", "\t")
 }
 
 func (person *PersonOrOrganization) ToCFF() ([]byte, error) {
diff --git a/codemeta/json_handlers.go b/codemeta/json_handlers.go
@@ -0,0 +1,50 @@
+package codemeta
+
+import (
+	"bytes"
+	"encoding/json"
+	"io"
+)
+
+// JSONUnmarshal is a custom JSON decoder so we can treat numbers easier
+func JSONUnmarshal(src []byte, data interface{}) error {
+        dec := json.NewDecoder(bytes.NewReader(src))
+        dec.UseNumber()
+        err := dec.Decode(&data)
+        if err != nil && err != io.EOF {
+                return err
+        }
+        return nil
+}
+
+// JSONMarshal provides provide a custom json encoder to solve a an issue with
+// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
+func JSONMarshal(data interface{}) ([]byte, error) {
+        buf := []byte{}
+        w := bytes.NewBuffer(buf)
+        enc := json.NewEncoder(w)
+        enc.SetEscapeHTML(false)
+        err := enc.Encode(data)
+        if err != nil {
+                return nil, err
+        }
+        return w.Bytes(), err
+}
+
+
+// JSONMarshalIndent provides provide a custom json encoder to solve a an issue with
+// HTML entities getting converted to UTF-8 code points by json.Marshal(), json.MarshalIndent().
+func JSONMarshalIndent(data interface{}, prefix string, indent string) ([]byte, error) {
+        buf := []byte{}
+        w := bytes.NewBuffer(buf)
+        enc := json.NewEncoder(w)
+        enc.SetEscapeHTML(false)
+        enc.SetIndent(prefix, indent)
+        err := enc.Encode(data)
+        if err != nil {
+                return nil, err
+        }
+        return w.Bytes(), err
+}
+
+
diff --git a/datatools.go b/datatools.go
@@ -23,7 +23,6 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/csv"
-	"encoding/json"
 	"fmt"
 	"io"
 	"strings"
@@ -168,7 +167,7 @@ func Text2Fields(r *bufio.Reader, options *Options) ([]byte, error) {
 	case AsCSV:
 		return CSVMarshal(words)
 	case AsJSON:
-		return json.Marshal(words)
+		return JSONMarshal(words)
 	default:
 		return []byte(strings.Join(words, options.Delimiter)), nil
 	}
diff --git a/how-to/2col.csv b/how-to/2col.csv
@@ -0,0 +1,2 @@
+one,three
+1,3
diff --git a/how-to/3col.csv b/how-to/3col.csv
@@ -0,0 +1,2 @@
+one,two,three
+1,2,3
diff --git a/how-to/expected2.txt b/how-to/expected2.txt
@@ -1,3 +1,3 @@
-42
 "Doe, Jane"
-"jane.doe@example.org"
+"jd@example.org"
+42
diff --git a/how-to/expected3-1.csv b/how-to/expected3-1.csv
@@ -0,0 +1,4 @@
+Number,Value
+one,1
+two,2
+three,3
diff --git a/how-to/expected3-3.txt b/how-to/expected3-3.txt
@@ -0,0 +1,2 @@
+My worksheet 1
+My worksheet 2
diff --git a/how-to/result3-1.txt b/how-to/result3-1.txt
@@ -0,0 +1 @@
+"Doe, Jane"
diff --git a/how-to/result3-2.txt b/how-to/result3-2.txt
@@ -0,0 +1 @@
+"Doe, Jane"|42
diff --git a/installer.sh b/installer.sh
@@ -4,7 +4,7 @@
 # Set the package name and version to install
 #
 PACKAGE="datatools"
-VERSION="1.2.3"
+VERSION="1.2.4"
 GIT_GROUP="caltechlibrary"
 RELEASE="https://github.com/$GIT_GROUP/$PACKAGE/releases/tag/v$VERSION"
 
diff --git a/json_handlers.go b/json_handlers.go
diff --git a/test_cmd.bash b/test_cmd.bash
diff --git a/version.go b/version.go

-Original file line number
+Diff line change
 # Project ignores
 *.json
 testout
 +temp.*
 +temp?.*
Original file line number	Diff line number	Diff line change
`@@ -7,7 +7,6 @@`
`7`	`7`	`package main`
`8`	`8`
`9`	`9`	`import (`
`10`		`- "encoding/json"`
`11`	`10`	`"flag"`
`12`	`11`	`"fmt"`
`13`	`12`	`"io/ioutil"`
`@@ -287,7 +286,7 @@ func mapVals(data map[string]interface{}, limit int) ([]string, error) {`
`287`	`286`	`if i == limit {`
`288`	`287`	`return result, nil`
`289`	`288`	`}`
`290`		`- outSrc, err := json.Marshal(val)`
	`289`	`+ outSrc, err := datatools.JSONMarshal(val)`
`291`	`290`	`if err != nil {`
`292`	`291`	`return nil, err`
`293`	`292`	`}`
`@@ -303,7 +302,7 @@ func arrayVals(data []interface{}, limit int) ([]string, error) {`
`303`	`302`	`if i == limit {`
`304`	`303`	`return result, nil`
`305`	`304`	`}`
`306`		`- outSrc, err := json.Marshal(val)`
	`305`	`+ outSrc, err := datatools.JSONMarshal(val)`
`307`	`306`	`if err != nil {`
`308`	`307`	`return nil, err`
`309`	`308`	`}`