added -use-lazy-quotes and -trim-leading-space to all csv commands for CSV input parsing

Author: rsdoiel

cmds/csv2json/csv2json.go

@@ -65,9 +65,11 @@ Convert data1.csv to JSON blobs, one line per blob
 	eol                  string
 
 	// Application Options
-	useHeader bool
-	asBlobs   bool
-	delimiter string
+	useHeader        bool
+	asBlobs          bool
+	delimiter        string
+	lazyQuotes       bool
+	trimLeadingSpace bool
 )
 
 func main() {
@@ -94,6 +96,8 @@ func main() {
 	app.BoolVar(&useHeader, "use-header", true, "treat the first row as field names")
 	app.BoolVar(&asBlobs, "as-blobs", false, "output as one JSON blob per line")
 	app.StringVar(&delimiter, "d,delimiter", "", "set the delimter character")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes for for CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in fields for CSV input")
 
 	// Parse environment and options
 	app.Parse()
@@ -140,6 +144,8 @@ func main() {
 	rowNo := 0
 	fieldNames := []string{}
 	r := csv.NewReader(app.In)
+	r.LazyQuotes = lazyQuotes
+	r.TrimLeadingSpace = trimLeadingSpace
 	if delimiter != "" {
 		r.Comma = datatools.NormalizeDelimiterRune(delimiter)
 	}

cmds/csv2mdtable/csv2mdtable.go

@@ -60,7 +60,9 @@ Convert data1.csv to data1.md using options.
 	eol                  string
 
 	// Application Options
-	delimiter string
+	delimiter        string
+	lazyQuotes       bool
+	trimLeadingSpace bool
 )
 
 func main() {
@@ -85,6 +87,8 @@ func main() {
 
 	// Application Options
 	app.StringVar(&delimiter, "d,delimiter", "", "set delimiter character")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "using lazy quotes for CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in field(s) for CSV input")
 
 	// Parse environment and options
 	app.Parse()
@@ -129,6 +133,9 @@ func main() {
 	}
 
 	r := csv.NewReader(app.In)
+	r.LazyQuotes = lazyQuotes
+	r.TrimLeadingSpace = trimLeadingSpace
+
 	if delimiter != "" {
 		r.Comma = datatools.NormalizeDelimiterRune(delimiter)
 	}

cmds/csv2xlsx/csv2xlsx.go

@@ -65,12 +65,14 @@ the workbook's 'My worksheet 2' sheet.
 	quiet                bool
 
 	// App Specific Options
-	workbookName string
-	sheetName    string
-	delimiter    string
+	workbookName     string
+	sheetName        string
+	delimiter        string
+	lazyQuotes       bool
+	trimLeadingSpace bool
 )
 
-func csv2XLSXSheet(in *os.File, workbookName string, sheetName string, delimiter string) error {
+func csv2XLSXSheet(in *os.File, workbookName string, sheetName string, delimiter string, lazyQuotes, trimLeadingSpace bool) error {
 	var workbook *xlsx.File
 
 	// Open the workbook
@@ -89,6 +91,9 @@ func csv2XLSXSheet(in *os.File, workbookName string, sheetName string, delimiter
 	}
 
 	r := csv.NewReader(in)
+	r.LazyQuotes = lazyQuotes
+	r.TrimLeadingSpace = trimLeadingSpace
+
 	if delimiter != "" {
 		r.Comma = datatools.NormalizeDelimiterRune(delimiter)
 	}
@@ -134,6 +139,8 @@ func main() {
 	app.StringVar(&workbookName, "workbook", "", "Workbook name")
 	app.StringVar(&sheetName, "sheet", "", "Sheet name to create/replace")
 	app.StringVar(&delimiter, "d,delimiter", "", "set delimiter character (input)")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes for CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in field(s) for CSV input")
 
 	// Parse environment and options
 	app.Parse()
@@ -196,6 +203,6 @@ func main() {
 	if len(sheetName) == 0 {
 		cli.ExitOnError(app.Eout, fmt.Errorf("Missing sheet name"), quiet)
 	}
-	err = csv2XLSXSheet(app.In, workbookName, sheetName, delimiter)
+	err = csv2XLSXSheet(app.In, workbookName, sheetName, delimiter, lazyQuotes, trimLeadingSpace)
 	cli.ExitOnError(app.Eout, err, quiet)
 }

cmds/csvcleaner/csvcleaner.go

@@ -48,15 +48,15 @@ Normalizing a spread sheet's column count to 5 padding columns as needed per row
 
     cat mysheet.csv | %s -field-per-row=5
 
-Trim leading spaces.
+Trim leading spaces from output.
 
     cat mysheet.csv | %s -left-trim
 
-Trim trailing spaces.
+Trim trailing spaces from output.
 
     cat mysheet.csv | %s -right-trim
 
-Trim leading and trailing spaces
+Trim leading and trailing spaces from output.
 
     cat mysheet.csv | %s -trim
 `
@@ -74,18 +74,19 @@ Trim leading and trailing spaces
 	//eol                  string
 
 	// App Options
-	comma             string
-	rowComment        string
-	fieldsPerRecord   int
-	lazyQuotes        bool
-	trailingComma     bool
-	trimSpace         bool
-	trimLeadingSpace  bool
-	trimTrailingSpace bool
-	reuseRecord       bool
-	commaOut          string
-	useCRLF           bool
-	stopOnError       bool
+	comma            string
+	rowComment       string
+	fieldsPerRecord  int
+	trailingComma    bool
+	trimSpace        bool
+	trimLeftSpace    bool
+	trimRightSpace   bool
+	reuseRecord      bool
+	commaOut         string
+	useCRLF          bool
+	stopOnError      bool
+	lazyQuotes       bool
+	trimLeadingSpace bool
 
 	verbose bool
 )
@@ -112,16 +113,17 @@ func main() {
 
 	// Application specific options
 	app.IntVar(&fieldsPerRecord, "fields-per-row", 0, "set the number of columns to output right padding empty cells as needed")
-	app.BoolVar(&lazyQuotes, "use-lazy-quoting", false, "If LazyQuotes is true, a quote may appear in an unquoted field and a non-doubled quote may appear in a quoted field.")
-	app.BoolVar(&trimSpace, "trim", false, "If set to true leading and trailing white space in a field is ignored.")
-	app.BoolVar(&trimLeadingSpace, "left-trim", false, "If set to true leading white space in a field is ignored.")
-	app.BoolVar(&trimTrailingSpace, "right-trim", false, "If set to true trailing white space in a field is ignored.")
+	app.BoolVar(&trimSpace, "trim", false, "trim spaces on CSV out")
+	app.BoolVar(&trimLeftSpace, "left-trim", false, "left trim spaces on CSV out")
+	app.BoolVar(&trimRightSpace, "right-trim", false, "right trim spaces on CSV out")
 	app.BoolVar(&reuseRecord, "reuse", true, "if false then a new array is allocated for each row processed, if true the array gets reused")
 	app.StringVar(&comma, "comma", "", "if set use this character in place of a comma for delimiting cells")
 	app.StringVar(&rowComment, "comment-char", "", "if set, rows starting with this character will be ignored as comments")
 	app.StringVar(&commaOut, "output-comma", "", "if set use this character in place of a comma for delimiting output cells")
 	app.BoolVar(&useCRLF, "use-crlf", false, "if set use a charage return and line feed in output")
 	app.BoolVar(&stopOnError, "stop-on-error", false, "exit on error, useful if you're trying to debug a problematic CSV file")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes for CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space from field(s) for CSV input")
 
 	app.BoolVar(&verbose, "V,verbose", false, "write verbose output to standard error")
 
@@ -162,16 +164,10 @@ func main() {
 		fmt.Fprintln(app.Out, app.Version())
 		os.Exit(0)
 	}
-	/*
-		if newLine {
-			eol = "\n"
-		}
-	*/
 
 	// Loop through input CSV, apply options, write to output CSV
-	if trimSpace == true {
-		trimLeadingSpace = true
-		trimTrailingSpace = true
+	if trimLeftSpace == true && trimRightSpace == true {
+		trimSpace = true
 	}
 
 	// Setup our CSV reader with any cli options
@@ -237,10 +233,17 @@ func main() {
 				}
 			}
 		}
-		if trimSpace {
+		if trimSpace || trimLeftSpace || trimRightSpace {
 			for i := range row {
 				s := row[i]
-				row[i] = strings.TrimSpace(s)
+				switch {
+				case trimSpace:
+					row[i] = strings.TrimSpace(s)
+				case trimRightSpace:
+					row[i] = strings.TrimRight(s, " \t\n\r")
+				case trimLeftSpace:
+					row[i] = strings.TrimLeft(s, " \t\n\r")
+				}
 			}
 		}
 		if err := w.Write(row); err != nil {

cmds/csvcols/csvcols.go

@@ -74,11 +74,13 @@ Using options filter a 3 column CSV file for columns 1,3 into 2col.csv
 	quiet                bool
 
 	// App Options
-	outputColumns   string
-	prefixUUID      bool
-	skipHeaderRow   bool
-	delimiter       string
-	outputDelimiter string
+	outputColumns    string
+	prefixUUID       bool
+	skipHeaderRow    bool
+	delimiter        string
+	outputDelimiter  string
+	lazyQuotes       bool
+	trimLeadingSpace bool
 )
 
 func selectedColumns(rowNo int, record []string, columnNos []int, prefixUUID bool, skipHeaderRow bool) []string {
@@ -106,10 +108,13 @@ func selectedColumns(rowNo int, record []string, columnNos []int, prefixUUID boo
 	return result
 }
 
-func CSVColumns(in *os.File, out *os.File, columnNos []int, prefixUUID bool, skipHeaderRow bool, delimiterIn string, delimiterOut string) {
+func CSVColumns(in *os.File, out *os.File, columnNos []int, prefixUUID bool, skipHeaderRow bool, delimiterIn string, delimiterOut string, lazyQuotes, trimLeadingSpace bool) {
 	var err error
 
 	r := csv.NewReader(in)
+	r.LazyQuotes = lazyQuotes
+	r.TrimLeadingSpace = trimLeadingSpace
+
 	w := csv.NewWriter(out)
 	if delimiterIn != "" {
 		r.Comma = datatools.NormalizeDelimiterRune(delimiterIn)
@@ -161,6 +166,8 @@ func main() {
 	app.StringVar(&outputDelimiter, "od,output-delimiter", "", "set the output delimiter character")
 	app.BoolVar(&skipHeaderRow, "skip-header-row", true, "skip the header row")
 	app.BoolVar(&prefixUUID, "uuid", false, "add a prefix row with generated UUID cell")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes on CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in field(s) for CSV input")
 
 	// Parse env and options
 	app.Parse()
@@ -212,7 +219,7 @@ func main() {
 				columnNos[i] = 0
 			}
 		}
-		CSVColumns(app.In, app.Out, columnNos, prefixUUID, skipHeaderRow, delimiter, outputDelimiter)
+		CSVColumns(app.In, app.Out, columnNos, prefixUUID, skipHeaderRow, delimiter, outputDelimiter, lazyQuotes, trimLeadingSpace)
 		os.Exit(0)
 	}
 

cmds/csvfind/csvfind.go

@@ -84,6 +84,8 @@ You can also search for phrases in columns.
 	trimSpaces         bool
 	allowDuplicates    bool
 	delimiter          string
+	lazyQuotes         bool
+	trimLeadingSpace   bool
 )
 
 func main() {
@@ -124,6 +126,8 @@ func main() {
 	app.BoolVar(&skipHeaderRow, "skip-header-row", true, "skip the header row")
 	app.BoolVar(&allowDuplicates, "allow-duplicates", true, "allow duplicates when searching for matches")
 	app.BoolVar(&trimSpaces, "trimspace,trimspaces", false, "trim spaces around cell values before comparing")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes on CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leadings space in field(s) for CSV input")
 
 	// Parse env and options
 	app.Parse()
@@ -190,6 +194,8 @@ func main() {
 	}
 
 	csvIn := csv.NewReader(app.In)
+	csvIn.LazyQuotes = lazyQuotes
+	csvIn.TrimLeadingSpace = trimLeadingSpace
 	csvOut := csv.NewWriter(app.Out)
 	if delimiter != "" {
 		csvIn.Comma = datatools.NormalizeDelimiterRune(delimiter)

cmds/csvjoin/csvjoin.go

@@ -58,23 +58,25 @@ merged-data.csv..
 	quiet                bool
 
 	// App Options
-	verbose         bool
-	csv1FName       string
-	csv2FName       string
-	col1            int
-	col2            int
-	trimSpaces      bool
-	caseSensitive   bool
-	useContains     bool
-	useLevenshtein  bool
-	insertCost      int
-	deleteCost      int
-	substituteCost  int
-	maxEditDistance int
-	stopWordsOption string
-	allowDuplicates bool
-	asInMemory      bool
-	delimiter       string
+	verbose          bool
+	csv1FName        string
+	csv2FName        string
+	col1             int
+	col2             int
+	trimSpaces       bool
+	caseSensitive    bool
+	useContains      bool
+	useLevenshtein   bool
+	insertCost       int
+	deleteCost       int
+	substituteCost   int
+	maxEditDistance  int
+	stopWordsOption  string
+	allowDuplicates  bool
+	asInMemory       bool
+	delimiter        string
+	lazyQuotes       bool
+	trimLeadingSpace bool
 )
 
 // cellsMatch checks if two cells' values match
@@ -185,6 +187,8 @@ func main() {
 	app.BoolVar(&trimSpaces, "trimspaces", false, "trim spaces around cell values before comparing")
 	app.BoolVar(&asInMemory, "in-memory", false, "if true read both CSV files")
 	app.StringVar(&delimiter, "d,delimiter", "", "set delimiter character")
+	app.BoolVar(&lazyQuotes, "use-lazy-quotes", false, "use lazy quotes for CSV input")
+	app.BoolVar(&trimLeadingSpace, "trim-leading-space", false, "trim leading space in field(s) for CSV input")
 
 	// Parse env and options
 	app.Parse()
@@ -261,11 +265,15 @@ func main() {
 	cli.ExitOnError(app.Eout, err, quiet)
 	defer fp1.Close()
 	csv1 := csv.NewReader(fp1)
+	csv1.LazyQuotes = lazyQuotes
+	csv1.TrimLeadingSpace = trimLeadingSpace
 
 	fp2, err := os.Open(csv2FName)
 	cli.ExitOnError(app.Eout, err, quiet)
 	defer fp2.Close()
 	csv2 := csv.NewReader(fp2)
+	csv2.LazyQuotes = lazyQuotes
+	csv2.TrimLeadingSpace = trimLeadingSpace
 
 	w := csv.NewWriter(app.Out)
 	if delimiter != "" {