refactor: Make keep-sorted debug output a bit more useful for figuring out why things were sorted a certain way. (#88)

The debug output now reports all of the tokens that were actually used
in comparisons. It also now reports all of the line groups in
their sorted order instead of the line groups in their original order,
but I think that should be fine.

If you want to see it in action, run something like

```sh
$ go run . -vv --id keep-sorted-test - < <(cat goldens/*.in) 1>/dev/null
```

Author: JeffFaer

goldens/by_regex.err

@@ -1,3 +1,3 @@
-WRN while parsing option "by_regex": error parsing regexp: missing argument to repetition operator: `*` line=99
-WRN by_regex cannot be used with ignore_prefixes (consider adding a non-capturing group to the start of your regex instead of ignore_prefixes: "(?:foo|bar)") line=106
+WRN while parsing option "by_regex": error parsing regexp: missing argument to repetition operator: `*` line=107
+WRN by_regex cannot be used with ignore_prefixes (consider adding a non-capturing group to the start of your regex instead of ignore_prefixes: "(?:foo|bar)") line=114
 exit status 1

goldens/by_regex.in

@@ -57,6 +57,14 @@ Multiple regexes
   int bar
   keep-sorted-test end
 
+One regex, multiple capturing groups
+  keep-sorted-test start by_regex=(.*)_(.*)
+  foo_bar
+  foo_baz
+  bar_baz
+  baz_qux
+  keep-sorted-test end
+
 Multiline blocks
   keep-sorted-test start block=yes newline_separated=yes by_regex=(\w+)\(\)\s+{
   bool func2() {

goldens/by_regex.out

@@ -57,6 +57,14 @@ Multiple regexes
   long foo
   keep-sorted-test end
 
+One regex, multiple capturing groups
+  keep-sorted-test start by_regex=(.*)_(.*)
+  bar_baz
+  baz_qux
+  foo_bar
+  foo_baz
+  keep-sorted-test end
+
 Multiline blocks
   keep-sorted-test start block=yes newline_separated=yes by_regex=(\w+)\(\)\s+{
   List<SomeReallyLongTypeParameterThatWouldForceTheFunctionNameOntoTheNextLine>

keepsorted/block.go

@@ -235,8 +235,8 @@ func (b block) sorted() (sorted []string, alreadySorted bool) {
 		}
 	}
 
+	log.Printf("Creating line groups for block at index %d (options %v)", b.start, b.metadata.opts)
 	groups := groupLines(lines, b.metadata)
-	log.Printf("Previous %d groups were for block at index %d are (options %v)", len(groups), b.start, b.metadata.opts)
 	trimTrailingComma := handleTrailingComma(groups)
 
 	numNewlines := int(b.metadata.opts.NewlineSeparated)
@@ -257,7 +257,7 @@ func (b block) sorted() (sorted []string, alreadySorted bool) {
 		seen := map[string]bool{}
 		var deduped []*lineGroup
 		for _, lg := range groups {
-			if s := lg.joinedLines() + "\n" + strings.Join(lg.comment, "\n"); !seen[s] {
+			if s := lg.String(); !seen[s] {
 				seen[s] = true
 				deduped = append(deduped, lg)
 			} else {
@@ -268,11 +268,22 @@ func (b block) sorted() (sorted []string, alreadySorted bool) {
 	}
 
 	if alreadySorted && wasNewlineSeparated && !removedDuplicate && slices.IsSortedFunc(groups, compareLineGroups) {
+		log.Printf("It was already sorted!")
+		if log.Debug().Enabled() {
+			for _, lg := range groups {
+				log.Print(lg.DebugString())
+			}
+		}
 		trimTrailingComma(groups)
 		return lines, true
 	}
 
 	slices.SortStableFunc(groups, compareLineGroups)
+	if log.Debug().Enabled() {
+		for _, lg := range groups {
+			log.Print(lg.DebugString())
+		}
+	}
 
 	trimTrailingComma(groups)
 

keepsorted/line_group.go

@@ -32,6 +32,10 @@ type lineGroup struct {
 
 	// The actual content of the lineGroup.
 	lineGroupContent
+
+	// Track which methods are used during sorting so we can filter debugging
+	// output to just the parts that are relevant.
+	access accessRecorder
 }
 
 var compareLineGroups = comparingFunc((*lineGroup).commentOnly, falseFirst()).
@@ -50,6 +54,13 @@ type lineGroupContent struct {
 	lines   []string
 }
 
+type accessRecorder struct {
+	commentOnly   bool
+	regexTokens   []regexTokenAccessRecorder
+	joinedLines   bool
+	joinedComment bool
+}
+
 // groupLines splits lines into one or more lineGroups based on the provided options.
 func groupLines(lines []string, metadata blockMetadata) []*lineGroup {
 	var groups []*lineGroup
@@ -113,7 +124,6 @@ func groupLines(lines []string, metadata blockMetadata) []*lineGroup {
 		commentRange = indexRange{}
 		lineRange = indexRange{}
 		block = codeBlock{}
-		log.Printf("%#v", groups[len(groups)-1])
 	}
 	for i, l := range lines {
 		if metadata.opts.Block && !lineRange.empty() && block.expectsContinuation() {
@@ -331,6 +341,7 @@ func findQuote(s string, i int) string {
 }
 
 func (lg *lineGroup) append(s string) {
+	lg.access = accessRecorder{}
 	lg.lines[len(lg.lines)-1] = lg.lines[len(lg.lines)-1] + s
 }
 
@@ -339,42 +350,53 @@ func (lg *lineGroup) hasSuffix(s string) bool {
 }
 
 func (lg *lineGroup) trimSuffix(s string) {
+	lg.access = accessRecorder{}
 	lg.lines[len(lg.lines)-1] = strings.TrimSuffix(lg.lines[len(lg.lines)-1], s)
 }
 
 func (lg *lineGroup) commentOnly() bool {
+	lg.access.commentOnly = true
 	return len(lg.lines) == 0
 }
 
 func (lg *lineGroup) regexTokens() []regexToken {
 	// TODO: jfaer - Should we match regexes on the original content?
-	regexMatches := lg.opts.matchRegexes(lg.joinedLines())
+	regexMatches := lg.opts.matchRegexes(lg.internalJoinedLines())
 	ret := make([]regexToken, len(regexMatches))
+	if lg.access.regexTokens == nil {
+		lg.access.regexTokens = make([]regexTokenAccessRecorder, len(regexMatches))
+	}
 	for i, match := range regexMatches {
 		if match == nil {
 			// Regex did not match.
 			continue
 		}
 
 		ret[i] = make(regexToken, len(match))
+		if lg.access.regexTokens[i] == nil {
+			lg.access.regexTokens[i] = make(regexTokenAccessRecorder, len(match))
+		}
 		for j, s := range match {
-			prefixOrder := lg.prefixOrder
+			order := lg.prefixOrder
 			if j != 0 {
 				// Only try to match PrefixOrder on the first capture group in a regex.
 				// TODO: jfaer - Should this just be the first capture group in the first regex match?
-				prefixOrder = nil
+				order = func() *prefixOrder { return nil }
 			}
 			ret[i][j] = &captureGroupToken{
 				opts:        &lg.opts,
-				prefixOrder: prefixOrder,
+				prefixOrder: order,
 				raw:         s,
+				access:      &lg.access.regexTokens[i][j],
 			}
 		}
 	}
 	return ret
 }
 
-func (lg *lineGroup) joinedLines() string {
+// internalJoinedLines calculates the same thing as joinedLines, except it
+// doesn't record that it was used in the accessRecorder.
+func (lg *lineGroup) internalJoinedLines() string {
 	if len(lg.lines) == 0 {
 		return ""
 	}
@@ -394,23 +416,58 @@ func (lg *lineGroup) joinedLines() string {
 	return s.String()
 }
 
+func (lg *lineGroup) joinedLines() string {
+	lg.access.joinedLines = true
+	return lg.internalJoinedLines()
+}
+
 func (lg *lineGroup) joinedComment() string {
+	lg.access.joinedComment = true
 	if len(lg.comment) == 0 {
 		return ""
 	}
 	return strings.Join(lg.comment, "\n")
 }
 
-func (lg *lineGroup) GoString() string {
-	var comment strings.Builder
-	for _, c := range lg.comment {
-		comment.WriteString(fmt.Sprintf("  %#v\n", c))
+func (lg *lineGroup) DebugString() string {
+	var s strings.Builder
+	s.WriteString("LineGroup{\n")
+	if len(lg.comment) > 0 {
+		s.WriteString("comment=\n")
+		for _, c := range lg.comment {
+			fmt.Fprintf(&s, "  %#v\n", c)
+		}
 	}
-	var lines strings.Builder
-	for _, l := range lg.lines {
-		lines.WriteString(fmt.Sprintf("  %#v\n", l))
+	if len(lg.lines) > 0 {
+		s.WriteString("lines=\n")
+		for _, l := range lg.lines {
+			fmt.Fprintf(&s, "  %#v\n", l)
+		}
 	}
-	return fmt.Sprintf("LineGroup{\ncomment=\n%slines=\n%s}", comment.String(), lines.String())
+	if lg.access.commentOnly {
+		fmt.Fprintf(&s, "commentOnly=%t\n", lg.commentOnly())
+	}
+	if lg.access.regexTokens != nil {
+		for i, regex := range lg.regexTokens() {
+			if regex.wasUsed() {
+				fmt.Fprintf(&s, "regex[%d]=%s\n", i, regex.DebugString())
+			}
+		}
+	}
+	if lg.access.joinedLines {
+		if len(lg.lines) > 1 {
+			// Only print the joinedLines when they're meaningfully different from the
+			// raw lines above.
+			fmt.Fprintf(&s, "joinedLines=%#v\n", lg.joinedLines())
+		} else if !lg.access.joinedComment {
+			s.WriteString("linesTiebreaker=true\n")
+		}
+	}
+	if lg.access.joinedComment {
+		s.WriteString("commentTiebreaker=true\n")
+	}
+	s.WriteString("}")
+	return s.String()
 }
 
 func (lg *lineGroup) allLines() []string {
@@ -426,22 +483,66 @@ func (lg *lineGroup) String() string {
 
 type regexToken []*captureGroupToken
 
+type regexTokenAccessRecorder []captureGroupTokenAccessRecorder
+
+func (t regexToken) wasUsed() bool {
+	if t == nil {
+		// Report that the regex didn't match.
+		return true
+	}
+	for _, cg := range t {
+		if cg.wasUsed() {
+			return true
+		}
+	}
+	return false
+}
+
+func (t regexToken) DebugString() string {
+	if t == nil {
+		return "<did not match>"
+	}
+
+	captureGroups := make([]string, len(t))
+	for i, cg := range t {
+		if cg.wasUsed() {
+			captureGroups[i] = cg.DebugString()
+		} else {
+			captureGroups[i] = "<unused>"
+		}
+	}
+
+	if len(captureGroups) == 1 {
+		return captureGroups[0]
+	}
+	return fmt.Sprintf("%v", captureGroups)
+}
+
 type captureGroupToken struct {
 	opts        *blockOptions
 	prefixOrder func() *prefixOrder
 
 	raw string
+
+	access *captureGroupTokenAccessRecorder
+}
+
+type captureGroupTokenAccessRecorder struct {
+	prefix    bool
+	transform bool
 }
 
 func (t *captureGroupToken) prefix() orderedPrefix {
-	if t.prefixOrder == nil {
+	ord := t.prefixOrder()
+	if ord == nil {
 		return orderedPrefix{}
 	}
-
-	return t.prefixOrder().match(t.raw)
+	t.access.prefix = true
+	return ord.match(t.raw)
 }
 
 func (t *captureGroupToken) transform() numericTokens {
+	t.access.transform = true
 	// Combinations of switches (for example, case-insensitive and numeric
 	// ordering) which must be applied to create a single comparison key,
 	// otherwise a sub-ordering can preempt a total ordering:
@@ -466,3 +567,28 @@ func (t *captureGroupToken) transform() numericTokens {
 	}
 	return t.opts.maybeParseNumeric(s)
 }
+
+func (t captureGroupToken) wasUsed() bool {
+	return t.access.prefix || t.access.transform
+}
+
+func (t captureGroupToken) DebugString() string {
+	var s []string
+	if t.access.prefix {
+		s = append(s, fmt.Sprintf("prefix:%q", t.prefix().prefix))
+	}
+	if t.access.transform {
+		var tokens strings.Builder
+		if len(s) > 0 {
+			tokens.WriteString("tokens:")
+		}
+		fmt.Fprintf(&tokens, "%s", t.transform().DebugString())
+		s = append(s, tokens.String())
+	}
+
+	ret := strings.Join(s, " ")
+	if len(s) > 1 {
+		ret = "[" + ret + "]"
+	}
+	return ret
+}

keepsorted/options.go

@@ -473,6 +473,25 @@ type numericTokens struct {
 	i []*big.Int
 }
 
+func (t numericTokens) DebugString() string {
+	s := make([]string, 0, t.len())
+	for i := 0; i < t.len(); i++ {
+		if i%2 == 0 {
+			val := t.s[i/2]
+			if i == 0 && val == "" {
+				continue
+			}
+			s = append(s, fmt.Sprintf("%#v", t.s[i/2]))
+		} else {
+			s = append(s, fmt.Sprintf("%#v", t.i[i/2]))
+		}
+	}
+	if len(s) == 1 {
+		return s[0]
+	}
+	return fmt.Sprintf("%v", s)
+}
+
 func (t numericTokens) len() int {
 	return len(t.s) + len(t.i)
 }
@@ -502,6 +521,10 @@ type prefixOrder struct {
 }
 
 func newPrefixOrder(opts blockOptions) *prefixOrder {
+	if len(opts.PrefixOrder) == 0 {
+		return nil
+	}
+
 	// Assign a weight to each prefix so that they will be sorted into their
 	// predetermined order.
 	// Weights are negative so that entries with matching prefixes are put before
@@ -520,7 +543,11 @@ func newPrefixOrder(opts blockOptions) *prefixOrder {
 	return &prefixOrder{opts, prefixWeights, prefixes}
 }
 
-func (o prefixOrder) match(s string) orderedPrefix {
+func (o *prefixOrder) match(s string) orderedPrefix {
+	if o == nil {
+		return orderedPrefix{}
+	}
+
 	pre, _ := o.opts.hasPrefix(s, slices.Values(o.prefixes))
 	return orderedPrefix{pre, o.prefixWeights[pre]}
 }