Add large, realistic tests and group folders that get too big during sort

Author: stilvoid

data/tosec-spectrum-sorted-100.txt

@@ -8,17 +8,6 @@ import (
 	"path/filepath"
 )
 
-// Sort converts a list of paths to files into a mapping from source paths
-// to destination paths where no directory in the destinations
-// contains any more than size files
-func Sort(sources []string, size int) map[string]string {
-	group := newGroup(sources)
-
-	groups := group.sort(size)
-
-	return groups.fileMap()
-}
-
 type group struct {
 	entries    []entry
 	prefixSize int
@@ -80,89 +69,6 @@ func (g group) name() string {
 	return fmt.Sprintf("%s-%s", a, b)
 }
 
-// split returns g split into sub-groups using the specific prefixSize.
-// the second return value indicates whether the group was able to meet the size constraint
-func (g group) split(prefixSize, size int) (groups, bool) {
-	success := true
-
-	counts := make(map[string]int)
-	prefixes := make([]string, 0)
-
-	// Fail if any individual prefix is too big
-	for _, entry := range g.entries {
-		prefix := entry.prefix(prefixSize)
-
-		if counts[prefix] == 0 {
-			prefixes = append(prefixes, prefix)
-		}
-
-		counts[prefix]++
-
-		if counts[prefix] > size {
-			success = false
-		}
-	}
-
-	groups := make(groups, 0)
-	cur := newGroup([]string{})
-	cur.prefixSize = prefixSize
-	cur.path = g.path
-	pos := 0
-
-	// Consolidate
-	for i, prefix := range prefixes {
-		// Copy
-		for j := 0; j < counts[prefix]; j++ {
-			cur.entries = append(cur.entries, g.entries[pos])
-			pos++
-		}
-
-		// Check if we split here
-		if i == len(prefixes)-1 || cur.Len()+counts[prefixes[i+1]] > size {
-			groups = append(groups, cur)
-			cur = newGroup([]string{})
-			cur.prefixSize = prefixSize
-			cur.path = g.path
-		}
-	}
-
-	if len(cur.entries) > 0 {
-		groups = append(groups, cur)
-	}
-
-	if len(groups) > size {
-		success = false
-	}
-
-	return groups, success
-}
-
-func (g group) sort(size int) groups {
-	if g.Len() <= size {
-		return groups{g}
-	}
-
-	gs, _ := g.split(g.prefixSize+1, size)
-
-	out := make(groups, 0)
-
-	for _, sub := range gs {
-		if sub.Len() <= size {
-			out = append(out, sub)
-		} else {
-			if len(gs) > 1 {
-				sub.path = filepath.Join(sub.path, sub.name())
-			}
-
-			for _, part := range sub.sort(size) {
-				out = append(out, part)
-			}
-		}
-	}
-
-	return out
-}
-
 func (g group) String() string {
 	path := filepath.Join(g.path, g.name())
 
@@ -186,9 +92,23 @@ type groups []group
 func (gs groups) fileMap() map[string]string {
 	out := make(map[string]string)
 
+	dedup := make(map[string]bool)
+
 	for _, g := range gs {
 		for src, dst := range g.fileMap() {
+			if dedup[dst] {
+				dir, fn := filepath.Split(dst)
+				ext := filepath.Ext(fn)
+				for i := 2; ; i++ {
+					dst = filepath.Join(dir, fmt.Sprintf("%s-%d%s", fn[:len(fn)-len(ext)], i, ext))
+					if !dedup[dst] {
+						break
+					}
+				}
+			}
+
 			out[src] = dst
+			dedup[dst] = true
 		}
 	}
 

data/tosec-spectrum-sorted-1000.txt

@@ -135,14 +135,14 @@ func TestGroupSortBasic(t *testing.T) {
 	}
 
 	if d := cmp.Diff(map[string]string{
-		"/does/not/matter/and a third": "a/and a third",
-		"/somewhere/a 4th file":        "a/a 4th file",
+		"/does/not/matter/and a third": "a-f/a/and a third",
+		"/somewhere/a 4th file":        "a-f/a/a 4th file",
 	}, groups[1].fileMap()); d != "" {
 		t.Error(d)
 	}
 
 	if d := cmp.Diff(map[string]string{
-		"/some/path/first": "f/first",
+		"/some/path/first": "a-f/f/first",
 	}, groups[2].fileMap()); d != "" {
 		t.Error(d)
 	}
@@ -189,9 +189,9 @@ func TestGroups(t *testing.T) {
 
 	if d := cmp.Diff(map[string]string{
 		"/another/path/2nd.file":       "#/2nd.file",
-		"/somewhere/a 4th file":        "a/a 4th file",
-		"/does/not/matter/and a third": "a/and a third",
-		"/some/path/first":             "f/first",
+		"/somewhere/a 4th file":        "a-f/a/a 4th file",
+		"/does/not/matter/and a third": "a-f/a/and a third",
+		"/some/path/first":             "a-f/f/first",
 	}, groups.fileMap()); d != "" {
 		t.Error(d)
 	}

data/tosec-spectrum-sorted-255.txt

@@ -0,0 +1,123 @@
+package retrosort
+
+import (
+	"math"
+	"path/filepath"
+)
+
+// Sort converts a list of paths to files into a mapping from source paths
+// to destination paths where no directory in the destinations
+// contains any more than size files
+func Sort(sources []string, size int) map[string]string {
+	group := newGroup(sources)
+
+	groups := group.sort(size)
+
+	return groups.fileMap()
+}
+
+// split returns g split into sub-groups using the specific prefixSize.
+// the second return value indicates whether the group was able to meet the size constraint
+func (g group) split(prefixSize, size int) (groups, bool) {
+	success := true
+
+	counts := make(map[string]int)
+	prefixes := make([]string, 0)
+
+	// Fail if any individual prefix is too big
+	for _, entry := range g.entries {
+		prefix := entry.prefix(prefixSize)
+
+		if counts[prefix] == 0 {
+			prefixes = append(prefixes, prefix)
+		}
+
+		counts[prefix]++
+
+		if counts[prefix] > size {
+			success = false
+		}
+	}
+
+	groups := make(groups, 0)
+	cur := newGroup([]string{})
+	cur.prefixSize = prefixSize
+	cur.path = g.path
+	pos := 0
+
+	// Consolidate
+	for i, prefix := range prefixes {
+		// Copy
+		for j := 0; j < counts[prefix]; j++ {
+			cur.entries = append(cur.entries, g.entries[pos])
+			pos++
+		}
+
+		// Check if we split here
+		if i == len(prefixes)-1 || cur.Len()+counts[prefixes[i+1]] > size {
+			groups = append(groups, cur)
+			cur = newGroup([]string{})
+			cur.prefixSize = prefixSize
+			cur.path = g.path
+		}
+	}
+
+	if len(cur.entries) > 0 {
+		groups = append(groups, cur)
+	}
+
+	if len(groups) > size {
+		success = false
+	}
+
+	// Group folders that got too big
+	if len(groups) > size {
+		parts := int(math.Ceil(float64(len(groups)) / float64(size)))
+
+		for i := 0; i < parts; i++ {
+			start := i * len(groups) / parts
+			end := (i+1)*len(groups)/parts - 1
+			if end > len(groups)-1 {
+				end = len(groups) - 1
+			}
+
+			if end-start > 0 {
+				a := groups[start].entries[0].prefix(prefixSize)
+				b := groups[end].entries[len(groups[end].entries)-1].prefix(prefixSize)
+				prefix := a + "-" + b
+
+				for j := start; j <= end; j++ {
+					groups[j].path = filepath.Join(groups[j].path, prefix)
+				}
+			}
+		}
+	}
+
+	return groups, success
+}
+
+func (g group) sort(size int) groups {
+	if g.Len() <= size {
+		return groups{g}
+	}
+
+	gs, _ := g.split(g.prefixSize+1, size)
+
+	out := make(groups, 0)
+
+	for _, sub := range gs {
+		if sub.Len() <= size {
+			out = append(out, sub)
+		} else {
+			if len(gs) > 1 {
+				sub.path = filepath.Join(sub.path, sub.name())
+			}
+
+			for _, part := range sub.sort(size) {
+				out = append(out, part)
+			}
+		}
+	}
+
+	return out
+}