handle deduplication in archives correctly

Author: bdon

main.go

@@ -71,7 +71,7 @@ var cli struct {
 
 	Merge struct {
 		Archives []string `arg:"" name:"inputs_then_output" help:"One or more input archives, followed by the output name last."`
-	} `cmd:"" help:"Merge multiple archives into a single archive" hidden:""`
+	} `cmd:"" help:"Merge multiple archives into a single archive"`
 
 	Convert struct {
 		Input           string `arg:"" help:"Input archive" type:"existingfile"`

pmtiles/merge.go

@@ -97,11 +97,16 @@ func Merge(logger *log.Logger, inputs []string) error {
 	acc := uint64(0)
 	addressedTiles := uint64(0)
 	tileContents := roaring64.New()
+	maxOffsets := make([]uint64, len(headers))
 	for idx := range mergedEntries {
-		// TODO: this algo is broken with any deduplication of tiles
-		// need to bookkeep on the max seen offset in each input archive
 		mergedEntries[idx].Entry.Offset = acc
-		acc += uint64(mergedEntries[idx].Entry.Length)
+
+		// skip any deduplication (backreferences)
+		if mergedEntries[idx].InputOffset >= maxOffsets[mergedEntries[idx].InputIdx] {
+			acc += uint64(mergedEntries[idx].Entry.Length)
+			maxOffsets[mergedEntries[idx].InputIdx] = mergedEntries[idx].InputOffset
+		}
+
 		addressedTiles += uint64(mergedEntries[idx].Entry.RunLength)
 		tileContents.Add(mergedEntries[idx].Entry.Offset)
 	}