First cut with basic set of features, well tested

Author: m-manu

.dockerignore

@@ -0,0 +1,6 @@
+/.idea/
+/.git/
+go-find-duplicates
+*.txt
+!default_exclusions.txt
+*.csv

.gitignore

@@ -0,0 +1,5 @@
+/.idea/
+go-find-duplicates
+*.txt
+!default_exclusions.txt
+*.csv

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.16.3-alpine3.13 as builder
+
+WORKDIR /opt/go-find-duplicates
+
+ADD . ./
+
+RUN go build
+
+FROM alpine:3.13
+
+RUN apk --no-cache add bash
+
+COPY --from=builder /opt/go-find-duplicates/go-find-duplicates /bin

README.md

@@ -0,0 +1,44 @@
+# Go Find Duplicates
+
+A blazingly-fast simple-to-use tool to find duplicate files (photos, videos, music, documents etc.) on your computer,
+portable hard drives etc.
+
+## How to install and use?
+
+Two ways: (one direct, one through docker)
+
+### Direct
+
+To install:
+
+1. Install Go version at least **1.16**
+    * On Ubuntu: `snap install go`
+    * On Mac: `brew install go`
+    * For any other OS: [Go downloads page](https://golang.org/dl/)
+2. Run command:
+   ```bash
+   go get github.com/m-manu/go-find-duplicates
+   ```
+
+To use:
+
+```bash
+go-find-duplicates {dir-1} {dir-2} ... {dir-n}
+```
+
+For more options and help, run:
+
+```bash
+go-find-duplicates -help
+```
+
+### Through Docker
+
+```bash
+docker run --rm -v /Volumes/PortableHD:/mnt/PortableHD manumk/go-find-duplicates:latest go-find-duplicates -output=print /mnt/PortableHD
+```
+
+In above command:
+
+* option `--rm` removes the container when it exits
+* option `-v` is mounts host directory `/Volumes/PortableHD` as `/mnt/PortableHD` inside the container

bytesutil/human_readable_size.go

@@ -0,0 +1,77 @@
+// Package bytesutil helps you convert byte sizes (such as  file size, data uploaded/downloaded etc.) to
+// human-readable strings. This allows conversion to decimal and binary formats.
+//
+// See: https://en.m.wikipedia.org/wiki/Byte#Multiple-byte_units
+package bytesutil
+
+import "fmt"
+
+// Constants for byte sizes in decimal and binary formats
+const (
+	KILO int64 = 1000        // 1000 power 1 (10 power 3)
+	KIBI int64 = 1024        // 1024 power 1 (2 power 10)
+	MEGA       = KILO * KILO // 1000 power 2 (10 power 6)
+	MEBI       = KIBI * KIBI // 1024 power 2 (2 power 20)
+	GIGA       = MEGA * KILO // 1000 power 3 (10 power 9)
+	GIBI       = MEBI * KIBI // 1024 power 3 (2 power 30)
+	TERA       = GIGA * KILO // 1000 power 4 (10 power 12)
+	TEBI       = GIBI * KIBI // 1024 power 4 (2 power 40)
+	PETA       = TERA * KILO // 1000 power 5 (10 power 15)
+	PEBI       = TEBI * KIBI // 1024 power 5 (2 power 50)
+	EXA        = PETA * KILO // 1000 power 6 (10 power 18)
+	EXBI       = PEBI * KIBI // 1024 power 6 (2 power 60)
+)
+
+// BinaryFormat formats a byte size to a human readable string in binary format.
+// Uses binary prefixes. See: https://en.m.wikipedia.org/wiki/Binary_prefix
+//
+// For example,
+//		fmt.Println(bytesutil.BinaryFormat(2140))
+// prints
+//		2.09 KiB
+func BinaryFormat(size int64) string {
+	if size < 0 {
+		return ""
+	} else if size < KIBI {
+		return fmt.Sprintf("%d B", size)
+	} else if size < MEBI {
+		return fmt.Sprintf("%.2f KiB", float64(size)/float64(KIBI))
+	} else if size < GIBI {
+		return fmt.Sprintf("%.2f MiB", float64(size)/float64(MEBI))
+	} else if size < TEBI {
+		return fmt.Sprintf("%.2f GiB", float64(size)/float64(GIBI))
+	} else if size < PEBI {
+		return fmt.Sprintf("%.2f TiB", float64(size)/float64(TEBI))
+	} else if size < EXBI {
+		return fmt.Sprintf("%.2f PiB", float64(size)/float64(PEBI))
+	} else {
+		return fmt.Sprintf("%.2f EiB", float64(size)/float64(EXBI))
+	}
+}
+
+// DecimalFormat formats a byte size to a human readable string in decimal format.
+// Uses metric prefixes. See: https://en.m.wikipedia.org/wiki/Metric_prefix
+//
+// For example,
+//		fmt.Println(bytesutil.DecimalFormat(2140))
+// prints
+//		2.14KB
+func DecimalFormat(size int64) string {
+	if size < 0 {
+		return ""
+	} else if size < KILO {
+		return fmt.Sprintf("%d B", size)
+	} else if size < MEGA {
+		return fmt.Sprintf("%.2f KB", float64(size)/float64(KILO))
+	} else if size < GIGA {
+		return fmt.Sprintf("%.2f MB", float64(size)/float64(MEGA))
+	} else if size < TERA {
+		return fmt.Sprintf("%.2f GB", float64(size)/float64(GIGA))
+	} else if size < PETA {
+		return fmt.Sprintf("%.2f TB", float64(size)/float64(TERA))
+	} else if size < EXA {
+		return fmt.Sprintf("%.2f PB", float64(size)/float64(PETA))
+	} else {
+		return fmt.Sprintf("%.2f EB", float64(size)/float64(EXA))
+	}
+}

bytesutil/human_readable_size_test.go

@@ -0,0 +1,21 @@
+package bytesutil
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+func TestFormats(t *testing.T) {
+	tests := map[int64][2]string{
+		-1:                        {"", ""},
+		0:                         {"0 B", "0 B"},
+		1_023:                     {"1023 B", "1.02 KB"},
+		2_140:                     {"2.09 KiB", "2.14 KB"},
+		2_828_382:                 {"2.70 MiB", "2.83 MB"},
+		2_341_234_123_412_341_234: {"2.03 EiB", "2.34 EB"},
+	}
+	for value, expectedValues := range tests {
+		assert.Equal(t, expectedValues[0], BinaryFormat(value))
+		assert.Equal(t, expectedValues[1], DecimalFormat(value))
+	}
+}

default_exclusions.txt

@@ -0,0 +1,12 @@
+.DS_Store
+System Volume Information
+$RECYCLE.BIN
+desktop.ini
+Thumbs.db
+.picasaoriginals
+.picasa.ini
+.Trashes
+.TemporaryItems
+.Spotlight-V100
+.fseventsd
+_PAlbTN

entity/digest_to_files.go

@@ -0,0 +1,42 @@
+package entity
+
+import (
+	"sync"
+)
+
+// DigestToFiles is a multi-map with FileDigest keys and string values.
+// Writes to this is goroutine-safe.
+type DigestToFiles struct {
+	mx   *sync.Mutex
+	data map[FileDigest][]string
+}
+
+// NewDigestToFiles creates new DigestToFiles
+func NewDigestToFiles(size int) (m *DigestToFiles) {
+	return &DigestToFiles{
+		data: make(map[FileDigest][]string, size),
+		mx:   &sync.Mutex{},
+	}
+}
+
+// Set sets a value for the key
+func (m *DigestToFiles) Set(key FileDigest, value string) {
+	m.mx.Lock()
+	m.data[key] = append(m.data[key], value)
+	m.mx.Unlock()
+}
+
+// Remove removes entry in the map
+func (m *DigestToFiles) Remove(fd FileDigest) {
+	delete(m.data, fd)
+}
+
+// Map returns internal map to iterate over
+func (m *DigestToFiles) Map() map[FileDigest][]string {
+	return m.data
+}
+
+// Size returns size of map
+func (m *DigestToFiles) Size() int {
+	return len(m.data)
+}

entity/file_digest.go

@@ -0,0 +1,17 @@
+package entity
+
+import (
+	"fmt"
+	"github.com/m-manu/go-find-duplicates/bytesutil"
+)
+
+// FileDigest contains properties of a file that makes the file unique to a very high degree of confidence
+type FileDigest struct {
+	FileExtension string
+	FileSize      int64
+	FileFuzzyHash string
+}
+
+func (f FileDigest) String() string {
+	return fmt.Sprintf("%v/%v/%v", f.FileExtension, f.FileFuzzyHash, bytesutil.BinaryFormat(f.FileSize))
+}

entity/file_ext_and_size.go

@@ -0,0 +1,16 @@
+package entity
+
+import "fmt"
+
+// FileExtAndSize is a struct of file extension and file size
+type FileExtAndSize struct {
+	FileExtension string
+	FileSize      int64
+}
+
+func (f FileExtAndSize) String() string {
+	return fmt.Sprintf("%v/%v", f.FileExtension, f.FileSize)
+}
+
+// FileExtAndSizeToFiles is a multi-map of FileExtAndSize key and string values
+type FileExtAndSizeToFiles map[FileExtAndSize][]string