feat(output): Implement gzipped output and make gzip-jsonl the default format

Author: Walz

cmd/src-fingerprint/main.go

@@ -47,12 +47,16 @@ func getProvider(providerStr string, token string, providerOptions provider.Opti
 	}
 }
 
-func getExporter(exporterStr string, output io.Writer) (exporter.Exporter, error) {
+func getExporter(exporterStr string, output io.WriteCloser) (exporter.Exporter, error) {
 	switch exporterStr {
 	case "json":
 		return exporter.NewJSONExporter(output), nil
+	case "gzip-json":
+		return exporter.NewGzipJSONExporter(output), nil
 	case "jsonl":
 		return exporter.NewJSONLExporter(output), nil
+	case "gzip-jsonl":
+		return exporter.NewGzipJSONLExporter(output), nil
 	default:
 		return nil, fmt.Errorf("invalid export format: %s", exporterStr)
 	}
@@ -105,6 +109,11 @@ func main() {
 				Value:   "-",
 				Usage:   "set output path to `FILE`. stdout by default",
 			},
+			&cli.StringFlag{
+				Name:  "export-format",
+				Value: "jsonl",
+				Usage: "export format: 'jsonl'/'gzip-jsonl'/'json'/'gzip-json'. 'jsonl' by default",
+			},
 			&cli.StringFlag{
 				Name:  "clone-dir",
 				Value: "-",
@@ -121,11 +130,6 @@ func main() {
 				Required: true,
 				Usage:    "vcs provider. options: 'gitlab'/'github'/'bitbucket'/'repository'",
 			},
-			&cli.StringFlag{
-				Name:  "export-format",
-				Value: "jsonl",
-				Usage: "export format: 'jsonl'/'json'. jsonl by default",
-			},
 			&cli.StringFlag{
 				Name:    "token",
 				Aliases: []string{"t"},
@@ -279,6 +283,5 @@ loop:
 	}
 
 	log.Infoln("Done")
-
 	return nil
 }

exporter/output.go

@@ -1,6 +1,7 @@
 package exporter
 
 import (
+	"compress/gzip"
 	"encoding/json"
 	"io"
 	"srcfingerprint"
@@ -20,12 +21,23 @@ type Exporter interface {
 type JSONExporter struct {
 	elements []*ExportGitFile
 	encoder  *json.Encoder
+	writer   io.WriteCloser
 }
 
-func NewJSONExporter(output io.Writer) Exporter {
+func NewJSONExporter(output io.WriteCloser) Exporter {
 	return &JSONExporter{
 		elements: []*ExportGitFile{},
 		encoder:  json.NewEncoder(output),
+		writer: output,
+	}
+}
+
+func NewGzipJSONExporter(output io.WriteCloser) Exporter {
+	compressedWriter := gzip.NewWriter(output)
+	return &JSONExporter{
+		elements: []*ExportGitFile{},
+		encoder:  json.NewEncoder(compressedWriter),
+		writer:   compressedWriter,
 	}
 }
 
@@ -36,16 +48,32 @@ func (e *JSONExporter) AddElement(gitFile *ExportGitFile) error {
 }
 
 func (e *JSONExporter) Close() error {
-	return e.encoder.Encode(e.elements)
+	if err1 := e.encoder.Encode(e.elements); err1 != nil {
+		return err1
+	}
+	if err2 := e.writer.Close(); err2 != nil {
+		return err2
+	}
+	return nil
 }
 
 type JSONLExporter struct {
 	encoder *json.Encoder
+	writer  io.WriteCloser
 }
 
-func NewJSONLExporter(output io.Writer) Exporter {
+func NewJSONLExporter(output io.WriteCloser) Exporter {
 	return &JSONLExporter{
 		encoder: json.NewEncoder(output),
+		writer:  output,
+	}
+}
+
+func NewGzipJSONLExporter(output io.WriteCloser) Exporter {
+	compressedWriter := gzip.NewWriter(output)
+	return &JSONLExporter{
+		encoder: json.NewEncoder(compressedWriter),
+		writer:  compressedWriter,
 	}
 }
 
@@ -54,5 +82,5 @@ func (e *JSONLExporter) AddElement(gitFile *ExportGitFile) error {
 }
 
 func (e *JSONLExporter) Close() error {
-	return nil
+	return e.writer.Close()
 }