remove older converter

Author: taigrr

cmd/converter/.gitignore

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/gob"
 	"flag"
 	"os"
 	"path/filepath"
@@ -48,36 +47,31 @@ func main() {
 	// Create the filesystem.
 	// The filesystem is created at the output path.
 	os.MkdirAll(*outputPath, 0o777)
-	saveState, err := os.Open(filepath.Join(*outputPath, "boundaries.gob"))
-	boundaries := []util.ZipBoundary{}
+	boundaries, err := util.DetermineZipBoundaries(*directoryPath, *thresholdSize)
 	if err != nil {
+		panic(err)
+	}
 
-		boundaries, err = util.DetermineZipBoundaries(*directoryPath, *thresholdSize)
+	for _, boundary := range boundaries {
+		lt, err := util.CreateInitialDJAFSManifest(boundary.Path, *outputPath, boundary.IncludeSubdirs)
 		if err != nil {
 			panic(err)
 		}
-		f, err := os.Create(filepath.Join(*outputPath, "boundaries.gob"))
+		subpath := strings.TrimPrefix(boundary.Path, *directoryPath)
+		newPath := filepath.Join(*outputPath, util.DataDir, subpath)
+		err = os.MkdirAll(newPath, 0o777)
 		if err != nil {
 			panic(err)
 		}
-		gob.NewEncoder(f).Encode(boundaries)
-		f.Close()
-	} else {
-		gob.NewDecoder(saveState).Decode(&boundaries)
-		saveState.Close()
-	}
-	for _, boundary := range boundaries {
-		lt, err := util.CreateInitialDJAFSManifest(boundary.Path, *outputPath, boundary.IncludeSubdirs)
+		err = util.WriteJSONFile(filepath.Join(newPath, "lookups.djfl"), lt)
 		if err != nil {
 			panic(err)
 		}
-		subpath := strings.TrimPrefix(boundary.Path, *directoryPath)
-		newPath := filepath.Join(*outputPath, util.MappingDir, subpath)
-		err = os.MkdirAll(newPath, 0o777)
+		metadata, err := lt.GenerateMetadata("")
 		if err != nil {
 			panic(err)
 		}
-		err = util.WriteJSONFile(filepath.Join(newPath, "lookups.djfl"), lt)
+		err = util.WriteJSONFile(filepath.Join(newPath, "metadata.djfm"), metadata)
 		if err != nil {
 			panic(err)
 		}

cmd/converter/main.go

@@ -31,11 +31,13 @@ func GetVersion() string {
 // metadata struct file to accompany the zip
 func (l LookupTable) GenerateMetadata(path string) (Metadata, error) {
 	var m Metadata
-	stat, err := os.Stat(path)
-	if err != nil {
-		return m, err
+	if path != "" {
+		stat, err := os.Stat(path)
+		if err != nil {
+			return m, err
+		}
+		m.CompressedSize = int(stat.Size())
 	}
-	m.CompressedSize = int(stat.Size())
 	m.DJAFSVersion = GetVersion()
 	m.NewestFileTS = l.GetNewestFileTS()
 	m.OldestFileTS = l.GetOldestFileTS()

cmd/uniconverter/main.go

@@ -12,10 +12,9 @@ import (
 )
 
 var (
-	gcLock     sync.Mutex
-	WorkDir    = ".work"
-	DataDir    = ".data"
-	MappingDir = ".mappings"
+	gcLock  sync.Mutex
+	WorkDir = ".work"
+	DataDir = ".data"
 )
 
 func CopyToWorkDir(path, workDirPath, hash string) (string, error) {
@@ -81,18 +80,20 @@ func WorkDirPathToZipPath(workDir string) string {
 	return filepath.Join(DataDir, wd+".djfz")
 }
 
-func worker(jobChan chan string, done chan struct{}) error {
+func worker(jobChan chan string, errChan chan error) error {
 	for workDir := range jobChan {
 		err := PackWorkDir(workDir)
 		if err != nil {
+			errChan <- err
 			return err
 		}
 		err = os.RemoveAll(workDir)
 		if err != nil {
+			errChan <- err
 			return err
 		}
 	}
-	done <- struct{}{}
+	errChan <- nil
 	return nil
 }
 
@@ -107,17 +108,28 @@ func GCWorkDirs(WorkDirPath string) error {
 		os.MkdirAll(DataDir, 0o755)
 	}
 
-	doneChan := make(chan struct{}, 1)
 	jobChan := make(chan string, 1)
+	errChan := make(chan error, runtime.NumCPU())
+	for i := 0; i < runtime.NumCPU(); i++ {
+		go worker(jobChan, errChan)
+	}
 	for _, workDir := range workDirs {
 		jobChan <- workDir
 	}
-	for i := 0; i < runtime.NumCPU(); i++ {
-		go worker(jobChan, doneChan)
-	}
+
 	close(jobChan)
+
+	// Collect errors
+	var errs []error
 	for i := 0; i < runtime.NumCPU(); i++ {
-		<-doneChan
+		if err := <-errChan; err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	if len(errs) > 0 {
+		// Handle or return errors
+		return errors.Join(errs...)
 	}
 
 	return nil