Skip to content

Commit e67d747

Browse files
committed
remove older converter
1 parent 2df476b commit e67d747

File tree

5 files changed

+40
-125
lines changed

5 files changed

+40
-125
lines changed

cmd/converter/.gitignore

Lines changed: 0 additions & 1 deletion
This file was deleted.

cmd/converter/main.go

Lines changed: 0 additions & 92 deletions
This file was deleted.

cmd/uniconverter/main.go

Lines changed: 11 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package main
22

33
import (
4-
"encoding/gob"
54
"flag"
65
"os"
76
"path/filepath"
@@ -48,36 +47,31 @@ func main() {
4847
// Create the filesystem.
4948
// The filesystem is created at the output path.
5049
os.MkdirAll(*outputPath, 0o777)
51-
saveState, err := os.Open(filepath.Join(*outputPath, "boundaries.gob"))
52-
boundaries := []util.ZipBoundary{}
50+
boundaries, err := util.DetermineZipBoundaries(*directoryPath, *thresholdSize)
5351
if err != nil {
52+
panic(err)
53+
}
5454

55-
boundaries, err = util.DetermineZipBoundaries(*directoryPath, *thresholdSize)
55+
for _, boundary := range boundaries {
56+
lt, err := util.CreateInitialDJAFSManifest(boundary.Path, *outputPath, boundary.IncludeSubdirs)
5657
if err != nil {
5758
panic(err)
5859
}
59-
f, err := os.Create(filepath.Join(*outputPath, "boundaries.gob"))
60+
subpath := strings.TrimPrefix(boundary.Path, *directoryPath)
61+
newPath := filepath.Join(*outputPath, util.DataDir, subpath)
62+
err = os.MkdirAll(newPath, 0o777)
6063
if err != nil {
6164
panic(err)
6265
}
63-
gob.NewEncoder(f).Encode(boundaries)
64-
f.Close()
65-
} else {
66-
gob.NewDecoder(saveState).Decode(&boundaries)
67-
saveState.Close()
68-
}
69-
for _, boundary := range boundaries {
70-
lt, err := util.CreateInitialDJAFSManifest(boundary.Path, *outputPath, boundary.IncludeSubdirs)
66+
err = util.WriteJSONFile(filepath.Join(newPath, "lookups.djfl"), lt)
7167
if err != nil {
7268
panic(err)
7369
}
74-
subpath := strings.TrimPrefix(boundary.Path, *directoryPath)
75-
newPath := filepath.Join(*outputPath, util.MappingDir, subpath)
76-
err = os.MkdirAll(newPath, 0o777)
70+
metadata, err := lt.GenerateMetadata("")
7771
if err != nil {
7872
panic(err)
7973
}
80-
err = util.WriteJSONFile(filepath.Join(newPath, "lookups.djfl"), lt)
74+
err = util.WriteJSONFile(filepath.Join(newPath, "metadata.djfm"), metadata)
8175
if err != nil {
8276
panic(err)
8377
}

util/metadata.go

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,13 @@ func GetVersion() string {
3131
// metadata struct file to accompany the zip
3232
func (l LookupTable) GenerateMetadata(path string) (Metadata, error) {
3333
var m Metadata
34-
stat, err := os.Stat(path)
35-
if err != nil {
36-
return m, err
34+
if path != "" {
35+
stat, err := os.Stat(path)
36+
if err != nil {
37+
return m, err
38+
}
39+
m.CompressedSize = int(stat.Size())
3740
}
38-
m.CompressedSize = int(stat.Size())
3941
m.DJAFSVersion = GetVersion()
4042
m.NewestFileTS = l.GetNewestFileTS()
4143
m.OldestFileTS = l.GetOldestFileTS()

util/repack.go

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,9 @@ import (
1212
)
1313

1414
var (
15-
gcLock sync.Mutex
16-
WorkDir = ".work"
17-
DataDir = ".data"
18-
MappingDir = ".mappings"
15+
gcLock sync.Mutex
16+
WorkDir = ".work"
17+
DataDir = ".data"
1918
)
2019

2120
func CopyToWorkDir(path, workDirPath, hash string) (string, error) {
@@ -81,18 +80,20 @@ func WorkDirPathToZipPath(workDir string) string {
8180
return filepath.Join(DataDir, wd+".djfz")
8281
}
8382

84-
func worker(jobChan chan string, done chan struct{}) error {
83+
func worker(jobChan chan string, errChan chan error) error {
8584
for workDir := range jobChan {
8685
err := PackWorkDir(workDir)
8786
if err != nil {
87+
errChan <- err
8888
return err
8989
}
9090
err = os.RemoveAll(workDir)
9191
if err != nil {
92+
errChan <- err
9293
return err
9394
}
9495
}
95-
done <- struct{}{}
96+
errChan <- nil
9697
return nil
9798
}
9899

@@ -107,17 +108,28 @@ func GCWorkDirs(WorkDirPath string) error {
107108
os.MkdirAll(DataDir, 0o755)
108109
}
109110

110-
doneChan := make(chan struct{}, 1)
111111
jobChan := make(chan string, 1)
112+
errChan := make(chan error, runtime.NumCPU())
113+
for i := 0; i < runtime.NumCPU(); i++ {
114+
go worker(jobChan, errChan)
115+
}
112116
for _, workDir := range workDirs {
113117
jobChan <- workDir
114118
}
115-
for i := 0; i < runtime.NumCPU(); i++ {
116-
go worker(jobChan, doneChan)
117-
}
119+
118120
close(jobChan)
121+
122+
// Collect errors
123+
var errs []error
119124
for i := 0; i < runtime.NumCPU(); i++ {
120-
<-doneChan
125+
if err := <-errChan; err != nil {
126+
errs = append(errs, err)
127+
}
128+
}
129+
130+
if len(errs) > 0 {
131+
// Handle or return errors
132+
return errors.Join(errs...)
121133
}
122134

123135
return nil

0 commit comments

Comments
 (0)