Skip to content

Commit f30710c

Browse files
ShawkyZShawkyZ
authored
fix: revert #87 (#93)
1 parent 3059282 commit f30710c

File tree

6 files changed

+24
-80
lines changed

6 files changed

+24
-80
lines changed

‎internal/bundle/bundle.go

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ type Bundle interface {
3030
UploadBatch(ctx context.Context, requestId string, batch *Batch) error
3131
GetBundleHash() string
3232
GetFiles() map[string]deepcode.BundleFile
33-
ClearFiles()
3433
GetMissingFiles() []string
3534
GetLimitToFiles() []string
3635
GetRootPath() string
@@ -82,10 +81,6 @@ func (b *deepCodeBundle) GetFiles() map[string]deepcode.BundleFile {
8281
return b.files
8382
}
8483

85-
func (b *deepCodeBundle) ClearFiles() {
86-
b.files = make(map[string]deepcode.BundleFile)
87-
}
88-
8984
func (b *deepCodeBundle) GetMissingFiles() []string {
9085
return b.missingFiles
9186
}
@@ -141,15 +136,15 @@ func NewBatch(documents map[string]deepcode.BundleFile) *Batch {
141136

142137
// todo simplify the size computation
143138
// maybe consider an addFile / canFitFile interface with proper error handling
144-
func (b *Batch) canFitFile(uri string, contentSize int) bool {
145-
docPayloadSize := b.getTotalDocPayloadSize(uri, contentSize)
139+
func (b *Batch) canFitFile(uri string, content []byte) bool {
140+
docPayloadSize := b.getTotalDocPayloadSize(uri, content)
146141
newSize := docPayloadSize + b.getSize()
147142
b.size += docPayloadSize
148143
return newSize < maxUploadBatchSize
149144
}
150145

151-
func (b *Batch) getTotalDocPayloadSize(documentURI string, contentSize int) int {
152-
return len(jsonHashSizePerFile) + len(jsonOverheadPerFile) + len([]byte(documentURI)) + contentSize
146+
func (b *Batch) getTotalDocPayloadSize(documentURI string, content []byte) int {
147+
return len(jsonHashSizePerFile) + len(jsonOverheadPerFile) + len([]byte(documentURI)) + len(content)
153148
}
154149

155150
func (b *Batch) getSize() int {

‎internal/bundle/bundle_manager.go

Lines changed: 12 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -105,25 +105,20 @@ func (b *bundleManager) Create(ctx context.Context,
105105
if !supported {
106106
continue
107107
}
108-
109-
fileInfo, fileErr := os.Stat(absoluteFilePath)
110-
if fileErr != nil {
111-
b.logger.Error().Err(err).Str("filePath", absoluteFilePath).Msg("Failed to read file info")
112-
continue
113-
}
114-
115-
if fileInfo.Size() == 0 || fileInfo.Size() > maxFileSize {
108+
var fileContent []byte
109+
fileContent, err = os.ReadFile(absoluteFilePath)
110+
if err != nil {
111+
b.logger.Error().Err(err).Str("filePath", absoluteFilePath).Msg("could not load content of file")
116112
continue
117113
}
118114

119-
fileContent, fileErr := os.ReadFile(absoluteFilePath)
120-
if fileErr != nil {
121-
b.logger.Error().Err(err).Str("filePath", absoluteFilePath).Msg("Failed to load content of file")
115+
if !(len(fileContent) > 0 && len(fileContent) <= maxFileSize) {
122116
continue
123117
}
124118

125-
relativePath, fileErr := util.ToRelativeUnixPath(rootPath, absoluteFilePath)
126-
if fileErr != nil {
119+
var relativePath string
120+
relativePath, err = util.ToRelativeUnixPath(rootPath, absoluteFilePath)
121+
if err != nil {
127122
b.errorReporter.CaptureError(err, observability.ErrorReporterOptions{ErrorDiagnosticPath: rootPath})
128123
}
129124
relativePath = util.EncodePath(relativePath)
@@ -186,44 +181,16 @@ func (b *bundleManager) Upload(
186181
if err := ctx.Err(); err != nil {
187182
return bundle, err
188183
}
189-
b.enrichBatchWithFileContent(batch, bundle.GetRootPath())
190184
err := bundle.UploadBatch(s.Context(), requestId, batch)
191185
if err != nil {
192186
return bundle, err
193187
}
194-
batch.documents = make(map[string]deepcode.BundleFile)
195188
}
196189
}
197190

198-
// bundle doesn't need file map anymore since they are already grouped and uploaded
199-
bundle.ClearFiles()
200191
return bundle, nil
201192
}
202193

203-
func (b *bundleManager) enrichBatchWithFileContent(batch *Batch, rootPath string) {
204-
for filePath, bundleFile := range batch.documents {
205-
absPath, err := util.DecodePath(util.ToAbsolutePath(rootPath, filePath))
206-
if err != nil {
207-
b.logger.Error().Err(err).Str("file", filePath).Msg("Failed to decode Path")
208-
continue
209-
}
210-
content, err := os.ReadFile(absPath)
211-
if err != nil {
212-
b.logger.Error().Err(err).Str("file", filePath).Msg("Failed to read bundle file")
213-
continue
214-
}
215-
216-
utf8Content, err := util.ConvertToUTF8(content)
217-
if err != nil {
218-
b.logger.Error().Err(err).Str("file", filePath).Msg("Failed to convert bundle file to UTF-8")
219-
continue
220-
}
221-
222-
bundleFile.Content = string(utf8Content)
223-
batch.documents[filePath] = bundleFile
224-
}
225-
}
226-
227194
func (b *bundleManager) groupInBatches(
228195
ctx context.Context,
229196
bundle Bundle,
@@ -245,11 +212,12 @@ func (b *bundleManager) groupInBatches(
245212
}
246213

247214
file := files[filePath]
248-
if batch.canFitFile(filePath, file.ContentSize) {
249-
b.logger.Trace().Str("path", filePath).Int("size", file.ContentSize).Msgf("added to deepCodeBundle #%v", len(batches))
215+
var fileContent = []byte(file.Content)
216+
if batch.canFitFile(filePath, fileContent) {
217+
b.logger.Trace().Str("path", filePath).Int("size", len(fileContent)).Msgf("added to deepCodeBundle #%v", len(batches))
250218
batch.documents[filePath] = file
251219
} else {
252-
b.logger.Trace().Str("path", filePath).Int("size", file.ContentSize).Msgf("created new deepCodeBundle - %v bundles in this upload so far", len(batches))
220+
b.logger.Trace().Str("path", filePath).Int("size", len(fileContent)).Msgf("created new deepCodeBundle - %v bundles in this upload so far", len(batches))
253221
newUploadBatch := NewBatch(map[string]deepcode.BundleFile{})
254222
newUploadBatch.documents[filePath] = file
255223
batches = append(batches, newUploadBatch)

‎internal/bundle/bundle_manager_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@ func createTempFileInDir(t *testing.T, name string, size int, temporaryDir strin
444444
t.Helper()
445445

446446
documentURI, fileContent := createFileOfSize(t, name, size, temporaryDir)
447-
return documentURI, deepcode.BundleFile{Hash: util.Hash(fileContent), ContentSize: size}
447+
return documentURI, deepcode.BundleFile{Hash: util.Hash(fileContent), Content: string(fileContent)}
448448
}
449449

450450
func Test_IsSupported_Extensions(t *testing.T) {

‎internal/bundle/mocks/bundle.go

Lines changed: 0 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎internal/deepcode/helpers.go

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -20,16 +20,14 @@ import (
2020
)
2121

2222
type BundleFile struct {
23-
Hash string `json:"hash"`
24-
Content string `json:"content"`
25-
ContentSize int `json:"size"`
23+
Hash string `json:"hash"`
24+
Content string `json:"content"`
2625
}
2726

2827
func BundleFileFrom(content []byte) BundleFile {
2928
file := BundleFile{
30-
Hash: util.Hash(content),
31-
Content: "", // We create the bundleFile empty, and enrich with content later.
32-
ContentSize: len(content),
29+
Hash: util.Hash(content),
30+
Content: string(content),
3331
}
3432
return file
3533
}

‎internal/util/hash.go

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -26,18 +26,13 @@ import (
2626
)
2727

2828
func Hash(content []byte) string {
29-
utf8content, err := ConvertToUTF8(content)
29+
byteReader := bytes.NewReader(content)
30+
reader, _ := charset.NewReaderLabel("UTF-8", byteReader)
31+
utf8content, err := io.ReadAll(reader)
3032
if err != nil {
3133
utf8content = content
3234
}
3335
b := sha256.Sum256(utf8content)
3436
sum256 := hex.EncodeToString(b[:])
3537
return sum256
3638
}
37-
38-
func ConvertToUTF8(content []byte) ([]byte, error) {
39-
byteReader := bytes.NewReader(content)
40-
reader, _ := charset.NewReaderLabel("UTF-8", byteReader)
41-
utf8content, err := io.ReadAll(reader)
42-
return utf8content, err
43-
}

0 commit comments

Comments
 (0)