fix: resolve hash mismatches when uploading files (#96)

Co-authored-by: Peter Schäfer <[email protected]>

Author: CatalinSnyk

bundle/bundle_manager.go

@@ -18,10 +18,11 @@ package bundle
 
 import (
 	"context"
-	"github.com/snyk/code-client-go/scan"
 	"os"
 	"path/filepath"
 
+	"github.com/snyk/code-client-go/scan"
+
 	"github.com/puzpuzpuz/xsync"
 	"github.com/rs/zerolog"
 
@@ -105,14 +106,20 @@ func (b *bundleManager) Create(ctx context.Context,
 		if !supported {
 			continue
 		}
-		var fileContent []byte
-		fileContent, err = os.ReadFile(absoluteFilePath)
+		var rawContent []byte
+		rawContent, err = os.ReadFile(absoluteFilePath)
 		if err != nil {
 			b.logger.Error().Err(err).Str("filePath", absoluteFilePath).Msg("could not load content of file")
 			continue
 		}
 
-		if !(len(fileContent) > 0 && len(fileContent) <= maxFileSize) {
+		bundleFile, bundleError := deepcode.BundleFileFrom(rawContent)
+		if bundleError != nil {
+			b.logger.Error().Err(bundleError).Str("filePath", absoluteFilePath).Msg("could not convert content of file to UTF-8")
+			continue
+		}
+
+		if !(len(bundleFile.Content) > 0 && len(bundleFile.Content) <= maxFileSize) {
 			continue
 		}
 
@@ -123,7 +130,6 @@ func (b *bundleManager) Create(ctx context.Context,
 		}
 		relativePath = util.EncodePath(relativePath)
 
-		bundleFile := deepcode.BundleFileFrom(fileContent)
 		bundleFiles[relativePath] = bundleFile
 		fileHashes[relativePath] = bundleFile.Hash
 		b.logger.Trace().Str("method", "BundleFileFrom").Str("hash", bundleFile.Hash).Str("filePath", absoluteFilePath).Msg("")

bundle/bundle_test.go

@@ -18,6 +18,9 @@ package bundle_test
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"os"
 	"testing"
 
 	"github.com/golang/mock/gomock"
@@ -104,3 +107,25 @@ func Test_UploadBatch(t *testing.T) {
 		assert.NotEqual(t, oldHash, newHash)
 	})
 }
+
+func Test_BundleEncoding(t *testing.T) {
+	t.Run("utf-8 encoded content", func(t *testing.T) {
+		content := []byte("hello")
+		bundle, err := deepcode.BundleFileFrom(content)
+		assert.NoError(t, err)
+
+		actualShasum := sha256.Sum256([]byte(bundle.Content))
+		assert.Equal(t, bundle.Hash, hex.EncodeToString(actualShasum[:]))
+	})
+
+	t.Run("non utf-8 / binary file", func(t *testing.T) {
+		content, err := os.ReadFile("testdata/rshell_font.php")
+		assert.NoError(t, err)
+
+		bundle, err := deepcode.BundleFileFrom(content)
+		assert.NoError(t, err)
+
+		actualShasum := sha256.Sum256([]byte(bundle.Content))
+		assert.Equal(t, bundle.Hash, hex.EncodeToString(actualShasum[:]))
+	})
+}

bundle/testdata/rshell_font.php

@@ -16,6 +16,8 @@
 package deepcode
 
 import (
+	"bytes"
+
 	"github.com/snyk/code-client-go/internal/util"
 )
 
@@ -24,10 +26,15 @@ type BundleFile struct {
 	Content string `json:"content"`
 }
 
-func BundleFileFrom(content []byte) BundleFile {
+func BundleFileFrom(rawContent []byte) (BundleFile, error) {
+	fileContent, err := util.ConvertToUTF8(bytes.NewReader(rawContent))
+	if err != nil {
+		return BundleFile{}, err
+	}
+
 	file := BundleFile{
-		Hash:    util.Hash(content),
-		Content: string(content),
+		Hash:    util.Hash(fileContent),
+		Content: string(fileContent),
 	}
-	return file
+	return file, nil
 }

internal/deepcode/helpers.go

@@ -17,7 +17,6 @@
 package util
 
 import (
-	"bytes"
 	"crypto/sha256"
 	"encoding/hex"
 	"io"
@@ -26,13 +25,16 @@ import (
 )
 
 func Hash(content []byte) string {
-	byteReader := bytes.NewReader(content)
-	reader, _ := charset.NewReaderLabel("UTF-8", byteReader)
-	utf8content, err := io.ReadAll(reader)
-	if err != nil {
-		utf8content = content
-	}
-	b := sha256.Sum256(utf8content)
+	b := sha256.Sum256(content)
 	sum256 := hex.EncodeToString(b[:])
 	return sum256
 }
+
+func ConvertToUTF8(reader io.Reader) ([]byte, error) {
+	utf8Reader, err := charset.NewReaderLabel("UTF-8", reader)
+	if err != nil {
+		return nil, err
+	}
+	utf8content, err := io.ReadAll(utf8Reader)
+	return utf8content, err
+}

internal/util/hash.go

@@ -17,11 +17,12 @@ package codeclient_test
 
 import (
 	"context"
-	"github.com/google/uuid"
 	"os"
 	"path/filepath"
 	"testing"
 
+	"github.com/google/uuid"
+
 	"github.com/golang/mock/gomock"
 	"github.com/rs/zerolog"
 	"github.com/stretchr/testify/assert"
@@ -44,9 +45,14 @@ import (
 func Test_UploadAndAnalyze(t *testing.T) {
 	baseDir, firstDocPath, secondDocPath, firstDocContent, secondDocContent := setupDocs(t)
 	docs := sliceToChannel([]string{firstDocPath, secondDocPath})
+	firstBundle, err := deepcode.BundleFileFrom(firstDocContent)
+	assert.NoError(t, err)
+	secondBundle, err := deepcode.BundleFileFrom(secondDocContent)
+	assert.NoError(t, err)
+
 	files := map[string]deepcode.BundleFile{
-		firstDocPath: deepcode.BundleFileFrom(firstDocContent),
-		firstDocPath: deepcode.BundleFileFrom(secondDocContent),
+		firstDocPath: firstBundle,
+		firstDocPath: secondBundle,
 	}
 
 	logger := zerolog.Nop()