Skip to content

Commit 7dc2328

Browse files
morremeyermorremeyer
authored
fix: import hashes should be a hex string of the bytes (#655)
This fixes a bug where the import hash was the string representation of a slice containing the bytes for the SHA256 hash. Import hashes are now always the hexadecimal string representation of the SHA256 hash.
1 parent 09a57fe commit 7dc2328

File tree

4 files changed

+104
-14
lines changed

4 files changed

+104
-14
lines changed

‎pkg/importer/helpers/sha256.go‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package helpers
2+
3+
import (
4+
"crypto/sha256"
5+
"fmt"
6+
)
7+
8+
// Sha256String calculates the SHA256 hash of a given string and returns its string representation.
9+
func Sha256String(input string) string {
10+
return fmt.Sprintf("%x", sha256.Sum256([]byte(input)))
11+
}

‎pkg/importer/helpers/sha256_test.go‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package helpers_test
2+
3+
import (
4+
"testing"
5+
6+
"github.com/envelope-zero/backend/v2/pkg/importer/helpers"
7+
"github.com/stretchr/testify/assert"
8+
)
9+
10+
func TestSha256(t *testing.T) {
11+
s := helpers.Sha256String("Envelope Zero")
12+
assert.Equal(t, "dbac4a4ba50e42b6e04b43c2c9b3619e3668dc0a8caf050b584bdafaebee1787", s, "SHA256 checksum calculation is wrong!")
13+
}

‎pkg/importer/parser/ynab4/parse.go‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package ynab4
22

33
import (
4-
"crypto/sha256"
54
"encoding/json"
65
"errors"
76
"fmt"
@@ -14,6 +13,7 @@ import (
1413
internal_types "github.com/envelope-zero/backend/v2/internal/types"
1514
"github.com/google/uuid"
1615

16+
"github.com/envelope-zero/backend/v2/pkg/importer/helpers"
1717
"github.com/envelope-zero/backend/v2/pkg/importer/types"
1818
"github.com/envelope-zero/backend/v2/pkg/models"
1919
"golang.org/x/exp/maps"
@@ -99,7 +99,7 @@ func parseAccounts(resources *types.ParsedResources, accounts []Account) IDToNam
9999
Note: account.Note,
100100
OnBudget: account.OnBudget,
101101
Hidden: account.Hidden,
102-
ImportHash: fmt.Sprint(sha256.Sum256([]byte(account.EntityID))),
102+
ImportHash: helpers.Sha256String(account.EntityID),
103103
},
104104
})
105105
}
@@ -128,7 +128,7 @@ func parsePayees(resources *types.ParsedResources, payees []Payee) IDToName {
128128
Name: payee.Name,
129129
OnBudget: false,
130130
External: true,
131-
ImportHash: fmt.Sprint(sha256.Sum256([]byte(payee.EntityID))),
131+
ImportHash: helpers.Sha256String(payee.EntityID),
132132
},
133133
})
134134
}
@@ -236,7 +236,7 @@ func parseTransactions(resources *types.ParsedResources, transactions []Transact
236236

237237
// We generate an import hash for the "YNAB 4 - No payee" account that might be added since
238238
// we create it and it therefore does not have a UUID
239-
noPayeeImportHash := fmt.Sprint(sha256.Sum256([]byte(uuid.New().String())))
239+
noPayeeImportHash := helpers.Sha256String(uuid.New().String())
240240

241241
// Add all transactions
242242
for _, transaction := range transactions {
@@ -266,7 +266,7 @@ func parseTransactions(resources *types.ParsedResources, transactions []Transact
266266
addNoPayee = true
267267
} else {
268268
// Use the payee ID from the transaction in all other cases
269-
payeeImportHash = fmt.Sprint(sha256.Sum256([]byte(transaction.PayeeID)))
269+
payeeImportHash = helpers.Sha256String(transaction.PayeeID)
270270
}
271271

272272
// Parse the date of the transaction
@@ -275,7 +275,7 @@ func parseTransactions(resources *types.ParsedResources, transactions []Transact
275275
return fmt.Errorf("could not parse date, the Budget.yfull file seems to be corrupt: %w", err)
276276
}
277277

278-
accountImportHash := fmt.Sprint(sha256.Sum256([]byte(transaction.AccountID)))
278+
accountImportHash := helpers.Sha256String(transaction.AccountID)
279279

280280
// Envelope Zero does not use a magic “Starting Balance” account, instead
281281
// every account has a field for the starting balance
@@ -296,7 +296,7 @@ func parseTransactions(resources *types.ParsedResources, transactions []Transact
296296
TransactionCreate: models.TransactionCreate{
297297
Date: date,
298298
Note: strings.TrimSpace(transaction.Memo),
299-
ImportHash: fmt.Sprint(sha256.Sum256([]byte(transaction.EntityID))),
299+
ImportHash: helpers.Sha256String(transaction.EntityID),
300300
},
301301
},
302302
}
@@ -385,7 +385,7 @@ func parseTransactions(resources *types.ParsedResources, transactions []Transact
385385

386386
// The transaction is a transfer
387387
if sub.TargetAccountID != "" {
388-
targetAccountImportHash := fmt.Sprint(sha256.Sum256([]byte(sub.TargetAccountID)))
388+
targetAccountImportHash := helpers.Sha256String(sub.TargetAccountID)
389389
if sub.Amount.IsPositive() {
390390
subTransaction.SourceAccountHash = targetAccountImportHash
391391
} else {

‎pkg/models/database.go‎

Lines changed: 72 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package models
22

33
import (
44
"fmt"
5+
"strconv"
6+
"strings"
57

68
"gorm.io/gorm"
79
)
@@ -13,21 +15,19 @@ func Migrate(db *gorm.DB) error {
1315
return fmt.Errorf("error during DB migration: %w", err)
1416
}
1517

16-
/*
17-
* Workaround for https://github.com/go-gorm/gorm/issues/5968
18-
*/
1918
queries := []*gorm.DB{
19+
/*
20+
* Workaround for https://github.com/go-gorm/gorm/issues/5968
21+
* Remove with 3.0.0
22+
*/
2023
// Account
2124
db.Unscoped().Model(&Account{}).Select("OnBudget").Where("accounts.on_budget IS NULL").Update("OnBudget", false),
2225
db.Unscoped().Model(&Account{}).Select("External").Where("accounts.external IS NULL").Update("External", false),
2326
db.Unscoped().Model(&Account{}).Select("Hidden").Where("accounts.hidden IS NULL").Update("Hidden", false),
24-
2527
// Category
2628
db.Unscoped().Model(&Category{}).Select("Hidden").Where("categories.hidden IS NULL").Update("Hidden", false),
27-
2829
// Envelope
2930
db.Unscoped().Model(&Envelope{}).Select("Hidden").Where("envelopes.hidden IS NULL").Update("Hidden", false),
30-
3131
// Transaction
3232
db.Unscoped().Model(&Transaction{}).Select("Reconciled").Where("transactions.reconciled IS NULL").Update("Reconciled", false),
3333
db.Unscoped().Model(&Transaction{}).Select("ReconciledSource").Where("transactions.reconciled_source IS NULL").Update("ReconciledSource", false),
@@ -40,5 +40,71 @@ func Migrate(db *gorm.DB) error {
4040
}
4141
}
4242

43+
/*
44+
* Complex migrations
45+
*/
46+
47+
// Migration for https://github.com/envelope-zero/backend/issues/628.
48+
// Remove with 3.0.0
49+
err = migrateImportHashString(db)
50+
if err != nil {
51+
return fmt.Errorf("error during migrateImportHashString: %w", err)
52+
}
53+
4354
return nil
4455
}
56+
57+
// migrateImportHashString migrates the string representation of the SHA256 hash as byte array
58+
// to a hex string representation of the hash to use the common way of representing SHA256 hashes.
59+
// See https://github.com/envelope-zero/backend/issues/628.
60+
func migrateImportHashString(db *gorm.DB) (err error) {
61+
var accounts []Account
62+
err = db.Unscoped().Where("import_hash LIKE '[%'").Find(&accounts).Error
63+
if err != nil {
64+
return err
65+
}
66+
67+
for _, account := range accounts {
68+
// The string looks like this: "[40 52 207 7 118 61 80 107 178 242 5 47 211 161 180 135 104 222 118 28 56 12 33 63 179 78 39 173 206 11 77 3]"
69+
// With trimming and splitting it, we get a slice containing every individual number
70+
bytes := strings.Split(strings.TrimRight(strings.TrimLeft(account.ImportHash, "["), "]"), " ")
71+
72+
// Assemble the slice back to a string. We pad with zeroes so that every byte takes two characters
73+
var b strings.Builder
74+
for _, part := range bytes {
75+
// Need to convert to int first so that it's interpreted correctly
76+
charAsInt, _ := strconv.Atoi(part)
77+
b.WriteString(fmt.Sprintf("%02x", byte(charAsInt)))
78+
}
79+
80+
// Save the record back to the DB
81+
account.ImportHash = b.String()
82+
db.Unscoped().Save(&account)
83+
}
84+
85+
var transactions []Transaction
86+
err = db.Unscoped().Where("import_hash LIKE '[%'").Find(&transactions).Error
87+
if err != nil {
88+
return err
89+
}
90+
91+
for _, transaction := range transactions {
92+
// The string looks like this: "[40 52 207 7 118 61 80 107 178 242 5 47 211 161 180 135 104 222 118 28 56 12 33 63 179 78 39 173 206 11 77 3]"
93+
// With trimming and splitting it, we get a slice containing every individual number
94+
bytes := strings.Split(strings.TrimRight(strings.TrimLeft(transaction.ImportHash, "["), "]"), " ")
95+
96+
// Assemble the slice back to a string. We pad with zeroes so that every byte takes two characters
97+
var b strings.Builder
98+
for _, part := range bytes {
99+
// Need to convert to int first so that it's interpreted correctly
100+
charAsInt, _ := strconv.Atoi(part)
101+
b.WriteString(fmt.Sprintf("%02x", byte(charAsInt)))
102+
}
103+
104+
// Save the record back to the DB
105+
transaction.ImportHash = b.String()
106+
db.Unscoped().Save(&transaction)
107+
}
108+
109+
return
110+
}

0 commit comments

Comments
 (0)