fix: remove all use of SUM() in SQL queries

All calculations are now done in the code instead of in the query since
the database will introduce floating point errors.

Author: morremeyer

pkg/controllers/account.go

@@ -340,8 +340,14 @@ func (co Controller) getAccountObject(c *gin.Context, id uuid.UUID) (Account, bo
 		return Account{}, false
 	}
 
+	account, err = account.WithCalculations(co.DB)
+	if err != nil {
+		httperrors.Handler(c, err)
+		return Account{}, false
+	}
+
 	return Account{
-		account.WithCalculations(co.DB),
+		account,
 		recentEnvelopes,
 		AccountLinks{
 			Self:         fmt.Sprintf("%s/v1/accounts/%s", c.GetString("baseURL"), account.ID),

pkg/controllers/budget.go

@@ -657,7 +657,13 @@ func (co Controller) getBudgetResource(c *gin.Context, id uuid.UUID) (models.Bud
 		return models.Budget{}, false
 	}
 
-	return budget.WithCalculations(co.DB), true
+	budget, err := budget.WithCalculations(co.DB)
+	if err != nil {
+		httperrors.Handler(c, err)
+		return models.Budget{}, false
+	}
+
+	return budget, true
 }
 
 func (co Controller) getBudgetObject(c *gin.Context, id uuid.UUID) (Budget, bool) {

pkg/models/account.go

@@ -29,11 +29,19 @@ type AccountCreate struct {
 	Hidden             bool            `json:"hidden" example:"true" default:"false"`
 }
 
-func (a Account) WithCalculations(db *gorm.DB) Account {
-	a.Balance = a.getBalance(db)
-	a.ReconciledBalance = a.SumReconciledTransactions(db)
+func (a Account) WithCalculations(db *gorm.DB) (Account, error) {
+	balance, _, err := a.GetBalanceMonth(db, types.NewMonth(1, 1))
+	if err != nil {
+		return Account{}, err
+	}
+	a.Balance = balance
+
+	a.ReconciledBalance, err = a.SumReconciled(db)
+	if err != nil {
+		return Account{}, err
+	}
 
-	return a
+	return a, nil
 }
 
 // BeforeSave sets OnBudget to false when External is true.
@@ -54,21 +62,33 @@ func (a Account) Transactions(db *gorm.DB) []Transaction {
 }
 
 // Transactions returns all transactions for this account.
-func (a Account) SumReconciledTransactions(db *gorm.DB) decimal.Decimal {
-	return a.InitialBalance.Add(TransactionsSum(db,
-		Transaction{
-			TransactionCreate: TransactionCreate{
-				Reconciled:           true,
-				DestinationAccountID: a.ID,
-			},
-		},
-		Transaction{
-			TransactionCreate: TransactionCreate{
-				Reconciled:      true,
-				SourceAccountID: a.ID,
-			},
-		},
-	))
+func (a Account) SumReconciled(db *gorm.DB) (balance decimal.Decimal, err error) {
+	var transactions []Transaction
+
+	err = db.
+		Preload("DestinationAccount").
+		Preload("SourceAccount").
+		Where(
+			db.Where(Transaction{TransactionCreate: TransactionCreate{DestinationAccountID: a.ID, Reconciled: true}}).
+				Or(db.Where(Transaction{TransactionCreate: TransactionCreate{SourceAccountID: a.ID, Reconciled: true}}))).
+		Find(&transactions).Error
+
+	if err != nil {
+		return decimal.Zero, err
+	}
+
+	balance = a.InitialBalance
+
+	// Add incoming transactions, subtract outgoing transactions
+	for _, t := range transactions {
+		if t.DestinationAccountID == a.ID {
+			balance = balance.Add(t.Amount)
+		} else {
+			balance = balance.Sub(t.Amount)
+		}
+	}
+
+	return
 }
 
 // GetBalanceMonth calculates the balance and available sums for a specific month.
@@ -78,20 +98,24 @@ func (a Account) SumReconciledTransactions(db *gorm.DB) decimal.Decimal {
 func (a Account) GetBalanceMonth(db *gorm.DB, month types.Month) (balance, available decimal.Decimal, err error) {
 	var transactions []Transaction
 
-	err = db.
+	query := db.
 		Preload("DestinationAccount").
 		Preload("SourceAccount").
-		Where("transactions.date < date(?)", month.AddDate(0, 1)).
 		Where(
 			db.Where(Transaction{TransactionCreate: TransactionCreate{DestinationAccountID: a.ID}}).
-				Or(db.Where(Transaction{TransactionCreate: TransactionCreate{SourceAccountID: a.ID}}))).
-		Find(&transactions).
-		Error
+				Or(db.Where(Transaction{TransactionCreate: TransactionCreate{SourceAccountID: a.ID}})))
+
+	// Limit to the month if it is specified
+	if !month.IsZero() {
+		query = query.Where("transactions.date < date(?)", month.AddDate(0, 1))
+	}
+
+	err = query.Find(&transactions).Error
 	if err != nil {
 		return decimal.Zero, decimal.Zero, err
 	}
 
-	if a.InitialBalanceDate != nil && month.AddDate(0, 1).AfterTime(*a.InitialBalanceDate) {
+	if month.IsZero() || (a.InitialBalanceDate != nil && month.AddDate(0, 1).AfterTime(*a.InitialBalanceDate)) {
 		balance = a.InitialBalance
 		available = a.InitialBalance
 	}
@@ -116,46 +140,6 @@ func (a Account) GetBalanceMonth(db *gorm.DB, month types.Month) (balance, avail
 	return
 }
 
-// getBalance returns the balance of the account calculated over all transactions.
-func (a Account) getBalance(db *gorm.DB) decimal.Decimal {
-	return a.InitialBalance.Add(TransactionsSum(db,
-		Transaction{
-			TransactionCreate: TransactionCreate{
-				DestinationAccountID: a.ID,
-			},
-		},
-		Transaction{
-			TransactionCreate: TransactionCreate{
-				SourceAccountID: a.ID,
-			},
-		},
-	))
-}
-
-// TransactionSums returns the sum of all transactions matching two Transaction structs
-//
-// The incoming Transactions fields is used to add the amount of all matching transactions to the overall sum
-// The outgoing Transactions fields is used to subtract the amount of all matching transactions from the overall sum.
-func TransactionsSum(db *gorm.DB, incoming, outgoing Transaction) decimal.Decimal {
-	var outgoingSum, incomingSum decimal.NullDecimal
-
-	_ = db.Table("transactions").
-		Where(&outgoing).
-		Where("deleted_at is NULL").
-		Select("SUM(amount)").
-		Row().
-		Scan(&outgoingSum)
-
-	_ = db.Table("transactions").
-		Where(&incoming).
-		Where("deleted_at is NULL").
-		Select("SUM(amount)").
-		Row().
-		Scan(&incomingSum)
-
-	return incomingSum.Decimal.Sub(outgoingSum.Decimal)
-}
-
 // RecentEnvelopes returns the most common envelopes used in the last 10
 // transactions where the account is the destination account.
 //

pkg/models/account_test.go

@@ -108,7 +108,8 @@ func (suite *TestSuiteStandard) TestAccountCalculations() {
 		suite.Assert().Fail("Resource could not be saved", err)
 	}
 
-	a := account.WithCalculations(suite.db)
+	a, err := account.WithCalculations(suite.db)
+	assert.Nil(suite.T(), err)
 
 	expected := incomingTransaction.Amount.Sub(outgoingTransaction.Amount).Add(a.InitialBalance).Add(decimal.NewFromFloat(100)) // Add 100 for futureIncomeTransaction
 	assert.True(suite.T(), a.Balance.Equal(expected), "Balance for account is not correct. Should be: %v but is %v", expected, a.Balance)
@@ -130,7 +131,9 @@ func (suite *TestSuiteStandard) TestAccountCalculations() {
 		suite.Assert().Fail("Resource could not be deleted", err)
 	}
 
-	a = account.WithCalculations(suite.db)
+	a, err = account.WithCalculations(suite.db)
+	assert.Nil(suite.T(), err)
+
 	expected = outgoingTransaction.Amount.Neg().Add(a.InitialBalance).Add(decimal.NewFromFloat(100)) // Add 100 for futureIncomeTransaction
 	assert.True(suite.T(), a.Balance.Equal(expected), "Balance for account is not correct. Should be: %v but is %v", expected, a.Balance)
 

pkg/models/budget.go

@@ -37,7 +37,7 @@ type BudgetMonth struct {
 }
 
 // WithCalculations computes all the calculated values.
-func (b Budget) WithCalculations(db *gorm.DB) Budget {
+func (b Budget) WithCalculations(db *gorm.DB) (Budget, error) {
 	b.Balance = decimal.Zero
 
 	// Get all OnBudget accounts for the budget
@@ -51,10 +51,15 @@ func (b Budget) WithCalculations(db *gorm.DB) Budget {
 
 	// Add all their balances to the budget's balance
 	for _, account := range accounts {
-		b.Balance = b.Balance.Add(account.WithCalculations(db).Balance)
+		account, err := account.WithCalculations(db)
+		if err != nil {
+			return Budget{}, err
+		}
+
+		b.Balance = b.Balance.Add(account.Balance)
 	}
 
-	return b
+	return b, nil
 }
 
 // Income returns the income for a budget in a given month.
@@ -68,8 +73,7 @@ func (b Budget) Income(db *gorm.DB, month types.Month) (decimal.Decimal, error)
 		Where("source_account.external = 1").
 		Where("destination_account.external = 0").
 		Where("transactions.envelope_id IS NULL").
-		Where("strftime('%m', transactions.date) = ?", fmt.Sprintf("%02d", time.Time(month).Month())).
-		Where("strftime('%Y', transactions.date) = ?", fmt.Sprintf("%d", time.Time(month).Year())).
+		Where("transactions.available_from >= date(?) AND transactions.available_from < date(?)", month, month.AddDate(0, 1)).
 		Where(&Transaction{
 			TransactionCreate: TransactionCreate{
 				BudgetID: b.ID,

pkg/models/budget_test.go

@@ -240,7 +240,8 @@ func (suite *TestSuiteStandard) TestBudgetCalculations() {
 		suite.Assert().Fail("Resource could not be saved", err)
 	}
 
-	budget = budget.WithCalculations(suite.db)
+	budget, err = budget.WithCalculations(suite.db)
+	assert.Nil(suite.T(), err)
 
 	shouldBalance := decimal.NewFromFloat(7269.38)
 	assert.True(suite.T(), budget.Balance.Equal(shouldBalance), "Balance for budget is not correct. Should be %s, is %s", shouldBalance, budget.Balance)
@@ -381,7 +382,7 @@ func (suite *TestSuiteStandard) TestMonth() {
 	budget := suite.createTestBudget(models.BudgetCreate{})
 	category := suite.createTestCategory(models.CategoryCreate{BudgetID: budget.ID, Name: "Upkeep"})
 	envelope := suite.createTestEnvelope(models.EnvelopeCreate{CategoryID: category.ID, Name: "Utilities"})
-	account := suite.createTestAccount(models.AccountCreate{BudgetID: budget.ID, OnBudget: true})
+	account := suite.createTestAccount(models.AccountCreate{BudgetID: budget.ID, OnBudget: true, Name: "TestMonth"})
 	externalAccount := suite.createTestAccount(models.AccountCreate{BudgetID: budget.ID, External: true})
 
 	allocationJanuary := suite.createTestAllocation(models.AllocationCreate{