Skip to content

Commit 6e4b256

Browse files
authored
feat: parser: update SBI parser to make use of newer excel format (#117)
1 parent 0ad46f7 commit 6e4b256

File tree

15 files changed

+512
-382
lines changed

15 files changed

+512
-382
lines changed

frontend/components/custom/Modal/Statement/ImportStatementModal.tsx

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,8 @@ export function ImportStatementModal({
7676
}, []);
7777

7878
const validateFile = (file: File, forBank: boolean): string | null => {
79-
if (file.size > 256 * 1024) {
80-
return "File size must be less than 256KB";
79+
if (file.size > 5 * 1024 * 1024) {
80+
return "File size must be less than 5MB";
8181
}
8282
const validExtensions = forBank
8383
? [".csv", ".xls", ".xlsx", ".txt"]

frontend/components/custom/Modal/Statement/steps/FallbackParsing.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ export function FallbackParsing({
189189
: "Drag & drop your bank statement here, or click to select"}
190190
</p>
191191
<p className="text-xs text-muted-foreground">
192-
Supports CSV, XLS, TXT files (max 256KB)
192+
Supports CSV, XLS, TXT files (max 5MB)
193193
</p>
194194
<Input
195195
id="file-input-fallback"

frontend/components/custom/Modal/Statement/steps/ImportFromBank.tsx

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ export function ImportFromBank({
135135
: "Drag & drop your bank statements here, or click to select"}
136136
</p>
137137
<p className="text-xs text-muted-foreground">
138-
Supports CSV, XLS, XLSX, TXT files (max 256KB each, up to 10
138+
Supports CSV, XLS, XLSX, TXT files (max 5MB each, up to 10
139139
files)
140140
</p>
141141
<Input

server/e2e_test.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
set -euo pipefail
33

44
export DB_SCHEMA="test"
5-
export DB_SEED_DIR=${DB_SEED_DIR:-./internal/database/seed/test} # ← no trailing space!
5+
export DB_SEED_DIR=${DB_SEED_DIR:-./internal/database/seed/test}
66
export ENV="test"
77

88
if [[ "$DB_SCHEMA" != "test" ]]; then

server/go.mod

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ require (
1010
github.com/jackc/pgx/v5 v5.8.0
1111
github.com/onsi/ginkgo/v2 v2.27.3
1212
github.com/onsi/gomega v1.38.3
13+
github.com/xuri/excelize/v2 v2.10.0
1314
go.uber.org/zap v1.27.1
1415
golang.org/x/crypto v0.46.0
1516
)
@@ -43,9 +44,14 @@ require (
4344
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
4445
github.com/quic-go/qpack v0.6.0 // indirect
4546
github.com/quic-go/quic-go v0.58.0 // indirect
47+
github.com/richardlehane/mscfb v1.0.5 // indirect
48+
github.com/richardlehane/msoleps v1.0.4 // indirect
4649
github.com/rogpeppe/go-internal v1.14.1 // indirect
50+
github.com/tiendc/go-deepcopy v1.7.2 // indirect
4751
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
4852
github.com/ugorji/go/codec v1.3.1 // indirect
53+
github.com/xuri/efp v0.0.1 // indirect
54+
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 // indirect
4955
go.uber.org/mock v0.6.0 // indirect
5056
go.uber.org/multierr v1.11.0 // indirect
5157
go.yaml.in/yaml/v3 v3.0.4 // indirect

server/go.sum

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,10 @@ github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8=
9393
github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII=
9494
github.com/quic-go/quic-go v0.58.0 h1:ggY2pvZaVdB9EyojxL1p+5mptkuHyX5MOSv4dgWF4Ug=
9595
github.com/quic-go/quic-go v0.58.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU=
96+
github.com/richardlehane/mscfb v1.0.5 h1:OoQkDV2Bf2bIoSacCfJhSwm7BJN05fYFkwFUpxExtdY=
97+
github.com/richardlehane/mscfb v1.0.5/go.mod h1:pe0+IUIc0AHh0+teNzBlJCtSyZdFOGgV4ZK9bsoV+Jo=
98+
github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00=
99+
github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg=
96100
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
97101
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
98102
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -115,10 +119,18 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
115119
github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
116120
github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
117121
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
122+
github.com/tiendc/go-deepcopy v1.7.2 h1:Ut2yYR7W9tWjTQitganoIue4UGxZwCcJy3orjrrIj44=
123+
github.com/tiendc/go-deepcopy v1.7.2/go.mod h1:4bKjNC2r7boYOkD2IOuZpYjmlDdzjbpTRyCx+goBCJQ=
118124
github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
119125
github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
120126
github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY=
121127
github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4=
128+
github.com/xuri/efp v0.0.1 h1:fws5Rv3myXyYni8uwj2qKjVaRP30PdjeYe2Y6FDsCL8=
129+
github.com/xuri/efp v0.0.1/go.mod h1:ybY/Jr0T0GTCnYjKqmdwxyxn2BQf2RcQIIvex5QldPI=
130+
github.com/xuri/excelize/v2 v2.10.0 h1:8aKsP7JD39iKLc6dH5Tw3dgV3sPRh8uRVXu/fMstfW4=
131+
github.com/xuri/excelize/v2 v2.10.0/go.mod h1:SC5TzhQkaOsTWpANfm+7bJCldzcnU/jrhqkTi/iBHBU=
132+
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9 h1:+C0TIdyyYmzadGaL/HBLbf3WdLgC29pgyhTjAT/0nuE=
133+
github.com/xuri/nfp v0.0.2-0.20250530014748-2ddeb826f9a9/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ=
122134
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
123135
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
124136
go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y=
@@ -133,6 +145,8 @@ golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg=
133145
golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A=
134146
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
135147
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
148+
golang.org/x/image v0.25.0 h1:Y6uW6rH1y5y/LK1J8BPWZtr6yZ7hrsy6hFrXjgsc2fQ=
149+
golang.org/x/image v0.25.0/go.mod h1:tCAmOEGthTtkalusGp1g3xa2gke8J6c2N565dTyl9Rs=
136150
golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
137151
golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
138152
golang.org/x/net v0.48.0 h1:zyQRTTrjc33Lhh0fBgT/H3oZq9WuvRR5gPC70xpDiQU=

server/internal/api/controller/statement_controller_test.go

Lines changed: 86 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package controller_test
22

33
import (
44
"expenses/internal/models"
5+
"expenses/pkg/utils"
56
"fmt"
67
"net/http"
78
"strconv"
@@ -38,18 +39,6 @@ func createAccount(testHelper *TestHelper, name string, balance float64) float64
3839
return response["data"].(map[string]any)["id"].(float64)
3940
}
4041

41-
func uploadStatement(testHelper *TestHelper, accountId float64, filename string, fileContent []byte) float64 {
42-
statementInput := map[string]any{
43-
"account_id": int64(accountId),
44-
"original_filename": filename,
45-
"file_type": "csv",
46-
"file": fileContent,
47-
}
48-
resp, response := testHelper.MakeMultipartRequest(http.MethodPost, "/statement", statementInput)
49-
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
50-
return response["data"].(map[string]any)["id"].(float64)
51-
}
52-
5342
func waitForStatementDone(testHelper *TestHelper, statementId float64) map[string]any {
5443
var status string
5544
var data map[string]any
@@ -408,9 +397,9 @@ var _ = Describe("StatementController", func() {
408397
Expect(response).To(HaveKey("message"))
409398
})
410399

411-
It("should return error for file larger than 256kb", func() {
412-
// Create a file >256kb
413-
bigFile := make([]byte, 257*1024)
400+
It("should return error for file larger than 5MB", func() {
401+
// Create a file >5MB
402+
bigFile := make([]byte, 5*1024*1024+1)
414403
for i := range bigFile {
415404
bigFile[i] = 'A'
416405
}
@@ -644,7 +633,7 @@ var _ = Describe("StatementController", func() {
644633

645634
It("should handle service error in PreviewStatement", func() {
646635
// Create a file that will cause service-level validation error
647-
fileContent := make([]byte, 300*1024) // File larger than 256KB
636+
fileContent := make([]byte, 5*1024*1024+1) // File larger than 5MB
648637
for i := range fileContent {
649638
fileContent[i] = 'A'
650639
}
@@ -732,19 +721,21 @@ var _ = Describe("StatementController", func() {
732721
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
733722
accountId := response["data"].(map[string]any)["id"].(float64)
734723

735-
// 4. Create a statement with file upload
736-
fileContent := []byte(
737-
"Txn Date Value Date Description Ref No. Debit Credit Balance\n" +
738-
"1 Aug 2022 1 Aug 2022 TO TRANSFER-UPI/DR/221356312527/RITIK S/SBIN/rs6321908@/UPI-- 123456 100.00 1000.00\n" +
739-
"BADROW\n" +
740-
"2 Aug 2022 2 Aug 2022 BY TRANSFER-NEFT*HDFC0000001*N215222062454075*QURIATE TECHNOLO-- 654321 200.00 1200.00\n" +
741-
"Computer Generated Statement")
742-
fileName := "integration_statement.csv"
724+
// 4. Create a statement with an XLSX upload so the SBI parser can handle it
725+
xlsxData := [][]string{
726+
{"Txn Date", "Details", "Ref No.", "Debit", "Credit", "Balance"},
727+
{"1 Aug 2022", "TO TRANSFER-UPI/DR/221356312527/RITIK S/SBIN/rs6321908@/UPI--", "123456", "100.00", "", "1000.00"},
728+
{"BADROW"},
729+
{"2 Aug 2022", "BY TRANSFER-NEFT*HDFC0000001*N215222062454075*QURIATE TECHNOLO--", "654321", "", "200.00", "1200.00"},
730+
{"Computer Generated Statement"},
731+
}
732+
fileBytes := utils.CreateXLSXFile(xlsxData)
733+
fileName := "integration_statement.xlsx"
743734
statementInput := map[string]any{
744735
"account_id": int64(accountId),
745736
"original_filename": fileName,
746-
"file_type": "csv",
747-
"file": fileContent,
737+
"file_type": "excel",
738+
"file": fileBytes,
748739
}
749740
resp, response = testHelper.MakeMultipartRequest(http.MethodPost, "/statement", statementInput)
750741
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
@@ -782,12 +773,24 @@ var _ = Describe("StatementController", func() {
782773
testHelper := createUniqueUser(baseURL)
783774
accountId := createAccount(testHelper, "Account 1", 1000.0)
784775

785-
fileContent4 := []byte("Txn Date\tValue Date\tDescription\tRef No.\tDebit\tCredit\tBalance\n" +
786-
"1 Aug 2022\t1 Aug 2022\tDesc1\t123\t100.00\t\t1000.00\n" +
787-
"2 Aug 2022\t2 Aug 2022\tDesc2\t124\t200.00\t\t1200.00\n" +
788-
"3 Aug 2022\t3 Aug 2022\tDesc3\t125\t300.00\t\t1500.00\n" +
789-
"4 Aug 2022\t4 Aug 2022\tDesc4\t126\t400.00\t\t1900.00\nComputer Generated Statement")
790-
statementId := uploadStatement(testHelper, accountId, "statement_4.csv", fileContent4)
776+
// Upload as XLSX so the SBI parser can parse rows reliably
777+
xlsx4 := [][]string{
778+
{"Txn Date", "Details", "Ref No.", "Debit", "Credit", "Balance"},
779+
{"1 Aug 2022", "Desc1", "123", "100.00", "", "1000.00"},
780+
{"2 Aug 2022", "Desc2", "124", "200.00", "", "1200.00"},
781+
{"3 Aug 2022", "Desc3", "125", "300.00", "", "1500.00"},
782+
{"4 Aug 2022", "Desc4", "126", "400.00", "", "1900.00"},
783+
}
784+
fileBytes4 := utils.CreateXLSXFile(xlsx4)
785+
statementInput4 := map[string]any{
786+
"account_id": int64(accountId),
787+
"original_filename": "statement_4.xlsx",
788+
"file_type": "excel",
789+
"file": fileBytes4,
790+
}
791+
resp, response := testHelper.MakeMultipartRequest(http.MethodPost, "/statement", statementInput4)
792+
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
793+
statementId := response["data"].(map[string]any)["id"].(float64)
791794
waitForStatementDone(testHelper, statementId)
792795

793796
// Fetch transactions for the first statement
@@ -799,15 +802,27 @@ var _ = Describe("StatementController", func() {
799802
transactions := txData["transactions"].([]any)
800803
Expect(transactions).To(HaveLen(4))
801804

802-
fileContent7 := []byte("Txn Date\tValue Date\tDescription\tRef No.\tDebit\tCredit\tBalance\n" +
803-
"1 Aug 2022\t1 Aug 2022\tDesc1\t123\t100.00\t\t1000.00\n" +
804-
"2 Aug 2022\t2 Aug 2022\tDesc2\t124\t200.00\t\t1200.00\n" +
805-
"3 Aug 2022\t3 Aug 2022\tDesc3\t125\t300.00\t\t1500.00\n" +
806-
"4 Aug 2022\t4 Aug 2022\tDesc4\t126\t400.00\t\t1900.00\n" +
807-
"5 Aug 2022\t5 Aug 2022\tDesc5\t127\t500.00\t\t2400.00\n" +
808-
"6 Aug 2022\t6 Aug 2022\tDesc6\t128\t600.00\t\t3000.00\n" +
809-
"7 Aug 2022\t7 Aug 2022\tDesc7\t129\t700.00\t\t3700.00\nComputer Generated Statement")
810-
statementId = uploadStatement(testHelper, accountId, "statement_7.csv", fileContent7)
805+
// Upload as XLSX for the 7-row statement
806+
xlsx7 := [][]string{
807+
{"Txn Date", "Details", "Ref No.", "Debit", "Credit", "Balance"},
808+
{"1 Aug 2022", "Desc1", "123", "100.00", "", "1000.00"},
809+
{"2 Aug 2022", "Desc2", "124", "200.00", "", "1200.00"},
810+
{"3 Aug 2022", "Desc3", "125", "300.00", "", "1500.00"},
811+
{"4 Aug 2022", "Desc4", "126", "400.00", "", "1900.00"},
812+
{"5 Aug 2022", "Desc5", "127", "500.00", "", "2400.00"},
813+
{"6 Aug 2022", "Desc6", "128", "600.00", "", "3000.00"},
814+
{"7 Aug 2022", "Desc7", "129", "700.00", "", "3700.00"},
815+
}
816+
fileBytes7 := utils.CreateXLSXFile(xlsx7)
817+
statementInput7 := map[string]any{
818+
"account_id": int64(accountId),
819+
"original_filename": "statement_7.xlsx",
820+
"file_type": "excel",
821+
"file": fileBytes7,
822+
}
823+
resp, response = testHelper.MakeMultipartRequest(http.MethodPost, "/statement", statementInput7)
824+
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
825+
statementId = response["data"].(map[string]any)["id"].(float64)
811826
waitForStatementDone(testHelper, statementId)
812827

813828
// Fetch transactions for the second statement
@@ -820,20 +835,41 @@ var _ = Describe("StatementController", func() {
820835
Expect(transactions).To(HaveLen(3))
821836
})
822837

823-
It("should parse statement with 1000 transactions", func() {
838+
It("should parse statement with 10000 transactions", func() {
824839
testHelper := createUniqueUser(baseURL)
825840
accountId := createAccount(testHelper, "Account 1", 1000.0)
826841

827-
var rows string
842+
// Build XLSX data (header + 10000 rows)
843+
data := [][]string{
844+
{"Txn Date", "Details", "Ref No.", "Debit", "Credit", "Balance"},
845+
}
828846
for i := 1; i <= 1000; i++ {
829-
rows += fmt.Sprintf("22 Aug 2022 22 Aug 2022 Desc%d %d %.2f %.2f\n", i, 1000+i, float64(i), float64(1000+i))
847+
row := []string{
848+
"22 Aug 2022",
849+
fmt.Sprintf("Desc%d", i),
850+
fmt.Sprintf("%d", 1000+i),
851+
fmt.Sprintf("%.2f", float64(i)),
852+
fmt.Sprintf("%.2f", float64(1000+i)),
853+
fmt.Sprintf("%.2f", float64(2000+i)),
854+
}
855+
data = append(data, row)
830856
}
831-
fileContent := []byte("Txn Date Value Date Description Ref No. Debit Credit Balance\n" + rows + "Computer Generated Statement")
832-
statementId := uploadStatement(testHelper, accountId, "statement_1000.csv", fileContent)
857+
858+
fileBytes := utils.CreateXLSXFile(data)
859+
// Upload as an Excel file
860+
statementInput := map[string]any{
861+
"account_id": int64(accountId),
862+
"original_filename": "statement_10000.xlsx",
863+
"file_type": "excel",
864+
"file": fileBytes,
865+
}
866+
resp, response := testHelper.MakeMultipartRequest(http.MethodPost, "/statement", statementInput)
867+
Expect(resp.StatusCode).To(Equal(http.StatusCreated))
868+
statementId := response["data"].(map[string]any)["id"].(float64)
833869
waitForStatementDone(testHelper, statementId)
834870

835871
// Fetch transactions for the statement
836-
resp, response := testHelper.MakeRequest(http.MethodGet, "/transaction?page_size=10&statement_id="+strconv.FormatFloat(statementId, 'f', 0, 64), nil)
872+
resp, response = testHelper.MakeRequest(http.MethodGet, "/transaction?page_size=10&statement_id="+strconv.FormatFloat(statementId, 'f', 0, 64), nil)
837873
Expect(resp.StatusCode).To(Equal(http.StatusOK))
838874
Expect(response).To(HaveKey("data"))
839875
txData := response["data"].(map[string]any)
@@ -872,11 +908,11 @@ var _ = Describe("StatementController", func() {
872908
Expect(resp.StatusCode).To(Equal(http.StatusNotFound))
873909
})
874910

875-
It("should fail to parse multipart file >256kb", func() {
911+
It("should fail to parse multipart file > 5MB", func() {
876912
testHelper := createUniqueUser(baseURL)
877913
accountId := createAccount(testHelper, "BigFileAccount", 1000.0)
878-
// Create a file >256kb
879-
bigFile := make([]byte, 257*1024)
914+
// Create a file >5MB
915+
bigFile := make([]byte, 5*1024*1024+1)
880916
for i := range bigFile {
881917
bigFile[i] = 'A'
882918
}

server/internal/mock/repository/statement_repository.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ func (m *MockStatementRepository) CreateStatement(ctx context.Context, input mod
3030
return models.StatementResponse{}, errors.New("filename cannot be empty")
3131
}
3232
fileType := input.FileType
33-
if fileType != "csv" && fileType != "xls" && fileType != "xlsx" {
33+
if fileType != "csv" && fileType != "excel" {
3434
return models.StatementResponse{}, errors.New("invalid file type")
3535
}
3636
m.mu.Lock()

0 commit comments

Comments
 (0)