Skip to content

Commit 559ccd1

Browse files
skynet2skynet2
authored
feat: skip duplicates (#105)
* feat: add skip duplicates * fix: lint * fix: lint * fix: new lint
1 parent 5abe52b commit 559ccd1

File tree

9 files changed

+88
-31
lines changed

9 files changed

+88
-31
lines changed

‎cmd/server/internal/handlers/transactions.go‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@ type TransactionApi struct {
1717
mapper MapperSvc
1818
}
1919

20-
func (a *TransactionApi) DeleteTransactions(ctx context.Context, c *connect.Request[transactionsv1.DeleteTransactionsRequest]) (*connect.Response[transactionsv1.DeleteTransactionsRequest], error) {
20+
func (a *TransactionApi) DeleteTransactions(
21+
ctx context.Context,
22+
c *connect.Request[transactionsv1.DeleteTransactionsRequest],
23+
) (*connect.Response[transactionsv1.DeleteTransactionsResponse], error) {
2124
jwtData := middlewares.FromContext(ctx)
2225
if jwtData.UserID == 0 {
2326
return nil, connect.NewError(connect.CodePermissionDenied, auth.ErrInvalidToken)
@@ -28,7 +31,9 @@ func (a *TransactionApi) DeleteTransactions(ctx context.Context, c *connect.Requ
2831
return nil, err
2932
}
3033

31-
return connect.NewResponse(c.Msg), nil
34+
return connect.NewResponse(&transactionsv1.DeleteTransactionsResponse{
35+
DeletedCount: int32(len(c.Msg.Ids)),
36+
}), nil
3237
}
3338

3439
func (a *TransactionApi) GetApplicableAccounts(

‎frontend/package-lock.json‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎frontend/src/app/pages/transactions/transactions-import.component.html‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,9 @@
2323

2424
<p-checkbox id="chkbox3" [(ngModel)]="treatDatesAsUtc" [binary]="true" />
2525
<label for="chkbox3">Treat Dates as UTC</label>
26+
27+
<p-checkbox id="chkbox4" [(ngModel)]="skipDuplicateReferenceCheck" [binary]="true" />
28+
<label for="chkbox4">Skip Duplicate Reference Check</label>
2629
</div>
2730
@if (this.isRawTextImport()) {
2831
<p-iftalabel>

‎frontend/src/app/pages/transactions/transactions-import.component.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ export class TransactionsImportComponent {
5050
public sources = EnumService.getImportTypes();
5151
public skipRules: boolean = false;
5252
public treatDatesAsUtc: boolean = false;
53+
public skipDuplicateReferenceCheck: boolean = false;
5354
public isLoading: boolean = false;
5455
public stagingMode: boolean = true;
5556

@@ -114,7 +115,8 @@ export class TransactionsImportComponent {
114115
create(ParseTransactionsRequestSchema, {
115116
content: contents,
116117
source: this.selectedSource,
117-
treatDatesAsUtc: this.treatDatesAsUtc
118+
treatDatesAsUtc: this.treatDatesAsUtc,
119+
skipDuplicateReferenceCheck: this.skipDuplicateReferenceCheck
118120
})
119121
);
120122

@@ -177,6 +179,7 @@ export class TransactionsImportComponent {
177179
create(ImportTransactionsRequestSchema, {
178180
skipRules: this.skipRules,
179181
treatDatesAsUtc: this.treatDatesAsUtc,
182+
skipDuplicateReferenceCheck: this.skipDuplicateReferenceCheck,
180183
source: this.selectedSource,
181184
content: contents
182185
})

‎go.mod‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ module github.com/ft-t/go-money
33
go 1.24.0
44

55
require (
6-
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2
7-
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1
6+
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2
7+
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1
88
connectrpc.com/connect v1.19.1
99
connectrpc.com/grpcreflect v1.3.0
1010
github.com/DATA-DOG/go-sqlmock v1.5.2
@@ -40,7 +40,7 @@ require (
4040
github.com/yuin/gopher-lua v1.1.1
4141
golang.org/x/crypto v0.41.0
4242
golang.org/x/net v0.43.0
43-
google.golang.org/protobuf v1.36.10
43+
google.golang.org/protobuf v1.36.11
4444
gorm.io/driver/postgres v1.6.0
4545
gorm.io/gorm v1.30.1
4646
layeh.com/gopher-luar v1.0.11

‎go.sum‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
al.essio.dev/pkg/shellescape v1.5.1/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
22
al.essio.dev/pkg/shellescape v1.6.0 h1:NxFcEqzFSEVCGN2yq7Huv/9hyCEGVa/TncnOOBBeXHA=
33
al.essio.dev/pkg/shellescape v1.6.0/go.mod h1:6sIqp7X2P6mThCQ7twERpZTuigpr6KbZWtls1U8I890=
4-
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2 h1:7kBnsqWCfiH07qhOTr5X0F8NFtYDRzNjAcSL9jQfxcE=
5-
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260104141704-8728d2c3d2bb.2/go.mod h1:gEifYXu//7jMqINMfR3bowTg4x0QVNEeOPG0aPCHqfA=
6-
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1 h1:zdfQtxzenede57tReYipiR3ZObDQnqujnnXAIlWRZsw=
7-
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.10-20260104141704-8728d2c3d2bb.1/go.mod h1:1m2p+4J6zxArUgz9MC5m6ViEir1z82cOu9Ub9Pgwa2Q=
4+
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2 h1:RKXNh2DP9OU7NJO1zIE7FPr1Hteg/5n5cKk0y7gDx3o=
5+
buf.build/gen/go/xskydev/go-money-pb/connectrpc/go v1.19.1-20260221145631-b1f442f55195.2/go.mod h1:VTTULP3xLDXiwW6AkXskKv7ZBgzqh4wgauKnA8Svumc=
6+
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1 h1:TCX0BllmWXkSzbWvPtsD2a7FBELmFhX5NIX1B5Vf6HI=
7+
buf.build/gen/go/xskydev/go-money-pb/protocolbuffers/go v1.36.11-20260221145631-b1f442f55195.1/go.mod h1:h6BlZCpceFTRzghEP6LSlTeYkE05pzKfQCnRYsP4kDA=
88
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
99
connectrpc.com/connect v1.19.1 h1:R5M57z05+90EfEvCY1b7hBxDVOUl45PrtXtAV2fOC14=
1010
connectrpc.com/connect v1.19.1/go.mod h1:tN20fjdGlewnSFeZxLKb0xwIZ6ozc3OQs2hTXy4du9w=
@@ -408,8 +408,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
408408
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
409409
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
410410
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
411-
google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
412-
google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
411+
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
412+
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
413413
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
414414
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
415415
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎pkg/importers/importer.go‎

Lines changed: 24 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package importers
22

33
import (
44
"context"
5+
"fmt"
56
"sort"
67
"strings"
78

@@ -49,6 +50,7 @@ func NewImporter(
4950
func (i *Importer) CheckDuplicates(
5051
ctx context.Context,
5152
requests []*transactionsv1.CreateTransactionRequest,
53+
skipDuplicateRefCheck bool,
5254
) ([]*DeduplicationItem, error) {
5355
var allRefs []string
5456
refToItem := map[string]*DeduplicationItem{}
@@ -67,13 +69,26 @@ func (i *Importer) CheckDuplicates(
6769
return nil, errors.New("all transactions must have at least one reference number for deduplication")
6870
}
6971

70-
for _, ref := range validRefs {
71-
if existing, exists := refToItem[ref]; exists {
72-
return nil, errors.Errorf("duplicate reference number found in import data: ref=%s. raw=%v", ref,
73-
existing.CreateRequest.Notes)
72+
for idx, ref := range validRefs {
73+
if _, exists := refToItem[ref]; exists {
74+
if !skipDuplicateRefCheck {
75+
return nil, errors.Errorf("duplicate reference number found in import data: ref=%s. raw=%v", ref,
76+
refToItem[ref].CreateRequest.Notes)
77+
}
78+
79+
counter := 1
80+
newRef := fmt.Sprintf("%s_%d", ref, counter)
81+
for _, exists := refToItem[newRef]; exists; _, exists = refToItem[newRef] {
82+
counter++
83+
newRef = fmt.Sprintf("%s_%d", ref, counter)
84+
}
85+
86+
validRefs[idx] = newRef
7487
}
7588
}
7689

90+
req.InternalReferenceNumbers = validRefs
91+
7792
item := &DeduplicationItem{CreateRequest: req}
7893
items = append(items, item)
7994

@@ -115,9 +130,10 @@ func (i *Importer) Import(
115130
req *importv1.ImportTransactionsRequest,
116131
) (*importv1.ImportTransactionsResponse, error) {
117132
parsed, err := i.ParseInternal(ctx, &importv1.ParseTransactionsRequest{
118-
Content: req.Content,
119-
Source: req.Source,
120-
TreatDatesAsUtc: req.TreatDatesAsUtc,
133+
Content: req.Content,
134+
Source: req.Source,
135+
TreatDatesAsUtc: req.TreatDatesAsUtc,
136+
SkipDuplicateReferenceCheck: req.SkipDuplicateReferenceCheck,
121137
})
122138
if err != nil {
123139
return nil, err
@@ -252,7 +268,7 @@ func (i *Importer) ParseInternal(
252268
r.Extra["import_batch_id"] = batchID
253269
}
254270

255-
items, err := i.CheckDuplicates(ctx, parsed.CreateRequests)
271+
items, err := i.CheckDuplicates(ctx, parsed.CreateRequests, req.SkipDuplicateReferenceCheck)
256272
if err != nil {
257273
log.Error().
258274
Err(err).

‎pkg/importers/importer_test.go‎

Lines changed: 37 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -506,7 +506,7 @@ func TestCheckDuplicates(t *testing.T) {
506506
},
507507
}
508508

509-
result, err := imp.CheckDuplicates(context.TODO(), requests)
509+
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
510510
assert.NoError(t, err)
511511
assert.Len(t, result, 2)
512512
assert.Equal(t, "Transaction 1", result[0].CreateRequest.Title)
@@ -530,7 +530,7 @@ func TestCheckDuplicates(t *testing.T) {
530530
},
531531
}
532532

533-
result, err := imp.CheckDuplicates(context.TODO(), requests)
533+
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
534534
assert.Error(t, err)
535535
assert.Nil(t, result)
536536
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
@@ -551,7 +551,7 @@ func TestCheckDuplicates(t *testing.T) {
551551
},
552552
}
553553

554-
result, err := imp.CheckDuplicates(context.TODO(), requests)
554+
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
555555
assert.Error(t, err)
556556
assert.Nil(t, result)
557557
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
@@ -572,7 +572,7 @@ func TestCheckDuplicates(t *testing.T) {
572572
},
573573
}
574574

575-
result, err := imp.CheckDuplicates(context.TODO(), requests)
575+
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
576576
assert.Error(t, err)
577577
assert.Nil(t, result)
578578
assert.Contains(t, err.Error(), "all transactions must have at least one reference number for deduplication")
@@ -597,12 +597,43 @@ func TestCheckDuplicates(t *testing.T) {
597597
},
598598
}
599599

600-
result, err := imp.CheckDuplicates(context.TODO(), requests)
600+
result, err := imp.CheckDuplicates(context.TODO(), requests, false)
601601
assert.Error(t, err)
602602
assert.Nil(t, result)
603603
assert.Contains(t, err.Error(), "duplicate reference number found in import data: ref=ref_duplicate")
604604
})
605605

606+
t.Run("skip duplicate reference check adds suffix", func(t *testing.T) {
607+
ctrl := gomock.NewController(t)
608+
defer ctrl.Finish()
609+
610+
impl := NewMockImplementation(ctrl)
611+
impl.EXPECT().Type().Return(importv1.ImportSource_IMPORT_SOURCE_FIREFLY)
612+
imp := importers.NewImporter(&importers.ImporterConfig{}, impl)
613+
614+
requests := []*transactionsv1.CreateTransactionRequest{
615+
{
616+
Title: "Transaction 1",
617+
InternalReferenceNumbers: []string{"ref_duplicate"},
618+
},
619+
{
620+
Title: "Transaction 2",
621+
InternalReferenceNumbers: []string{"ref_duplicate"},
622+
},
623+
{
624+
Title: "Transaction 3",
625+
InternalReferenceNumbers: []string{"ref_duplicate"},
626+
},
627+
}
628+
629+
result, err := imp.CheckDuplicates(context.TODO(), requests, true)
630+
assert.NoError(t, err)
631+
assert.Len(t, result, 3)
632+
assert.Equal(t, []string{"ref_duplicate"}, result[0].CreateRequest.InternalReferenceNumbers)
633+
assert.Equal(t, []string{"ref_duplicate_1"}, result[1].CreateRequest.InternalReferenceNumbers)
634+
assert.Equal(t, []string{"ref_duplicate_2"}, result[2].CreateRequest.InternalReferenceNumbers)
635+
})
636+
606637
t.Run("db error on check", func(t *testing.T) {
607638
ctrl := gomock.NewController(t)
608639
defer ctrl.Finish()
@@ -625,7 +656,7 @@ func TestCheckDuplicates(t *testing.T) {
625656
},
626657
}
627658

628-
result, err := imp.CheckDuplicates(ctx, requests)
659+
result, err := imp.CheckDuplicates(ctx, requests, false)
629660
assert.Error(t, err)
630661
assert.Nil(t, result)
631662
assert.Contains(t, err.Error(), "failed to check existing transactions")

‎pkg/testingutils/gorm.go‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -97,8 +97,7 @@ func flushInternal(config boilerplate.DbConfig, tables []string) error {
9797
}
9898

9999
if _, ok := existing[strings.ToLower(table)]; ok {
100-
101-
builder.WriteString(fmt.Sprintf(" truncate table %v CASCADE; ", strings.ToLower(table)))
100+
_, _ = fmt.Fprintf(&builder, " truncate table %v CASCADE; ", strings.ToLower(table))
102101
} else {
103102
log.Warn().Msgf("table %v does not exists", table)
104103
}
@@ -108,7 +107,7 @@ func flushInternal(config boilerplate.DbConfig, tables []string) error {
108107
if strings.ToLower(name) == "public.migrations" {
109108
continue
110109
}
111-
builder.WriteString(fmt.Sprintf(" truncate table %v CASCADE; ", strings.ToLower(name)))
110+
_, _ = fmt.Fprintf(&builder, " truncate table %v CASCADE; ", strings.ToLower(name))
112111
}
113112
}
114113

0 commit comments

Comments
 (0)