Skip to content

Commit 62d3f3d

Browse files
CufeeCufee
committed
using pools for snapshots
1 parent a1e11a2 commit 62d3f3d

File tree

11 files changed

+85
-75
lines changed

11 files changed

+85
-75
lines changed

‎cmd/core/server/handlers/private/accounts.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ func LoadAccountsHandler(client core.Client) http.HandlerFunc {
8787
return
8888
}
8989

90-
var inserts []models.Account
90+
var inserts []*models.Account
9191
for id, account := range data {
9292
if id == "" && account.ID == 0 {
9393
log.Warn().Str("reason", "id is blank").Msg("wargaming returned a bad account")
@@ -103,10 +103,12 @@ func LoadAccountsHandler(client core.Client) http.HandlerFunc {
103103
account.Nickname = "@Hidden"
104104
private = true
105105
}
106-
inserts = append(inserts, fetch.WargamingToAccount(realm, account, types.ClanMember{}, private))
106+
107+
update := fetch.WargamingToAccount(realm, account, types.ClanMember{}, private)
108+
inserts = append(inserts, &update)
107109
}
108110

109-
accErr, err := client.Database().UpsertAccounts(ctx, inserts)
111+
accErr, err := client.Database().UpsertAccounts(ctx, inserts...)
110112
if err != nil {
111113
log.Err(err).Msg("failed to upsert accounts")
112114
}

‎cmd/core/tasks/cleanup.go‎

Lines changed: 0 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -44,27 +44,6 @@ func init() {
4444
return errors.Wrap(err, "failed to delete expired snapshots")
4545
}
4646
}
47-
{
48-
leaderboardsExpiration, err := time.Parse(time.RFC3339, task.Data["expiration_leaderboards_daily"])
49-
if err != nil {
50-
return errors.Wrap(err, "failed to parse expiration_snapshots to time")
51-
}
52-
err = client.Database().DeleteExpiredLeaderboardScores(ctx, leaderboardsExpiration, models.LeaderboardScoreDaily)
53-
if err != nil && !database.IsNotFound(err) {
54-
return errors.Wrap(err, "failed to delete expired leaderboard scores")
55-
}
56-
}
57-
{
58-
leaderboardsExpiration, err := time.Parse(time.RFC3339, task.Data["expiration_leaderboards_hourly"])
59-
if err != nil {
60-
return errors.Wrap(err, "failed to parse expiration_snapshots to time")
61-
}
62-
err = client.Database().DeleteExpiredLeaderboardScores(ctx, leaderboardsExpiration, models.LeaderboardScoreHourly)
63-
if err != nil && !database.IsNotFound(err) {
64-
return errors.Wrap(err, "failed to delete expired leaderboard scores")
65-
}
66-
}
67-
6847
return nil
6948
},
7049
}

‎internal/database/accounts.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func (c *client) GetAccounts(ctx context.Context, ids []string) ([]models.Accoun
6060
return accounts, nil
6161
}
6262

63-
func (c *client) UpsertAccounts(ctx context.Context, accounts []models.Account) (map[string]error, error) {
63+
func (c *client) UpsertAccounts(ctx context.Context, accounts ...*models.Account) (map[string]error, error) {
6464
if len(accounts) < 1 {
6565
return nil, nil
6666
}
@@ -69,7 +69,7 @@ func (c *client) UpsertAccounts(ctx context.Context, accounts []models.Account)
6969
accountsMap := make(map[string]*models.Account)
7070
for _, a := range accounts {
7171
ids = append(ids, a.ID)
72-
accountsMap[a.ID] = &a
72+
accountsMap[a.ID] = a
7373
}
7474

7575
records, err := c.db.Account.Query().Where(account.IDIn(ids...)).All(ctx)

‎internal/database/client.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ type AccountsClient interface {
3636
GetAccountByID(ctx context.Context, id string) (models.Account, error)
3737
GetRealmAccountIDs(ctx context.Context, realm string) ([]string, error)
3838
AccountSetPrivate(ctx context.Context, id string, value bool) error
39-
UpsertAccounts(ctx context.Context, accounts []models.Account) (map[string]error, error)
39+
UpsertAccounts(ctx context.Context, accounts ...*models.Account) (map[string]error, error)
4040
}
4141

4242
type GlossaryClient interface {
@@ -85,19 +85,12 @@ type SnapshotsClient interface {
8585
GetAccountLastBattleTimes(ctx context.Context, accountIDs []string, kind models.SnapshotType, options ...Query) (map[string]time.Time, error)
8686
GetVehicleLastBattleTimes(ctx context.Context, accountID string, vehicleIDs []string, kind models.SnapshotType, options ...Query) (map[string]time.Time, error)
8787

88-
CreateAccountSnapshots(ctx context.Context, snapshots ...models.AccountSnapshot) error
89-
CreateAccountVehicleSnapshots(ctx context.Context, accountID string, snapshots ...models.VehicleSnapshot) error
88+
CreateAccountSnapshots(ctx context.Context, snapshots ...*models.AccountSnapshot) error
89+
CreateAccountVehicleSnapshots(ctx context.Context, accountID string, snapshots ...*models.VehicleSnapshot) error
9090

9191
DeleteExpiredSnapshots(ctx context.Context, expiration time.Time) error
9292
}
9393

94-
type LeaderboardsClient interface {
95-
CreateLeaderboardScores(ctx context.Context, scores ...models.LeaderboardScore) error
96-
GetLeaderboardScores(ctx context.Context, leaderboardID string, scoreType models.ScoreType, options ...Query) ([]models.LeaderboardScore, error)
97-
98-
DeleteExpiredLeaderboardScores(ctx context.Context, expiration time.Time, scoreType models.ScoreType) error
99-
}
100-
10194
type TasksClient interface {
10295
CreateTasks(ctx context.Context, tasks ...models.Task) error
10396
GetTasks(ctx context.Context, ids ...string) ([]models.Task, error)
@@ -141,7 +134,6 @@ type Client interface {
141134
GlossaryClient
142135
AccountsClient
143136
SnapshotsClient
144-
LeaderboardsClient
145137

146138
TasksClient
147139

‎internal/database/client_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,10 +43,10 @@ func TestConcurrentWrites(t *testing.T) {
4343
v.AccountID = id
4444
v.ReferenceID = id
4545
fn = append(fn, func() {
46-
_, err := client.UpsertAccounts(ctx, []models.Account{{ID: id, Realm: "test", Nickname: "test_account"}})
46+
_, err := client.UpsertAccounts(ctx, &models.Account{ID: id, Realm: "test", Nickname: "test_account"})
4747
is.NoErr(err)
4848

49-
err = client.CreateAccountVehicleSnapshots(ctx, id, v)
49+
err = client.CreateAccountVehicleSnapshots(ctx, id, &v)
5050
is.NoErr(err)
5151
})
5252
}

‎internal/database/snapshots.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -103,7 +103,7 @@ func (c *client) GetVehicleLastBattleTimes(ctx context.Context, accountID string
103103
}
104104

105105
// create vehicle snapshots for a specific account
106-
func (c *client) CreateAccountVehicleSnapshots(ctx context.Context, accountID string, snapshots ...models.VehicleSnapshot) error {
106+
func (c *client) CreateAccountVehicleSnapshots(ctx context.Context, accountID string, snapshots ...*models.VehicleSnapshot) error {
107107
if len(snapshots) < 1 {
108108
return nil
109109
}
@@ -249,7 +249,7 @@ func (c *client) GetAccountLastBattleTimes(ctx context.Context, accountIDs []str
249249
return lastBattles, nil
250250
}
251251

252-
func (c *client) CreateAccountSnapshots(ctx context.Context, snapshots ...models.AccountSnapshot) error {
252+
func (c *client) CreateAccountSnapshots(ctx context.Context, snapshots ...*models.AccountSnapshot) error {
253253
if len(snapshots) < 1 {
254254
return nil
255255
}

‎internal/database/snapshots_test.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func TestGetVehicleSnapshots(t *testing.T) {
2626
client.db.VehicleSnapshot.Delete().Where().Exec(ctx)
2727
defer client.db.VehicleSnapshot.Delete().Exec(ctx)
2828

29-
_, err = client.UpsertAccounts(ctx, []models.Account{{ID: "a1", Realm: "test", Nickname: "test_account"}})
29+
_, err = client.UpsertAccounts(ctx, &models.Account{ID: "a1", Realm: "test", Nickname: "test_account"})
3030
assert.NoError(t, err, "failed to upsert an account")
3131

3232
createdAtVehicle1 := time.Date(2023, 6, 1, 0, 0, 0, 0, time.UTC)
@@ -92,7 +92,7 @@ func TestGetVehicleSnapshots(t *testing.T) {
9292
}
9393

9494
{ // create snapshots
95-
snapshots := []models.VehicleSnapshot{vehicle1, vehicle2, vehicle3, vehicle4, vehicle5, vehicle6}
95+
snapshots := []*models.VehicleSnapshot{&vehicle1, &vehicle2, &vehicle3, &vehicle4, &vehicle5, &vehicle6}
9696
err := client.CreateAccountVehicleSnapshots(ctx, "a1", snapshots...)
9797
assert.NoError(t, err, "create vehicle snapshot should not error")
9898
}

‎internal/logic/pool.go‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
package logic
2+
3+
import "sync"
4+
5+
type pool[T any] struct {
6+
pool *sync.Pool
7+
}
8+
9+
func (p *pool[T]) Get() *T {
10+
item, _ := p.pool.Get().(*T)
11+
if item == nil {
12+
item = new(T)
13+
} else {
14+
// Clear object
15+
var zero T
16+
*item = zero
17+
}
18+
return item
19+
}
20+
21+
func (p *pool[T]) Put(s *T) {
22+
p.pool.Put(s)
23+
}
24+
25+
func newPool[T any]() *pool[T] {
26+
return &pool[T]{
27+
pool: &sync.Pool{
28+
New: func() any {
29+
return new(T)
30+
},
31+
},
32+
}
33+
}

‎internal/logic/snapshots.go‎

Lines changed: 29 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ import (
1616
"github.com/pkg/errors"
1717
)
1818

19+
var (
20+
accountsPool = newPool[models.Account]()
21+
vehicleSnapshotsPool = newPool[models.VehicleSnapshot]()
22+
accountSnapshotsPool = newPool[models.AccountSnapshot]()
23+
)
24+
1925
type AccountsWithReference map[string]string
2026

2127
func WithReference(accountID string, referenceID string) AccountsWithReference {
@@ -143,10 +149,10 @@ func RecordAccountSnapshots(ctx context.Context, wgClient wargaming.Client, dbCl
143149
group.Wait()
144150

145151
var accountErrors = make(map[string]error)
146-
var accountUpdates = make(map[string]models.Account)
152+
var accountUpdates = make(map[string]*models.Account)
147153

148-
var accountSnapshots []models.AccountSnapshot
149-
var vehicleSnapshots = make(map[string][]models.VehicleSnapshot)
154+
var accountSnapshots []*models.AccountSnapshot
155+
var vehicleSnapshots = make(map[string][]*models.VehicleSnapshot)
150156

151157
for _, accountID := range accountsNeedAnUpdate {
152158
vehicles := accountVehicles[accountID]
@@ -162,16 +168,25 @@ func RecordAccountSnapshots(ctx context.Context, wgClient wargaming.Client, dbCl
162168

163169
snapshotStats := fetch.WargamingToStats(realm, accounts[accountID], clans[accountID], vehicles.Data)
164170
{ // account snapshot
165-
accountSnapshots = append(accountSnapshots, models.AccountSnapshot{
171+
sht := accountSnapshotsPool.Get()
172+
defer accountSnapshotsPool.Put(sht)
173+
174+
*sht = models.AccountSnapshot{
166175
Type: models.SnapshotTypeDaily,
167176
CreatedAt: createdAt,
168177
ReferenceID: accountRefID,
169178
AccountID: snapshotStats.Account.ID,
170179
LastBattleTime: snapshotStats.LastBattleTime,
171180
RatingBattles: snapshotStats.RatingBattles.StatsFrame,
172181
RegularBattles: snapshotStats.RegularBattles.StatsFrame,
173-
})
174-
accountUpdates[accountID] = snapshotStats.Account
182+
}
183+
accountSnapshots = append(accountSnapshots, sht)
184+
185+
asht := accountsPool.Get()
186+
defer accountsPool.Put(asht)
187+
188+
*asht = snapshotStats.Account
189+
accountUpdates[accountID] = asht
175190
}
176191

177192
// vehicle snapshots
@@ -180,15 +195,20 @@ func RecordAccountSnapshots(ctx context.Context, wgClient wargaming.Client, dbCl
180195
if len(vehicles.Data) > 0 {
181196
for id, vehicle := range vehicleStats {
182197
vehicleLastBattleTimes[id] = vehicleBattleData{vehicle.LastBattleTime, int(vehicle.Battles.Float())}
183-
vehicleSnapshots[accountID] = append(vehicleSnapshots[accountID], models.VehicleSnapshot{
198+
199+
sht := vehicleSnapshotsPool.Get()
200+
defer vehicleSnapshotsPool.Put(sht)
201+
202+
*sht = models.VehicleSnapshot{
184203
Type: models.SnapshotTypeDaily,
185204
LastBattleTime: vehicle.LastBattleTime,
186205
Stats: *vehicle.StatsFrame,
187206
VehicleID: vehicle.VehicleID,
188207
ReferenceID: accountRefID,
189208
AccountID: accountID,
190209
CreatedAt: createdAt,
191-
})
210+
}
211+
vehicleSnapshots[accountID] = append(vehicleSnapshots[accountID], sht)
192212
}
193213
}
194214
}
@@ -212,7 +232,7 @@ func RecordAccountSnapshots(ctx context.Context, wgClient wargaming.Client, dbCl
212232
}
213233

214234
// update account cache, non critical and should not fail the flow
215-
_, err = dbClient.UpsertAccounts(ctx, []models.Account{accountUpdates[accountSnapshot.AccountID]})
235+
_, err = dbClient.UpsertAccounts(ctx, accountUpdates[accountSnapshot.AccountID])
216236
if err != nil {
217237
log.Err(err).Str("accountId", accountSnapshot.AccountID).Msg("failed to upsert account")
218238
}

‎internal/stats/fetch/v1/multisource.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ func (c *multiSourceClient) Account(ctx context.Context, id string) (models.Acco
151151
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
152152
defer cancel()
153153

154-
aErr, err := c.database.UpsertAccounts(ctx, []models.Account{account})
154+
aErr, err := c.database.UpsertAccounts(ctx, &account)
155155
if err != nil {
156156
log.Err(err).Msg("failed to update account cache")
157157
}
@@ -243,7 +243,7 @@ func (c *multiSourceClient) CurrentStats(ctx context.Context, id string, opts ..
243243
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
244244
defer cancel()
245245

246-
aErr, err := c.database.UpsertAccounts(ctx, []models.Account{account})
246+
aErr, err := c.database.UpsertAccounts(ctx, &account)
247247
if err != nil {
248248
log.Err(err).Msg("failed to update account cache")
249249
}

0 commit comments

Comments
 (0)