Skip to content

Commit bb8cdd7

Browse files
skrasekmichaelskrasekmichael
authored
Improve seeding tool (#68)
- add CLI parameters - level up experience with timers and progress logging - improve performance with batched inserting
1 parent 683af9c commit bb8cdd7

File tree

8 files changed

+379
-42
lines changed

8 files changed

+379
-42
lines changed

‎Directory.Packages.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
<PackageVersion Include="Asp.Versioning.Mvc.ApiExplorer" Version="8.1.0" />
88
<PackageVersion Include="Bogus" Version="35.6.3" />
99
<PackageVersion Include="coverlet.collector" Version="6.0.4" />
10+
<PackageVersion Include="EFCore.BulkExtensions.PostgreSql" Version="9.0.1" />
1011
<PackageVersion Include="FluentValidation" Version="12.0.0" />
1112
<PackageVersion Include="FluentValidation.DependencyInjectionExtensions" Version="12.0.0" />
1213
<PackageVersion Include="MailKit" Version="4.12.0" />

‎tests/TeamUp.Tests.Common/DataGenerators/EventGeneratorExtensions.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,29 @@ public static EventGenerator WithRandomEventResponses(this EventGenerator genera
2424
.ToList());
2525
}
2626

27+
public static EventGenerator WithRandomEventResponses(this EventGenerator generator, IEnumerable<TeamMember> members, int randomMembersCount)
28+
{
29+
var randomMembers = members
30+
.OrderBy(_ => Guid.NewGuid())
31+
.Take(randomMembersCount)
32+
.ToList();
33+
34+
return generator
35+
.RuleFor(EventGenerators.EVENT_RESPONSES_FIELD, (f, e) => randomMembers
36+
.Select(member => EventGenerators.Response
37+
.RuleForBackingField(er => er.EventId, e.Id)
38+
.RuleForBackingField(er => er.TeamMemberId, member.Id)
39+
.RuleFor(er => er.TimeStampUtc, f => f.Date
40+
.Between(DateTime.UtcNow.AddDays(-2), e.FromUtc - e.MeetTime - e.ReplyClosingTimeBeforeMeetTime)
41+
.DropMicroSeconds()
42+
.AsUtc())
43+
.RuleFor(er => er.ReplyType, f => f.PickRandom(EventGenerators.ReplyTypes))
44+
.RuleFor(er => er.Message, (f, er) => er.ReplyType == ReplyType.Yes ? string.Empty : f.Random.Text(1, EventConstants.EVENT_REPLY_MESSAGE_MAX_SIZE))
45+
.Generate())
46+
.OrderBy(er => er.TimeStampUtc)
47+
.ToList());
48+
}
49+
2750
public static EventGenerator WithEventResponses(this EventGenerator generator, List<(TeamMember Member, ReplyType Type)> responses)
2851
{
2952
return generator

‎tools/TeamUp.Tools.Seeder/BatchedInsert.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
using EFCore.BulkExtensions;
2+
using TeamUp.Infrastructure.Persistence;
3+
4+
namespace TeamUp.Tools.Seeder;
5+
6+
internal static class BatchedInsert
7+
{
8+
public static async Task BatchedInsertAsync<T>(this ApplicationDbContext dbContext, List<T> input, BulkConfig bulkConfig, Action<double> progress, CancellationToken ct = default) where T : class
9+
{
10+
if (input.Count > bulkConfig.BatchSize)
11+
{
12+
progress(0);
13+
}
14+
15+
double done = 0;
16+
double step = 1.0 / ((double)input.Count / bulkConfig.BatchSize);
17+
18+
int i = 0;
19+
IEnumerable<T> enumerable = input;
20+
while (i < input.Count)
21+
{
22+
await dbContext.BulkInsertAsync(enumerable.Take(bulkConfig.BatchSize), bulkConfig, cancellationToken: ct);
23+
24+
done += step;
25+
progress(Math.Min(done, 1.0));
26+
27+
enumerable = enumerable.Skip(bulkConfig.BatchSize);
28+
i += bulkConfig.BatchSize;
29+
}
30+
}
31+
}

‎tools/TeamUp.Tools.Seeder/ConsoleTimer.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
using System.Diagnostics;
2+
3+
namespace TeamUp.Tools.Seeder;
4+
5+
internal sealed class ConsoleTimer : IAsyncDisposable
6+
{
7+
private readonly int _timerX, _timerY;
8+
private readonly long _startTimestamp;
9+
private readonly Task _timerTask;
10+
private double? _progress = null;
11+
private bool _isRunning;
12+
13+
public ConsoleTimer(CancellationToken ct = default)
14+
{
15+
(_timerX, _timerY) = Console.GetCursorPosition();
16+
_isRunning = true;
17+
_startTimestamp = Stopwatch.GetTimestamp();
18+
19+
_timerTask = Task.Run(async () =>
20+
{
21+
while (!ct.IsCancellationRequested)
22+
{
23+
var elapsed = Stopwatch.GetElapsedTime(_startTimestamp);
24+
25+
Console.CursorVisible = false;
26+
var (currX, currY) = Console.GetCursorPosition();
27+
Console.SetCursorPosition(_timerX, _timerY);
28+
29+
if (!_isRunning)
30+
{
31+
Console.WriteLine($"DONE [{elapsed:hh\\:mm\\:ss}] ");
32+
return;
33+
}
34+
35+
var progress = _progress is not null ? $"{_progress * 100,2:#0} % " : null;
36+
Console.Write($"{progress}[{elapsed:hh\\:mm\\:ss}] ");
37+
Console.SetCursorPosition(currX, currY);
38+
Console.CursorVisible = true;
39+
await Task.Delay(1000, ct);
40+
}
41+
}, ct);
42+
}
43+
44+
public void SetProgress(double progress)
45+
{
46+
_progress = progress;
47+
}
48+
49+
public async ValueTask DisposeAsync()
50+
{
51+
_isRunning = false;
52+
await _timerTask;
53+
}
54+
55+
public static Task<T> CallAsync<T>(Func<T> call, CancellationToken ct = default)
56+
{
57+
return CallAsync(ct => Task.Run(call, ct), ct);
58+
}
59+
60+
public static async Task CallAsync(Func<CancellationToken, Task> asyncCall, CancellationToken ct = default)
61+
{
62+
await using var timer = new ConsoleTimer(ct);
63+
await asyncCall(ct);
64+
}
65+
66+
public static async Task<T> CallAsync<T>(Func<CancellationToken, Task<T>> asyncCall, CancellationToken ct = default)
67+
{
68+
await using var timer = new ConsoleTimer(ct);
69+
return await asyncCall(ct);
70+
}
71+
}

‎tools/TeamUp.Tools.Seeder/Program.cs‎

Lines changed: 63 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,79 @@
11
using Microsoft.EntityFrameworkCore;
2+
using Npgsql;
3+
using Respawn;
4+
using Respawn.Graph;
25
using TeamUp.Infrastructure.Persistence;
36

47
namespace TeamUp.Tools.Seeder;
58

6-
class Program
9+
static class Program
710
{
811
/// <param name="connectionString">DB connection string</param>
9-
static async Task<int> Main(string connectionString)
12+
/// <param name="seedingInstructionsJSON">Expects seeding instructions in JSON format, if nothing is passed, the tool will use default seeding instructions</param>
13+
/// <param name="seedDb">true means the tool will try to seed the database according to seeding strategy</param>
14+
/// <param name="clearDb">true means the tool will clear the database</param>
15+
static async Task<int> Main(
16+
string connectionString,
17+
string? seedingInstructionsJSON = null,
18+
bool seedDb = true,
19+
bool clearDb = false)
1020
{
11-
var options = new DbContextOptionsBuilder<ApplicationDbContext>()
12-
.UseNpgsql(connectionString)
13-
.Options;
21+
if (!SeedingInstructions.TryParse(seedingInstructionsJSON, out var seedingInstructions))
22+
{
23+
Console.Error.WriteLine("Failed to parse seeding instructions");
24+
return 1;
25+
}
26+
27+
var validator = new SeedingInstructions.SeedingInstructionsValidator();
28+
var result = validator.Validate(seedingInstructions);
1429

15-
await using (var dbContext = new ApplicationDbContext(options))
30+
if (!result.IsValid)
1631
{
17-
await dbContext.Database.MigrateAsync();
32+
Console.Error.WriteLine("Invalid seeding instructions:");
33+
foreach (var error in result.Errors)
34+
{
35+
Console.Error.WriteLine($"- {error.ErrorMessage}");
36+
}
1837

19-
var seeder = new Seeder(dbContext);
20-
await seeder.SeedAsync();
38+
return 1;
39+
}
40+
41+
if (clearDb)
42+
{
43+
Console.Write("Clearing database...");
44+
await ConsoleTimer.CallAsync(() => ClearDatabaseAsync(connectionString), default);
45+
}
46+
47+
if (seedDb)
48+
{
49+
var options = new DbContextOptionsBuilder<ApplicationDbContext>()
50+
.UseNpgsql(connectionString)
51+
.Options;
52+
53+
await using (var dbContext = new ApplicationDbContext(options))
54+
{
55+
Console.Write("Migrating database...");
56+
await ConsoleTimer.CallAsync(dbContext.Database.MigrateAsync, default);
57+
58+
var seeder = new Seeder(dbContext, seedingInstructions);
59+
await seeder.SeedAsync();
60+
}
2161
}
2262

2363
return 0;
2464
}
65+
66+
private static async Task ClearDatabaseAsync(string connectionString)
67+
{
68+
await using var connection = new NpgsqlConnection(connectionString);
69+
await connection.OpenAsync();
70+
71+
var respawner = await Respawner.CreateAsync(connection, new()
72+
{
73+
DbAdapter = DbAdapter.Postgres,
74+
TablesToIgnore = [new Table("public", "__EFMigrationsHistory")]
75+
});
76+
77+
await respawner.ResetAsync(connection);
78+
}
2579
}

‎tools/TeamUp.Tools.Seeder/Seeder.cs‎

Lines changed: 110 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,135 @@
1-
using TeamUp.Contracts.Teams;
1+
using EFCore.BulkExtensions;
2+
using TeamUp.Contracts.Teams;
3+
using TeamUp.Domain.Aggregates.Teams;
4+
using TeamUp.Domain.Aggregates.Users;
25
using TeamUp.Infrastructure.Persistence;
36
using TeamUp.Tests.Common.DataGenerators;
47

58
namespace TeamUp.Tools.Seeder;
69

7-
internal sealed class Seeder(ApplicationDbContext dbContext) : IAsyncDisposable, IDisposable
10+
internal sealed class Seeder : IDisposable, IAsyncDisposable
811
{
9-
private readonly ApplicationDbContext _dbContext = dbContext;
12+
private readonly ApplicationDbContext _dbContext;
13+
private readonly SeedingInstructions _instructions;
14+
private readonly int _usersWithoutTeamCount, _usersWithTeamCount, _invitationsCount, _numberOfResponsesPerEvent;
15+
16+
internal Seeder(ApplicationDbContext dbContext, SeedingInstructions instructions)
17+
{
18+
_dbContext = dbContext;
19+
_instructions = instructions;
20+
21+
_usersWithoutTeamCount = (int)(instructions.TotalUsers * instructions.UsersWithoutTeamRatio);
22+
_usersWithTeamCount = instructions.TotalUsers - _usersWithoutTeamCount;
23+
_invitationsCount = (int)(instructions.InvitationsRate * _usersWithoutTeamCount);
24+
_numberOfResponsesPerEvent = (int)(instructions.EventResponseRate * instructions.EventsPerTeam);
25+
}
1026

1127
public async Task SeedAsync(CancellationToken ct = default)
1228
{
13-
var users = UserGenerators.DistinctUser.Generate(3500);
14-
var usersWithoutTeam = UserGenerators.DistinctUser.Generate(500);
29+
Console.WriteLine(_instructions);
30+
Console.WriteLine();
31+
32+
var (users, usersWithoutTeam) = await SeedUsersAsync(ct);
33+
Console.WriteLine();
34+
35+
var teams = await SeedTeamsAsync(users, ct);
36+
Console.WriteLine();
37+
38+
await SeedEventsAsync(teams, ct);
39+
Console.WriteLine();
40+
41+
await SeedInvitationsAsync(usersWithoutTeam, teams, ct);
42+
Console.WriteLine();
43+
}
44+
45+
private async Task<(List<User> users, List<User> usersWithoutTeam)> SeedUsersAsync(CancellationToken ct)
46+
{
47+
Console.Write($"Generating {_instructions.TotalUsers} users...");
48+
var (users, usersWithoutTeam) = await ConsoleTimer.CallAsync(() =>
49+
{
50+
var users = UserGenerators.DistinctUser.Generate(_usersWithTeamCount);
51+
var usersWithoutTeam = UserGenerators.DistinctUser.Generate(_usersWithoutTeamCount);
52+
return (users, usersWithoutTeam);
53+
}, ct);
1554

1655
_dbContext.Users.AddRange(users);
1756
_dbContext.Users.AddRange(usersWithoutTeam);
1857

19-
Console.Write("Adding 4.000 users...");
20-
await _dbContext.SaveChangesAsync(ct);
21-
Console.WriteLine("DONE");
58+
Console.Write($"Inserting {_instructions.TotalUsers} users...");
59+
await ConsoleTimer.CallAsync(_dbContext.SaveChangesAsync, ct);
60+
Console.WriteLine($"Total users: {_instructions.TotalUsers}, users with no team: {_usersWithoutTeamCount}");
61+
return (users, usersWithoutTeam);
62+
}
2263

23-
var teams = TeamGenerators.Team
24-
.WithRandomMembers(TeamConstants.MAX_TEAM_CAPACITY, users)
25-
.WithEventTypes(10)
26-
.Generate(500);
64+
private async Task<List<Team>> SeedTeamsAsync(List<User> users, CancellationToken ct)
65+
{
66+
Console.Write($"Generating {_instructions.TotalTeams} teams...");
67+
var teams = await ConsoleTimer.CallAsync(() =>
68+
{
69+
return TeamGenerators.Team
70+
.WithRandomMembers(TeamConstants.MAX_TEAM_CAPACITY, users)
71+
.WithEventTypes(_instructions.EventTypesPerTeam)
72+
.Generate(_instructions.TotalTeams);
73+
}, ct);
2774

2875
_dbContext.Teams.AddRange(teams);
2976

30-
Console.Write("Adding 500 teams each with 30 members and 10 event types...");
31-
await _dbContext.SaveChangesAsync(ct);
32-
Console.WriteLine("DONE");
33-
34-
var events = teams
35-
.Select(team => EventGenerators.Event
36-
.ForTeam(team.Id)
37-
.WithRandomEventType(team.EventTypes)
38-
.WithRandomEventResponses(team.Members)
39-
.Generate(100))
40-
.SelectMany(teamEvents => teamEvents)
41-
.ToList();
42-
43-
_dbContext.Events.AddRange(events);
77+
Console.Write($"Inserting {_instructions.TotalTeams} teams...");
78+
await ConsoleTimer.CallAsync(_dbContext.SaveChangesAsync, ct);
79+
Console.WriteLine($"Total teams: {_instructions.TotalTeams}, each with {TeamConstants.MAX_TEAM_CAPACITY} members and {_instructions.EventTypesPerTeam} event types");
80+
Console.WriteLine($"Total members: {TeamConstants.MAX_TEAM_CAPACITY * _instructions.TotalTeams}");
81+
Console.WriteLine($"Total event types: {_instructions.EventTypesPerTeam * _instructions.TotalTeams}");
82+
return teams;
83+
}
4484

45-
Console.Write("Adding 50.000 events and 1.500.000 event responses...");
46-
await _dbContext.SaveChangesAsync(ct);
47-
Console.WriteLine("DONE");
85+
private async Task SeedEventsAsync(List<Team> teams, CancellationToken ct)
86+
{
87+
Console.Write($"Generating {_instructions.EventsPerTeam * teams.Count} events...");
88+
var events = await ConsoleTimer.CallAsync(() =>
89+
{
90+
return teams
91+
.Select(team => EventGenerators.Event
92+
.ForTeam(team.Id)
93+
.WithRandomEventType(team.EventTypes)
94+
.WithRandomEventResponses(team.Members, _numberOfResponsesPerEvent)
95+
.Generate(_instructions.EventsPerTeam))
96+
.SelectMany(teamEvents => teamEvents)
97+
.ToList();
98+
}, ct);
99+
100+
var bulkConfig = new BulkConfig
101+
{
102+
BatchSize = 50_000,
103+
SetOutputIdentity = false
104+
};
105+
106+
Console.Write($"Inserting {events.Count} events...");
107+
await using (var timer = new ConsoleTimer(ct))
108+
{
109+
await _dbContext.BatchedInsertAsync(events, bulkConfig, timer.SetProgress, ct);
110+
}
111+
112+
var eventResponses = events.SelectMany(e => e.EventResponses).ToList();
113+
Console.Write($"Inserting {eventResponses.Count} event responses...");
114+
await using (var timer = new ConsoleTimer(ct))
115+
{
116+
await _dbContext.BatchedInsertAsync(eventResponses, bulkConfig, timer.SetProgress, ct);
117+
}
118+
}
48119

49-
var invitations = InvitationGenerators.GenerateRandomInvitations(DateTime.UtcNow, usersWithoutTeam, teams);
120+
private async Task SeedInvitationsAsync(List<User> usersWithoutTeam, List<Team> teams, CancellationToken ct)
121+
{
122+
Console.Write($"Generating {_invitationsCount} invitations...");
123+
var invitations = await ConsoleTimer.CallAsync(() =>
124+
{
125+
var usersToInvite = usersWithoutTeam.OrderBy(_ => Guid.NewGuid()).Take(_invitationsCount).ToList();
126+
return InvitationGenerators.GenerateRandomInvitations(DateTime.UtcNow, usersToInvite, teams);
127+
}, ct);
50128

51129
_dbContext.Invitations.AddRange(invitations);
52130

53-
Console.Write("Adding 500 invitations...");
54-
await _dbContext.SaveChangesAsync(ct);
55-
Console.WriteLine("DONE");
131+
Console.Write($"Inserting {invitations.Count} invitations...");
132+
await ConsoleTimer.CallAsync(_dbContext.SaveChangesAsync, ct);
56133
}
57134

58135
public ValueTask DisposeAsync() => _dbContext.DisposeAsync();

0 commit comments

Comments
 (0)