Skip to content

Commit 3c91cdb

Browse files
isoosisoos
committed
New field for moderation subjects: retentionUntil.
1 parent 60290d7 commit 3c91cdb

14 files changed

+232
-27
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ AppEngine version, listed here to ease deployment and troubleshooting.
44
## Next Release (replace with git tag when deployed)
55
* Bump runtimeVersion to `2025.05.15`.
66
* Upgraded stable Dart analysis SDK to `3.8.0`
7+
* Note: started to use (and backfill) `retentionUntil` fields for moderation subjects.
78

89
## `20250515t085900-all`
910

‎app/lib/account/models.dart‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import 'package:clock/clock.dart';
66
import 'package:json_annotation/json_annotation.dart';
77
import 'package:pub_dev/admin/actions/actions.dart';
8+
import 'package:pub_dev/admin/models.dart';
89
import 'package:pub_dev/shared/utils.dart';
910
import 'package:ulid/ulid.dart';
1011

@@ -61,6 +62,10 @@ class User extends db.ExpandoModel<String> {
6162
@db.StringProperty()
6263
String? moderatedReason;
6364

65+
/// The timestamp when the user entry will be cleared to the bare minimum information.
66+
@db.DateTimeProperty()
67+
DateTime? retentionUntil;
68+
6469
User();
6570
User.init() {
6671
isDeleted = false;
@@ -86,8 +91,14 @@ class User extends db.ExpandoModel<String> {
8691
}
8792

8893
this.isModerated = isModerated;
89-
moderatedAt = isModerated ? clock.now().toUtc() : null;
9094
this.moderatedReason = moderatedReason;
95+
if (isModerated) {
96+
moderatedAt = clock.now().toUtc();
97+
retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
98+
} else {
99+
moderatedAt = null;
100+
retentionUntil = null;
101+
}
91102
}
92103
}
93104

‎app/lib/admin/actions/moderate_package.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,11 +101,13 @@ Note: the action may take a longer time to complete as the public archive bucket
101101
'before': {
102102
'isModerated': p.isModerated,
103103
'moderatedAt': p.moderatedAt?.toIso8601String(),
104+
'retentionUntil': p.retentionUntil?.toIso8601String(),
104105
},
105106
if (p2 != null)
106107
'after': {
107108
'isModerated': p2.isModerated,
108109
'moderatedAt': p2.moderatedAt?.toIso8601String(),
110+
'retentionUntil': p2.retentionUntil?.toIso8601String(),
109111
},
110112
};
111113
},

‎app/lib/admin/actions/moderate_package_versions.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,11 +136,13 @@ Set the moderated flag on a package version (updating the flag and the timestamp
136136
'before': {
137137
'isModerated': pv.isModerated,
138138
'moderatedAt': pv.moderatedAt?.toIso8601String(),
139+
'retentionUntil': pv.retentionUntil?.toIso8601String(),
139140
},
140141
if (pv2 != null)
141142
'after': {
142143
'isModerated': pv2.isModerated,
143144
'moderatedAt': pv2.moderatedAt?.toIso8601String(),
145+
'retentionUntil': pv2.retentionUntil?.toIso8601String(),
144146
},
145147
};
146148
},

‎app/lib/admin/actions/moderate_publisher.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,11 +83,13 @@ can't be updated, administrators must not be able to update publisher options.
8383
'before': {
8484
'isModerated': publisher.isModerated,
8585
'moderatedAt': publisher.moderatedAt?.toIso8601String(),
86+
'retentionUntil': publisher.retentionUntil?.toIso8601String(),
8687
},
8788
if (publisher2 != null)
8889
'after': {
8990
'isModerated': publisher2.isModerated,
9091
'moderatedAt': publisher2.moderatedAt?.toIso8601String(),
92+
'retentionUntil': publisher2.retentionUntil?.toIso8601String(),
9193
},
9294
};
9395
},

‎app/lib/admin/actions/moderate_user.dart‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,12 +145,14 @@ The active web sessions of the user will be expired.
145145
'isModerated': user.isModerated,
146146
'moderatedAt': user.moderatedAt?.toIso8601String(),
147147
'moderatedReason': user.moderatedReason,
148+
'retentionUntil': user.retentionUntil?.toIso8601String(),
148149
},
149150
if (user2 != null)
150151
'after': {
151152
'isModerated': user2.isModerated,
152153
'moderatedAt': user2.moderatedAt?.toIso8601String(),
153154
'moderatedReason': user2.moderatedReason,
155+
'retentionUntil': user2.retentionUntil?.toIso8601String(),
154156
},
155157
};
156158
},

‎app/lib/admin/models.dart‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,11 @@ import '../shared/urls.dart' as urls;
1313

1414
part 'models.g.dart';
1515

16+
/// The default time period while the moderated subject's (package, version,
17+
/// publisher or user) database entry is kept. Once the time is over, we are
18+
/// deleting the database entry and related assets.
19+
final defaultModeratedKeptUntil = Duration(days: 3 * 366); // extra buffer days
20+
1621
/// Tracks the status of the moderation or appeal case.
1722
@db.Kind(name: 'ModerationCase', idType: db.IdType.String)
1823
class ModerationCase extends db.ExpandoModel<String> {

‎app/lib/package/models.dart‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import 'package:_pub_shared/search/tags.dart';
99
import 'package:clock/clock.dart';
1010
import 'package:json_annotation/json_annotation.dart';
1111
import 'package:pana/models.dart';
12+
import 'package:pub_dev/admin/models.dart';
1213
import 'package:pub_dev/service/download_counts/download_counts.dart';
1314
import 'package:pub_semver/pub_semver.dart';
1415

@@ -129,6 +130,10 @@ class Package extends db.ExpandoModel<String> {
129130
@db.DateTimeProperty()
130131
DateTime? moderatedAt;
131132

133+
/// The timestamp when the package entry will be deleted.
134+
@db.DateTimeProperty()
135+
DateTime? retentionUntil;
136+
132137
/// Tags that are assigned to this package.
133138
///
134139
/// The permissions required to assign a tag typically depends on the tag.
@@ -399,9 +404,15 @@ class Package extends db.ExpandoModel<String> {
399404
void updateIsModerated({
400405
required bool isModerated,
401406
}) {
402-
this.isModerated = isModerated;
403-
moderatedAt = isModerated ? clock.now().toUtc() : null;
404407
updated = clock.now().toUtc();
408+
this.isModerated = isModerated;
409+
if (isModerated) {
410+
moderatedAt = clock.now().toUtc();
411+
retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
412+
} else {
413+
moderatedAt = null;
414+
retentionUntil = null;
415+
}
405416
}
406417
}
407418

@@ -586,6 +597,10 @@ class PackageVersion extends db.ExpandoModel<String> {
586597
@db.DateTimeProperty()
587598
DateTime? moderatedAt;
588599

600+
/// The timestamp when the version entry will be deleted.
601+
@db.DateTimeProperty()
602+
DateTime? retentionUntil;
603+
589604
PackageVersion();
590605

591606
PackageVersion.init() {
@@ -652,7 +667,13 @@ class PackageVersion extends db.ExpandoModel<String> {
652667
required bool isModerated,
653668
}) {
654669
this.isModerated = isModerated;
655-
moderatedAt = isModerated ? clock.now().toUtc() : null;
670+
if (isModerated) {
671+
moderatedAt = clock.now().toUtc();
672+
retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
673+
} else {
674+
moderatedAt = null;
675+
retentionUntil = null;
676+
}
656677
}
657678
}
658679

‎app/lib/publisher/models.dart‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
import 'package:clock/clock.dart';
66
import 'package:json_annotation/json_annotation.dart';
7+
import 'package:pub_dev/admin/models.dart';
78

89
import '../shared/datastore.dart' as db;
910

@@ -62,6 +63,10 @@ class Publisher extends db.ExpandoModel<String> {
6263
@db.DateTimeProperty()
6364
DateTime? moderatedAt;
6465

66+
/// The timestamp when the publisher entry will be deleted.
67+
@db.DateTimeProperty()
68+
DateTime? retentionUntil;
69+
6570
Publisher();
6671

6772
Publisher.init({
@@ -91,9 +96,15 @@ class Publisher extends db.ExpandoModel<String> {
9196
bool get isVisible => !isUnlisted;
9297

9398
void updateIsModerated({required bool isModerated}) {
94-
this.isModerated = isModerated;
95-
moderatedAt = isModerated ? clock.now().toUtc() : null;
9699
updated = clock.now().toUtc();
100+
this.isModerated = isModerated;
101+
if (isModerated) {
102+
moderatedAt = clock.now().toUtc();
103+
retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
104+
} else {
105+
moderatedAt = null;
106+
retentionUntil = null;
107+
}
97108
}
98109
}
99110

‎app/lib/tool/backfill/backfill_new_fields.dart‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
// BSD-style license that can be found in the LICENSE file.
44

55
import 'package:logging/logging.dart';
6+
import 'package:pub_dev/account/models.dart';
7+
import 'package:pub_dev/admin/models.dart';
8+
import 'package:pub_dev/package/models.dart';
9+
import 'package:pub_dev/publisher/models.dart';
10+
import 'package:pub_dev/shared/datastore.dart';
611

712
final _logger = Logger('backfill_new_fields');
813

@@ -12,5 +17,63 @@ final _logger = Logger('backfill_new_fields');
1217
/// CHANGELOG.md must be updated with the new fields, and the next
1318
/// release could remove the backfill from here.
1419
Future<void> backfillNewFields() async {
15-
_logger.info('No new fields.');
20+
_logger.info('Backfill Package.retentionUntil...');
21+
final pkgQuery = dbService.query<Package>()..filter('isModerated =', true);
22+
await for (final e in pkgQuery.run()) {
23+
if (e.retentionUntil != null) continue;
24+
await withRetryTransaction(dbService, (tx) async {
25+
final pkg = await tx.lookupValue<Package>(e.key);
26+
if (!pkg.isModerated || pkg.retentionUntil != null) {
27+
return;
28+
}
29+
pkg.retentionUntil = pkg.moderatedAt!.add(defaultModeratedKeptUntil);
30+
tx.insert(pkg);
31+
});
32+
}
33+
34+
_logger.info('Backfill PackageVersion.retentionUntil...');
35+
final versionQuery = dbService.query<PackageVersion>()
36+
..filter('isModerated =', true);
37+
await for (final e in versionQuery.run()) {
38+
if (e.retentionUntil != null) continue;
39+
await withRetryTransaction(dbService, (tx) async {
40+
final pv = await tx.lookupValue<PackageVersion>(e.key);
41+
if (!pv.isModerated || pv.retentionUntil != null) {
42+
return;
43+
}
44+
pv.retentionUntil = pv.moderatedAt!.add(defaultModeratedKeptUntil);
45+
tx.insert(pv);
46+
});
47+
}
48+
49+
_logger.info('Backfill Publisher.retentionUntil...');
50+
final publisherQuery = dbService.query<Publisher>()
51+
..filter('isModerated =', true);
52+
await for (final e in publisherQuery.run()) {
53+
if (e.retentionUntil != null) continue;
54+
await withRetryTransaction(dbService, (tx) async {
55+
final p = await tx.lookupValue<Publisher>(e.key);
56+
if (!p.isModerated || p.retentionUntil != null) {
57+
return;
58+
}
59+
p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil);
60+
tx.insert(p);
61+
});
62+
}
63+
64+
_logger.info('Backfill User.retentionUntil...');
65+
final userQuery = dbService.query<User>()..filter('isModerated =', true);
66+
await for (final e in userQuery.run()) {
67+
if (e.retentionUntil != null) continue;
68+
await withRetryTransaction(dbService, (tx) async {
69+
final p = await tx.lookupValue<User>(e.key);
70+
if (!p.isModerated || p.retentionUntil != null) {
71+
return;
72+
}
73+
p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil);
74+
tx.insert(p);
75+
});
76+
}
77+
78+
_logger.info('Backfill completed.');
1679
}

0 commit comments

Comments
 (0)