New field for moderation subjects: retentionUntil.

CHANGELOG.md

@@ -2,6 +2,7 @@ Important changes to data models, configuration, and migrations between each
 AppEngine version, listed here to ease deployment and troubleshooting.
 
 ## Next Release (replace with git tag when deployed)
+ * Note: started to use (and backfill) `retentionUntil` fields for moderation subjects.
 
 ## `20250516t132100-all`
  * Bump runtimeVersion to `2025.05.15`.

app/lib/account/models.dart

@@ -5,6 +5,7 @@
 import 'package:clock/clock.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:pub_dev/admin/actions/actions.dart';
+import 'package:pub_dev/admin/models.dart';
 import 'package:pub_dev/shared/utils.dart';
 import 'package:ulid/ulid.dart';
 
@@ -61,6 +62,10 @@ class User extends db.ExpandoModel<String> {
   @db.StringProperty()
   String? moderatedReason;
 
+  /// The timestamp when the user entry will be cleared to the bare minimum information.
+  @db.DateTimeProperty()
+  DateTime? retentionUntil;
+
   User();
   User.init() {
     isDeleted = false;
@@ -86,8 +91,14 @@ class User extends db.ExpandoModel<String> {
     }
 
     this.isModerated = isModerated;
-    moderatedAt = isModerated ? clock.now().toUtc() : null;
     this.moderatedReason = moderatedReason;
+    if (isModerated) {
+      moderatedAt = clock.now().toUtc();
+      retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
+    } else {
+      moderatedAt = null;
+      retentionUntil = null;
+    }
   }
 }
 

app/lib/admin/actions/moderate_package.dart

@@ -101,11 +101,13 @@ Note: the action may take a longer time to complete as the public archive bucket
       'before': {
         'isModerated': p.isModerated,
         'moderatedAt': p.moderatedAt?.toIso8601String(),
+        'retentionUntil': p.retentionUntil?.toIso8601String(),
       },
       if (p2 != null)
         'after': {
           'isModerated': p2.isModerated,
           'moderatedAt': p2.moderatedAt?.toIso8601String(),
+          'retentionUntil': p2.retentionUntil?.toIso8601String(),
         },
     };
   },

app/lib/admin/actions/moderate_package_versions.dart

@@ -136,11 +136,13 @@ Set the moderated flag on a package version (updating the flag and the timestamp
       'before': {
         'isModerated': pv.isModerated,
         'moderatedAt': pv.moderatedAt?.toIso8601String(),
+        'retentionUntil': pv.retentionUntil?.toIso8601String(),
       },
       if (pv2 != null)
         'after': {
           'isModerated': pv2.isModerated,
           'moderatedAt': pv2.moderatedAt?.toIso8601String(),
+          'retentionUntil': pv2.retentionUntil?.toIso8601String(),
         },
     };
   },

app/lib/admin/actions/moderate_publisher.dart

@@ -83,11 +83,13 @@ can't be updated, administrators must not be able to update publisher options.
       'before': {
         'isModerated': publisher.isModerated,
         'moderatedAt': publisher.moderatedAt?.toIso8601String(),
+        'retentionUntil': publisher.retentionUntil?.toIso8601String(),
       },
       if (publisher2 != null)
         'after': {
           'isModerated': publisher2.isModerated,
           'moderatedAt': publisher2.moderatedAt?.toIso8601String(),
+          'retentionUntil': publisher2.retentionUntil?.toIso8601String(),
         },
     };
   },

app/lib/admin/actions/moderate_user.dart

@@ -145,12 +145,14 @@ The active web sessions of the user will be expired.
         'isModerated': user.isModerated,
         'moderatedAt': user.moderatedAt?.toIso8601String(),
         'moderatedReason': user.moderatedReason,
+        'retentionUntil': user.retentionUntil?.toIso8601String(),
       },
       if (user2 != null)
         'after': {
           'isModerated': user2.isModerated,
           'moderatedAt': user2.moderatedAt?.toIso8601String(),
           'moderatedReason': user2.moderatedReason,
+          'retentionUntil': user2.retentionUntil?.toIso8601String(),
         },
     };
   },

app/lib/admin/models.dart

@@ -13,6 +13,11 @@ import '../shared/urls.dart' as urls;
 
 part 'models.g.dart';
 
+/// The default time period while the moderated subject's (package, version,
+/// publisher or user) database entry is kept. Once the time is over, we are
+/// deleting the database entry and related assets.
+final defaultModeratedKeptUntil = Duration(days: 3 * 366); // extra buffer days
+
 /// Tracks the status of the moderation or appeal case.
 @db.Kind(name: 'ModerationCase', idType: db.IdType.String)
 class ModerationCase extends db.ExpandoModel<String> {

app/lib/package/models.dart

@@ -9,6 +9,7 @@ import 'package:_pub_shared/search/tags.dart';
 import 'package:clock/clock.dart';
 import 'package:json_annotation/json_annotation.dart';
 import 'package:pana/models.dart';
+import 'package:pub_dev/admin/models.dart';
 import 'package:pub_dev/service/download_counts/download_counts.dart';
 import 'package:pub_semver/pub_semver.dart';
 
@@ -129,6 +130,10 @@ class Package extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime? moderatedAt;
 
+  /// The timestamp when the package entry will be deleted.
+  @db.DateTimeProperty()
+  DateTime? retentionUntil;
+
   /// Tags that are assigned to this package.
   ///
   /// The permissions required to assign a tag typically depends on the tag.
@@ -399,9 +404,15 @@ class Package extends db.ExpandoModel<String> {
   void updateIsModerated({
     required bool isModerated,
   }) {
-    this.isModerated = isModerated;
-    moderatedAt = isModerated ? clock.now().toUtc() : null;
     updated = clock.now().toUtc();
+    this.isModerated = isModerated;
+    if (isModerated) {
+      moderatedAt = clock.now().toUtc();
+      retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
+    } else {
+      moderatedAt = null;
+      retentionUntil = null;
+    }
   }
 }
 
@@ -586,6 +597,10 @@ class PackageVersion extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime? moderatedAt;
 
+  /// The timestamp when the version entry will be deleted.
+  @db.DateTimeProperty()
+  DateTime? retentionUntil;
+
   PackageVersion();
 
   PackageVersion.init() {
@@ -652,7 +667,13 @@ class PackageVersion extends db.ExpandoModel<String> {
     required bool isModerated,
   }) {
     this.isModerated = isModerated;
-    moderatedAt = isModerated ? clock.now().toUtc() : null;
+    if (isModerated) {
+      moderatedAt = clock.now().toUtc();
+      retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
+    } else {
+      moderatedAt = null;
+      retentionUntil = null;
+    }
   }
 }
 

app/lib/publisher/models.dart

@@ -4,6 +4,7 @@
 
 import 'package:clock/clock.dart';
 import 'package:json_annotation/json_annotation.dart';
+import 'package:pub_dev/admin/models.dart';
 
 import '../shared/datastore.dart' as db;
 
@@ -62,6 +63,10 @@ class Publisher extends db.ExpandoModel<String> {
   @db.DateTimeProperty()
   DateTime? moderatedAt;
 
+  /// The timestamp when the publisher entry will be deleted.
+  @db.DateTimeProperty()
+  DateTime? retentionUntil;
+
   Publisher();
 
   Publisher.init({
@@ -91,9 +96,15 @@ class Publisher extends db.ExpandoModel<String> {
   bool get isVisible => !isUnlisted;
 
   void updateIsModerated({required bool isModerated}) {
-    this.isModerated = isModerated;
-    moderatedAt = isModerated ? clock.now().toUtc() : null;
     updated = clock.now().toUtc();
+    this.isModerated = isModerated;
+    if (isModerated) {
+      moderatedAt = clock.now().toUtc();
+      retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil);
+    } else {
+      moderatedAt = null;
+      retentionUntil = null;
+    }
   }
 }
 

app/lib/tool/backfill/backfill_new_fields.dart

@@ -3,6 +3,11 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'package:logging/logging.dart';
+import 'package:pub_dev/account/models.dart';
+import 'package:pub_dev/admin/models.dart';
+import 'package:pub_dev/package/models.dart';
+import 'package:pub_dev/publisher/models.dart';
+import 'package:pub_dev/shared/datastore.dart';
 
 final _logger = Logger('backfill_new_fields');
 
@@ -12,5 +17,63 @@ final _logger = Logger('backfill_new_fields');
 /// CHANGELOG.md must be updated with the new fields, and the next
 /// release could remove the backfill from here.
 Future<void> backfillNewFields() async {
-  _logger.info('No new fields.');
+  _logger.info('Backfill Package.retentionUntil...');
+  final pkgQuery = dbService.query<Package>()..filter('isModerated =', true);
+  await for (final e in pkgQuery.run()) {
+    if (e.retentionUntil != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final pkg = await tx.lookupValue<Package>(e.key);
+      if (!pkg.isModerated || pkg.retentionUntil != null) {
+        return;
+      }
+      pkg.retentionUntil = pkg.moderatedAt!.add(defaultModeratedKeptUntil);
+      tx.insert(pkg);
+    });
+  }
+
+  _logger.info('Backfill PackageVersion.retentionUntil...');
+  final versionQuery = dbService.query<PackageVersion>()
+    ..filter('isModerated =', true);
+  await for (final e in versionQuery.run()) {
+    if (e.retentionUntil != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final pv = await tx.lookupValue<PackageVersion>(e.key);
+      if (!pv.isModerated || pv.retentionUntil != null) {
+        return;
+      }
+      pv.retentionUntil = pv.moderatedAt!.add(defaultModeratedKeptUntil);
+      tx.insert(pv);
+    });
+  }
+
+  _logger.info('Backfill Publisher.retentionUntil...');
+  final publisherQuery = dbService.query<Publisher>()
+    ..filter('isModerated =', true);
+  await for (final e in publisherQuery.run()) {
+    if (e.retentionUntil != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final p = await tx.lookupValue<Publisher>(e.key);
+      if (!p.isModerated || p.retentionUntil != null) {
+        return;
+      }
+      p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil);
+      tx.insert(p);
+    });
+  }
+
+  _logger.info('Backfill User.retentionUntil...');
+  final userQuery = dbService.query<User>()..filter('isModerated =', true);
+  await for (final e in userQuery.run()) {
+    if (e.retentionUntil != null) continue;
+    await withRetryTransaction(dbService, (tx) async {
+      final p = await tx.lookupValue<User>(e.key);
+      if (!p.isModerated || p.retentionUntil != null) {
+        return;
+      }
+      p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil);
+      tx.insert(p);
+    });
+  }
+
+  _logger.info('Backfill completed.');
 }

app/test/admin/moderate_package_test.dart

@@ -73,15 +73,27 @@ void main() {
       final r1 = await _moderate('oxygen', caseId: mc.caseId);
       expect(r1.output, {
         'package': 'oxygen',
-        'before': {'isModerated': false, 'moderatedAt': null},
+        'before': {
+          'isModerated': false,
+          'moderatedAt': null,
+          'retentionUntil': null,
+        },
       });
       await expectModerationActions(mc.caseId, actions: []);
 
       final r2 = await _moderate('oxygen', state: true, caseId: mc.caseId);
       expect(r2.output, {
         'package': 'oxygen',
-        'before': {'isModerated': false, 'moderatedAt': null},
-        'after': {'isModerated': true, 'moderatedAt': isNotEmpty},
+        'before': {
+          'isModerated': false,
+          'moderatedAt': null,
+          'retentionUntil': null,
+        },
+        'after': {
+          'isModerated': true,
+          'moderatedAt': isNotEmpty,
+          'retentionUntil': isNotEmpty,
+        },
       });
       final p2 = await packageBackend.lookupPackage('oxygen');
       expect(p2!.isModerated, isTrue);
@@ -118,15 +130,27 @@ void main() {
       final r1 = await _moderate('oxygen', caseId: mc.caseId);
       expect(r1.output, {
         'package': 'oxygen',
-        'before': {'isModerated': false, 'moderatedAt': null},
+        'before': {
+          'isModerated': false,
+          'moderatedAt': null,
+          'retentionUntil': null,
+        },
       });
       await expectModerationActions(mc.caseId, actions: []);
 
       final r2 = await _moderate('oxygen', state: true, caseId: mc.caseId);
       expect(r2.output, {
         'package': 'oxygen',
-        'before': {'isModerated': false, 'moderatedAt': null},
-        'after': {'isModerated': true, 'moderatedAt': isNotEmpty},
+        'before': {
+          'isModerated': false,
+          'moderatedAt': null,
+          'retentionUntil': null,
+        },
+        'after': {
+          'isModerated': true,
+          'moderatedAt': isNotEmpty,
+          'retentionUntil': isNotEmpty,
+        },
       });
       final p2 = await packageBackend.lookupPackage('oxygen');
       expect(p2!.isModerated, isTrue);
@@ -137,8 +161,16 @@ void main() {
       final r3 = await _moderate('oxygen', state: false, caseId: mc.caseId);
       expect(r3.output, {
         'package': 'oxygen',
-        'before': {'isModerated': true, 'moderatedAt': isNotEmpty},
-        'after': {'isModerated': false, 'moderatedAt': null},
+        'before': {
+          'isModerated': true,
+          'moderatedAt': isNotEmpty,
+          'retentionUntil': isNotEmpty
+        },
+        'after': {
+          'isModerated': false,
+          'moderatedAt': null,
+          'retentionUntil': null,
+        },
       });
       final p3 = await packageBackend.lookupPackage('oxygen');
       expect(p3!.isModerated, isFalse);