diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ef06936d7..a5e6bf6ee7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ Important changes to data models, configuration, and migrations between each AppEngine version, listed here to ease deployment and troubleshooting. ## Next Release (replace with git tag when deployed) + * Note: started to use (and backfill) `retentionUntil` fields for moderation subjects. ## `20250516t132100-all` * Bump runtimeVersion to `2025.05.15`. diff --git a/app/lib/account/models.dart b/app/lib/account/models.dart index f9fd6708c2..1ad391010c 100644 --- a/app/lib/account/models.dart +++ b/app/lib/account/models.dart @@ -5,6 +5,7 @@ import 'package:clock/clock.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:pub_dev/admin/actions/actions.dart'; +import 'package:pub_dev/admin/models.dart'; import 'package:pub_dev/shared/utils.dart'; import 'package:ulid/ulid.dart'; @@ -61,6 +62,10 @@ class User extends db.ExpandoModel { @db.StringProperty() String? moderatedReason; + /// The timestamp when the user entry will be cleared to the bare minimum information. + @db.DateTimeProperty() + DateTime? retentionUntil; + User(); User.init() { isDeleted = false; @@ -86,8 +91,14 @@ class User extends db.ExpandoModel { } this.isModerated = isModerated; - moderatedAt = isModerated ? clock.now().toUtc() : null; this.moderatedReason = moderatedReason; + if (isModerated) { + moderatedAt = clock.now().toUtc(); + retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil); + } else { + moderatedAt = null; + retentionUntil = null; + } } } diff --git a/app/lib/admin/actions/moderate_package.dart b/app/lib/admin/actions/moderate_package.dart index 3e3203ce97..98b25ce516 100644 --- a/app/lib/admin/actions/moderate_package.dart +++ b/app/lib/admin/actions/moderate_package.dart @@ -101,11 +101,13 @@ Note: the action may take a longer time to complete as the public archive bucket 'before': { 'isModerated': p.isModerated, 'moderatedAt': p.moderatedAt?.toIso8601String(), + 'retentionUntil': p.retentionUntil?.toIso8601String(), }, if (p2 != null) 'after': { 'isModerated': p2.isModerated, 'moderatedAt': p2.moderatedAt?.toIso8601String(), + 'retentionUntil': p2.retentionUntil?.toIso8601String(), }, }; }, diff --git a/app/lib/admin/actions/moderate_package_versions.dart b/app/lib/admin/actions/moderate_package_versions.dart index 9a653fbacc..9f5c6c2659 100644 --- a/app/lib/admin/actions/moderate_package_versions.dart +++ b/app/lib/admin/actions/moderate_package_versions.dart @@ -136,11 +136,13 @@ Set the moderated flag on a package version (updating the flag and the timestamp 'before': { 'isModerated': pv.isModerated, 'moderatedAt': pv.moderatedAt?.toIso8601String(), + 'retentionUntil': pv.retentionUntil?.toIso8601String(), }, if (pv2 != null) 'after': { 'isModerated': pv2.isModerated, 'moderatedAt': pv2.moderatedAt?.toIso8601String(), + 'retentionUntil': pv2.retentionUntil?.toIso8601String(), }, }; }, diff --git a/app/lib/admin/actions/moderate_publisher.dart b/app/lib/admin/actions/moderate_publisher.dart index 02d379b14c..985618f92b 100644 --- a/app/lib/admin/actions/moderate_publisher.dart +++ b/app/lib/admin/actions/moderate_publisher.dart @@ -83,11 +83,13 @@ can't be updated, administrators must not be able to update publisher options. 'before': { 'isModerated': publisher.isModerated, 'moderatedAt': publisher.moderatedAt?.toIso8601String(), + 'retentionUntil': publisher.retentionUntil?.toIso8601String(), }, if (publisher2 != null) 'after': { 'isModerated': publisher2.isModerated, 'moderatedAt': publisher2.moderatedAt?.toIso8601String(), + 'retentionUntil': publisher2.retentionUntil?.toIso8601String(), }, }; }, diff --git a/app/lib/admin/actions/moderate_user.dart b/app/lib/admin/actions/moderate_user.dart index 4cebe49bf7..ce2ac26a55 100644 --- a/app/lib/admin/actions/moderate_user.dart +++ b/app/lib/admin/actions/moderate_user.dart @@ -145,12 +145,14 @@ The active web sessions of the user will be expired. 'isModerated': user.isModerated, 'moderatedAt': user.moderatedAt?.toIso8601String(), 'moderatedReason': user.moderatedReason, + 'retentionUntil': user.retentionUntil?.toIso8601String(), }, if (user2 != null) 'after': { 'isModerated': user2.isModerated, 'moderatedAt': user2.moderatedAt?.toIso8601String(), 'moderatedReason': user2.moderatedReason, + 'retentionUntil': user2.retentionUntil?.toIso8601String(), }, }; }, diff --git a/app/lib/admin/models.dart b/app/lib/admin/models.dart index e2ebf3aac5..5fc394c859 100644 --- a/app/lib/admin/models.dart +++ b/app/lib/admin/models.dart @@ -13,6 +13,11 @@ import '../shared/urls.dart' as urls; part 'models.g.dart'; +/// The default time period while the moderated subject's (package, version, +/// publisher or user) database entry is kept. Once the time is over, we are +/// deleting the database entry and related assets. +final defaultModeratedKeptUntil = Duration(days: 3 * 366); // extra buffer days + /// Tracks the status of the moderation or appeal case. @db.Kind(name: 'ModerationCase', idType: db.IdType.String) class ModerationCase extends db.ExpandoModel { diff --git a/app/lib/package/models.dart b/app/lib/package/models.dart index f6178686b7..99ca53e73a 100644 --- a/app/lib/package/models.dart +++ b/app/lib/package/models.dart @@ -9,6 +9,7 @@ import 'package:_pub_shared/search/tags.dart'; import 'package:clock/clock.dart'; import 'package:json_annotation/json_annotation.dart'; import 'package:pana/models.dart'; +import 'package:pub_dev/admin/models.dart'; import 'package:pub_dev/service/download_counts/download_counts.dart'; import 'package:pub_semver/pub_semver.dart'; @@ -129,6 +130,10 @@ class Package extends db.ExpandoModel { @db.DateTimeProperty() DateTime? moderatedAt; + /// The timestamp when the package entry will be deleted. + @db.DateTimeProperty() + DateTime? retentionUntil; + /// Tags that are assigned to this package. /// /// The permissions required to assign a tag typically depends on the tag. @@ -399,9 +404,15 @@ class Package extends db.ExpandoModel { void updateIsModerated({ required bool isModerated, }) { - this.isModerated = isModerated; - moderatedAt = isModerated ? clock.now().toUtc() : null; updated = clock.now().toUtc(); + this.isModerated = isModerated; + if (isModerated) { + moderatedAt = clock.now().toUtc(); + retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil); + } else { + moderatedAt = null; + retentionUntil = null; + } } } @@ -586,6 +597,10 @@ class PackageVersion extends db.ExpandoModel { @db.DateTimeProperty() DateTime? moderatedAt; + /// The timestamp when the version entry will be deleted. + @db.DateTimeProperty() + DateTime? retentionUntil; + PackageVersion(); PackageVersion.init() { @@ -652,7 +667,13 @@ class PackageVersion extends db.ExpandoModel { required bool isModerated, }) { this.isModerated = isModerated; - moderatedAt = isModerated ? clock.now().toUtc() : null; + if (isModerated) { + moderatedAt = clock.now().toUtc(); + retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil); + } else { + moderatedAt = null; + retentionUntil = null; + } } } diff --git a/app/lib/publisher/models.dart b/app/lib/publisher/models.dart index 58e82f61a1..7e751a1b81 100644 --- a/app/lib/publisher/models.dart +++ b/app/lib/publisher/models.dart @@ -4,6 +4,7 @@ import 'package:clock/clock.dart'; import 'package:json_annotation/json_annotation.dart'; +import 'package:pub_dev/admin/models.dart'; import '../shared/datastore.dart' as db; @@ -62,6 +63,10 @@ class Publisher extends db.ExpandoModel { @db.DateTimeProperty() DateTime? moderatedAt; + /// The timestamp when the publisher entry will be deleted. + @db.DateTimeProperty() + DateTime? retentionUntil; + Publisher(); Publisher.init({ @@ -91,9 +96,15 @@ class Publisher extends db.ExpandoModel { bool get isVisible => !isUnlisted; void updateIsModerated({required bool isModerated}) { - this.isModerated = isModerated; - moderatedAt = isModerated ? clock.now().toUtc() : null; updated = clock.now().toUtc(); + this.isModerated = isModerated; + if (isModerated) { + moderatedAt = clock.now().toUtc(); + retentionUntil = moderatedAt!.add(defaultModeratedKeptUntil); + } else { + moderatedAt = null; + retentionUntil = null; + } } } diff --git a/app/lib/tool/backfill/backfill_new_fields.dart b/app/lib/tool/backfill/backfill_new_fields.dart index 6b471dd8fb..4638f9ca79 100644 --- a/app/lib/tool/backfill/backfill_new_fields.dart +++ b/app/lib/tool/backfill/backfill_new_fields.dart @@ -3,6 +3,11 @@ // BSD-style license that can be found in the LICENSE file. import 'package:logging/logging.dart'; +import 'package:pub_dev/account/models.dart'; +import 'package:pub_dev/admin/models.dart'; +import 'package:pub_dev/package/models.dart'; +import 'package:pub_dev/publisher/models.dart'; +import 'package:pub_dev/shared/datastore.dart'; final _logger = Logger('backfill_new_fields'); @@ -12,5 +17,63 @@ final _logger = Logger('backfill_new_fields'); /// CHANGELOG.md must be updated with the new fields, and the next /// release could remove the backfill from here. Future backfillNewFields() async { - _logger.info('No new fields.'); + _logger.info('Backfill Package.retentionUntil...'); + final pkgQuery = dbService.query()..filter('isModerated =', true); + await for (final e in pkgQuery.run()) { + if (e.retentionUntil != null) continue; + await withRetryTransaction(dbService, (tx) async { + final pkg = await tx.lookupValue(e.key); + if (!pkg.isModerated || pkg.retentionUntil != null) { + return; + } + pkg.retentionUntil = pkg.moderatedAt!.add(defaultModeratedKeptUntil); + tx.insert(pkg); + }); + } + + _logger.info('Backfill PackageVersion.retentionUntil...'); + final versionQuery = dbService.query() + ..filter('isModerated =', true); + await for (final e in versionQuery.run()) { + if (e.retentionUntil != null) continue; + await withRetryTransaction(dbService, (tx) async { + final pv = await tx.lookupValue(e.key); + if (!pv.isModerated || pv.retentionUntil != null) { + return; + } + pv.retentionUntil = pv.moderatedAt!.add(defaultModeratedKeptUntil); + tx.insert(pv); + }); + } + + _logger.info('Backfill Publisher.retentionUntil...'); + final publisherQuery = dbService.query() + ..filter('isModerated =', true); + await for (final e in publisherQuery.run()) { + if (e.retentionUntil != null) continue; + await withRetryTransaction(dbService, (tx) async { + final p = await tx.lookupValue(e.key); + if (!p.isModerated || p.retentionUntil != null) { + return; + } + p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil); + tx.insert(p); + }); + } + + _logger.info('Backfill User.retentionUntil...'); + final userQuery = dbService.query()..filter('isModerated =', true); + await for (final e in userQuery.run()) { + if (e.retentionUntil != null) continue; + await withRetryTransaction(dbService, (tx) async { + final p = await tx.lookupValue(e.key); + if (!p.isModerated || p.retentionUntil != null) { + return; + } + p.retentionUntil = p.moderatedAt!.add(defaultModeratedKeptUntil); + tx.insert(p); + }); + } + + _logger.info('Backfill completed.'); } diff --git a/app/test/admin/moderate_package_test.dart b/app/test/admin/moderate_package_test.dart index bff6ee16df..c7830eb2e4 100644 --- a/app/test/admin/moderate_package_test.dart +++ b/app/test/admin/moderate_package_test.dart @@ -73,15 +73,27 @@ void main() { final r1 = await _moderate('oxygen', caseId: mc.caseId); expect(r1.output, { 'package': 'oxygen', - 'before': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); await expectModerationActions(mc.caseId, actions: []); final r2 = await _moderate('oxygen', state: true, caseId: mc.caseId); expect(r2.output, { 'package': 'oxygen', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }); final p2 = await packageBackend.lookupPackage('oxygen'); expect(p2!.isModerated, isTrue); @@ -118,15 +130,27 @@ void main() { final r1 = await _moderate('oxygen', caseId: mc.caseId); expect(r1.output, { 'package': 'oxygen', - 'before': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); await expectModerationActions(mc.caseId, actions: []); final r2 = await _moderate('oxygen', state: true, caseId: mc.caseId); expect(r2.output, { 'package': 'oxygen', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }); final p2 = await packageBackend.lookupPackage('oxygen'); expect(p2!.isModerated, isTrue); @@ -137,8 +161,16 @@ void main() { final r3 = await _moderate('oxygen', state: false, caseId: mc.caseId); expect(r3.output, { 'package': 'oxygen', - 'before': {'isModerated': true, 'moderatedAt': isNotEmpty}, - 'after': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty + }, + 'after': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); final p3 = await packageBackend.lookupPackage('oxygen'); expect(p3!.isModerated, isFalse); diff --git a/app/test/admin/moderate_package_version_test.dart b/app/test/admin/moderate_package_version_test.dart index 3f32570d2a..3b176a84bc 100644 --- a/app/test/admin/moderate_package_version_test.dart +++ b/app/test/admin/moderate_package_version_test.dart @@ -72,7 +72,11 @@ void main() { expect(r1.output, { 'package': 'oxygen', 'version': '1.0.0', - 'before': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); final r2 = @@ -80,8 +84,16 @@ void main() { expect(r2.output, { 'package': 'oxygen', 'version': '1.0.0', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }); final p1 = await packageBackend.lookupPackage('oxygen'); expect(p1!.isModerated, isFalse); @@ -118,8 +130,16 @@ void main() { expect(r1.output, { 'package': 'oxygen', 'version': '1.0.0', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }); final p1 = await packageBackend.lookupPackage('oxygen'); expect(p1!.isModerated, isFalse); @@ -131,8 +151,16 @@ void main() { expect(r2.output, { 'package': 'oxygen', 'version': '1.0.0', - 'before': {'isModerated': true, 'moderatedAt': isNotEmpty}, - 'after': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, + 'after': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); final p2 = await packageBackend.lookupPackage('oxygen'); expect(p2!.isModerated, isFalse); diff --git a/app/test/admin/moderate_publisher_test.dart b/app/test/admin/moderate_publisher_test.dart index 104b1a5259..597f7d08a6 100644 --- a/app/test/admin/moderate_publisher_test.dart +++ b/app/test/admin/moderate_publisher_test.dart @@ -59,14 +59,26 @@ void main() { final r1 = await _moderate('example.com', caseId: mc.caseId); expect(r1.output, { 'publisherId': 'example.com', - 'before': {'isModerated': false, 'moderatedAt': null}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, }); final r2 = await _moderate('example.com', caseId: mc.caseId, state: true); expect(r2.output, { 'publisherId': 'example.com', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }); final p2 = await publisherBackend.lookupPublisher('example.com'); expect(p2!.isModerated, isTrue); @@ -75,8 +87,16 @@ void main() { await _moderate('example.com', caseId: mc.caseId, state: false); expect(r3.output, { 'publisherId': 'example.com', - 'before': {'isModerated': true, 'moderatedAt': isNotEmpty}, - 'after': {'isModerated': false, 'moderatedAt': isNull}, + 'before': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, + 'after': { + 'isModerated': false, + 'moderatedAt': isNull, + 'retentionUntil': null, + }, }); final p3 = await publisherBackend.lookupPublisher('example.com'); expect(p3!.isModerated, isFalse); diff --git a/app/test/admin/moderate_user_test.dart b/app/test/admin/moderate_user_test.dart index 246a318322..7273f9b941 100644 --- a/app/test/admin/moderate_user_test.dart +++ b/app/test/admin/moderate_user_test.dart @@ -69,6 +69,7 @@ void main() { 'isModerated': false, 'moderatedAt': null, 'moderatedReason': null, + 'retentionUntil': null, }, }); @@ -84,11 +85,13 @@ void main() { 'isModerated': false, 'moderatedAt': null, 'moderatedReason': null, + 'retentionUntil': null, }, 'after': { 'isModerated': true, 'moderatedAt': isNotEmpty, 'moderatedReason': 'policy-violation', + 'retentionUntil': isNotEmpty, }, }); final u2 = await accountBackend.lookupUserByEmail('user@pub.dev'); @@ -104,11 +107,13 @@ void main() { 'isModerated': true, 'moderatedAt': isNotEmpty, 'moderatedReason': 'policy-violation', + 'retentionUntil': isNotEmpty, }, 'after': { 'isModerated': false, 'moderatedAt': isNull, 'moderatedReason': null, + 'retentionUntil': null, }, }); final u3 = await accountBackend.lookupUserByEmail('user@pub.dev'); diff --git a/pkg/pub_integration/test/report_test.dart b/pkg/pub_integration/test/report_test.dart index 886450ebaa..c5c4342338 100644 --- a/pkg/pub_integration/test/report_test.dart +++ b/pkg/pub_integration/test/report_test.dart @@ -122,8 +122,16 @@ void main() { moderateRs.output, { 'package': 'oxygen', - 'before': {'isModerated': false, 'moderatedAt': null}, - 'after': {'isModerated': true, 'moderatedAt': isNotEmpty}, + 'before': { + 'isModerated': false, + 'moderatedAt': null, + 'retentionUntil': null, + }, + 'after': { + 'isModerated': true, + 'moderatedAt': isNotEmpty, + 'retentionUntil': isNotEmpty, + }, }, );