diff --git a/app/lib/admin/actions/moderate_package_versions.dart b/app/lib/admin/actions/moderate_package_versions.dart index f9e4fad43e..21524c606e 100644 --- a/app/lib/admin/actions/moderate_package_versions.dart +++ b/app/lib/admin/actions/moderate_package_versions.dart @@ -38,110 +38,134 @@ Set the moderated flag on a package version (updating the flag and the timestamp final caseId = options['case']; final package = options['package']; - InvalidInputException.check( - package != null && package.isNotEmpty, - 'package must be given', - ); final version = options['version']; - InvalidInputException.check( - version != null && version.isNotEmpty, - 'version must be given', - ); - final state = options['state']; - bool? valueToSet; - switch (state) { - case 'true': - valueToSet = true; - break; - case 'false': - valueToSet = false; - break; - } - final note = options['note']; final refCase = await adminBackend.loadAndVerifyModerationCaseForAdminAction(caseId); - final p = await packageBackend.lookupPackage(package!); - if (p == null) { - throw NotFoundException.resource(package); - } - final pv = await packageBackend.lookupPackageVersion(package, version!); - if (pv == null) { - throw NotFoundException.resource('$package $version'); - } - - PackageVersion? pv2; - if (valueToSet != null) { - final currentDartSdk = await getCachedDartSdkVersion( - lastKnownStable: toolStableDartSdkVersion); - final currentFlutterSdk = await getCachedFlutterSdkVersion( - lastKnownStable: toolStableFlutterSdkVersion); - pv2 = await withRetryTransaction(dbService, (tx) async { - final v = await tx.lookupValue(pv.key); - v.updateIsModerated(isModerated: valueToSet!); - tx.insert(v); - - // Update references to latest versions. - final pkg = await tx.lookupValue(p.key); - final versions = await tx.query(pkg.key).run().toList(); - pkg.updateVersions( - versions, - dartSdkVersion: currentDartSdk.semanticVersion, - flutterSdkVersion: currentFlutterSdk.semanticVersion, - replaced: v, - ); - if (pkg.latestVersionKey == null) { - throw InvalidInputException('xx'); - } - pkg.updated = clock.now().toUtc(); - tx.insert(pkg); + return await adminMarkPackageVersionVisibility( + package, + version, + state: state, + whenUpdating: (tx, v, valueToSet) async { + v.updateIsModerated(isModerated: valueToSet); if (refCase != null) { final mc = await tx.lookupValue(refCase.key); mc.addActionLogEntry( - ModerationSubject.package(package, version).fqn, + ModerationSubject.package(package!, version!).fqn, valueToSet ? ModerationAction.apply : ModerationAction.revert, note, ); tx.insert(mc); } - - return v; - }); - - // make sure visibility cache is updated immediately - await purgePackageCache(package); - - // sync exported API(s) - await apiExporter.synchronizePackage(package, forceDelete: true); - - // retract or re-populate public archive files - await packageBackend.tarballStorage.updatePublicArchiveBucket( - package: package, - ageCheckThreshold: Duration.zero, - deleteIfOlder: Duration.zero, - ); - - await taskBackend.trackPackage(package); - await purgePackageCache(package); - await purgeScorecardData(package, version, isLatest: true); - } - - return { - 'package': p.name, - 'version': pv.version, - 'before': { - 'isModerated': pv.isModerated, - 'moderatedAt': pv.moderatedAt?.toIso8601String(), }, - if (pv2 != null) - 'after': { - 'isModerated': pv2.isModerated, - 'moderatedAt': pv2.moderatedAt?.toIso8601String(), - }, - }; + valueFn: (v) => { + 'isModerated': v.isModerated, + 'moderatedAt': v.moderatedAt?.toIso8601String(), + }, + ); }, ); + +/// Changes the moderated or the admin-deleted flag and timestamp on a [package] [version]. +Future> adminMarkPackageVersionVisibility( + String? package, + String? version, { + /// `true`, `false` or `null` + required String? state, + + /// The updates to apply during the transaction. + required Future Function( + TransactionWrapper tx, + PackageVersion v, + bool valueToSet, + ) whenUpdating, + + /// The debug information to return. + required Map Function(PackageVersion v) valueFn, +}) async { + InvalidInputException.check( + package != null && package.isNotEmpty, + 'package must be given', + ); + InvalidInputException.check( + version != null && version.isNotEmpty, + 'version must be given', + ); + + bool? valueToSet; + switch (state) { + case 'true': + valueToSet = true; + break; + case 'false': + valueToSet = false; + break; + } + + final p = await packageBackend.lookupPackage(package!); + if (p == null) { + throw NotFoundException.resource(package); + } + final pv = await packageBackend.lookupPackageVersion(package, version!); + if (pv == null) { + throw NotFoundException.resource('$package $version'); + } + + PackageVersion? pv2; + if (valueToSet != null) { + final currentDartSdk = await getCachedDartSdkVersion( + lastKnownStable: toolStableDartSdkVersion); + final currentFlutterSdk = await getCachedFlutterSdkVersion( + lastKnownStable: toolStableFlutterSdkVersion); + pv2 = await withRetryTransaction(dbService, (tx) async { + final v = await tx.lookupValue(pv.key); + await whenUpdating(tx, v, valueToSet!); + tx.insert(v); + + // Update references to latest versions. + final pkg = await tx.lookupValue(p.key); + final versions = await tx.query(pkg.key).run().toList(); + pkg.updateVersions( + versions, + dartSdkVersion: currentDartSdk.semanticVersion, + flutterSdkVersion: currentFlutterSdk.semanticVersion, + replaced: v, + ); + if (pkg.latestVersionKey == null) { + throw InvalidInputException('xx'); + } + pkg.updated = clock.now().toUtc(); + tx.insert(pkg); + + return v; + }); + + // make sure visibility cache is updated immediately + await purgePackageCache(package); + + // sync exported API(s) + await apiExporter.synchronizePackage(package, forceDelete: true); + + // retract or re-populate public archive files + await packageBackend.tarballStorage.updatePublicArchiveBucket( + package: package, + ageCheckThreshold: Duration.zero, + deleteIfOlder: Duration.zero, + ); + + await taskBackend.trackPackage(package); + await purgePackageCache(package); + await purgeScorecardData(package, version, isLatest: true); + } + + return { + 'package': p.name, + 'version': pv.version, + 'before': valueFn(pv), + if (pv2 != null) 'after': valueFn(pv2), + }; +} diff --git a/app/lib/admin/actions/package_version_delete.dart b/app/lib/admin/actions/package_version_delete.dart index 1eb459239f..56848737e5 100644 --- a/app/lib/admin/actions/package_version_delete.dart +++ b/app/lib/admin/actions/package_version_delete.dart @@ -4,47 +4,38 @@ import '../../account/backend.dart'; import '../../shared/configuration.dart'; -import '../backend.dart'; import 'actions.dart'; +import 'moderate_package_versions.dart'; final packageVersionDelete = AdminAction( - name: 'package-version-delete', - options: { - 'package': 'name of package to delete', - 'version': 'version of package', - }, - summary: 'Deletes package version .', - description: ''' -Deletes package version . - -Deletes all associated resources: - -* PackageVersions -* PackageVersionAsset -* archives (might be retrievable from backup) - -The package version will be "tombstoned" and same version cannot be published -later. + name: 'package-version-delete', + summary: + 'Set the admin-deleted flag on a package version (making it not visible).', + description: ''' +Set the admin-deleted flag on a package version (updating the flag and the timestamp). After 2 months it will be fully deleted. ''', - invoke: (args) async { - await requireAuthenticatedAdmin(AdminPermission.removePackage); - final packageName = args['package']; - if (packageName == null) { - throw InvalidInputException('Missing `package` argument'); - } - final version = args['version']; - if (version == null) { - throw InvalidInputException('Missing `version` argument'); - } - final result = - await adminBackend.removePackageVersion(packageName, version); - - return { - 'message': 'Package version and all associated resources deleted.', - 'package': packageName, - 'version': version, - 'deletedPackageVersions': result.deletedPackageVersions, - 'deletedPackageVersionInfos': result.deletedPackageVersionInfos, - 'deletedPackageVersionAssets': result.deletedPackageVersionAssets, - }; - }); + options: { + 'package': 'The package name to be deleted', + 'version': 'The version to be deleted', + 'state': + 'Set admin-deleted state true / false. Returns current state if omitted.', + }, + invoke: (args) async { + await requireAuthenticatedAdmin(AdminPermission.removePackage); + final package = args['package']; + final version = args['version']; + final state = args['state']; + return await adminMarkPackageVersionVisibility( + package, + version, + state: state, + whenUpdating: (tx, v, valueToSet) async { + v.updateIsAdminDeleted(isAdminDeleted: valueToSet); + }, + valueFn: (v) => { + 'isAdminDeleted': v.isAdminDeleted, + 'adminDeletedAt': v.adminDeletedAt?.toIso8601String(), + }, + ); + }, +); diff --git a/app/lib/admin/backend.dart b/app/lib/admin/backend.dart index e857d8b9a7..ab6f447ee6 100644 --- a/app/lib/admin/backend.dart +++ b/app/lib/admin/backend.dart @@ -455,6 +455,7 @@ class AdminBackend { /// Removes the specific package version from the Datastore and updates other /// related entities. It is safe to call [removePackageVersion] on an already /// removed version, as the call is idempotent. + @visibleForTesting Future< ({ int deletedPackageVersions, @@ -466,6 +467,20 @@ class AdminBackend { lastKnownStable: toolStableDartSdkVersion); final currentFlutterSdk = await getCachedFlutterSdkVersion( lastKnownStable: toolStableFlutterSdkVersion); + + _logger.info('Removing GCS objects ...'); + await packageBackend.removePackageTarball(packageName, version); + + final deletedPackageVersionInfos = await _db.deleteWithQuery( + _db.query()..filter('package =', packageName), + where: (PackageVersionInfo info) => info.version == version, + ); + + final deletedPackageVersionAssets = await _db.deleteWithQuery( + _db.query()..filter('package =', packageName), + where: (PackageVersionAsset asset) => asset.version == version, + ); + await withRetryTransaction(_db, (tx) async { final packageKey = _db.emptyKey.append(Package, id: packageName); final package = await tx.lookupOrNull(packageKey); @@ -502,19 +517,6 @@ class AdminBackend { tx.insert(package); }); - _logger.info('Removing GCS objects ...'); - await packageBackend.removePackageTarball(packageName, version); - - final deletedPackageVersionInfos = await _db.deleteWithQuery( - _db.query()..filter('package =', packageName), - where: (PackageVersionInfo info) => info.version == version, - ); - - final deletedPackageVersionAssets = await _db.deleteWithQuery( - _db.query()..filter('package =', packageName), - where: (PackageVersionAsset asset) => asset.version == version, - ); - await purgePackageCache(packageName); await purgeScorecardData(packageName, version, isLatest: true); // trigger (eventual) re-analysis @@ -813,48 +815,7 @@ class AdminBackend { _logger.info( 'Deleting moderated package version: ${version.qualifiedVersionKey}'); - - // deleting from canonical bucket - await packageBackend.tarballStorage - .deleteArchiveFromCanonicalBucket(version.package, version.version!); - - // deleting from datastore - await withRetryTransaction(_db, (tx) async { - final pv = await tx.lookupOrNull(version.key); - if (pv == null) { - return null; - } - final p = await tx.lookupOrNull(version.packageKey!); - if (p == null) { - return; - } - final pvi = await tx.lookupOrNull(_db.emptyKey - .append(PackageVersionInfo, - id: version.qualifiedVersionKey.qualifiedVersion)); - - p.deletedVersions ??= []; - p.deletedVersions!.add(version.version!); - p.deletedVersions!.sort(); - p.updated = clock.now().toUtc(); - tx.insert(p); - - // delete version + info + assets - tx.delete(pv.key); - if (pvi != null) { - tx.delete(pvi.key); - - for (final assetKind in pvi.assets) { - tx.delete( - _db.emptyKey.append(PackageVersionAsset, - id: Uri(pathSegments: [ - version.package, - version.version!, - assetKind - ]).path), - ); - } - } - }); + await removePackageVersion(version.package, version.version!); _logger.info( 'Deleted moderated package version: ${version.qualifiedVersionKey}'); } @@ -913,6 +874,31 @@ class AdminBackend { } } + /// Scans datastore and deletes admin-deleted entities where the last action + /// was more than 2 months ago. + Future deleteAdminDeletedEntities({ + @visibleForTesting DateTime? before, + }) async { + before ??= clock.ago(days: 62).toUtc(); // extra buffer days + + // delete package versions + final pvQuery = _db.query() + ..filter('adminDeletedAt <', before) + ..order('adminDeletedAt'); + await for (final version in pvQuery.run()) { + // sanity check + if (!(version.isAdminDeleted ?? false)) { + continue; + } + + _logger.info( + 'Deleting admin-deleted package version: ${version.qualifiedVersionKey}'); + await removePackageVersion(version.package, version.version!); + _logger.info( + 'Deleted moderated package version: ${version.qualifiedVersionKey}'); + } + } + /// Whether the [ModerationCase] has been appealed by [email] already. Future isModerationCaseAppealedByEmail({ required String caseId, diff --git a/app/lib/package/models.dart b/app/lib/package/models.dart index cba3c033b3..a3b1babb92 100644 --- a/app/lib/package/models.dart +++ b/app/lib/package/models.dart @@ -412,6 +412,14 @@ class Package extends db.ExpandoModel { moderatedAt = isModerated ? clock.now().toUtc() : null; updated = clock.now().toUtc(); } + + void updateIsAdminDeleted({ + required bool isAdminDeleted, + }) { + this.isAdminDeleted = isAdminDeleted; + adminDeletedAt = isAdminDeleted ? clock.now().toUtc() : null; + updated = clock.now().toUtc(); + } } /// Describes the various categories of latest releases. @@ -676,6 +684,13 @@ class PackageVersion extends db.ExpandoModel { this.isModerated = isModerated; moderatedAt = isModerated ? clock.now().toUtc() : null; } + + void updateIsAdminDeleted({ + required bool isAdminDeleted, + }) { + this.isAdminDeleted = isAdminDeleted; + adminDeletedAt = isAdminDeleted ? clock.now().toUtc() : null; + } } /// A derived entity that holds derived/cleaned content of [PackageVersion]. diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index 62089f6f92..30bd099405 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -136,6 +136,13 @@ List createPeriodicTaskSchedulers({ task: () async => await apiExporter.synchronizeExportedApi(), ), + // Deletes admin-deleted entities. + _weekly( + name: 'delete-admin-deleted-entities', + isRuntimeVersioned: false, + task: () async => adminBackend.deleteAdminDeletedEntities(), + ), + // Deletes moderated packages, versions, publishers and users. _weekly( name: 'delete-moderated-subjects', diff --git a/app/test/admin/api_test.dart b/app/test/admin/api_test.dart index 3372d0dae9..b9dc7ea802 100644 --- a/app/test/admin/api_test.dart +++ b/app/test/admin/api_test.dart @@ -240,7 +240,105 @@ void main() { AdminInvokeActionArguments( arguments: {'package': 'oxygen', 'version': '1.2.0'}))); - testWithProfile('OK', processJobsWithFakeRunners: true, fn: () async { + testWithProfile( + 'marking and reverting', + expectedLogMessages: [ + 'SHOUT Deleting object from public bucket: "packages/oxygen-1.2.0.tar.gz".', + ], + fn: () async { + final client = createPubApiClient(authToken: siteAdminToken); + final removeVersion = '1.2.0'; + + // mark for deletion + await client.adminInvokeAction( + 'package-version-delete', + AdminInvokeActionArguments( + arguments: { + 'package': 'oxygen', + 'version': removeVersion, + 'state': 'true', + }, + ), + ); + + // repeated call updates the timestamp + final rsAgain = await client.adminInvokeAction( + 'package-version-delete', + AdminInvokeActionArguments( + arguments: { + 'package': 'oxygen', + 'version': removeVersion, + 'state': 'true', + }, + ), + ); + expect(rsAgain.output, { + 'package': 'oxygen', + 'version': '1.2.0', + 'before': {'isAdminDeleted': true, 'adminDeletedAt': isNotNull}, + 'after': {'isAdminDeleted': true, 'adminDeletedAt': isNotNull}, + }); + expect( + (rsAgain.output['before'] as Map)['adminDeletedAt'] == + (rsAgain.output['after'] as Map)['adminDeletedAt'], + isFalse, + ); + + // version is no longer visible + final pvAfterMarked = await packageBackend.lookupPackageVersion( + 'oxygen', removeVersion); + expect(pvAfterMarked!.isVisible, false); + final archiveAfterMarked = await packageBackend.tarballStorage + .getPublicBucketArchiveInfo('oxygen', removeVersion); + expect(archiveAfterMarked, isNull); + final versionsAfterMarked = + await packageBackend.listVersions('oxygen'); + expect( + versionsAfterMarked.versions + .where((v) => v.version == removeVersion), + isEmpty, + ); + + // revert + final rsRevert = await client.adminInvokeAction( + 'package-version-delete', + AdminInvokeActionArguments( + arguments: { + 'package': 'oxygen', + 'version': removeVersion, + 'state': 'false', + }, + ), + ); + expect(rsRevert.output, { + 'package': 'oxygen', + 'version': '1.2.0', + 'before': {'isAdminDeleted': true, 'adminDeletedAt': isNotNull}, + 'after': {'isAdminDeleted': false, 'adminDeletedAt': null}, + }); + + // version is visible again + final pvAfterReverted = await packageBackend.lookupPackageVersion( + 'oxygen', removeVersion); + expect(pvAfterReverted!.isVisible, true); + final archiveAfterReverted = await packageBackend.tarballStorage + .getPublicBucketArchiveInfo('oxygen', removeVersion); + expect(archiveAfterReverted, isNotNull); + final versionsAfterReverted = + await packageBackend.listVersions('oxygen'); + expect( + versionsAfterReverted.versions + .where((v) => v.version == removeVersion), + isNotEmpty, + ); + }, + ); + + testWithProfile('OK', + processJobsWithFakeRunners: true, + expectedLogMessages: [ + 'SHOUT Deleting object from public bucket: "packages/oxygen-1.2.0.tar.gz".', + ], fn: () async { final client = createPubApiClient(authToken: siteAdminToken); final removeVersion = '1.2.0'; @@ -281,19 +379,41 @@ void main() { final timeBeforeRemoval = clock.now().toUtc(); final rs = await client.adminInvokeAction( - 'package-version-delete', - AdminInvokeActionArguments( - arguments: {'package': 'oxygen', 'version': removeVersion})); + 'package-version-delete', + AdminInvokeActionArguments( + arguments: { + 'package': 'oxygen', + 'version': removeVersion, + 'state': 'true', + }, + ), + ); expect(rs.output, { - 'message': 'Package version and all associated resources deleted.', 'package': 'oxygen', 'version': '1.2.0', - 'deletedPackageVersions': 1, - 'deletedPackageVersionInfos': 1, - 'deletedPackageVersionAssets': 5 + 'before': {'isAdminDeleted': false, 'adminDeletedAt': null}, + 'after': {'isAdminDeleted': true, 'adminDeletedAt': isNotEmpty}, }); + // entity is present, but public archive is not accessible + final pvAfterMarked = + await packageBackend.lookupPackageVersion('oxygen', removeVersion); + expect(pvAfterMarked!.isVisible, false); + final archiveAfterMarked = await packageBackend.tarballStorage + .getPublicBucketArchiveInfo('oxygen', removeVersion); + expect(archiveAfterMarked, isNull); + final versionsAfterMarked = await packageBackend.listVersions('oxygen'); + expect( + versionsAfterMarked.versions.where((v) => v.version == removeVersion), + isEmpty, + ); + + // trigger removal after 60+ days + await withClock(Clock.fixed(clock.daysFromNow(63)), + () => adminBackend.deleteAdminDeletedEntities()); + + // checks after actual removal final pkgAfter1stRemoval = await dbService.lookupOrNull(pkgKey); expect(pkgAfter1stRemoval, isNotNull); @@ -319,39 +439,6 @@ void main() { await dbService.lookupOrNull(moderatedPkgKey); expect(moderatedPkg, isNull); - // calling remove second time must not affect updated or version count - final rs2 = await client.adminInvokeAction( - 'package-version-delete', - AdminInvokeActionArguments( - arguments: {'package': 'oxygen', 'version': removeVersion})); - expect(rs2.output, { - 'message': 'Package version and all associated resources deleted.', - 'package': 'oxygen', - 'version': '1.2.0', - 'deletedPackageVersions': 0, - 'deletedPackageVersionInfos': 0, - 'deletedPackageVersionAssets': 0, - }); - final pkgAfter2ndRemoval = - await dbService.lookupOrNull(pkgKey); - expect(pkgAfter2ndRemoval, isNotNull); - expect( - pkgAfter2ndRemoval!.latestVersion, - pkgAfter1stRemoval.latestVersion, - ); - expect( - pkgAfter2ndRemoval.updated, - pkgAfter1stRemoval.updated, - ); - expect( - pkgAfter2ndRemoval.versionCount, - pkgAfter1stRemoval.versionCount, - ); - expect( - pkgAfter2ndRemoval.deletedVersions, - pkgAfter1stRemoval.deletedVersions, - ); - // sanity check that scorecard is being loaded final sc2 = await scoreCardBackend.getLatestFinishedScoreCardData('oxygen');