diff --git a/app/lib/admin/actions/actions.dart b/app/lib/admin/actions/actions.dart index 833ea85fa1..425840f859 100644 --- a/app/lib/admin/actions/actions.dart +++ b/app/lib/admin/actions/actions.dart @@ -17,6 +17,7 @@ import 'moderation_case_info.dart'; import 'moderation_case_list.dart'; import 'moderation_case_resolve.dart'; import 'moderation_case_update.dart'; +import 'moderation_transparency_metrics.dart'; import 'package_info.dart'; import 'package_version_info.dart'; import 'package_version_retraction.dart'; @@ -97,6 +98,7 @@ final class AdminAction { moderationCaseList, moderationCaseResolve, moderationCaseUpdate, + moderationTransparencyMetrics, packageInfo, packageVersionInfo, packageVersionRetraction, diff --git a/app/lib/admin/actions/moderation_transparency_metrics.dart b/app/lib/admin/actions/moderation_transparency_metrics.dart new file mode 100644 index 0000000000..dea76138be --- /dev/null +++ b/app/lib/admin/actions/moderation_transparency_metrics.dart @@ -0,0 +1,332 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:pub_dev/account/models.dart'; +import 'package:pub_dev/admin/models.dart'; +import 'package:pub_dev/shared/datastore.dart'; + +import 'actions.dart'; + +final moderationTransparencyMetrics = AdminAction( + name: 'moderation-transparency-metrics', + summary: 'Collects and provides transparency metrics.', + description: ''' +Scans ModerationCase and User entities and collects statistics +required for the transparency metrics report. +''', + options: { + 'start': + 'The inclusive start date of the reported period in the format of `YYYY-MM-DD`.', + 'end': + 'The inclusive end date of the reported period in the format of `YYYY-MM-DD`.', + }, + invoke: (options) async { + final dateRegExp = RegExp(r'^\d{4}\-\d{2}\-\d{2}$'); + + DateTime parseDate(String key) { + final param = options[key] ?? ''; + InvalidInputException.check( + dateRegExp.matchAsPrefix(param) != null, + '`$key` must be a valid date in YYYY-MM-DD format', + ); + final parsed = DateTime.tryParse(param); + InvalidInputException.check( + parsed != null, + '`$key` must be a valid date in YYYY-MM-DD format', + ); + return parsed!; + } + + final start = parseDate('start'); + final end = parseDate('end'); + + // The number of cases where a moderation action has been done. + var totalModerationCount = 0; + // The grouped counts of moderation actions by violations. + final violations = {}; + // The grouped counts of moderation actions by detection sources. + final sources = {}; + // The grouped counts of moderation actions by applied restrictions. + final restrictions = {}; + + // The number of appeals. + var totalAppealCount = 0; + // The number of appeals where the person responding is a known content owner. + var contentOwnerAppealCount = 0; + // The grouped counts of appeals by outcomes. + final appealOutcomes = {}; + // The list of time-to-action days - required for reporting the median. + final appealTimeToActionDays = []; + + final mcQuery = dbService.query() + // timestamp start on 00:00:00 on the day + ..filter('resolved >=', start) + // adding an extra day to make sure the end day is fully included + ..filter('resolved <', end.add(Duration(days: 1))); + await for (final mc in mcQuery.run()) { + // sanity check + if (mc.resolved == null) { + continue; + } + + // Report group #1: case has moderated action. Whether the case was + // a notification or appeal won't make a difference for this group. + if (mc.status == ModerationStatus.moderationApplied || + mc.status == ModerationStatus.noActionReverted) { + totalModerationCount++; + violations.increment(mc.violation ?? ''); + sources.increment(mc.source); + + final hasUserRestriction = mc + .getActionLog() + .entries + .any((e) => ModerationSubject.tryParse(e.subject)!.isUser); + // If actions resulted in a user being blocked, then we count it as + // "provision", even if packages were also removed. + // Reasoning that it's natural that blocking a user would also + // remove some of the users content. + if (hasUserRestriction) { + restrictions.increment('provision'); + } else { + restrictions.increment('visibility'); + } + } + + // Report group #2: appeals. + if (mc.appealedCaseId != null) { + totalAppealCount++; + if (mc.isSubjectOwner) { + contentOwnerAppealCount++; + } + + switch (mc.status) { + case ModerationStatus.noActionUpheld: + case ModerationStatus.moderationUpheld: + appealOutcomes.increment('upheld'); + break; + case ModerationStatus.noActionReverted: + case ModerationStatus.moderationReverted: + appealOutcomes.increment('reverted'); + break; + default: + appealOutcomes.increment('omitted'); + break; + } + + final timeToActionDays = mc.resolved!.difference(mc.opened).inDays + 1; + appealTimeToActionDays.add(timeToActionDays); + } + } + + appealTimeToActionDays.sort(); + final appealMedianTimeToActionDays = appealTimeToActionDays.isEmpty + ? 0 + : appealTimeToActionDays[appealTimeToActionDays.length ~/ 2]; + + final userQuery = dbService.query() + // timestamp start on 00:00:00 on the day + ..filter('moderatedAt >=', start) + // adding an extra day to make sure the end day is fully included + ..filter('moderatedAt <', end.add(Duration(days: 1))); + final reasonCounts = {}; + await for (final user in userQuery.run()) { + // sanity check + if (user.moderatedAt == null) { + continue; + } + + // Report group #3: user restrictions. + reasonCounts.increment(user.moderatedReason ?? ''); + } + + final text = toCsV([ + // --------------------------------------- + ['Restrictive actions', ''], + ['Total number of actions taken', totalModerationCount], + [ + 'Number of actions taken, by type of illegal content or violation of terms and conditions', + '', + ], + [ + 'VIOLATION_CATEGORY_ANIMAL_WELFARE', + violations[ModerationViolation.animalWelfare] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_DATA_PROTECTION_AND_PRIVACY_VIOLATIONS', + violations[ModerationViolation.dataProtectionAndPrivacyViolations] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_ILLEGAL_OR_HARMFUL_SPEECH', + violations[ModerationViolation.illegalAndHatefulSpeech] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_INTELLECTUAL_PROPERTY_INFRINGEMENTS', + violations[ModerationViolation.intellectualPropertyInfringements] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_NEGATIVE_EFFECTS_ON_CIVIC_DISCOURSE_OR_ELECTIONS', + violations[ModerationViolation + .negativeEffectsOnCivicDiscourseOrElections] ?? + 0, + ], + [ + 'VIOLATION_CATEGORY_NON_CONSENSUAL_BEHAVIOUR', + violations[ModerationViolation.nonConsensualBehavior] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_PORNOGRAPHY_OR_SEXUALIZED_CONTENT', + violations[ModerationViolation.pornographyOrSexualizedContent] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_PROTECTION_OF_MINORS', + violations[ModerationViolation.protectionOfMinors] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_RISK_FOR_PUBLIC_SECURITY', + violations[ModerationViolation.riskForPublicSecurity] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_SCAMS_AND_FRAUD', + violations[ModerationViolation.scamsAndFraud] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_SELF_HARM', + violations[ModerationViolation.selfHarm] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_SCOPE_OF_PLATFORM_SERVICE', + violations[ModerationViolation.scopeOfPlatformService] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_UNSAFE_AND_ILLEGAL_PRODUCTS', + violations[ModerationViolation.unsafeAndIllegalProducts] ?? 0, + ], + [ + 'VIOLATION_CATEGORY_VIOLENCE', + violations[ModerationViolation.violence] ?? 0, + ], + ['Number of actions taken, by detection method', ''], + [ + 'Automated detection', + sources[ModerationSource.automatedDetection] ?? 0 + ], + [ + 'Non-automated detection', + sources.entries + .where((e) => e.key != ModerationSource.automatedDetection) + .map((e) => e.value) + .fold(0, (a, b) => a + b) + ], + ['Number of actions taken, by type of restriction applied', ''], + [ + 'Restrictions of Visibility', + restrictions['visibility'] ?? 0, + ], + [ + 'Restrictions of Monetisation', + restrictions['monetisation'] ?? 0, + ], + [ + 'Restrictions of Provision of the Service', + restrictions['provision'] ?? 0, + ], + [ + 'Restrictions of an Account', + restrictions['account'] ?? 0, + ], + + // --------------------------------------- + ['Complaints received through internal complaint handling systems', ''], + ['Total number of complaints received', totalAppealCount], + ['Number of complaints received, by reason for complaint', ''], + ['CONTENT_ACCOUNT_OWNER_APPEAL', contentOwnerAppealCount], + ['REPORTER_APPEAL', totalAppealCount - contentOwnerAppealCount], + ['Number of complaints received, by outcome', ''], + [ + 'Initial decision upheld', + appealOutcomes['upheld'] ?? 0, + ], + [ + 'Initial decision reversed', + appealOutcomes['reverted'] ?? 0, + ], + [ + 'Decision omitted', + appealOutcomes['omitted'] ?? 0, + ], + [ + 'Median time to action a complaint (days)', + appealMedianTimeToActionDays, + ], + + // --------------------------------------- + ['Suspensions imposed to protect against misuse', ''], + [ + 'Number of suspensions for manifestly illegal content imposed pursuant to Article 23', + reasonCounts[UserModeratedReason.illegalContent] ?? 0, + ], + [ + 'Number of suspensions for manifestly unfounded notices imposed pursuant to Article 23', + reasonCounts[UserModeratedReason.unfoundedNotifications] ?? 0, + ], + [ + 'Number of suspensions for manifestly unfounded complaints imposed pursuant to Article 23', + reasonCounts[UserModeratedReason.unfoundedAppeals] ?? 0, + ], + ]); + + return { + 'text': text, + 'moderations': { + 'total': totalModerationCount, + 'violations': violations, + 'sources': sources, + 'restrictions': restrictions, + }, + 'appeals': { + 'total': totalAppealCount, + 'contentOwner': contentOwnerAppealCount, + 'outcomes': appealOutcomes, + 'medianTimeToActionDays': appealMedianTimeToActionDays, + }, + 'users': { + 'suspensions': reasonCounts, + } + }; + }, +); + +/// Loose implementation of RFC 4180 writing tabular data into Comma Separated Values. +/// The current implementation supports only String and int values. +String toCsV(List> data) { + final sb = StringBuffer(); + for (final row in data) { + for (var i = 0; i < row.length; i++) { + if (i > 0) { + sb.write(','); + } + final value = row[i]; + if (value is int) { + sb.write(value); + } else if (value is String) { + final mustEscape = value.contains(',') || + value.contains('"') || + value.contains('\r') || + value.contains('\n'); + sb.write(mustEscape ? '"${value.replaceAll('"', '""')}"' : value); + } else { + throw UnimplementedError( + 'Unhandled CSV type: ${value.runtimeType}/$value'); + } + } + sb.write('\r\n'); + } + return sb.toString(); +} + +extension on Map { + void increment(String key) { + this[key] = (this[key] ?? 0) + 1; + } +} diff --git a/app/lib/admin/models.dart b/app/lib/admin/models.dart index ff4ebf8e41..7ef51049b2 100644 --- a/app/lib/admin/models.dart +++ b/app/lib/admin/models.dart @@ -268,22 +268,40 @@ abstract class ModerationGrounds { abstract class ModerationViolation { static const none = 'none'; + static const animalWelfare = 'animal_welfare'; + static const dataProtectionAndPrivacyViolations = + 'data_protection_and_privacy_violations'; + static const illegalAndHatefulSpeech = 'illegal_or_harmful_speech'; + static const intellectualPropertyInfringements = + 'intellectual_property_infringements'; + static const negativeEffectsOnCivicDiscourseOrElections = + 'negative_effects_on_civic_discourse_or_elections'; + static const nonConsensualBehavior = 'non_consensual_behaviour'; + static const pornographyOrSexualizedContent = + 'pornography_or_sexualized_content'; + static const protectionOfMinors = 'protection_of_minors'; + static const riskForPublicSecurity = 'risk_for_public_security'; + static const scamsAndFraud = 'scams_and_fraud'; + static const selfHarm = 'self_harm'; + static const scopeOfPlatformService = 'scope_of_platform_service'; + static const unsafeAndIllegalProducts = 'unsafe_and_illegal_products'; + static const violence = 'violence'; static const violationValues = [ - 'animal_welfare', - 'data_protection_and_privacy_violations', - 'illegal_or_harmful_speech', - 'intellectual_property_infringements', - 'negative_effects_on_civic_discourse_or_elections', - 'non_consensual_behaviour', - 'pornography_or_sexualized_content', - 'protection_of_minors', - 'risk_for_public_security', - 'scams_and_fraud', - 'self_harm', - 'scope_of_platform_service', - 'unsafe_and_illegal_products', - 'violence', + animalWelfare, + dataProtectionAndPrivacyViolations, + illegalAndHatefulSpeech, + intellectualPropertyInfringements, + negativeEffectsOnCivicDiscourseOrElections, + nonConsensualBehavior, + pornographyOrSexualizedContent, + protectionOfMinors, + riskForPublicSecurity, + scamsAndFraud, + selfHarm, + scopeOfPlatformService, + unsafeAndIllegalProducts, + violence, ]; } diff --git a/app/test/admin/moderation_transparency_metrics_test.dart b/app/test/admin/moderation_transparency_metrics_test.dart new file mode 100644 index 0000000000..8983c67afd --- /dev/null +++ b/app/test/admin/moderation_transparency_metrics_test.dart @@ -0,0 +1,280 @@ +// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:_pub_shared/data/account_api.dart' as account_api; +import 'package:_pub_shared/data/admin_api.dart'; +import 'package:clock/clock.dart'; +import 'package:pub_dev/account/models.dart'; +import 'package:pub_dev/admin/backend.dart'; +import 'package:pub_dev/admin/models.dart'; +import 'package:pub_dev/shared/datastore.dart'; +import 'package:test/test.dart'; + +import '../shared/test_models.dart'; +import '../shared/test_services.dart'; + +void main() { + group('moderation transparency metrics', () { + Future _report( + String package, { + String? email, + String? caseId, + }) async { + await withHttpPubApiClient( + fn: (client) async { + await client.postReport(account_api.ReportForm( + email: email ?? 'user@pub.dev', + subject: 'package:$package', + caseId: caseId, + message: 'Huston, we have a problem.', + )); + }, + ); + final list = await dbService.query().run().toList(); + return list.reduce((a, b) => a.opened.isAfter(b.opened) ? a : b); + } + + Future _moderatePkg( + String package, { + required String caseId, + bool? state, + }) async { + final api = createPubApiClient(authToken: siteAdminToken); + return await api.adminInvokeAction( + 'moderate-package', + AdminInvokeActionArguments(arguments: { + 'case': caseId, + 'package': package, + if (state != null) 'state': state.toString(), + }), + ); + } + + Future _moderateUser( + String email, { + required String caseId, + bool? state, + String? reason, + }) async { + final api = createPubApiClient(authToken: siteAdminToken); + return await api.adminInvokeAction( + 'moderate-user', + AdminInvokeActionArguments(arguments: { + 'case': caseId, + 'user': email, + if (reason != null) 'reason': reason, + if (state != null) 'state': state.toString(), + }), + ); + } + + Future _resolve( + String caseId, { + String? grounds, + String? violation, + String? reason, + }) async { + // infer grounds and violation for test purposes + if (reason != null) { + grounds ??= ModerationGrounds.policy; + violation ??= 'scope_of_platform_service'; + } + final api = createPubApiClient(authToken: siteAdminToken); + await api.adminInvokeAction( + 'moderation-case-resolve', + AdminInvokeActionArguments(arguments: { + 'case': caseId, + if (grounds != null) 'grounds': grounds, + if (violation != null) 'violation': violation, + if (reason != null) 'reason': reason, + }), + ); + final mc = await adminBackend.lookupModerationCase(caseId); + return mc!.status!; + } + + testWithProfile('nothing to report', fn: () async { + final api = createPubApiClient(authToken: siteAdminToken); + final rs = await api.adminInvokeAction( + 'moderation-transparency-metrics', + AdminInvokeActionArguments(arguments: { + 'start': '2000-01-01', + 'end': '2000-01-01', + }), + ); + expect(rs.output, { + 'text': isNotEmpty, + 'moderations': { + 'total': 0, + 'violations': {}, + 'sources': {}, + 'restrictions': {}, + }, + 'appeals': { + 'total': 0, + 'contentOwner': 0, + 'outcomes': {}, + 'medianTimeToActionDays': 0, + }, + 'users': { + 'suspensions': {}, + }, + }); + final text = rs.output['text'] as String; + expect(text, contains('Total number of actions taken,0\r\n')); + }); + + testWithProfile('moderated package', fn: () async { + final mc = await _report('oxygen'); + await _moderatePkg('oxygen', caseId: mc.caseId, state: true); + await _resolve( + mc.caseId, + grounds: ModerationGrounds.policy, + violation: ModerationViolation.scamsAndFraud, + reason: 'package contains scam', + ); + + final api = createPubApiClient(authToken: siteAdminToken); + final rs = await api.adminInvokeAction( + 'moderation-transparency-metrics', + AdminInvokeActionArguments(arguments: { + 'start': clock.daysAgo(30).toIso8601String().split('T').first, + 'end': clock.now().toIso8601String().split('T').first, + }), + ); + expect(rs.output, { + 'text': isNotEmpty, + 'moderations': { + 'total': 1, + 'violations': {'scams_and_fraud': 1}, + 'sources': {'external-notification': 1}, + 'restrictions': {'visibility': 1} + }, + 'appeals': { + 'total': 0, + 'contentOwner': 0, + 'outcomes': {}, + 'medianTimeToActionDays': 0 + }, + 'users': { + 'suspensions': {}, + }, + }); + final text = rs.output['text'] as String; + expect(text, contains('Total number of actions taken,1\r\n')); + expect( + text, contains('VIOLATION_CATEGORY_SCOPE_OF_PLATFORM_SERVICE,0\r\n')); + expect(text, contains('VIOLATION_CATEGORY_SCAMS_AND_FRAUD,1\r\n')); + }); + + testWithProfile('moderated user', fn: () async { + final mc = await _report('oxygen'); + await _moderateUser( + 'user@pub.dev', + caseId: mc.caseId, + state: true, + reason: UserModeratedReason.illegalContent, + ); + await _resolve( + mc.caseId, + grounds: ModerationGrounds.illegal, + violation: ModerationViolation.unsafeAndIllegalProducts, + reason: 'controlled substances', + ); + + final api = createPubApiClient(authToken: siteAdminToken); + final rs = await api.adminInvokeAction( + 'moderation-transparency-metrics', + AdminInvokeActionArguments(arguments: { + 'start': clock.daysAgo(30).toIso8601String().split('T').first, + 'end': clock.now().toIso8601String().split('T').first, + }), + ); + expect(rs.output, { + 'text': isNotEmpty, + 'moderations': { + 'total': 1, + 'violations': {'unsafe_and_illegal_products': 1}, + 'sources': {'external-notification': 1}, + 'restrictions': {'provision': 1} + }, + 'appeals': { + 'total': 0, + 'contentOwner': 0, + 'outcomes': {}, + 'medianTimeToActionDays': 0 + }, + 'users': { + 'suspensions': {'illegal-content': 1}, + }, + }); + final text = rs.output['text'] as String; + expect( + text, + allOf([ + contains('Total number of actions taken,1\r\n'), + contains('Automated detection,0\r\n'), + contains('Non-automated detection,1\r\n'), + contains('VIOLATION_CATEGORY_UNSAFE_AND_ILLEGAL_PRODUCTS,1\r\n'), + contains('VIOLATION_CATEGORY_SCOPE_OF_PLATFORM_SERVICE,0\r\n'), + contains('Restrictions of Provision of the Service,1\r\n'), + ])); + }); + + testWithProfile('appeal', fn: () async { + final mc = await _report('oxygen'); + await _moderatePkg('oxygen', caseId: mc.caseId, state: true); + await _resolve( + mc.caseId, + grounds: ModerationGrounds.policy, + violation: ModerationViolation.animalWelfare, + reason: 'abused pet photos', + ); + + final appeal = await _report( + 'oxygen', + caseId: mc.caseId, + email: 'admin@pub.dev', + ); + await _moderatePkg('oxygen', caseId: appeal.caseId, state: false); + await _resolve(appeal.caseId); + + final api = createPubApiClient(authToken: siteAdminToken); + final rs = await api.adminInvokeAction( + 'moderation-transparency-metrics', + AdminInvokeActionArguments(arguments: { + 'start': clock.daysAgo(30).toIso8601String().split('T').first, + 'end': clock.now().toIso8601String().split('T').first, + }), + ); + expect(rs.output, { + 'text': isNotEmpty, + 'moderations': { + 'total': 1, + 'violations': {'animal_welfare': 1}, + 'sources': {'external-notification': 1}, + 'restrictions': {'visibility': 1} + }, + 'appeals': { + 'total': 1, + 'contentOwner': 0, + 'outcomes': {'reverted': 1}, + 'medianTimeToActionDays': 1, + }, + 'users': { + 'suspensions': {}, + }, + }); + final text = rs.output['text'] as String; + expect( + text, + allOf([ + contains('Total number of actions taken,1\r\n'), + contains('REPORTER_APPEAL,1\r\n'), + contains('Initial decision reversed,1\r\n'), + contains('Median time to action a complaint (days),1\r\n'), + ])); + }); + }); +}