From 4c0668a02802a7a346c727ca78a83c5c0f9753c5 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 18 Dec 2024 08:55:21 +0100 Subject: [PATCH 1/3] Remove advisory sync task from post-test verifications. --- app/lib/tool/neat_task/pub_dev_tasks.dart | 19 +++++++++++++------ app/test/shared/test_services.dart | 3 ++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index dcc1c53e32..31eb63b096 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -7,6 +7,7 @@ import 'dart:io'; import 'package:gcloud/service_scope.dart' as ss; import 'package:logging/logging.dart'; +import 'package:meta/meta.dart'; import 'package:neat_periodic_task/neat_periodic_task.dart'; import 'package:pub_dev/service/download_counts/computations.dart'; @@ -43,7 +44,9 @@ void setupPeriodTaskSchedulers() { } /// List of periodic task schedulers. -List createPeriodicTaskSchedulers() { +List createPeriodicTaskSchedulers({ + @visibleForTesting bool isPostTestVerification = false, +}) { return [ // Tries to send pending outgoing emails. _15mins( @@ -203,11 +206,15 @@ List createPeriodicTaskSchedulers() { task: countTopics, ), - _daily( - name: 'sync-security-advisories', - isRuntimeVersioned: false, - task: syncSecurityAdvisories, - ), + // NOTE: This task will fetch the advisories from a public endpoint, + // running it on every test is not worth it. + // TODO: Consider injecting a fake data source for unit test. + if (!isPostTestVerification) + _daily( + name: 'sync-security-advisories', + isRuntimeVersioned: false, + task: syncSecurityAdvisories, + ), // Checks the Datastore integrity of the model objects. _weekly( diff --git a/app/test/shared/test_services.dart b/app/test/shared/test_services.dart index 6ee0a8e8ab..382300a862 100644 --- a/app/test/shared/test_services.dart +++ b/app/test/shared/test_services.dart @@ -142,7 +142,8 @@ Future _postTestVerification({ } // run all background tasks here - for (final scheduler in createPeriodicTaskSchedulers()) { + final schedulers = createPeriodicTaskSchedulers(isPostTestVerification: true); + for (final scheduler in schedulers) { await scheduler.trigger(); } From 7916af1ab8cd6861d405c08716a86c2055b0a463 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 18 Dec 2024 11:04:20 +0100 Subject: [PATCH 2/3] Also skip GC of cloud instances. --- app/lib/tool/neat_task/pub_dev_tasks.dart | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index 31eb63b096..9ab383acbc 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -180,13 +180,19 @@ List createPeriodicTaskSchedulers({ ), // Delete very old instances that have been abandoned - _daily( - name: 'garbage-collect-old-instances', - isRuntimeVersioned: false, - task: () async => await deleteAbandonedInstances( - project: activeConfiguration.taskWorkerProject!, + // + // NOTE: This task will use Google Cloud API to remove worker instances. + // The client is not configured for fake environment, we should skip + // this task in post-test verifications. + // TODO: Write fake cloud abstractions to improve code coverage here. + if (!isPostTestVerification) + _daily( + name: 'garbage-collect-old-instances', + isRuntimeVersioned: false, + task: () async => await deleteAbandonedInstances( + project: activeConfiguration.taskWorkerProject!, + ), ), - ), _daily( name: 'sync-download-counts', From b021374ae224d5699bece32224613ddd03a100f4 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 18 Dec 2024 11:08:55 +0100 Subject: [PATCH 3/3] Also skip download counts sync. --- app/lib/tool/neat_task/pub_dev_tasks.dart | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/app/lib/tool/neat_task/pub_dev_tasks.dart b/app/lib/tool/neat_task/pub_dev_tasks.dart index 9ab383acbc..f566118100 100644 --- a/app/lib/tool/neat_task/pub_dev_tasks.dart +++ b/app/lib/tool/neat_task/pub_dev_tasks.dart @@ -194,11 +194,16 @@ List createPeriodicTaskSchedulers({ ), ), - _daily( - name: 'sync-download-counts', - isRuntimeVersioned: false, - task: syncDownloadCounts, - ), + // Syncs download counts from storage bucket. + // + // NOTE: This task reports missing files in the logs. + // TODO: Provide fake download data so that the task does not fail here. + if (!isPostTestVerification) + _daily( + name: 'sync-download-counts', + isRuntimeVersioned: false, + task: syncDownloadCounts, + ), _daily( name: 'compute-download-counts-30-days-totals',