TestProfile packages: explicit imported and generated fields. (#8634)

Author: isoos

app/lib/tool/test_profile/import_source.dart

@@ -46,49 +46,21 @@ abstract class ImportSource {
   static ImportSource autoGenerated() => _autoGeneratedImportSource;
 }
 
-/// Resolves and downloads data from pub.dev.
-class _PubDevImportSource implements ImportSource {
-  final String archiveCachePath;
+abstract class _ImportSource implements ImportSource {
   final _client = Client();
 
-  _PubDevImportSource({
-    required this.archiveCachePath,
-  });
-
-  @override
-  Future<List<ResolvedVersion>> resolveVersions(TestProfile profile) =>
-      resolver.resolveVersions(_client, profile);
-
-  @override
-  Future<List<int>> getArchiveBytes(String package, String version) async {
-    final archiveName = '$package-$version.tar.gz';
-    final file = File(p.join(archiveCachePath, archiveName));
-    // download package archive if not already in the cache
-    if (!await file.exists()) {
-      // TODO: Use archive_url from version-listing, that is stable!
-      final rs = await _client.get(Uri.parse(
-        '${urls.siteRoot}/packages/$package/versions/$version.tar.gz',
-      ));
-      await file.parent.create(recursive: true);
-      await file.writeAsBytes(rs.bodyBytes);
-    }
-    return await file.readAsBytes();
-  }
-
   @override
-  Future<void> close() async {
-    _client.close();
+  Future<List<ResolvedVersion>> resolveVersions(TestProfile profile) async {
+    return <ResolvedVersion>[
+      ...await resolver.resolveVersions(_client, profile),
+      ...await _resolveGeneratedVersions(profile),
+    ];
   }
-}
-
-/// Generates data based on random seed, without any network (or file) access.
-class _AutoGeneratedImportSource implements ImportSource {
-  final _archives = <String, List<int>>{};
 
-  @override
-  Future<List<ResolvedVersion>> resolveVersions(TestProfile profile) async {
+  Future<List<ResolvedVersion>> _resolveGeneratedVersions(
+      TestProfile profile) async {
     final versions = <ResolvedVersion>[];
-    profile.packages.forEach((p) {
+    profile.generatedPackages.forEach((p) {
       final vs = <String>[
         ...?p.versions?.map((v) => v.version),
       ];
@@ -119,6 +91,41 @@ class _AutoGeneratedImportSource implements ImportSource {
     return versions;
   }
 
+  @override
+  Future<void> close() async {
+    _client.close();
+  }
+}
+
+/// Resolves and downloads data from pub.dev.
+class _PubDevImportSource extends _ImportSource {
+  final String archiveCachePath;
+
+  _PubDevImportSource({
+    required this.archiveCachePath,
+  });
+
+  @override
+  Future<List<int>> getArchiveBytes(String package, String version) async {
+    final archiveName = '$package-$version.tar.gz';
+    final file = File(p.join(archiveCachePath, archiveName));
+    // download package archive if not already in the cache
+    if (!await file.exists()) {
+      // TODO: Use archive_url from version-listing, that is stable!
+      final rs = await _client.get(Uri.parse(
+        '${urls.siteRoot}/packages/$package/versions/$version.tar.gz',
+      ));
+      await file.parent.create(recursive: true);
+      await file.writeAsBytes(rs.bodyBytes);
+    }
+    return await file.readAsBytes();
+  }
+}
+
+/// Generates data based on random seed, without any network (or file) access.
+class _AutoGeneratedImportSource extends _ImportSource {
+  final _archives = <String, List<int>>{};
+
   @override
   Future<List<int>> getArchiveBytes(String package, String version) async {
     final key = '$package/$version';
@@ -189,9 +196,6 @@ enum TypeEnum { a, b }
     _archives[key] = content;
     return content;
   }
-
-  @override
-  Future<void> close() async {}
 }
 
 @visibleForTesting

app/lib/tool/test_profile/importer.dart

@@ -9,6 +9,7 @@ import 'dart:io';
 import 'package:_pub_shared/data/admin_api.dart';
 import 'package:_pub_shared/data/package_api.dart';
 import 'package:_pub_shared/search/tags.dart';
+import 'package:collection/collection.dart';
 import 'package:meta/meta.dart';
 import 'package:tar/tar.dart';
 
@@ -37,7 +38,11 @@ Future<void> importProfile({
   // expand profile with resolved version information
   profile = normalize(profile, resolvedVersions: resolvedVersions);
 
-  if (profile.packages
+  if (profile.importedPackages
+      .any((p) => p.uploaders != null && p.uploaders!.length > 1)) {
+    throw UnimplementedError('More than one uploader is not implemented.');
+  }
+  if (profile.generatedPackages
       .any((p) => p.uploaders != null && p.uploaders!.length > 1)) {
     throw UnimplementedError('More than one uploader is not implemented.');
   }
@@ -116,7 +121,7 @@ Future<void> importProfile({
         '\n$lastException\n$lastStackTrace');
   }
 
-  for (final testPackage in profile.packages) {
+  Future<void> updatePackage(TestPackage testPackage) async {
     final packageName = testPackage.name;
     final activeEmail = lastActiveUploaderEmails[packageName];
 
@@ -167,6 +172,13 @@ Future<void> importProfile({
     }
   }
 
+  for (final p in profile.importedPackages) {
+    await updatePackage(p);
+  }
+  for (final p in profile.generatedPackages) {
+    await updatePackage(p);
+  }
+
   final createLikeCounts = <String, int>{};
   // create users
   for (final u in profile.users) {
@@ -187,10 +199,10 @@ Future<void> importProfile({
     );
   }
   // fill in missing likes
-  for (final p in profile.packages) {
+  Future<void> createMissingLike(TestPackage p) async {
     if (p.likeCount != null) {
       final likesMissing = p.likeCount! - (createLikeCounts[p.name] ?? 0);
-      if (likesMissing <= 0) continue;
+      if (likesMissing <= 0) return;
 
       for (var i = 0; i < likesMissing; i++) {
         final userEmail = 'like-$i@pub.dev';
@@ -205,12 +217,21 @@ Future<void> importProfile({
     }
   }
 
+  for (final p in profile.importedPackages) {
+    await createMissingLike(p);
+  }
+  for (final p in profile.generatedPackages) {
+    await createMissingLike(p);
+  }
+
   await source.close();
   await asyncQueue.ongoingProcessing;
 }
 
 List<String> _potentialActiveEmails(TestProfile profile, String packageName) {
-  final testPackage = profile.packages.firstWhere((p) => p.name == packageName);
+  final testPackage =
+      profile.importedPackages.firstWhereOrNull((p) => p.name == packageName) ??
+          profile.generatedPackages.firstWhere((p) => p.name == packageName);
 
   // uploaders
   if (testPackage.publisher == null) return testPackage.uploaders!;

app/lib/tool/test_profile/models.dart

@@ -9,23 +9,33 @@ import 'package:yaml/yaml.dart' as yaml;
 
 part 'models.g.dart';
 
-/// The configuration to use when creating a local (partial) mirror of pub.dev
-/// in order to us it in tests.
+/// The configuration to use when creating a local database for testing and
+/// development, containing a partial mirror of pub.dev and also synthetic
+/// generated package contents.
 @JsonSerializable(explicitToJson: true, includeIfNull: false)
 class TestProfile {
-  final List<TestPackage> packages;
+  /// Packages that will be imported from pub.dev.
+  ///
+  /// The archive of the package will be used without modification.
+  /// The package options may be updated as provided by the [TestPackage].
+  final List<TestPackage> importedPackages;
+
+  /// Packages that will be generated locally using the provided parameters and semi-random templates.
+  final List<TestPackage> generatedPackages;
   final List<TestPublisher> publishers;
   final List<TestUser> users;
 
   /// The email address of the default user.
   final String? defaultUser;
 
   TestProfile({
-    required List<TestPackage>? packages,
+    List<TestPackage>? importedPackages,
+    List<TestPackage>? generatedPackages,
     List<TestPublisher>? publishers,
     List<TestUser>? users,
     this.defaultUser,
-  })  : packages = packages ?? <TestPackage>[],
+  })  : importedPackages = importedPackages ?? <TestPackage>[],
+        generatedPackages = generatedPackages ?? <TestPackage>[],
         publishers = publishers ?? <TestPublisher>[],
         users = users ?? <TestUser>[];
 
@@ -41,7 +51,8 @@ class TestProfile {
 
   TestProfile changeDefaultUser(String email) {
     return TestProfile(
-      packages: packages,
+      importedPackages: importedPackages,
+      generatedPackages: generatedPackages,
       publishers: publishers,
       users: users,
       defaultUser: email,

app/lib/tool/test_profile/models.g.dart

@@ -12,7 +12,8 @@ TestProfile normalize(
 }) {
   final users = <String, TestUser>{};
   final publishers = <String, TestPublisher>{};
-  final packages = <String, TestPackage>{};
+  final importedPackages = <String, TestPackage>{};
+  final generatedPackages = <String, TestPackage>{};
 
   profile.users.forEach((user) {
     users[user.email] = user;
@@ -28,8 +29,14 @@ TestProfile normalize(
     });
   });
 
-  profile.packages.forEach((package) {
-    packages[package.name] = package;
+  profile.importedPackages.forEach((package) {
+    importedPackages[package.name] = package;
+    package.uploaders?.forEach((uploader) {
+      _createUserIfNeeded(users, uploader);
+    });
+  });
+  profile.generatedPackages.forEach((package) {
+    generatedPackages[package.name] = package;
     package.uploaders?.forEach((uploader) {
       _createUserIfNeeded(users, uploader);
     });
@@ -40,7 +47,10 @@ TestProfile normalize(
   // add missing packages from resolved versions
   if (resolvedVersions != null) {
     resolvedVersions.forEach((rv) {
-      packages.putIfAbsent(
+      if (generatedPackages.containsKey(rv.package)) {
+        return;
+      }
+      importedPackages.putIfAbsent(
           rv.package,
           () => TestPackage(
                 name: rv.package,
@@ -51,8 +61,8 @@ TestProfile normalize(
               ));
     });
     // update versions from resolved versions
-    packages.values.toList().forEach((p) {
-      final versions = resolvedVersions
+    List<TestVersion> getUpdateVersions(TestPackage p) {
+      return resolvedVersions
           .where((rv) => rv.package == p.name)
           .map((rv) => rv.version)
           .toSet()
@@ -61,27 +71,49 @@ TestProfile normalize(
               created:
                   resolvedVersions.firstWhere((x) => x.version == v).created))
           .toList();
-      packages[p.name] = p.change(versions: versions);
+    }
+
+    importedPackages.values.toList().forEach((p) {
+      importedPackages[p.name] = p.change(versions: getUpdateVersions(p));
+    });
+    generatedPackages.values.toList().forEach((p) {
+      generatedPackages[p.name] = p.change(versions: getUpdateVersions(p));
     });
   }
 
-  packages.values.toList().forEach((package) {
-    if (package.publisher != null) {
-      _createPublisherIfNeeded(
-        publishers,
-        package.publisher!,
-        memberEmail: defaultUser,
-      );
-    } else if (package.uploaders == null || package.uploaders!.isEmpty) {
-      packages[package.name] = package.change(uploaders: [defaultUser]);
+  final publishersToCreate = {
+    ...importedPackages.values.map((p) => p.publisher).nonNulls,
+    ...generatedPackages.values.map((p) => p.publisher).nonNulls,
+  };
+  for (final publisher in publishersToCreate) {
+    _createPublisherIfNeeded(
+      publishers,
+      publisher,
+      memberEmail: defaultUser,
+    );
+  }
+
+  for (final package in importedPackages.values.toList()) {
+    if (package.publisher == null &&
+        (package.uploaders == null || package.uploaders!.isEmpty)) {
+      importedPackages[package.name] = package.change(uploaders: [defaultUser]);
     }
-  });
+  }
+
+  for (final package in generatedPackages.values.toList()) {
+    if (package.publisher == null &&
+        (package.uploaders == null || package.uploaders!.isEmpty)) {
+      generatedPackages[package.name] =
+          package.change(uploaders: [defaultUser]);
+    }
+  }
 
   return TestProfile(
     defaultUser: defaultUser,
     users: users.values.toList(),
     publishers: publishers.values.toList(),
-    packages: packages.values.toList(),
+    importedPackages: importedPackages.values.toList(),
+    generatedPackages: generatedPackages.values.toList(),
   );
 }
 

app/lib/tool/test_profile/normalizer.dart

@@ -24,6 +24,9 @@ import 'models.dart';
 /// than the profile originally specified).
 Future<List<ResolvedVersion>> resolveVersions(
     http.Client client, TestProfile profile) async {
+  if (profile.importedPackages.isEmpty) {
+    return [];
+  }
   return await withTempDirectory((temp) async {
     final pubCacheDir = Directory(p.join(temp.path, 'pub-cache'));
     await pubCacheDir.create();
@@ -40,7 +43,7 @@ Future<List<ResolvedVersion>> resolveVersions(
       pubCacheDir: pubCacheDir.path,
     );
 
-    for (final package in profile.packages) {
+    for (final package in profile.importedPackages) {
       final versions = package.versions == null || package.versions!.isEmpty
           ? <TestVersion>[TestVersion(version: 'any', created: null)]
           : package.versions;