Merge branch 'master' into misc/recommended-lint-progress

Author: parlough

CHANGELOG.md

@@ -2,6 +2,11 @@ Important changes to data models, configuration, and migrations between each
 AppEngine version, listed here to ease deployment and troubleshooting.
 
 ## Next Release (replace with git tag when deployed)
+
+## `20241210t143600-all`
+ * Bump runtimeVersion to `2024.12.09`.
+ * Upgraded stable Dart analysis SDK to `3.6.0`
+ * Upgraded pana to `0.22.17`.
  * Note: `search` isolate renewal is randomized.
 
 ## `20241205t082000-all`

Dockerfile.worker

@@ -23,7 +23,7 @@ RUN mkdir -p /home/worker/config/dart-stable
 RUN mkdir -p /home/worker/config/flutter-stable
 
 # Setup Dart SDK into /home/worker/dart/{stable,preview}/
-RUN XDG_CONFIG_HOME=/home/worker/config/dart-stable tool/setup-dart.sh /home/worker/dart/stable 3.6.0-334.4.beta beta
+RUN XDG_CONFIG_HOME=/home/worker/config/dart-stable tool/setup-dart.sh /home/worker/dart/stable stable/raw/hash/ae7ca5199a0559db0ae60533e9cedd3ce0d6ab04
 
 # Setup Flutter SDK into /home/worker/flutter/{stable,preview}/
 RUN XDG_CONFIG_HOME=/home/worker/config/flutter-stable tool/setup-flutter.sh /home/worker/flutter/stable 3.27.0-0.2.pre beta

app/bin/tools/search_benchmark.dart

@@ -5,6 +5,7 @@
 import 'dart:convert';
 import 'dart:io';
 
+import 'package:_pub_shared/search/search_form.dart';
 import 'package:pub_dev/package/overrides.dart';
 import 'package:pub_dev/search/mem_index.dart';
 import 'package:pub_dev/search/models.dart';
@@ -24,6 +25,7 @@ Future<void> main(List<String> args) async {
 
   // NOTE: please add more queries to this list, especially if there is a performance bottleneck.
   final queries = [
+    'chart',
     'json',
     'camera',
     'android camera',
@@ -33,7 +35,10 @@ Future<void> main(List<String> args) async {
   final sw = Stopwatch()..start();
   var count = 0;
   for (var i = 0; i < 100; i++) {
-    index.search(ServiceSearchQuery.parse(query: queries[i % queries.length]));
+    index.search(ServiceSearchQuery.parse(
+      query: queries[i % queries.length],
+      tagsPredicate: TagsPredicate.regularSearch(),
+    ));
     count++;
   }
   sw.stop();

app/lib/frontend/handlers/experimental.dart

@@ -95,7 +95,9 @@ class ExperimentalFlags {
   bool get isDarkModeEnabled => isEnabled('dark');
   bool get isDarkModeDefault => isEnabled('dark-as-default');
 
-  bool get showDownloadCounts => isEnabled('download-counts');
+  bool get showDownloadCountsVersionChart =>
+      isEnabled('download-counts-version-chart');
+  bool get showDownloadCounts => true;
 
   String encodedAsCookie() => _enabled.join(':');
 

app/lib/frontend/templates/views/pkg/labeled_scores.dart

@@ -29,8 +29,7 @@ d.Node labeledScoresNode({
       ),
       d.div(
         classes: ['packages-score', 'packages-score-health'],
-        child:
-            _labeledScore('pub points', grantedPubPoints?.toString(), sign: ''),
+        child: _labeledScore('points', grantedPubPoints?.toString(), sign: ''),
       ),
       requestContext.experimentalFlags.showDownloadCounts
           ? d.div(

app/lib/frontend/templates/views/pkg/score_tab.dart

@@ -80,7 +80,7 @@ d.Node scoreTabNode({
         ),
         _reportNode(report),
         if (card.weeklyVersionDownloads != null &&
-            requestContext.experimentalFlags.showDownloadCounts)
+            requestContext.experimentalFlags.showDownloadCountsVersionChart)
           _downloadsChart(card.weeklyVersionDownloads!),
         if (toolEnvInfo != null) toolEnvInfo,
       ]),

app/lib/search/mem_index.dart

@@ -28,6 +28,8 @@ class InMemoryPackageIndex {
   late final TokenIndex<String> _readmeIndex;
   late final TokenIndex<IndexedApiDocPage> _apiSymbolIndex;
   late final _scorePool = ScorePool(_packageNameIndex._packageNames);
+  final _tagIds = <String, int>{};
+  final _documentTagIds = <List<int>>[];
 
   /// Adjusted score takes the overall score and transforms
   /// it linearly into the [0.4-1.0] range.
@@ -58,6 +60,14 @@ class InMemoryPackageIndex {
       final doc = _documents[i];
       _documentsByName[doc.package] = doc;
 
+      // transform tags into numberical IDs
+      final tagIds = <int>[];
+      for (final tag in doc.tags) {
+        tagIds.add(_tagIds.putIfAbsent(tag, () => _tagIds.length));
+      }
+      tagIds.sort();
+      _documentTagIds.add(tagIds);
+
       final apiDocPages = doc.apiDocPages;
       if (apiDocPages != null) {
         for (final page in apiDocPages) {
@@ -122,6 +132,14 @@ class InMemoryPackageIndex {
   }
 
   PackageSearchResult search(ServiceSearchQuery query) {
+    // prevent any work if offset is outside of the range
+    if ((query.offset ?? 0) > _documents.length) {
+      return PackageSearchResult(
+        timestamp: clock.now(),
+        totalCount: 0,
+        packageHits: [],
+      );
+    }
     return _scorePool.withScore(
       value: 1.0,
       fn: (score) {
@@ -144,8 +162,49 @@ class InMemoryPackageIndex {
     final combinedTagsPredicate =
         query.tagsPredicate.appendPredicate(query.parsedQuery.tagsPredicate);
     if (combinedTagsPredicate.isNotEmpty) {
-      packageScores.retainWhere(
-          (i, _) => combinedTagsPredicate.matches(_documents[i].tagsForLookup));
+      // The list of predicate tag entries, converted to tag IDs (or -1 if there is no indexed tag),
+      // sorted by their id.
+      final entriesToCheck = combinedTagsPredicate.entries
+          .map((e) => MapEntry(_tagIds[e.key] ?? -1, e.value))
+          .toList()
+        ..sort((a, b) => a.key.compareTo(b.key));
+
+      packageScores.retainWhere((docIndex, _) {
+        // keeping track of tag id iteration with the `nextTagIndex`
+        final tagIds = _documentTagIds[docIndex];
+        var nextTagIndex = 0;
+
+        for (final entry in entriesToCheck) {
+          if (entry.key == -1) {
+            // no tag id is present for this predicate
+            if (entry.value) {
+              // the predicate is required, no document will match it
+              return false;
+            } else {
+              // the predicate is prohibited, no document has it, always a match
+              continue;
+            }
+          }
+
+          // skipping the present tag ids until the currently matched predicate tag id
+          while (nextTagIndex < tagIds.length &&
+              tagIds[nextTagIndex] < entry.key) {
+            nextTagIndex++;
+          }
+
+          // checking presence
+          late bool present;
+          if (nextTagIndex == tagIds.length) {
+            present = false;
+          } else {
+            present = tagIds[nextTagIndex] == entry.key;
+          }
+
+          if (entry.value && !present) return false;
+          if (!entry.value && present) return false;
+        }
+        return true;
+      });
     }
 
     // filter on dependency
@@ -213,10 +272,12 @@ class InMemoryPackageIndex {
         /// it linearly into the [0.4-1.0] range, to allow better
         /// multiplication outcomes.
         packageScores.multiplyAllFromValues(_adjustedOverallScores);
-        indexedHits = _rankWithValues(packageScores);
+        indexedHits = _rankWithValues(packageScores,
+            requiredLengthThreshold: query.offset);
         break;
       case SearchOrder.text:
-        indexedHits = _rankWithValues(packageScores);
+        indexedHits = _rankWithValues(packageScores,
+            requiredLengthThreshold: query.offset);
         break;
       case SearchOrder.created:
         indexedHits = _createdOrderedHits.whereInScores(packageScores);
@@ -395,21 +456,29 @@ class InMemoryPackageIndex {
     return null;
   }
 
-  List<IndexedPackageHit> _rankWithValues(IndexedScore<String> score) {
+  List<IndexedPackageHit> _rankWithValues(
+    IndexedScore<String> score, {
+    // if the item count is fewer than this threshold, an empty list will be returned
+    int? requiredLengthThreshold,
+  }) {
     final list = <IndexedPackageHit>[];
     for (var i = 0; i < score.length; i++) {
       final value = score.getValue(i);
       if (value <= 0.0) continue;
       list.add(IndexedPackageHit(
           i, PackageHit(package: score.keys[i], score: value)));
     }
+    if ((requiredLengthThreshold ?? 0) > list.length) {
+      // There is no point to sort or even keep the results, as the search query offset ignores these anyway.
+      return [];
+    }
     list.sort((a, b) {
       final scoreCompare = -a.hit.score!.compareTo(b.hit.score!);
       if (scoreCompare != 0) return scoreCompare;
       // if two packages got the same score, order by last updated
       return _compareUpdated(_documents[a.index], _documents[b.index]);
     });
-    return list.toList();
+    return list;
   }
 
   List<IndexedPackageHit> _rankWithComparator(

app/lib/search/search_service.dart

@@ -135,9 +135,6 @@ class PackageDocument {
 
   Map<String, dynamic> toJson() => _$PackageDocumentToJson(this);
 
-  @JsonKey(includeFromJson: false, includeToJson: false)
-  late final Set<String> tagsForLookup = Set.of(tags);
-
   late final packageNameLowerCased = package.toLowerCase();
 }
 

app/lib/search/text_utils.dart

@@ -102,23 +102,23 @@ Map<String, double>? tokenize(String? originalText, {bool isSplit = false}) {
 
     // Scan for CamelCase phrases and extract Camel and Case separately.
     final wordCodeUnits = word.codeUnits;
-    final changeIndex = <int>[0];
     bool prevLower = _isLower(wordCodeUnits[0]);
-    for (int i = 1; i < word.length; i++) {
-      final lower = _isLower(wordCodeUnits[i]);
-      if (!lower && prevLower) {
-        changeIndex.add(i);
+    int prevIndex = 0;
+    for (int i = 1; i <= word.length; i++) {
+      if (i < word.length) {
+        final lower = _isLower(wordCodeUnits[i]);
+        final isChanging = !lower && prevLower;
+        prevLower = lower;
+        if (!isChanging) continue;
       }
-      prevLower = lower;
-    }
-    changeIndex.add(word.length);
-    for (int i = 1; i < changeIndex.length; i++) {
-      final token = normalizeBeforeIndexing(
-          word.substring(changeIndex[i - 1], changeIndex[i]));
+
+      final token = normalizeBeforeIndexing(word.substring(prevIndex, i));
       final weight = math.pow((token.length / word.length), 0.5).toDouble();
       if ((tokens[token] ?? 0.0) < weight) {
         tokens[token] = weight;
       }
+
+      prevIndex = i;
     }
   }
   return tokens;

app/lib/shared/versions.dart

@@ -24,10 +24,10 @@ final RegExp runtimeVersionPattern = RegExp(r'^\d{4}\.\d{2}\.\d{2}$');
 /// when the version switch happens.
 const _acceptedRuntimeVersions = <String>[
   // The current [runtimeVersion].
-  '2024.12.04',
+  '2024.12.09',
   // Fallback runtime versions.
+  '2024.12.04',
   '2024.11.21',
-  '2024.11.18',
 ];
 
 /// Sets the current runtime versions.
@@ -62,7 +62,7 @@ bool shouldGCVersion(String version) =>
 
 // keep in-sync with SDK version in .mono_repo.yml and Dockerfile
 final String runtimeSdkVersion = '3.5.0';
-final String toolStableDartSdkVersion = '3.6.0-334.4.beta';
+final String toolStableDartSdkVersion = '3.6.0';
 final String toolStableFlutterSdkVersion = '3.27.0-0.2.pre';
 
 final semanticToolStableDartSdkVersion =