Convert the completion metrics tool to use candidate suggestions

The tool was previously using the old implementation, which is now only
used by the Cider support. As a result, it wasn't providing valid timing
measurements for the new implementation. That's been fixed.

The map/reduce functionality is not yet supported, but it's also rarely
used. There's some information missing, but most of the functionality
works as expected, and I'd like to tackle the rest in a separate CL
(though I don't know exactly when I'll get to it).

Change-Id: I98150c1d5273a8ffaccf7858f9588e52e11636c8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/444183
Reviewed-by: Samuel Rawlins <[email protected]>
Commit-Queue: Brian Wilkerson <[email protected]>

Author: bwilkerson

pkg/analysis_server/lib/src/services/completion/dart/suggestion_builder.dart

@@ -7,6 +7,7 @@ import 'package:analysis_server/src/protocol_server.dart'
     hide Element, ElementKind;
 import 'package:analysis_server/src/protocol_server.dart' as protocol;
 import 'package:analysis_server/src/provisional/completion/dart/completion_dart.dart';
+import 'package:analysis_server/src/services/completion/dart/candidate_suggestion.dart';
 import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
 import 'package:analysis_server/src/services/completion/dart/dart_completion_suggestion.dart';
 import 'package:analysis_server/src/services/completion/dart/relevance_computer.dart';
@@ -1585,6 +1586,9 @@ class SuggestionBuilder {
 }
 
 abstract class SuggestionListener {
+  /// Invoked when a candidate suggestion has had its relevance score computed.
+  void builtCandidate(CandidateSuggestion candidate);
+
   /// Invoked when a suggestion has been built.
   void builtSuggestion(CompletionSuggestionBuilder suggestionBuilder);
 

pkg/analysis_server/lib/src/services/completion/dart/suggestion_collector.dart

@@ -93,6 +93,7 @@ class SuggestionCollector {
     computer.completionLocation = completionLocation;
     for (var candidate in suggestions) {
       candidate.relevanceScore = computer.computeRelevance(candidate);
+      computer.listener?.builtCandidate(candidate);
     }
     // Sort the suggestions based on both the matcher score and the relevance
     // score.

pkg/analysis_server/tool/code_completion/completion_metrics.dart

@@ -11,14 +11,13 @@ import 'dart:io' show stdout;
 import 'dart:math' as math;
 
 import 'package:_fe_analyzer_shared/src/base/syntactic_entity.dart';
-import 'package:analysis_server/src/protocol/protocol_internal.dart';
 import 'package:analysis_server/src/protocol_server.dart' as protocol;
+import 'package:analysis_server/src/services/completion/dart/candidate_suggestion.dart';
 import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
 import 'package:analysis_server/src/services/completion/dart/feature_computer.dart';
 import 'package:analysis_server/src/services/completion/dart/probability_range.dart';
 import 'package:analysis_server/src/services/completion/dart/relevance_tables.g.dart';
 import 'package:analysis_server/src/services/completion/dart/suggestion_builder.dart';
-import 'package:analysis_server/src/services/completion/dart/utilities.dart';
 import 'package:analysis_server/src/status/pages.dart';
 import 'package:analyzer/dart/analysis/analysis_context.dart';
 import 'package:analyzer/dart/analysis/results.dart';
@@ -577,7 +576,7 @@ class CompletionMetrics {
   void recordShadowedCompletion(
     String? completionLocation,
     ExpectedCompletion expectedCompletion,
-    CompletionSuggestionLite closeMatchSuggestion,
+    CandidateSuggestion closeMatchSuggestion,
   ) {
     shadowedCompletions
         .putIfAbsent(completionLocation ?? 'unknown', () => [])
@@ -935,7 +934,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
     MetricsSuggestionListener listener,
     ExpectedCompletion expectedCompletion,
     String? completionLocation,
-    List<CompletionSuggestionLite> suggestions,
+    List<CandidateSuggestion> suggestions,
     CompletionMetrics metrics,
     int elapsedMS,
   ) {
@@ -965,7 +964,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
                 .toList();
         precedingRelevanceCounts = <int, int>{};
         for (var i = 0; i < rank - 1; i++) {
-          var relevance = suggestions[i].relevance;
+          var relevance = suggestions[i].relevanceScore;
           precedingRelevanceCounts[relevance] =
               (precedingRelevanceCounts[relevance] ?? 0) + 1;
         }
@@ -1008,7 +1007,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
 
       if (options.printMissedCompletionDetails ||
           options.printShadowedCompletionDetails) {
-        CompletionSuggestionLite? closeMatchSuggestion;
+        CandidateSuggestion? closeMatchSuggestion;
         for (var suggestion in suggestions) {
           if (suggestion.completion == expectedCompletion.completion) {
             closeMatchSuggestion = suggestion;
@@ -1488,7 +1487,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
 
   int _computeCharsBeforeTop(
     ExpectedCompletion target,
-    List<CompletionSuggestionLite> suggestions, {
+    List<CandidateSuggestion> suggestions, {
     int minRank = 1,
   }) {
     var rank = placementInSuggestionList(suggestions, target).rank;
@@ -1509,34 +1508,28 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
 
   /// Computes completion suggestions for [dartRequest], and returns the
   /// suggestions, sorted by rank and then by completion text.
-  Future<List<CompletionSuggestionLite>> _computeCompletionSuggestions(
+  Future<List<CandidateSuggestion>> _computeCompletionSuggestions(
     MetricsSuggestionListener listener,
     OperationPerformanceImpl performance,
     DartCompletionRequest dartRequest,
     NotImportedSuggestions notImportedSuggestions,
   ) async {
     var budget = CompletionBudget(Duration(seconds: 30));
-    var suggestions = await DartCompletionManager(
+    var collector = await DartCompletionManager(
       budget: budget,
       listener: listener,
       notImportedSuggestions: notImportedSuggestions,
-    ).computeSuggestions(
-      dartRequest,
-      performance,
+    ).computeFinalizedCandidateSuggestions(
+      request: dartRequest,
+      performance: performance,
       maxSuggestions: -1,
-      useFilter: true,
     );
-
-    // Note that some routes sort suggestions before responding differently.
-    // The Cider and legacy handlers use [fuzzyFilterSort], which does not match
-    // [completionComparator].
-    suggestions.sort(completionComparator);
-    return suggestions.map(CompletionSuggestionLite.fromBuilder).toList();
+    return collector.suggestions;
   }
 
-  List<CompletionSuggestionLite> _filterSuggestions(
+  List<CandidateSuggestion> _filterSuggestions(
     String prefix,
-    List<CompletionSuggestionLite> suggestions,
+    List<CandidateSuggestion> suggestions,
   ) {
     // TODO(brianwilkerson): Replace this with a more realistic filtering
     //  algorithm.
@@ -1550,7 +1543,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
       var suggestion = data.suggestion;
       return [
         rank.toString(),
-        suggestion.relevance.toString(),
+        suggestion.relevanceScore.toString(),
         suggestion.completion,
         suggestion.kind.toString(),
       ];
@@ -1635,7 +1628,7 @@ class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
   ///
   /// If [expectedCompletion] is not found, `Place.none()` is returned.
   static Place placementInSuggestionList(
-    List<CompletionSuggestionLite> suggestions,
+    List<CandidateSuggestion> suggestions,
     ExpectedCompletion expectedCompletion,
   ) {
     for (var i = 0; i < suggestions.length; i++) {
@@ -1824,79 +1817,6 @@ class CompletionResult {
   }
 }
 
-/// Enough data from [CompletionSuggestionBuilder] to compute metrics.
-///
-/// It excludes expensive parts that are not necessary, such as element
-/// properties.
-class CompletionSuggestionLite {
-  final String completion;
-  final protocol.ElementKind? elementKind;
-  final int relevance;
-  final protocol.CompletionSuggestionKind kind;
-
-  CompletionSuggestionLite({
-    required this.completion,
-    required this.elementKind,
-    required this.relevance,
-    required this.kind,
-  });
-
-  factory CompletionSuggestionLite.fromBuilder(
-    CompletionSuggestionBuilder builder,
-  ) {
-    return CompletionSuggestionLite(
-      completion: builder.completion,
-      elementKind: builder.elementKind,
-      relevance: builder.relevance,
-      kind: builder.kind,
-    );
-  }
-
-  factory CompletionSuggestionLite.fromJson(Map<String, Object?> map) {
-    var elementKindStr = map['elementKind'] as String?;
-
-    return CompletionSuggestionLite(
-      completion: map['completion'] as String,
-      elementKind:
-          elementKindStr != null
-              ? protocol.ElementKind.fromJson(
-                ResponseDecoder(null),
-                '',
-                elementKindStr,
-              )
-              : null,
-      relevance: map['relevance'] as int,
-      kind: protocol.CompletionSuggestionKind.fromJson(
-        ResponseDecoder(null),
-        '',
-        map['kind'] as String,
-      ),
-    );
-  }
-
-  @override
-  int get hashCode => Object.hash(completion, kind);
-
-  @override
-  bool operator ==(Object other) {
-    return other is CompletionSuggestionLite &&
-        other.completion == completion &&
-        other.kind == kind &&
-        other.elementKind == elementKind &&
-        other.relevance == relevance;
-  }
-
-  /// Return a map used to represent this suggestion data in a JSON structure.
-  Map<String, Object?> toJson() {
-    return {
-      'completion': completion,
-      if (elementKind case var elementKind?) 'elementKind': elementKind.name,
-      'relevance': relevance,
-      'kind': kind.name,
-    };
-  }
-}
-
 /// The data to be printed on a single line in the table of mrr values per
 /// completion location.
 class LocationTableLine {
@@ -1931,7 +1851,7 @@ class MetricsSuggestionListener implements SuggestionListener {
     0.0,
   ];
 
-  Map<CompletionSuggestionLite, List<double>> featureMap = Map.identity();
+  Map<CandidateSuggestion, List<double>> featureMap = Map.identity();
 
   List<double> cachedFeatures = noFeatures;
 
@@ -1940,12 +1860,16 @@ class MetricsSuggestionListener implements SuggestionListener {
   String? missingCompletionLocationTable;
 
   @override
-  void builtSuggestion(CompletionSuggestionBuilder builder) {
-    var suggestion = CompletionSuggestionLite.fromBuilder(builder);
-    featureMap[suggestion] = cachedFeatures;
+  void builtCandidate(CandidateSuggestion candidate) {
+    featureMap[candidate] = cachedFeatures;
     cachedFeatures = noFeatures;
   }
 
+  @override
+  void builtSuggestion(CompletionSuggestionBuilder builder) {
+    throw UnsupportedError('Unexpected use of SuggestionBuilder');
+  }
+
   @override
   void computedFeatures({
     double contextType = 0.0,
@@ -2020,15 +1944,15 @@ class RelevanceTables {
 class ShadowedCompletion {
   final ExpectedCompletion expectedCompletion;
 
-  final CompletionSuggestionLite closeMatchSuggestion;
+  final CandidateSuggestion closeMatchSuggestion;
 
   ShadowedCompletion(this.expectedCompletion, this.closeMatchSuggestion);
 }
 
 /// The information being remembered about an individual suggestion.
 class SuggestionData {
   /// The suggestion that was produced.
-  CompletionSuggestionLite suggestion;
+  CandidateSuggestion suggestion;
 
   /// The values of the features used to compute the suggestion.
   List<double> features;
@@ -2037,17 +1961,17 @@ class SuggestionData {
 
   /// Return an instance extracted from the decoded JSON [map].
   factory SuggestionData.fromJson(Map<String, dynamic> map) {
-    return SuggestionData(
-      CompletionSuggestionLite.fromJson(
-        map['suggestion'] as Map<String, Object?>,
-      ),
-      (map['features'] as List<dynamic>).cast<double>(),
-    );
+    throw UnimplementedError();
+    // return SuggestionData(
+    //   CandidateSuggestion.fromJson(map['suggestion'] as Map<String, Object?>),
+    //   (map['features'] as List<dynamic>).cast<double>(),
+    // );
   }
 
   /// Return a map used to represent this suggestion data in a JSON structure.
   Map<String, dynamic> toJson() {
-    return {'suggestion': suggestion.toJson(), 'features': features};
+    throw UnimplementedError();
+    // return {'suggestion': suggestion.toJson(), 'features': features};
   }
 }
 
@@ -2094,6 +2018,14 @@ extension on CompletionGroup {
   }
 }
 
+extension on CandidateSuggestion {
+  /// The kind of element being suggested.
+  protocol.CompletionSuggestionKind? get kind {
+    // TODO(brianwilkerson): Implement this.
+    return null;
+  }
+}
+
 extension on num {
   String asPercentage([int fractionDigits = 1]) =>
       '${(this * 100).toStringAsFixed(fractionDigits)}%'.padLeft(

pkg/analysis_server/tool/code_completion/visitors.dart

@@ -4,6 +4,7 @@
 
 import 'package:analysis_server/src/protocol/protocol_internal.dart';
 import 'package:analysis_server/src/protocol_server.dart' as protocol;
+import 'package:analysis_server/src/services/completion/dart/candidate_suggestion.dart';
 import 'package:analyzer/dart/analysis/results.dart';
 import 'package:analyzer/dart/ast/ast.dart';
 import 'package:analyzer/dart/ast/syntactic_entity.dart';
@@ -13,8 +14,6 @@ import 'package:analyzer/dart/element/element.dart' as element;
 import 'package:analyzer/dart/element/type.dart';
 import 'package:analyzer/source/line_info.dart';
 
-import 'completion_metrics.dart';
-
 class ExpectedCompletion {
   final String _filePath;
 
@@ -105,19 +104,11 @@ class ExpectedCompletion {
 
   SyntacticEntity get syntacticEntity => _entity;
 
-  bool matches(CompletionSuggestionLite completionSuggestion) {
-    if (completionSuggestion.completion == completion) {
-      if (kind != null && completionSuggestion.kind != kind) {
-        return false;
-      }
-      if (elementKind != null &&
-          completionSuggestion.elementKind != null &&
-          completionSuggestion.elementKind != elementKind) {
-        return false;
-      }
-      return true;
-    }
-    return false;
+  bool matches(CandidateSuggestion completionSuggestion) {
+    // This method used to use the `_kind` and `_elementKind` to further
+    // validate that the right suggestion had been found, but that was removed
+    // because candidate suggestions don't have that information.
+    return completionSuggestion.completion == completion;
   }
 
   /// Return a map used to represent this expected completion in a JSON structure.