Simpler SDK index content loading: library descriptions are now part of the index.json file. (#8679)

Author: isoos

app/lib/search/backend.dart

@@ -13,7 +13,6 @@ import 'package:clock/clock.dart';
 import 'package:collection/collection.dart';
 import 'package:gcloud/service_scope.dart' as ss;
 import 'package:gcloud/storage.dart';
-import 'package:html/parser.dart' as html_parser;
 import 'package:logging/logging.dart';
 import 'package:meta/meta.dart';
 // ignore: implementation_imports
@@ -441,47 +440,6 @@ class SearchBackend {
     return rs.body;
   }
 
-  /// Downloads the remote SDK page and tries to extract the first paragraph of the content.
-  Future<String?> _fetchSdkLibraryDescription({
-    required Uri baseUri,
-    required String relativePath,
-  }) async {
-    try {
-      final content = await fetchSdkIndexContentAsString(
-          baseUri: baseUri, relativePath: relativePath);
-      final parsed = html_parser.parse(content);
-      final descr = parsed.body
-          ?.querySelector('section.desc.markdown')
-          ?.querySelector('p')
-          ?.text
-          .trim();
-      return descr == null ? null : compactDescription(descr);
-    } catch (e) {
-      _logger.info(
-          'Unable to fetch SDK library description $baseUri $relativePath', e);
-      return null;
-    }
-  }
-
-  /// Downloads the remote SDK page and tries to extract the first paragraph of the content
-  /// for each library in [libraryRelativeUrls].
-  Future<Map<String, String>> fetchSdkLibraryDescriptions({
-    required Uri baseUri,
-    required Map<String, String> libraryRelativeUrls,
-  }) async {
-    final values = <String, String>{};
-    for (final library in libraryRelativeUrls.keys) {
-      final descr = await _fetchSdkLibraryDescription(
-        baseUri: baseUri,
-        relativePath: libraryRelativeUrls[library]!,
-      );
-      if (descr != null) {
-        values[library] = descr;
-      }
-    }
-    return values;
-  }
-
   Future<List<PackageDocument>?> fetchSnapshotDocuments() async {
     try {
       final map = await _snapshotStorage.getContentAsJsonMap();

app/lib/search/dart_sdk_mem_index.dart

@@ -38,19 +38,11 @@ SdkMemIndex? get dartSdkMemIndex =>
 Future<SdkMemIndex?> createDartSdkMemIndex() async {
   try {
     final index = await SdkMemIndex.dart();
-    final content = DartdocIndex.parseJsonText(
-      await searchBackend.fetchSdkIndexContentAsString(
-        baseUri: index.baseUri,
-        relativePath: 'index.json',
-      ),
-    );
-    await index.addDartdocIndex(content);
-    index.addLibraryDescriptions(
-      await searchBackend.fetchSdkLibraryDescriptions(
-        baseUri: index.baseUri,
-        libraryRelativeUrls: content.libraryRelativeUrls,
-      ),
+    final content = await searchBackend.fetchSdkIndexContentAsString(
+      baseUri: index.baseUri,
+      relativePath: 'index.json',
     );
+    await index.addDartdocIndex(DartdocIndex.parseJsonText(content));
     index.updateWeights(
       libraryWeights: dartSdkLibraryWeights,
       apiPageDirWeights: {},

app/lib/search/flutter_sdk_mem_index.dart

@@ -55,19 +55,12 @@ SdkMemIndex? get flutterSdkMemIndex =>
 Future<SdkMemIndex?> createFlutterSdkMemIndex() async {
   try {
     final index = SdkMemIndex.flutter();
-    final content = DartdocIndex.parseJsonText(
-      await searchBackend.fetchSdkIndexContentAsString(
-        baseUri: index.baseUri,
-        relativePath: 'index.json',
-      ),
-    );
-    await index.addDartdocIndex(content, allowedLibraries: _allowedLibraries);
-    index.addLibraryDescriptions(
-      await searchBackend.fetchSdkLibraryDescriptions(
-        baseUri: index.baseUri,
-        libraryRelativeUrls: content.libraryRelativeUrls,
-      ),
+    final content = await searchBackend.fetchSdkIndexContentAsString(
+      baseUri: index.baseUri,
+      relativePath: 'index.json',
     );
+    await index.addDartdocIndex(DartdocIndex.parseJsonText(content),
+        allowedLibraries: _allowedLibraries);
     index.updateWeights(
       libraryWeights: {},
       apiPageDirWeights: _flutterApiPageDirWeights,

app/lib/search/sdk_mem_index.dart

@@ -5,6 +5,7 @@
 import 'dart:math';
 
 import 'package:_pub_shared/utils/http.dart';
+import 'package:meta/meta.dart';
 // ignore: implementation_imports
 import 'package:pana/src/dartdoc/dartdoc_index.dart';
 import 'package:path/path.dart' as p;
@@ -94,6 +95,11 @@ class SdkMemIndex {
       }
       if (f.isLibrary) {
         _baseUriPerLibrary[library] = _baseUri.resolve(f.href!).toString();
+
+        final desc = f.desc?.replaceAll(RegExp(r'\s+'), ' ').trim();
+        if (desc != null && desc.isNotEmpty) {
+          _descriptionPerLibrary[library] = desc;
+        }
       }
 
       final text = f.qualifiedName?.replaceAll('.', ' ').replaceAll(':', ' ');
@@ -107,11 +113,6 @@ class SdkMemIndex {
     }
   }
 
-  /// Adds the descriptions for each library.
-  void addLibraryDescriptions(Map<String, String> descriptions) {
-    _descriptionPerLibrary.addAll(descriptions);
-  }
-
   /// Updates the non-default weight for libraries.
   void updateWeights({
     required Map<String, double> libraryWeights,
@@ -183,6 +184,10 @@ class SdkMemIndex {
             ))
         .toList();
   }
+
+  @visibleForTesting
+  String? getLibraryDescription(String library) =>
+      _descriptionPerLibrary[library];
 }
 
 class _Hit {

app/test/search/backend_test.dart

@@ -14,16 +14,15 @@ void main() {
   group('search backend', () {
     testWithProfile('fetch SDK library description', fn: () async {
       final index = await SdkMemIndex.dart();
-      final descr = await searchBackend.fetchSdkLibraryDescriptions(
+      final content = await searchBackend.fetchSdkIndexContentAsString(
         baseUri: index.baseUri,
-        libraryRelativeUrls: {
-          'dart:async': 'dart-async/index.html',
-        },
+        relativePath: 'index.json',
+      );
+      await index.addDartdocIndex(DartdocIndex.parseJsonText(content));
+      expect(
+        index.getLibraryDescription('dart:async'),
+        'Support for asynchronous programming, with classes such as Future and Stream.',
       );
-      expect(descr, {
-        'dart:async':
-            'Support for asynchronous programming, with classes such as Future and Stream.',
-      });
     });
 
     testWithProfile('updates snapshot storage', fn: () async {

app/test/search/dartdoc_index_parsing_test.dart

@@ -97,6 +97,7 @@ void main() {
         {
           'sdk': 'flutter',
           'library': 'widgets',
+          'description': 'The Flutter widgets framework.',
           'url': contains('https://api.flutter.dev/flutter/widgets'),
           'score': closeTo(0.98, 0.01),
           'apiPages': [

app/test/search/sdk_mem_index_test.dart

@@ -32,6 +32,7 @@ void main() {
           'kind': 8,
           'overriddenDepth': 0,
           'packageName': 'Dart',
+          'desc': 'async description',
         },
         {
           'name': 'AsyncError',
@@ -60,6 +61,15 @@ void main() {
           'packageName': 'Dart',
           'enclosedBy': {'name': 'AsyncError', 'kind': 3},
         },
+        {
+          'name': 'dart:html',
+          'qualifiedName': 'dart:html',
+          'href': 'dart-html',
+          'kind': 8,
+          'overriddenDepth': 0,
+          'packageName': 'HTML',
+          'desc': 'html description',
+        },
         {
           'name': 'Window',
           'qualifiedName': 'dart:html.Window',
@@ -79,8 +89,6 @@ void main() {
           'enclosedBy': {'name': 'dart:html.FakeIcons', 'kind': 3},
         },
       ]));
-      index.addLibraryDescriptions({'dart:async': 'async description'});
-      index.addLibraryDescriptions({'dart:html': 'html description'});
     });
 
     test('AsyncError', () async {