From 74531b0ccb48a19953468229e1868e8a1763c060 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Thu, 21 Nov 2024 20:21:26 +0100 Subject: [PATCH 1/3] Integeration test to use pub get with the exported API bucket (+ minimal content rewrite). --- .../fake/server/fake_server_entrypoint.dart | 2 + app/lib/fake/server/fake_storage_server.dart | 11 +- pkg/pub_integration/pubspec.yaml | 1 + .../test/exported_bucket_test.dart | 138 ++++++++++++++++++ 4 files changed, 149 insertions(+), 3 deletions(-) create mode 100644 pkg/pub_integration/test/exported_bucket_test.dart diff --git a/app/lib/fake/server/fake_server_entrypoint.dart b/app/lib/fake/server/fake_server_entrypoint.dart index ccec27de98..fbe18e0694 100644 --- a/app/lib/fake/server/fake_server_entrypoint.dart +++ b/app/lib/fake/server/fake_server_entrypoint.dart @@ -14,6 +14,7 @@ import 'package:pub_dev/fake/server/fake_search_service.dart'; import 'package:pub_dev/fake/server/fake_storage_server.dart'; import 'package:pub_dev/fake/server/local_server_state.dart'; import 'package:pub_dev/frontend/static_files.dart'; +import 'package:pub_dev/package/api_export/api_exporter.dart'; import 'package:pub_dev/shared/configuration.dart'; import 'package:pub_dev/shared/handlers.dart'; import 'package:pub_dev/shared/logging.dart'; @@ -177,6 +178,7 @@ Future _testProfile(shelf.Request rq) async { source: ImportSource.autoGenerated(), ); final analysis = (map['analysis'] as String?) ?? 'fake'; + await apiExporter!.synchronizeExportedApi(); await processTaskFakeLocalOrWorker(analysis); return shelf.Response.ok('{}'); } diff --git a/app/lib/fake/server/fake_storage_server.dart b/app/lib/fake/server/fake_storage_server.dart index 541adc48f1..b30aa519cf 100644 --- a/app/lib/fake/server/fake_storage_server.dart +++ b/app/lib/fake/server/fake_storage_server.dart @@ -74,9 +74,14 @@ class FakeStorageServer { if (exists == null) { return Response.notFound('404 Not Found'); } - final contentType = lookupMimeType(objectName); - return Response.ok(bucket.read(objectName), - headers: {if (contentType != null) 'Content-Type': contentType}); + final contentType = + exists.metadata.contentType ?? lookupMimeType(objectName); + final contentEncoding = exists.metadata.contentEncoding; + + return Response.ok(bucket.read(objectName), headers: { + if (contentType != null) 'Content-Type': contentType, + if (contentEncoding != null) 'Content-Encoding': contentEncoding, + }); } else if (request.method == 'POST') { _logger.info('Uploading: ${request.requestedUri.path}'); final contentHeader = _parse(request.headers['content-type']); diff --git a/pkg/pub_integration/pubspec.yaml b/pkg/pub_integration/pubspec.yaml index 696f0765f3..48ba0a3892 100644 --- a/pkg/pub_integration/pubspec.yaml +++ b/pkg/pub_integration/pubspec.yaml @@ -17,4 +17,5 @@ dependencies: dev_dependencies: coverage: any # test already depends on it + shelf: ^1.4.0 test: ^1.16.5 diff --git a/pkg/pub_integration/test/exported_bucket_test.dart b/pkg/pub_integration/test/exported_bucket_test.dart new file mode 100644 index 0000000000..915e50bb9a --- /dev/null +++ b/pkg/pub_integration/test/exported_bucket_test.dart @@ -0,0 +1,138 @@ +// Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:convert'; + +import 'dart:io'; + +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as p; +import 'package:pub_integration/src/fake_test_context_provider.dart'; +import 'package:shelf/shelf.dart' as shelf; +import 'package:shelf/shelf_io.dart' as shelf_io; +import 'package:test/test.dart'; + +void main() { + group( + 'exported bucket works with pub get', + () { + late final TestContextProvider fakeTestScenario; + final httpClient = http.Client(); + late final Directory temp; + late final HttpServer proxy; + + setUpAll(() async { + fakeTestScenario = await TestContextProvider.start(); + temp = await Directory.systemTemp.createTemp('exported-bucket'); + + final pubUri = Uri.parse(fakeTestScenario.pubHostedUrl); + final storageUri = pubUri.replace( + port: pubUri.port + 1, path: '/fake-exported-apis/latest'); + + // read-only proxy server with minimal content rewrite + proxy = await shelf_io.serve((rq) async { + try { + // the proxy url with the appropriate path prefix + final proxyUri = storageUri.replace(pathSegments: [ + ...storageUri.pathSegments, + ...rq.requestedUri.pathSegments + ]); + + // archive files are served from the storage bucket directly + if (rq.requestedUri.path.endsWith('tar.gz')) { + return shelf.Response.seeOther(proxyUri); + } + + // other requests are proxied + final rs = await http.get(proxyUri); + + Map copyHeaders(List keys) { + return Map.fromEntries( + rs.headers.entries.where((e) => keys.contains(e.key))); + } + + // package listing requests need content rewrite (+ keeping the condition broad for future JSON APIs) + if (rs.statusCode == 200 && + rq.requestedUri.toString().contains('/api/packages/')) { + return shelf.Response( + rs.statusCode, + body: rs.body.replaceAll( + pubUri.toString(), 'http://localhost:${proxy.port}'), + headers: copyHeaders(['content-type']), + ); + } + + // otherwise return the result as-is + return shelf.Response( + rs.statusCode, + body: rs.bodyBytes, + headers: copyHeaders(['content-type', 'content-encoding']), + ); + } catch (e, st) { + print(e); + print(st); + return shelf.Response.internalServerError(); + } + }, InternetAddress.loopbackIPv4, 0); + }); + + tearDownAll(() async { + await proxy.close(); + await temp.delete(recursive: true); + await fakeTestScenario.close(); + httpClient.close(); + }); + + test('bulk tests', () async { + final origin = fakeTestScenario.pubHostedUrl; + // init server data + await httpClient.post( + Uri.parse('$origin/fake-test-profile'), + body: json.encode( + { + 'testProfile': { + 'defaultUser': 'admin@pub.dev', + 'packages': [ + { + 'name': 'exported_api_pkg', + 'versions': [ + {'version': '1.0.0'}, + {'version': '1.0.2-dev+2'}, + ], + }, + ], + }, + }, + ), + ); + + // create a local project with dependency on exported_api_pkg:1.0.2-dev+2 + final pubspec = File(p.join(temp.path, 'pubspec.yaml')); + await pubspec.writeAsString(json.encode({ + 'name': 'test_pkg', + 'environment': { + 'sdk': '>=3.0.0 <4.0.0', + }, + 'dependencies': { + 'exported_api_pkg': '^1.0.1', + }, + })); + + // run pub get and verify its success + final pr = await Process.run( + 'dart', + ['pub', 'get', '-v'], + environment: {'PUB_HOSTED_URL': 'http://localhost:${proxy.port}'}, + workingDirectory: temp.path, + ); + expect(pr.exitCode, 0, reason: [pr.stdout, pr.stderr].join('\n')); + + final resolvedFile = + File(p.join(temp.path, '.dart_tool', 'package_config.json')); + expect(resolvedFile.existsSync(), true); + }); + }, + timeout: Timeout.factor(testTimeoutFactor), + ); +} From 54bb19a0c0f1d3c7262f0769222ba32d6a4c75c2 Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 27 Nov 2024 14:55:34 +0100 Subject: [PATCH 2/3] remove storage bucket redirect. --- pkg/pub_integration/test/exported_bucket_test.dart | 5 ----- 1 file changed, 5 deletions(-) diff --git a/pkg/pub_integration/test/exported_bucket_test.dart b/pkg/pub_integration/test/exported_bucket_test.dart index 915e50bb9a..097727a544 100644 --- a/pkg/pub_integration/test/exported_bucket_test.dart +++ b/pkg/pub_integration/test/exported_bucket_test.dart @@ -39,11 +39,6 @@ void main() { ...rq.requestedUri.pathSegments ]); - // archive files are served from the storage bucket directly - if (rq.requestedUri.path.endsWith('tar.gz')) { - return shelf.Response.seeOther(proxyUri); - } - // other requests are proxied final rs = await http.get(proxyUri); From ff483b37cb55e6d4cde17cffa80ea652f09defaf Mon Sep 17 00:00:00 2001 From: Istvan Soos Date: Wed, 27 Nov 2024 15:17:33 +0100 Subject: [PATCH 3/3] fix test --- app/test/admin/moderate_package_version_test.dart | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/app/test/admin/moderate_package_version_test.dart b/app/test/admin/moderate_package_version_test.dart index b42f4ca0cc..a5a1de7c1c 100644 --- a/app/test/admin/moderate_package_version_test.dart +++ b/app/test/admin/moderate_package_version_test.dart @@ -3,7 +3,6 @@ // BSD-style license that can be found in the LICENSE file. import 'dart:convert'; -import 'dart:io'; import 'dart:typed_data'; import 'package:_pub_shared/data/account_api.dart'; @@ -242,8 +241,8 @@ void main() { '/$prefix/api/packages/oxygen'; final rs = await http.get(Uri.parse(url)); expect(rs.statusCode, 200); - final data = json.decode(utf8.decode(gzip.decode(rs.bodyBytes))) - as Map; + final data = + json.decode(utf8.decode(rs.bodyBytes)) as Map; final versions = (data['versions'] as List) .map((i) => (i as Map)['version']) .toSet();