diff --git a/Sources/App/Commands/Reconcile.swift b/Sources/App/Commands/Reconcile.swift index 10500fc9e..072bfc15a 100644 --- a/Sources/App/Commands/Reconcile.swift +++ b/Sources/App/Commands/Reconcile.swift @@ -67,12 +67,13 @@ func reconcile(client: Client, database: Database) async throws { func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] { - async let sourcePackageList = try Current.fetchPackageList(client) - async let sourcePackageDenyList = try Current.fetchPackageDenyList(client) + @Dependency(\.packageListRepository) var packageListRepository + + let sourcePackageList = try await packageListRepository.fetchPackageList(client: client) + let sourcePackageDenyList = try await packageListRepository.fetchPackageDenyList(client: client) async let currentList = try fetchCurrentPackageList(database) - let packageList = processPackageDenyList(packageList: try await sourcePackageList, - denyList: try await sourcePackageDenyList) + let packageList = processPackageDenyList(packageList: sourcePackageList, denyList: sourcePackageDenyList) try await reconcileLists(db: database, source: packageList, @@ -82,33 +83,6 @@ func reconcileMainPackageList(client: Client, database: Database) async throws - } -func liveFetchPackageList(_ client: Client) async throws -> [URL] { - try await client - .get(Constants.packageListUri) - .content - .decode([String].self, using: JSONDecoder()) - .compactMap(URL.init(string:)) -} - - -func liveFetchPackageDenyList(_ client: Client) async throws -> [URL] { - struct DeniedPackage: Decodable { - var packageUrl: String - - enum CodingKeys: String, CodingKey { - case packageUrl = "package_url" - } - } - - return try await client - .get(Constants.packageDenyListUri) - .content - .decode([DeniedPackage].self, using: JSONDecoder()) - .map(\.packageUrl) - .compactMap(URL.init(string:)) -} - - func fetchCurrentPackageList(_ db: Database) async throws -> [URL] { try await Package.query(on: db) .field(Package.self, \.$url) diff --git a/Sources/App/Core/AppEnvironment.swift b/Sources/App/Core/AppEnvironment.swift index 6889c169e..54efd59c1 100644 --- a/Sources/App/Core/AppEnvironment.swift +++ b/Sources/App/Core/AppEnvironment.swift @@ -43,8 +43,6 @@ struct AppEnvironment: Sendable { var environment: @Sendable () -> Environment var fetchDocumentation: @Sendable (_ client: Client, _ url: URI) async throws -> ClientResponse var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus - var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] - var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL] var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License? var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme? @@ -165,8 +163,6 @@ extension AppEnvironment { environment: { (try? Environment.detect()) ?? .development }, fetchDocumentation: { client, url in try await client.get(url) }, fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) }, - fetchPackageList: { client in try await liveFetchPackageList(client) }, - fetchPackageDenyList: { client in try await liveFetchPackageDenyList(client) }, fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) }, fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) }, fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) }, diff --git a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift index 8b68a3f4b..c92548545 100644 --- a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift +++ b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift @@ -19,15 +19,39 @@ import Vapor @DependencyClient struct PackageListRepositoryClient { + var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL] + var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL] var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL] var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO] - // TODO: move other package list dependencies here } extension PackageListRepositoryClient: DependencyKey { static var liveValue: PackageListRepositoryClient { .init( + fetchPackageList: { client in + try await client + .get(Constants.packageListUri) + .content + .decode([String].self, using: JSONDecoder()) + .compactMap(URL.init(string:)) + }, + fetchPackageDenyList: { client in + struct DeniedPackage: Decodable { + var packageUrl: String + + enum CodingKeys: String, CodingKey { + case packageUrl = "package_url" + } + } + + return try await client + .get(Constants.packageDenyListUri) + .content + .decode([DeniedPackage].self, using: JSONDecoder()) + .map(\.packageUrl) + .compactMap(URL.init(string:)) + }, fetchCustomCollection: { client, url in try await client .get(URI(string: url.absoluteString)) diff --git a/Tests/AppTests/MastodonTests.swift b/Tests/AppTests/MastodonTests.swift index 1f5b0ccf7..a161885e0 100644 --- a/Tests/AppTests/MastodonTests.swift +++ b/Tests/AppTests/MastodonTests.swift @@ -34,7 +34,6 @@ final class MastodonTests: AppTestCase { let url = "https://github.com/foo/bar" Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in [url.url] } Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in .t0 } @@ -58,6 +57,8 @@ final class MastodonTests: AppTestCase { try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } } operation: { diff --git a/Tests/AppTests/MetricsTests.swift b/Tests/AppTests/MetricsTests.swift index 49fe99ac7..3addc215b 100644 --- a/Tests/AppTests/MetricsTests.swift +++ b/Tests/AppTests/MetricsTests.swift @@ -100,11 +100,10 @@ class MetricsTests: AppTestCase { func test_reconcileDurationSeconds() async throws { try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "2", "3"].asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } } operation: { - // setup - Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs } - // MUT try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/Mocks/AppEnvironment+mock.swift b/Tests/AppTests/Mocks/AppEnvironment+mock.swift index 46bdd71b0..e36ad2490 100644 --- a/Tests/AppTests/Mocks/AppEnvironment+mock.swift +++ b/Tests/AppTests/Mocks/AppEnvironment+mock.swift @@ -42,13 +42,6 @@ extension AppEnvironment { environment: { .development }, fetchDocumentation: { _, _ in .init(status: .ok) }, fetchHTTPStatusCode: { _ in .ok }, - fetchPackageList: { _ in - ["https://github.com/finestructure/Gala", - "https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server"].asURLs - }, - fetchPackageDenyList: { _ in - ["https://github.com/daveverwer/LeftPad"].asURLs - }, fetchLicense: { _, _, _ in .init(htmlUrl: "https://github.com/foo/bar/blob/main/LICENSE") }, fetchMetadata: { _, _, _ in .mock }, fetchReadme: { _, _, _ in .init(html: "readme html", htmlUrl: "readme html url", imagesToCache: []) }, diff --git a/Tests/AppTests/PackageTests.swift b/Tests/AppTests/PackageTests.swift index 518a28cfa..36269265b 100644 --- a/Tests/AppTests/PackageTests.swift +++ b/Tests/AppTests/PackageTests.swift @@ -295,14 +295,15 @@ final class PackageTests: AppTestCase { } func test_isNew() async throws { + let url = "1".asGithubUrl try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } } operation: { // setup - let url = "1".asGithubUrl Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in [url.url] } Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in Date(timeIntervalSince1970: 0) } Current.git.getTags = { @Sendable _ in [] } diff --git a/Tests/AppTests/PipelineTests.swift b/Tests/AppTests/PipelineTests.swift index c66c3884e..0895f80d0 100644 --- a/Tests/AppTests/PipelineTests.swift +++ b/Tests/AppTests/PipelineTests.swift @@ -158,17 +158,17 @@ class PipelineTests: AppTestCase { } func test_processing_pipeline() async throws { + let urls = ["1", "2", "3"].asGithubUrls try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } } operation: { // Test pipeline pick-up end to end // setup - let urls = ["1", "2", "3"].asGithubUrls Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) } - Current.fetchPackageList = { _ in urls.asURLs } - Current.git.commitCount = { @Sendable _ in 12 } Current.git.firstCommitDate = { @Sendable _ in .t0 } Current.git.lastCommitDate = { @Sendable _ in .t1 } @@ -224,76 +224,78 @@ class PipelineTests: AppTestCase { XCTAssertEqual(packages.map(\.isNew), [false, false, false]) } - // Now we've got a new package and a deletion - Current.fetchPackageList = { _ in ["1", "3", "4"].asGithubUrls.asURLs } - - // MUT - reconcile again - try await reconcile(client: app.client, database: app.db) - - do { // validate - only new package moves to .reconciliation stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) - } - - // MUT - ingest again - try await ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) - } - - // MUT - analyze again - let lastAnalysis = Date.now - try await Analyze.analyze(client: app.client, - database: app.db, - mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) - XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) - } - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + // Now we've got a new package and a deletion + $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs } } operation: { - // MUT - ingest yet again + // MUT - reconcile again + try await reconcile(client: app.client, database: app.db) + + do { // validate - only new package moves to .reconciliation stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation]) + XCTAssertEqual(packages.map(\.isNew), [false, false, true]) + } + + // MUT - ingest again try await ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - now all three packages should have been updated + + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion]) + XCTAssertEqual(packages.map(\.isNew), [false, false, true]) } - - // MUT - re-run analysis to complete the sequence + + // MUT - analyze again + let lastAnalysis = Date.now try await Analyze.analyze(client: app.client, database: app.db, mode: .limit(10)) - + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) + XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true]) XCTAssertEqual(packages.map(\.isNew), [false, false, false]) } - - // at this point we've ensured that retriggering ingestion after the deadtime will - // refresh analysis as expected + + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + // MUT - ingest yet again + try await ingest(client: app.client, database: app.db, mode: .limit(10)) + + do { // validate - now all three packages should have been updated + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) + XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) + XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + } + + // MUT - re-run analysis to complete the sequence + try await Analyze.analyze(client: app.client, + database: app.db, + mode: .limit(10)) + + do { // validate - only new package moves to .ingestion stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) + XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) + XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) + XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + } + + // at this point we've ensured that retriggering ingestion after the deadtime will + // refresh analysis as expected + } } } } diff --git a/Tests/AppTests/ReconcilerTests.swift b/Tests/AppTests/ReconcilerTests.swift index 0eb97a18d..1a7de3bb5 100644 --- a/Tests/AppTests/ReconcilerTests.swift +++ b/Tests/AppTests/ReconcilerTests.swift @@ -36,12 +36,14 @@ class ReconcilerTests: AppTestCase { } func test_reconcileMainPackageList() async throws { - // setup let urls = ["1", "2", "3"] - Current.fetchPackageList = { _ in urls.asURLs } - - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -63,10 +65,14 @@ class ReconcilerTests: AppTestCase { // new package list drops 2, 3, adds 4, 5 let urls = ["1", "4", "5"] - Current.fetchPackageList = { _ in urls.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -81,14 +87,17 @@ class ReconcilerTests: AppTestCase { // New list adds two new packages 4, 5 let packageList = ["1", "2", "3", "4", "5"] - Current.fetchPackageList = { _ in packageList.asURLs } // Deny list denies 2 and 4 (one existing and one new) let packageDenyList = ["2", "4"] - Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in packageDenyList.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -103,14 +112,17 @@ class ReconcilerTests: AppTestCase { // New list adds no new packages let packageList = ["https://example.com/one/one", "https://example.com/two/two"] - Current.fetchPackageList = { _ in packageList.asURLs } // Deny list denies one/one, but with incorrect casing. let packageDenyList = ["https://example.com/OnE/oNe"] - Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } - // MUT - _ = try await reconcileMainPackageList(client: app.client, database: app.db) + try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in packageList.asURLs } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in packageDenyList.asURLs } + } operation: { + // MUT + _ = try await reconcileMainPackageList(client: app.client, database: app.db) + } // validate let packages = try await Package.query(on: app.db).all() @@ -211,6 +223,8 @@ class ReconcilerTests: AppTestCase { struct TestError: Error { var message: String } try await withDependencies { + $0.packageListRepository.fetchPackageList = { @Sendable _ in fullPackageList } + $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] } $0.packageListRepository.fetchCustomCollection = { @Sendable _, url in if url == "collectionURL" { return [URL("2")] @@ -222,9 +236,6 @@ class ReconcilerTests: AppTestCase { [.init(name: "List", url: "collectionURL")] } } operation: { - // setup - Current.fetchPackageList = { _ in fullPackageList } - // MUT _ = try await reconcile(client: app.client, database: app.db)