Complete package list repository client

Sources/App/Commands/Reconcile.swift

@@ -67,12 +67,13 @@ func reconcile(client: Client, database: Database) async throws {
 
 
 func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] {
-    async let sourcePackageList = try Current.fetchPackageList(client)
-    async let sourcePackageDenyList = try Current.fetchPackageDenyList(client)
+    @Dependency(\.packageListRepository) var packageListRepository
+
+    let sourcePackageList = try await packageListRepository.fetchPackageList(client: client)
+    let sourcePackageDenyList = try await packageListRepository.fetchPackageDenyList(client: client)
     async let currentList = try fetchCurrentPackageList(database)
 
-    let packageList = processPackageDenyList(packageList: try await sourcePackageList,
-                                             denyList: try await sourcePackageDenyList)
+    let packageList = processPackageDenyList(packageList: sourcePackageList, denyList: sourcePackageDenyList)
 
     try await reconcileLists(db: database,
                              source: packageList,
@@ -82,33 +83,6 @@ func reconcileMainPackageList(client: Client, database: Database) async throws -
 }
 
 
-func liveFetchPackageList(_ client: Client) async throws -> [URL] {
-   try await client
-        .get(Constants.packageListUri)
-        .content
-        .decode([String].self, using: JSONDecoder())
-        .compactMap(URL.init(string:))
-}
-
-
-func liveFetchPackageDenyList(_ client: Client) async throws -> [URL] {
-    struct DeniedPackage: Decodable {
-        var packageUrl: String
-
-        enum CodingKeys: String, CodingKey {
-            case packageUrl = "package_url"
-        }
-    }
-
-    return try await client
-        .get(Constants.packageDenyListUri)
-        .content
-        .decode([DeniedPackage].self, using: JSONDecoder())
-        .map(\.packageUrl)
-        .compactMap(URL.init(string:))
-}
-
-
 func fetchCurrentPackageList(_ db: Database) async throws -> [URL] {
     try await Package.query(on: db)
         .field(Package.self, \.$url)

Sources/App/Core/AppEnvironment.swift

@@ -43,8 +43,6 @@ struct AppEnvironment: Sendable {
     var environment: @Sendable () -> Environment
     var fetchDocumentation: @Sendable (_ client: Client, _ url: URI) async throws -> ClientResponse
     var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus
-    var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL]
-    var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL]
     var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License?
     var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata
     var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme?
@@ -165,8 +163,6 @@ extension AppEnvironment {
         environment: { (try? Environment.detect()) ?? .development },
         fetchDocumentation: { client, url in try await client.get(url) },
         fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) },
-        fetchPackageList: { client in try await liveFetchPackageList(client) },
-        fetchPackageDenyList: { client in try await liveFetchPackageDenyList(client) },
         fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) },
         fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) },
         fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) },

Sources/App/Core/Dependencies/PackageListRepositoryClient.swift

@@ -19,15 +19,39 @@ import Vapor
 
 @DependencyClient
 struct PackageListRepositoryClient {
+    var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL]
+    var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL]
     var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL]
     var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO]
-    // TODO: move other package list dependencies here
 }
 
 
 extension PackageListRepositoryClient: DependencyKey {
     static var liveValue: PackageListRepositoryClient {
         .init(
+            fetchPackageList: { client in
+                try await client
+                    .get(Constants.packageListUri)
+                    .content
+                    .decode([String].self, using: JSONDecoder())
+                    .compactMap(URL.init(string:))
+            },
+            fetchPackageDenyList: { client in
+                struct DeniedPackage: Decodable {
+                    var packageUrl: String
+
+                    enum CodingKeys: String, CodingKey {
+                        case packageUrl = "package_url"
+                    }
+                }
+
+                return try await client
+                    .get(Constants.packageDenyListUri)
+                    .content
+                    .decode([DeniedPackage].self, using: JSONDecoder())
+                    .map(\.packageUrl)
+                    .compactMap(URL.init(string:))
+            },
             fetchCustomCollection: { client, url in
                 try await client
                     .get(URI(string: url.absoluteString))

Tests/AppTests/MastodonTests.swift

@@ -34,7 +34,6 @@ final class MastodonTests: AppTestCase {
 
         let url = "https://github.com/foo/bar"
         Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
-        Current.fetchPackageList = { _ in [url.url] }
 
         Current.git.commitCount = { @Sendable _ in 12 }
         Current.git.firstCommitDate = { @Sendable _ in .t0 }
@@ -58,6 +57,8 @@ final class MastodonTests: AppTestCase {
 
         try await withDependencies {
             $0.date.now = .now
+            $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] }
+            $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] }
         } operation: {

Tests/AppTests/MetricsTests.swift

@@ -100,11 +100,10 @@ class MetricsTests: AppTestCase {
 
     func test_reconcileDurationSeconds() async throws {
         try await withDependencies {
+            $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "2", "3"].asURLs }
+            $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
         } operation: {
-            // setup
-            Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs }
-
             // MUT
             try await reconcile(client: app.client, database: app.db)
 

Tests/AppTests/Mocks/AppEnvironment+mock.swift

@@ -42,13 +42,6 @@ extension AppEnvironment {
             environment: { .development },
             fetchDocumentation: { _, _ in .init(status: .ok) },
             fetchHTTPStatusCode: { _ in .ok },
-            fetchPackageList: { _ in
-                ["https://github.com/finestructure/Gala",
-                 "https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server"].asURLs
-            },
-            fetchPackageDenyList: { _ in
-                ["https://github.com/daveverwer/LeftPad"].asURLs
-            },
             fetchLicense: { _, _, _ in .init(htmlUrl: "https://github.com/foo/bar/blob/main/LICENSE") },
             fetchMetadata: { _, _, _ in .mock },
             fetchReadme: { _,  _, _ in .init(html: "readme html", htmlUrl: "readme html url", imagesToCache: []) },

Tests/AppTests/PackageTests.swift

@@ -295,14 +295,15 @@ final class PackageTests: AppTestCase {
     }
 
     func test_isNew() async throws {
+        let url = "1".asGithubUrl
         try await withDependencies {
             $0.date.now = .now
+            $0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] }
+            $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
         } operation: {
             // setup
-            let url = "1".asGithubUrl
             Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
-            Current.fetchPackageList = { _ in [url.url] }
             Current.git.commitCount = { @Sendable _ in 12 }
             Current.git.firstCommitDate = { @Sendable _ in Date(timeIntervalSince1970: 0) }
             Current.git.getTags = { @Sendable _ in [] }

Tests/AppTests/PipelineTests.swift

@@ -158,17 +158,17 @@ class PipelineTests: AppTestCase {
     }
 
     func test_processing_pipeline() async throws {
+        let urls = ["1", "2", "3"].asGithubUrls
         try await withDependencies {
             $0.date.now = .now
+            $0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs }
+            $0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
             $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] }
         } operation: {
             // Test pipeline pick-up end to end
             // setup
-            let urls = ["1", "2", "3"].asGithubUrls
             Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
-            Current.fetchPackageList = { _ in urls.asURLs }
-
             Current.git.commitCount = { @Sendable _ in 12 }
             Current.git.firstCommitDate = { @Sendable _ in .t0 }
             Current.git.lastCommitDate = { @Sendable _ in .t1 }
@@ -224,76 +224,78 @@ class PipelineTests: AppTestCase {
                 XCTAssertEqual(packages.map(\.isNew), [false, false, false])
             }
 
-            // Now we've got a new package and a deletion
-            Current.fetchPackageList = { _ in ["1", "3", "4"].asGithubUrls.asURLs }
-
-            // MUT - reconcile again
-            try await reconcile(client: app.client, database: app.db)
-
-            do {  // validate - only new package moves to .reconciliation stage
-                let packages = try await Package.query(on: app.db).sort(\.$url).all()
-                XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
-                XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
-                XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation])
-                XCTAssertEqual(packages.map(\.isNew), [false, false, true])
-            }
-
-            // MUT - ingest again
-            try await ingest(client: app.client, database: app.db, mode: .limit(10))
-
-            do {  // validate - only new package moves to .ingestion stage
-                let packages = try await Package.query(on: app.db).sort(\.$url).all()
-                XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
-                XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
-                XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion])
-                XCTAssertEqual(packages.map(\.isNew), [false, false, true])
-            }
-
-            // MUT - analyze again
-            let lastAnalysis = Date.now
-            try await Analyze.analyze(client: app.client,
-                                      database: app.db,
-                                      mode: .limit(10))
-
-            do {  // validate - only new package moves to .ingestion stage
-                let packages = try await Package.query(on: app.db).sort(\.$url).all()
-                XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
-                XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
-                XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
-                XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true])
-                XCTAssertEqual(packages.map(\.isNew), [false, false, false])
-            }
-
             try await withDependencies {
-                // fast forward our clock by the deadtime interval
-                $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
+                // Now we've got a new package and a deletion
+                $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs }
             } operation: {
-                // MUT - ingest yet again
+                // MUT - reconcile again
+                try await reconcile(client: app.client, database: app.db)
+
+                do {  // validate - only new package moves to .reconciliation stage
+                    let packages = try await Package.query(on: app.db).sort(\.$url).all()
+                    XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
+                    XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
+                    XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation])
+                    XCTAssertEqual(packages.map(\.isNew), [false, false, true])
+                }
+
+                // MUT - ingest again
                 try await ingest(client: app.client, database: app.db, mode: .limit(10))
-                
-                do {  // validate - now all three packages should have been updated
+
+                do {  // validate - only new package moves to .ingestion stage
                     let packages = try await Package.query(on: app.db).sort(\.$url).all()
                     XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
-                    XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
-                    XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion])
-                    XCTAssertEqual(packages.map(\.isNew), [false, false, false])
+                    XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
+                    XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion])
+                    XCTAssertEqual(packages.map(\.isNew), [false, false, true])
                 }
-
-                // MUT - re-run analysis to complete the sequence
+
+                // MUT - analyze again
+                let lastAnalysis = Date.now
                 try await Analyze.analyze(client: app.client,
                                           database: app.db,
                                           mode: .limit(10))
-                
+
                 do {  // validate - only new package moves to .ingestion stage
                     let packages = try await Package.query(on: app.db).sort(\.$url).all()
                     XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
                     XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
                     XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
+                    XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true])
                     XCTAssertEqual(packages.map(\.isNew), [false, false, false])
                 }
-
-                // at this point we've ensured that retriggering ingestion after the deadtime will
-                // refresh analysis as expected
+
+                try await withDependencies {
+                    // fast forward our clock by the deadtime interval
+                    $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
+                } operation: {
+                    // MUT - ingest yet again
+                    try await ingest(client: app.client, database: app.db, mode: .limit(10))
+
+                    do {  // validate - now all three packages should have been updated
+                        let packages = try await Package.query(on: app.db).sort(\.$url).all()
+                        XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
+                        XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
+                        XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion])
+                        XCTAssertEqual(packages.map(\.isNew), [false, false, false])
+                    }
+
+                    // MUT - re-run analysis to complete the sequence
+                    try await Analyze.analyze(client: app.client,
+                                              database: app.db,
+                                              mode: .limit(10))
+
+                    do {  // validate - only new package moves to .ingestion stage
+                        let packages = try await Package.query(on: app.db).sort(\.$url).all()
+                        XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
+                        XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
+                        XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
+                        XCTAssertEqual(packages.map(\.isNew), [false, false, false])
+                    }
+
+                    // at this point we've ensured that retriggering ingestion after the deadtime will
+                    // refresh analysis as expected
+                }
             }
         }
     }