Skip to content

Commit a949e12

Browse files
finestructurefinestructure
authored
Merge pull request #3438 from SwiftPackageIndex/complete-PackageListRepositoryClient
2 parents 8f6aa3a + bbd6d68 commit a949e12

File tree

9 files changed

+126
-125
lines changed

9 files changed

+126
-125
lines changed

‎Sources/App/Commands/Reconcile.swift‎

Lines changed: 5 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -67,12 +67,13 @@ func reconcile(client: Client, database: Database) async throws {
6767

6868

6969
func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] {
70-
async let sourcePackageList = try Current.fetchPackageList(client)
71-
async let sourcePackageDenyList = try Current.fetchPackageDenyList(client)
70+
@Dependency(\.packageListRepository) var packageListRepository
71+
72+
let sourcePackageList = try await packageListRepository.fetchPackageList(client: client)
73+
let sourcePackageDenyList = try await packageListRepository.fetchPackageDenyList(client: client)
7274
async let currentList = try fetchCurrentPackageList(database)
7375

74-
let packageList = processPackageDenyList(packageList: try await sourcePackageList,
75-
denyList: try await sourcePackageDenyList)
76+
let packageList = processPackageDenyList(packageList: sourcePackageList, denyList: sourcePackageDenyList)
7677

7778
try await reconcileLists(db: database,
7879
source: packageList,
@@ -82,33 +83,6 @@ func reconcileMainPackageList(client: Client, database: Database) async throws -
8283
}
8384

8485

85-
func liveFetchPackageList(_ client: Client) async throws -> [URL] {
86-
try await client
87-
.get(Constants.packageListUri)
88-
.content
89-
.decode([String].self, using: JSONDecoder())
90-
.compactMap(URL.init(string:))
91-
}
92-
93-
94-
func liveFetchPackageDenyList(_ client: Client) async throws -> [URL] {
95-
struct DeniedPackage: Decodable {
96-
var packageUrl: String
97-
98-
enum CodingKeys: String, CodingKey {
99-
case packageUrl = "package_url"
100-
}
101-
}
102-
103-
return try await client
104-
.get(Constants.packageDenyListUri)
105-
.content
106-
.decode([DeniedPackage].self, using: JSONDecoder())
107-
.map(\.packageUrl)
108-
.compactMap(URL.init(string:))
109-
}
110-
111-
11286
func fetchCurrentPackageList(_ db: Database) async throws -> [URL] {
11387
try await Package.query(on: db)
11488
.field(Package.self, \.$url)

‎Sources/App/Core/AppEnvironment.swift‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,6 @@ struct AppEnvironment: Sendable {
4343
var environment: @Sendable () -> Environment
4444
var fetchDocumentation: @Sendable (_ client: Client, _ url: URI) async throws -> ClientResponse
4545
var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus
46-
var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL]
47-
var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL]
4846
var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License?
4947
var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata
5048
var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme?
@@ -165,8 +163,6 @@ extension AppEnvironment {
165163
environment: { (try? Environment.detect()) ?? .development },
166164
fetchDocumentation: { client, url in try await client.get(url) },
167165
fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) },
168-
fetchPackageList: { client in try await liveFetchPackageList(client) },
169-
fetchPackageDenyList: { client in try await liveFetchPackageDenyList(client) },
170166
fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) },
171167
fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) },
172168
fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) },

‎Sources/App/Core/Dependencies/PackageListRepositoryClient.swift‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,39 @@ import Vapor
1919

2020
@DependencyClient
2121
struct PackageListRepositoryClient {
22+
var fetchPackageList: @Sendable (_ client: Client) async throws -> [URL]
23+
var fetchPackageDenyList: @Sendable (_ client: Client) async throws -> [URL]
2224
var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL]
2325
var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO]
24-
// TODO: move other package list dependencies here
2526
}
2627

2728

2829
extension PackageListRepositoryClient: DependencyKey {
2930
static var liveValue: PackageListRepositoryClient {
3031
.init(
32+
fetchPackageList: { client in
33+
try await client
34+
.get(Constants.packageListUri)
35+
.content
36+
.decode([String].self, using: JSONDecoder())
37+
.compactMap(URL.init(string:))
38+
},
39+
fetchPackageDenyList: { client in
40+
struct DeniedPackage: Decodable {
41+
var packageUrl: String
42+
43+
enum CodingKeys: String, CodingKey {
44+
case packageUrl = "package_url"
45+
}
46+
}
47+
48+
return try await client
49+
.get(Constants.packageDenyListUri)
50+
.content
51+
.decode([DeniedPackage].self, using: JSONDecoder())
52+
.map(\.packageUrl)
53+
.compactMap(URL.init(string:))
54+
},
3155
fetchCustomCollection: { client, url in
3256
try await client
3357
.get(URI(string: url.absoluteString))

‎Tests/AppTests/MastodonTests.swift‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@ final class MastodonTests: AppTestCase {
3434

3535
let url = "https://github.com/foo/bar"
3636
Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
37-
Current.fetchPackageList = { _ in [url.url] }
3837

3938
Current.git.commitCount = { @Sendable _ in 12 }
4039
Current.git.firstCommitDate = { @Sendable _ in .t0 }
@@ -58,6 +57,8 @@ final class MastodonTests: AppTestCase {
5857

5958
try await withDependencies {
6059
$0.date.now = .now
60+
$0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] }
61+
$0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
6162
$0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
6263
$0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] }
6364
} operation: {

‎Tests/AppTests/MetricsTests.swift‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,10 @@ class MetricsTests: AppTestCase {
100100

101101
func test_reconcileDurationSeconds() async throws {
102102
try await withDependencies {
103+
$0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "2", "3"].asURLs }
104+
$0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
103105
$0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
104106
} operation: {
105-
// setup
106-
Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs }
107-
108107
// MUT
109108
try await reconcile(client: app.client, database: app.db)
110109

‎Tests/AppTests/Mocks/AppEnvironment+mock.swift‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,6 @@ extension AppEnvironment {
4242
environment: { .development },
4343
fetchDocumentation: { _, _ in .init(status: .ok) },
4444
fetchHTTPStatusCode: { _ in .ok },
45-
fetchPackageList: { _ in
46-
["https://github.com/finestructure/Gala",
47-
"https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server"].asURLs
48-
},
49-
fetchPackageDenyList: { _ in
50-
["https://github.com/daveverwer/LeftPad"].asURLs
51-
},
5245
fetchLicense: { _, _, _ in .init(htmlUrl: "https://github.com/foo/bar/blob/main/LICENSE") },
5346
fetchMetadata: { _, _, _ in .mock },
5447
fetchReadme: { _, _, _ in .init(html: "readme html", htmlUrl: "readme html url", imagesToCache: []) },

‎Tests/AppTests/PackageTests.swift‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -295,14 +295,15 @@ final class PackageTests: AppTestCase {
295295
}
296296

297297
func test_isNew() async throws {
298+
let url = "1".asGithubUrl
298299
try await withDependencies {
299300
$0.date.now = .now
301+
$0.packageListRepository.fetchPackageList = { @Sendable _ in [url.url] }
302+
$0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
300303
$0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
301304
} operation: {
302305
// setup
303-
let url = "1".asGithubUrl
304306
Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
305-
Current.fetchPackageList = { _ in [url.url] }
306307
Current.git.commitCount = { @Sendable _ in 12 }
307308
Current.git.firstCommitDate = { @Sendable _ in Date(timeIntervalSince1970: 0) }
308309
Current.git.getTags = { @Sendable _ in [] }

‎Tests/AppTests/PipelineTests.swift‎

Lines changed: 59 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -158,17 +158,17 @@ class PipelineTests: AppTestCase {
158158
}
159159

160160
func test_processing_pipeline() async throws {
161+
let urls = ["1", "2", "3"].asGithubUrls
161162
try await withDependencies {
162163
$0.date.now = .now
164+
$0.packageListRepository.fetchPackageList = { @Sendable _ in urls.asURLs }
165+
$0.packageListRepository.fetchPackageDenyList = { @Sendable _ in [] }
163166
$0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] }
164167
$0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] }
165168
} operation: {
166169
// Test pipeline pick-up end to end
167170
// setup
168-
let urls = ["1", "2", "3"].asGithubUrls
169171
Current.fetchMetadata = { _, owner, repository in .mock(owner: owner, repository: repository) }
170-
Current.fetchPackageList = { _ in urls.asURLs }
171-
172172
Current.git.commitCount = { @Sendable _ in 12 }
173173
Current.git.firstCommitDate = { @Sendable _ in .t0 }
174174
Current.git.lastCommitDate = { @Sendable _ in .t1 }
@@ -224,76 +224,78 @@ class PipelineTests: AppTestCase {
224224
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
225225
}
226226

227-
// Now we've got a new package and a deletion
228-
Current.fetchPackageList = { _ in ["1", "3", "4"].asGithubUrls.asURLs }
229-
230-
// MUT - reconcile again
231-
try await reconcile(client: app.client, database: app.db)
232-
233-
do { // validate - only new package moves to .reconciliation stage
234-
let packages = try await Package.query(on: app.db).sort(\.$url).all()
235-
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
236-
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
237-
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation])
238-
XCTAssertEqual(packages.map(\.isNew), [false, false, true])
239-
}
240-
241-
// MUT - ingest again
242-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
243-
244-
do { // validate - only new package moves to .ingestion stage
245-
let packages = try await Package.query(on: app.db).sort(\.$url).all()
246-
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
247-
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
248-
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion])
249-
XCTAssertEqual(packages.map(\.isNew), [false, false, true])
250-
}
251-
252-
// MUT - analyze again
253-
let lastAnalysis = Date.now
254-
try await Analyze.analyze(client: app.client,
255-
database: app.db,
256-
mode: .limit(10))
257-
258-
do { // validate - only new package moves to .ingestion stage
259-
let packages = try await Package.query(on: app.db).sort(\.$url).all()
260-
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
261-
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
262-
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
263-
XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true])
264-
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
265-
}
266-
267227
try await withDependencies {
268-
// fast forward our clock by the deadtime interval
269-
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
228+
// Now we've got a new package and a deletion
229+
$0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs }
270230
} operation: {
271-
// MUT - ingest yet again
231+
// MUT - reconcile again
232+
try await reconcile(client: app.client, database: app.db)
233+
234+
do { // validate - only new package moves to .reconciliation stage
235+
let packages = try await Package.query(on: app.db).sort(\.$url).all()
236+
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
237+
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
238+
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation])
239+
XCTAssertEqual(packages.map(\.isNew), [false, false, true])
240+
}
241+
242+
// MUT - ingest again
272243
try await ingest(client: app.client, database: app.db, mode: .limit(10))
273-
274-
do { // validate - now all three packages should have been updated
244+
245+
do { // validate - only new package moves to .ingestion stage
275246
let packages = try await Package.query(on: app.db).sort(\.$url).all()
276247
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
277-
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
278-
XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion])
279-
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
248+
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new])
249+
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion])
250+
XCTAssertEqual(packages.map(\.isNew), [false, false, true])
280251
}
281-
282-
// MUT - re-run analysis to complete the sequence
252+
253+
// MUT - analyze again
254+
let lastAnalysis = Date.now
283255
try await Analyze.analyze(client: app.client,
284256
database: app.db,
285257
mode: .limit(10))
286-
258+
287259
do { // validate - only new package moves to .ingestion stage
288260
let packages = try await Package.query(on: app.db).sort(\.$url).all()
289261
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
290262
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
291263
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
264+
XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true])
292265
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
293266
}
294-
295-
// at this point we've ensured that retriggering ingestion after the deadtime will
296-
// refresh analysis as expected
267+
268+
try await withDependencies {
269+
// fast forward our clock by the deadtime interval
270+
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
271+
} operation: {
272+
// MUT - ingest yet again
273+
try await ingest(client: app.client, database: app.db, mode: .limit(10))
274+
275+
do { // validate - now all three packages should have been updated
276+
let packages = try await Package.query(on: app.db).sort(\.$url).all()
277+
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
278+
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
279+
XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion])
280+
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
281+
}
282+
283+
// MUT - re-run analysis to complete the sequence
284+
try await Analyze.analyze(client: app.client,
285+
database: app.db,
286+
mode: .limit(10))
287+
288+
do { // validate - only new package moves to .ingestion stage
289+
let packages = try await Package.query(on: app.db).sort(\.$url).all()
290+
XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls)
291+
XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok])
292+
XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis])
293+
XCTAssertEqual(packages.map(\.isNew), [false, false, false])
294+
}
295+
296+
// at this point we've ensured that retriggering ingestion after the deadtime will
297+
// refresh analysis as expected
298+
}
297299
}
298300
}
299301
}

0 commit comments

Comments
 (0)