diff --git a/Tests/AppTests/PipelineTests.swift b/Tests/AppTests/PipelineTests.swift index 23481bc9f..44b4cc11e 100644 --- a/Tests/AppTests/PipelineTests.swift +++ b/Tests/AppTests/PipelineTests.swift @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -import XCTest - @testable import App import Dependencies import SQLKit +import Testing import Vapor @@ -25,139 +24,155 @@ import Vapor // - candidate selection at each stage // - processing stage recording // - error recording -class PipelineTests: AppTestCase { +@Suite struct PipelineTests { - func test_fetchCandidates_ingestion_fifo() async throws { + @Test func fetchCandidates_ingestion_fifo() async throws { // oldest first - try await [ - Package(url: "1", status: .ok, processingStage: .reconciliation), - Package(url: "2", status: .ok, processingStage: .reconciliation), - ].save(on: app.db) + try await withApp { app in + try await [ + Package(url: "1", status: .ok, processingStage: .reconciliation), + Package(url: "2", status: .ok, processingStage: .reconciliation), + ].save(on: app.db) - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["1", "2"]) + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) + #expect(batch.map(\.model.url) == ["1", "2"]) + } } } - func test_fetchCandidates_ingestion_limit() async throws { - try await [ - Package(url: "1", status: .ok, processingStage: .reconciliation), - Package(url: "2", status: .ok, processingStage: .reconciliation), - ].save(on: app.db) + @Test func fetchCandidates_ingestion_limit() async throws { + try await withApp { app in + try await [ + Package(url: "1", status: .ok, processingStage: .reconciliation), + Package(url: "2", status: .ok, processingStage: .reconciliation), + ].save(on: app.db) - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 1) - XCTAssertEqual(batch.map(\.model.url), ["1"]) + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 1) + #expect(batch.map(\.model.url) == ["1"]) + } } } - func test_fetchCandidates_ingestion_correct_stage() async throws { + @Test func fetchCandidates_ingestion_correct_stage() async throws { // only pick up from reconciliation stage - try await [ - Package(url: "1", status: .ok, processingStage: nil), - Package(url: "2", status: .ok, processingStage: .reconciliation), - Package(url: "3", status: .ok, processingStage: .analysis), - ].save(on: app.db) + try await withApp { app in + try await [ + Package(url: "1", status: .ok, processingStage: nil), + Package(url: "2", status: .ok, processingStage: .reconciliation), + Package(url: "3", status: .ok, processingStage: .analysis), + ].save(on: app.db) - try await withDependencies { - $0.date.now = .now - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["2"]) + try await withDependencies { + $0.date.now = .now + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) + #expect(batch.map(\.model.url) == ["2"]) + } } } - func test_fetchCandidates_ingestion_prefer_new() async throws { + @Test func fetchCandidates_ingestion_prefer_new() async throws { // make sure records with status = new come first, then least recent - try await [ - Package(url: "1", status: .notFound, processingStage: .reconciliation), - Package(url: "2", status: .new, processingStage: .reconciliation), - Package(url: "3", status: .ok, processingStage: .reconciliation), - ].save(on: app.db) + try await withApp { app in + try await [ + Package(url: "1", status: .notFound, processingStage: .reconciliation), + Package(url: "2", status: .new, processingStage: .reconciliation), + Package(url: "3", status: .ok, processingStage: .reconciliation), + ].save(on: app.db) - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["2", "1", "3"]) + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) + #expect(batch.map(\.model.url) == ["2", "1", "3"]) + } } } - func test_fetchCandidates_ingestion_eventual_refresh() async throws { + @Test func fetchCandidates_ingestion_eventual_refresh() async throws { // Make sure packages in .analysis stage get re-ingested after a while to // check for upstream package changes - try await [ - Package(url: "1", status: .ok, processingStage: .analysis), - Package(url: "2", status: .ok, processingStage: .analysis), - ].save(on: app.db) - let p2 = try await Package.query(on: app.db).filter(by: "2").first()! - try await (app.db as! SQLDatabase).raw( - "update packages set updated_at = updated_at - interval '91 mins' where id = \(bind: p2.id)" - ).run() + try await withApp { app in + try await [ + Package(url: "1", status: .ok, processingStage: .analysis), + Package(url: "2", status: .ok, processingStage: .analysis), + ].save(on: app.db) + let p2 = try await Package.query(on: app.db).filter(by: "2").first()! + try await (app.db as! SQLDatabase).raw( + "update packages set updated_at = updated_at - interval '91 mins' where id = \(bind: p2.id)" + ).run() - try await withDependencies { - $0.date.now = .now - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["2"]) + try await withDependencies { + $0.date.now = .now + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) + #expect(batch.map(\.model.url) == ["2"]) + } } } - func test_fetchCandidates_ingestion_refresh_analysis_only() async throws { + @Test func fetchCandidates_ingestion_refresh_analysis_only() async throws { // Ensure we only pick up .analysis stage records on the refresh cycle *) - we don't // want to refresh .ingestion stage records that have lagged in analysis, because it // resets their `.new` state prematurely. // // *) in addition to the .reconciliation ones, which we always pick up, regardless of // ingestion dead time. - try await [ - Package(url: "1", status: .new, processingStage: .reconciliation), - Package(url: "2", status: .new, processingStage: .ingestion), - Package(url: "3", status: .new, processingStage: .analysis), - ].save(on: app.db) + try await withApp { app in + try await [ + Package(url: "1", status: .new, processingStage: .reconciliation), + Package(url: "2", status: .new, processingStage: .ingestion), + Package(url: "3", status: .new, processingStage: .analysis), + ].save(on: app.db) - try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) - } operation: { - let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["1", "3"]) + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + let batch = try await Package.fetchCandidates(app.db, for: .ingestion, limit: 10) + #expect(batch.map(\.model.url) == ["1", "3"]) + } } } - func test_fetchCandidates_analysis_correct_stage() async throws { + @Test func fetchCandidates_analysis_correct_stage() async throws { // only pick up from ingestion stage - try await [ - Package(url: "1", status: .ok, processingStage: nil), - Package(url: "2", status: .ok, processingStage: .reconciliation), - Package(url: "3", status: .ok, processingStage: .ingestion), - Package(url: "4", status: .ok, processingStage: .analysis), - ].save(on: app.db) - let batch = try await Package.fetchCandidates(app.db, for: .analysis, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["3"]) + try await withApp { app in + try await [ + Package(url: "1", status: .ok, processingStage: nil), + Package(url: "2", status: .ok, processingStage: .reconciliation), + Package(url: "3", status: .ok, processingStage: .ingestion), + Package(url: "4", status: .ok, processingStage: .analysis), + ].save(on: app.db) + let batch = try await Package.fetchCandidates(app.db, for: .analysis, limit: 10) + #expect(batch.map(\.model.url) == ["3"]) + } } - func test_fetchCandidates_analysis_prefer_new() async throws { + @Test func fetchCandidates_analysis_prefer_new() async throws { // Test pick up from ingestion stage with status = new first, then FIFO - try await [ - Package(url: "1", status: .notFound, processingStage: .ingestion), - Package(url: "2", status: .ok, processingStage: .ingestion), - Package(url: "3", status: .analysisFailed, processingStage: .ingestion), - Package(url: "4", status: .new, processingStage: .ingestion), - ].save(on: app.db) - let batch = try await Package.fetchCandidates(app.db, for: .analysis, limit: 10) - XCTAssertEqual(batch.map(\.model.url), ["4", "1", "2", "3"]) + try await withApp { app in + try await [ + Package(url: "1", status: .notFound, processingStage: .ingestion), + Package(url: "2", status: .ok, processingStage: .ingestion), + Package(url: "3", status: .analysisFailed, processingStage: .ingestion), + Package(url: "4", status: .new, processingStage: .ingestion), + ].save(on: app.db) + let batch = try await Package.fetchCandidates(app.db, for: .analysis, limit: 10) + #expect(batch.map(\.model.url) == ["4", "1", "2", "3"]) + } } - func test_processing_pipeline() async throws { + @Test func processing_pipeline() async throws { // Test pipeline pick-up end to end let urls = ["1", "2", "3"].asGithubUrls try await withDependencies { @@ -190,112 +205,114 @@ class PipelineTests: AppTestCase { return "" } } operation: { - // MUT - first stage - try await reconcile(client: app.client, database: app.db) - - do { // validate - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "2", "3"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.new, .new, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.reconciliation, .reconciliation, .reconciliation]) - XCTAssertEqual(packages.map(\.isNew), [true, true, true]) - } - - // MUT - second stage - try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "2", "3"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.new, .new, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [true, true, true]) - } - - // MUT - third stage - try await Analyze.analyze(client: app.client, - database: app.db, - mode: .limit(10)) - - do { // validate - let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "2", "3"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) - } - - try await withDependencies { - // Now we've got a new package and a deletion - $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs } - } operation: { - // MUT - reconcile again + try await withApp { app in + // MUT - first stage try await reconcile(client: app.client, database: app.db) - - do { // validate - only new package moves to .reconciliation stage + + do { // validate let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .reconciliation]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) + #expect(packages.map(\.url) == ["1", "2", "3"].asGithubUrls) + #expect(packages.map(\.status) == [.new, .new, .new]) + #expect(packages.map(\.processingStage) == [.reconciliation, .reconciliation, .reconciliation]) + #expect(packages.map(\.isNew) == [true, true, true]) } - - // MUT - ingest again + + // MUT - second stage try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage + + do { // validate let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .new]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, true]) + #expect(packages.map(\.url) == ["1", "2", "3"].asGithubUrls) + #expect(packages.map(\.status) == [.new, .new, .new]) + #expect(packages.map(\.processingStage) == [.ingestion, .ingestion, .ingestion]) + #expect(packages.map(\.isNew) == [true, true, true]) } - - // MUT - analyze again - let lastAnalysis = Date.now + + // MUT - third stage try await Analyze.analyze(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - only new package moves to .ingestion stage + + do { // validate let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) - XCTAssertEqual(packages.map { $0.updatedAt! > lastAnalysis }, [false, false, true]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + #expect(packages.map(\.url) == ["1", "2", "3"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .ok]) + #expect(packages.map(\.processingStage) == [.analysis, .analysis, .analysis]) + #expect(packages.map(\.isNew) == [false, false, false]) } - + try await withDependencies { - // fast forward our clock by the deadtime interval - $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + // Now we've got a new package and a deletion + $0.packageListRepository.fetchPackageList = { @Sendable _ in ["1", "3", "4"].asGithubUrls.asURLs } } operation: { - // MUT - ingest yet again + // MUT - reconcile again + try await reconcile(client: app.client, database: app.db) + + do { // validate - only new package moves to .reconciliation stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + #expect(packages.map(\.url) == ["1", "3", "4"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .new]) + #expect(packages.map(\.processingStage) == [.analysis, .analysis, .reconciliation]) + #expect(packages.map(\.isNew) == [false, false, true]) + } + + // MUT - ingest again try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10)) - - do { // validate - now all three packages should have been updated + + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.ingestion, .ingestion, .ingestion]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + #expect(packages.map(\.url) == ["1", "3", "4"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .new]) + #expect(packages.map(\.processingStage) == [.analysis, .analysis, .ingestion]) + #expect(packages.map(\.isNew) == [false, false, true]) } - - // MUT - re-run analysis to complete the sequence + + // MUT - analyze again + let lastAnalysis = Date.now try await Analyze.analyze(client: app.client, database: app.db, mode: .limit(10)) - + do { // validate - only new package moves to .ingestion stage let packages = try await Package.query(on: app.db).sort(\.$url).all() - XCTAssertEqual(packages.map(\.url), ["1", "3", "4"].asGithubUrls) - XCTAssertEqual(packages.map(\.status), [.ok, .ok, .ok]) - XCTAssertEqual(packages.map(\.processingStage), [.analysis, .analysis, .analysis]) - XCTAssertEqual(packages.map(\.isNew), [false, false, false]) + #expect(packages.map(\.url) == ["1", "3", "4"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .ok]) + #expect(packages.map(\.processingStage) == [.analysis, .analysis, .analysis]) + #expect(packages.map { $0.updatedAt! > lastAnalysis } == [false, false, true]) + #expect(packages.map(\.isNew) == [false, false, false]) + } + + try await withDependencies { + // fast forward our clock by the deadtime interval + $0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime) + } operation: { + // MUT - ingest yet again + try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10)) + + do { // validate - now all three packages should have been updated + let packages = try await Package.query(on: app.db).sort(\.$url).all() + #expect(packages.map(\.url) == ["1", "3", "4"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .ok]) + #expect(packages.map(\.processingStage) == [.ingestion, .ingestion, .ingestion]) + #expect(packages.map(\.isNew) == [false, false, false]) + } + + // MUT - re-run analysis to complete the sequence + try await Analyze.analyze(client: app.client, + database: app.db, + mode: .limit(10)) + + do { // validate - only new package moves to .ingestion stage + let packages = try await Package.query(on: app.db).sort(\.$url).all() + #expect(packages.map(\.url) == ["1", "3", "4"].asGithubUrls) + #expect(packages.map(\.status) == [.ok, .ok, .ok]) + #expect(packages.map(\.processingStage) == [.analysis, .analysis, .analysis]) + #expect(packages.map(\.isNew) == [false, false, false]) + } + + // at this point we've ensured that retriggering ingestion after the deadtime will + // refresh analysis as expected } - - // at this point we've ensured that retriggering ingestion after the deadtime will - // refresh analysis as expected } } } diff --git a/Tests/AppTests/PlausibleTests.swift b/Tests/AppTests/PlausibleTests.swift index cc111921c..8cf46548a 100644 --- a/Tests/AppTests/PlausibleTests.swift +++ b/Tests/AppTests/PlausibleTests.swift @@ -12,60 +12,61 @@ // See the License for the specific language governing permissions and // limitations under the License. -import XCTest +import Foundation @testable import App import Dependencies +import Testing -final class PlausibleTests: XCTestCase { +@Suite struct PlausibleTests { - func test_User_identifier() throws { - XCTAssertEqual(User.api(for: "token"), .init(name: "api", identifier: "3c469e9d")) + @Test func User_identifier() throws { + #expect(User.api(for: "token") == .init(name: "api", identifier: "3c469e9d")) } - func test_props() throws { - XCTAssertEqual(Plausible.props(for: nil), ["user": "none"]) - XCTAssertEqual(Plausible.props(for: .init(name: "api", identifier: "foo")), ["user": "foo"]) + @Test func props() throws { + #expect(Plausible.props(for: nil) == ["user": "none"]) + #expect(Plausible.props(for: .init(name: "api", identifier: "foo")) == ["user": "foo"]) } - func test_postEvent_anonymous() async throws { + @Test func postEvent_anonymous() async throws { let called = ActorIsolated(false) try await withDependencies { $0.environment.plausibleBackendReportingSiteID = { "foo.bar" } $0.httpClient.post = { @Sendable _, _, body in await called.withValue { $0 = true } // validate - let body = try XCTUnwrap(body) - XCTAssertEqual(try? JSONDecoder().decode(Plausible.Event.self, from: body), - .init(name: .pageview, - url: "https://foo.bar/api/search", - domain: "foo.bar", - props: ["user": "none"])) + let body = try #require(body) + #expect(try JSONDecoder().decode(Plausible.Event.self, from: body) + == .init(name: .pageview, + url: "https://foo.bar/api/search", + domain: "foo.bar", + props: ["user": "none"])) return .ok } } operation: { // MUT _ = try await Plausible.postEvent(kind: .pageview, path: .search, user: nil) - await called.withValue { XCTAssertTrue($0) } + await called.withValue { #expect($0) } } } - func test_postEvent_package() async throws { + @Test func postEvent_package() async throws { let called = ActorIsolated(false) try await withDependencies { $0.environment.plausibleBackendReportingSiteID = { "foo.bar" } $0.httpClient.post = { @Sendable _, _, body in await called.withValue { $0 = true } // validate - let body = try XCTUnwrap(body) - XCTAssertEqual(try? JSONDecoder().decode(Plausible.Event.self, from: body), - .init(name: .pageview, - url: "https://foo.bar/api/packages/{owner}/{repository}", - domain: "foo.bar", - props: ["user": "3c469e9d"])) + let body = try #require(body) + #expect(try JSONDecoder().decode(Plausible.Event.self, from: body) + == .init(name: .pageview, + url: "https://foo.bar/api/packages/{owner}/{repository}", + domain: "foo.bar", + props: ["user": "3c469e9d"])) return .ok } } operation: { @@ -74,7 +75,7 @@ final class PlausibleTests: XCTestCase { // MUT _ = try await Plausible.postEvent(kind: .pageview, path: .package, user: user) - await called.withValue { XCTAssertTrue($0) } + await called.withValue { #expect($0) } } } } diff --git a/Tests/AppTests/ProductTests.swift b/Tests/AppTests/ProductTests.swift index 8ec8389dc..e0588a817 100644 --- a/Tests/AppTests/ProductTests.swift +++ b/Tests/AppTests/ProductTests.swift @@ -12,93 +12,93 @@ // See the License for the specific language governing permissions and // limitations under the License. +import Foundation + @testable import App -import XCTVapor +import Testing -class ProductTests: AppTestCase { +@Suite struct ProductTests { - func test_ProductType_Codable() throws { + @Test func ProductType_Codable() throws { // Ensure ProductType is Codable in a way that's forward compatible with Swift 5.5's Codable synthesis for enums with associated types (SE-0295) let exe: ProductType = .executable let lib: ProductType = .library(.automatic) let test: ProductType = .test do { // encoding - XCTAssertEqual( - String(decoding: try JSONEncoder().encode(exe), as: UTF8.self), - #"{"executable":{}}"# + #expect( + String(decoding: try JSONEncoder().encode(exe), as: UTF8.self) == #"{"executable":{}}"# ) - XCTAssertEqual( - String(decoding: try JSONEncoder().encode(lib), as: UTF8.self), - #"{"library":{"_0":"automatic"}}"# + #expect( + String(decoding: try JSONEncoder().encode(lib), as: UTF8.self) == #"{"library":{"_0":"automatic"}}"# ) - XCTAssertEqual( - String(decoding: try JSONEncoder().encode(test), as: UTF8.self), - #"{"test":{}}"# + #expect( + String(decoding: try JSONEncoder().encode(test), as: UTF8.self) == #"{"test":{}}"# ) } do { // decoding - XCTAssertEqual( + #expect( try JSONDecoder().decode( ProductType.self, - from: Data(#"{"executable":{}}"#.utf8)), - exe) - XCTAssertEqual( + from: Data(#"{"executable":{}}"#.utf8)) == exe) + #expect( try JSONDecoder().decode( ProductType.self, - from: Data(#"{"library":{"_0":"automatic"}}"#.utf8)), - lib + from: Data(#"{"library":{"_0":"automatic"}}"#.utf8)) == lib ) - XCTAssertEqual( + #expect( try JSONDecoder().decode( ProductType.self, - from: Data(#"{"test":{}}"#.utf8)), - test) + from: Data(#"{"test":{}}"#.utf8)) == test) } } - func test_Product_save() async throws { - let pkg = Package(id: UUID(), url: "1") - let ver = try Version(id: UUID(), package: pkg) - let prod = try Product(id: UUID(), - version: ver, - type: .library(.automatic), - name: "p1", - targets: ["t1", "t2"]) - try await pkg.save(on: app.db) - try await ver.save(on: app.db) - try await prod.save(on: app.db) - do { - let p = try await XCTUnwrapAsync(try await Product.find(prod.id, on: app.db)) - XCTAssertEqual(p.$version.id, ver.id) - XCTAssertEqual(p.type, .library(.automatic)) - XCTAssertEqual(p.name, "p1") - XCTAssertEqual(p.targets, ["t1", "t2"]) + @Test func Product_save() async throws { + try await withApp { app in + let pkg = Package(id: UUID(), url: "1") + let ver = try Version(id: UUID(), package: pkg) + let prod = try Product(id: UUID(), + version: ver, + type: .library(.automatic), + name: "p1", + targets: ["t1", "t2"]) + try await pkg.save(on: app.db) + try await ver.save(on: app.db) + try await prod.save(on: app.db) + do { + let p = try #require(try await Product.find(prod.id, on: app.db)) + #expect(p.$version.id == ver.id) + #expect(p.type == .library(.automatic)) + #expect(p.name == "p1") + #expect(p.targets == ["t1", "t2"]) + } } } - func test_delete_cascade() async throws { + @Test func delete_cascade() async throws { // delete version must delete products - let pkg = Package(id: UUID(), url: "1") - let ver = try Version(id: UUID(), package: pkg) - let prod = try Product(id: UUID(), version: ver, type: .library(.automatic), name: "p1") - try await pkg.save(on: app.db) - try await ver.save(on: app.db) - try await prod.save(on: app.db) - let db = app.db + try await withApp { app in + let pkg = Package(id: UUID(), url: "1") + let ver = try Version(id: UUID(), package: pkg) + let prod = try Product(id: UUID(), version: ver, type: .library(.automatic), name: "p1") + try await pkg.save(on: app.db) + try await ver.save(on: app.db) + try await prod.save(on: app.db) + let db = app.db - try await XCTAssertEqualAsync(try await Package.query(on: db).count(), 1) - try await XCTAssertEqualAsync(try await Version.query(on: db).count(), 1) - try await XCTAssertEqualAsync(try await Product.query(on: db).count(), 1) + try await XCTAssertEqualAsync(try await Package.query(on: db).count(), 1) + try await XCTAssertEqualAsync(try await Version.query(on: db).count(), 1) + try await XCTAssertEqualAsync(try await Product.query(on: db).count(), 1) - // MUT - try await ver.delete(on: app.db) + // MUT + try await ver.delete(on: app.db) - // version and product should be deleted - try await XCTAssertEqualAsync(try await Package.query(on: db).count(), 1) - try await XCTAssertEqualAsync(try await Version.query(on: db).count(), 0) - try await XCTAssertEqualAsync(try await Product.query(on: db).count(), 0) + // version and product should be deleted + try await XCTAssertEqualAsync(try await Package.query(on: db).count(), 1) + try await XCTAssertEqualAsync(try await Version.query(on: db).count(), 0) + try await XCTAssertEqualAsync(try await Product.query(on: db).count(), 0) + } } } diff --git a/Tests/AppTests/QueryPerformanceTests.swift b/Tests/AppTests/QueryPerformanceTests.swift index 9cfce8b05..ffd17e281 100644 --- a/Tests/AppTests/QueryPerformanceTests.swift +++ b/Tests/AppTests/QueryPerformanceTests.swift @@ -14,153 +14,172 @@ @testable import App +import Dependencies import SQLKit +import Testing import Vapor -import XCTest -class QueryPerformanceTests: XCTestCase { - var app: Application! - +@Suite( + .disabled(if: !runQueryPerformanceTests()) +) +struct QueryPerformanceTests { // Set this to true when running locally to convert warnings to test failures for easier updating of values. static let failOnWarning = false - override func setUp() async throws { - try await super.setUp() - - try XCTSkipUnless(runQueryPerformanceTests) - + func withStagingApp(_ test: (Application) async throws -> Void) async throws { // Update db settings for CI runs in // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/settings/secrets/actions // or in `.env.staging` for local runs. - self.app = try await Application.make(.staging) - self.app.logger.logLevel = Environment.get("LOG_LEVEL") + let app = try await Application.make(.staging) + app.logger.logLevel = Environment.get("LOG_LEVEL") .flatMap(Logger.Level.init(rawValue:)) ?? .warning let host = try await configure(app) - XCTAssert(host.hasPrefix("spi-dev-db"), "was: \(host)") - XCTAssert(host.hasSuffix("postgres.database.azure.com"), "was: \(host)") + try #require(host.hasPrefix("spi-dev-db"), "was: \(host)") + try #require(host.hasSuffix("postgres.database.azure.com"), "was: \(host)") + + return try await run { + try await test(app) + } defer: { + try await app.asyncShutdown() + } } - func test_01_Search_packageMatchQuery() async throws { - let query = Search.packageMatchQueryBuilder(on: app.db, terms: ["a"], filters: []) - try await assertQueryPerformance(query, expectedCost: 1800, variation: 150) + @Test func queryPerformance_01_Search_packageMatchQuery() async throws { + try await withStagingApp { app in + let query = Search.packageMatchQueryBuilder(on: app.db, terms: ["a"], filters: []) + try await assertQueryPerformance(query, expectedCost: 1800, variation: 150) + } } - func test_02_Search_keywordMatchQuery() async throws { - let query = Search.keywordMatchQueryBuilder(on: app.db, terms: ["a"]) - try await assertQueryPerformance(query, expectedCost: 5900, variation: 200) + @Test func queryPerformance_02_Search_keywordMatchQuery() async throws { + try await withStagingApp { app in + let query = Search.keywordMatchQueryBuilder(on: app.db, terms: ["a"]) + try await assertQueryPerformance(query, expectedCost: 5900, variation: 200) + } } - func test_03_Search_authorMatchQuery() async throws { - let query = Search.authorMatchQueryBuilder(on: app.db, terms: ["a"]) - try await assertQueryPerformance(query, expectedCost: 1100, variation: 50) + @Test func queryPerformance_03_Search_authorMatchQuery() async throws { + try await withStagingApp { app in + let query = Search.authorMatchQueryBuilder(on: app.db, terms: ["a"]) + try await assertQueryPerformance(query, expectedCost: 1100, variation: 50) + } } - func test_04_Search_query_noFilter() async throws { - let query = try Search.query(app.db, ["a"], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 8100, variation: 200) + @Test func queryPerformance_04_Search_query_noFilter() async throws { + try await withStagingApp { app in + let query = try Search.query(app.db, ["a"], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 8100, variation: 200) + } } - func test_05_Search_query_authorFilter() async throws { - let filter = try AuthorSearchFilter(expression: .init(operator: .is, value: "apple")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 7700, variation: 200) + @Test func queryPerformance_05_Search_query_authorFilter() async throws { + try await withStagingApp { app in + let filter = try AuthorSearchFilter(expression: .init(operator: .is, value: "apple")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 7700, variation: 200) + } } - func test_06_Search_query_keywordFilter() async throws { - let filter = try KeywordSearchFilter(expression: .init(operator: .is, value: "apple")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 7800, variation: 200) + @Test func queryPerformance_06_Search_query_keywordFilter() async throws { + try await withStagingApp { app in + let filter = try KeywordSearchFilter(expression: .init(operator: .is, value: "apple")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 7800, variation: 200) + } } - func test_07_Search_query_lastActicityFilter() async throws { - let filter = try LastActivitySearchFilter(expression: .init(operator: .greaterThan, value: "2000-01-01")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 8100, variation: 200) + @Test func queryPerformance_07_Search_query_lastActicityFilter() async throws { + try await withStagingApp { app in + let filter = try LastActivitySearchFilter(expression: .init(operator: .greaterThan, value: "2000-01-01")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 8100, variation: 200) + } } - func test_08_Search_query_licenseFilter() async throws { - let filter = try LicenseSearchFilter(expression: .init(operator: .is, value: "mit")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 8000, variation: 200) + @Test func queryPerformance_08_Search_query_licenseFilter() async throws { + try await withStagingApp { app in + let filter = try LicenseSearchFilter(expression: .init(operator: .is, value: "mit")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 8000, variation: 200) + } } - func test_09_Search_query_platformFilter() async throws { - let filter = try PlatformSearchFilter(expression: .init(operator: .is, value: "macos,ios")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 7900, variation: 200) + @Test func queryPerformance_09_Search_query_platformFilter() async throws { + try await withStagingApp { app in + let filter = try PlatformSearchFilter(expression: .init(operator: .is, value: "macos,ios")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 7900, variation: 200) + } } - func test_10_Search_query_productTypeFilter() async throws { - let filter = try ProductTypeSearchFilter(expression: .init(operator: .is, value: "plugin")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 7700, variation: 200) + @Test func queryPerformance_10_Search_query_productTypeFilter() async throws { + try await withStagingApp { app in + let filter = try ProductTypeSearchFilter(expression: .init(operator: .is, value: "plugin")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 7700, variation: 200) + } } - func test_11_Search_query_starsFilter() async throws { - let filter = try StarsSearchFilter(expression: .init(operator: .greaterThan, value: "5")) - let query = try Search.query(app.db, ["a"], filters: [filter], page: 1) - .unwrap() - try await assertQueryPerformance(query, expectedCost: 7900, variation: 300) + @Test func queryPerformance_11_Search_query_starsFilter() async throws { + try await withStagingApp { app in + let filter = try StarsSearchFilter(expression: .init(operator: .greaterThan, value: "5")) + let query = try Search.query(app.db, ["a"], filters: [filter], page: 1).unwrap() + try await assertQueryPerformance(query, expectedCost: 7900, variation: 300) + } } - func test_12_Search_refresh() async throws { + @Test func queryPerformance_12_Search_refresh() async throws { // We can't "explain analyze" the refresh itself so we need to measure the underlying // query. // Unfortunately, this means it'll need to be kept in sync when updating the search // view. - guard let db = app.db as? SQLDatabase else { - XCTFail() - return + try await withStagingApp { app in + guard let db = app.db as? SQLDatabase else { + Issue.record() + return + } + let query = db.raw(""" + -- v12 + SELECT + p.id AS package_id, + p.platform_compatibility, + p.score, + r.keywords, + r.last_commit_date, + r.license, + r.name AS repo_name, + r.owner AS repo_owner, + r.stars, + r.last_activity_at, + r.summary, + v.package_name, + ( + ARRAY_LENGTH(doc_archives, 1) >= 1 + OR spi_manifest->'external_links'->'documentation' IS NOT NULL + ) AS has_docs, + ARRAY( + SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id + UNION + SELECT * FROM ( + SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets + WHERE targets.version_id = v.id) AS macro_targets + WHERE type = 'macro' + ) AS product_types, + ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names, + TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector + FROM packages p + JOIN repositories r ON r.package_id = p.id + JOIN versions v ON v.package_id = p.id + WHERE v.reference ->> 'branch' = r.default_branch + """) + try await assertQueryPerformance(query, expectedCost: 132_000, variation: 5000) } - let query = db.raw(""" - -- v12 - SELECT - p.id AS package_id, - p.platform_compatibility, - p.score, - r.keywords, - r.last_commit_date, - r.license, - r.name AS repo_name, - r.owner AS repo_owner, - r.stars, - r.last_activity_at, - r.summary, - v.package_name, - ( - ARRAY_LENGTH(doc_archives, 1) >= 1 - OR spi_manifest->'external_links'->'documentation' IS NOT NULL - ) AS has_docs, - ARRAY( - SELECT DISTINCT JSONB_OBJECT_KEYS(type) FROM products WHERE products.version_id = v.id - UNION - SELECT * FROM ( - SELECT DISTINCT JSONB_OBJECT_KEYS(type) AS "type" FROM targets - WHERE targets.version_id = v.id) AS macro_targets - WHERE type = 'macro' - ) AS product_types, - ARRAY(SELECT DISTINCT name FROM products WHERE products.version_id = v.id) AS product_names, - TO_TSVECTOR(CONCAT_WS(' ', COALESCE(v.package_name, ''), r.name, COALESCE(r.summary, ''), ARRAY_TO_STRING(r.keywords, ' '))) AS tsvector - FROM packages p - JOIN repositories r ON r.package_id = p.id - JOIN versions v ON v.package_id = p.id - WHERE v.reference ->> 'branch' = r.default_branch - """) - try await assertQueryPerformance(query, expectedCost: 132_000, variation: 5000) } } - // MARK: - Query plan helpers @@ -169,6 +188,11 @@ private extension Environment { } +private func runQueryPerformanceTests() -> Bool { + ProcessInfo.processInfo.environment.keys.contains("RUN_QUERY_PERFORMANCE_TESTS") +} + + final class SQLQueryExplainer: SQLQueryFetcher { var query: any SQLExpression var database: any SQLDatabase @@ -201,7 +225,8 @@ private extension QueryPerformanceTests { variation: Double = 0, filePath: StaticString = #filePath, lineNumber: UInt = #line, - testName: String = #function) async throws { + testName: String = #function, + sourceLocation: Testing.SourceLocation = #_sourceLocation) async throws { let queryPlan = try await query.explain() let parsedPlan = try QueryPlan(queryPlan) print("ℹ️ TEST: \(testName)") @@ -209,11 +234,12 @@ private extension QueryPerformanceTests { print("ℹ️ COST: \(parsedPlan.cost.total)") } else { if Self.failOnWarning { - XCTFail(""" - Total cost of \(parsedPlan.cost.total) above the expected cost of \(expectedCost) - """, - file: filePath, - line: lineNumber) + Issue.record( + """ + Total cost of \(parsedPlan.cost.total) above the expected cost of \(expectedCost) + """, + sourceLocation: sourceLocation + ) } else { print("⚠️ COST: \(parsedPlan.cost.total)") } @@ -223,35 +249,37 @@ private extension QueryPerformanceTests { switch parsedPlan.cost.total { case ..<10.0: - if isRunningInCI { + if isRunningInCI() { print("::error file=\(filePath),line=\(lineNumber),title=\(testName)::Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database?") } - XCTFail(""" - Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database? - - \(queryPlan) - """, - file: filePath, - line: lineNumber) + Issue.record( + """ + Cost very low \(parsedPlan.cost.total) - did you run the query against an empty database? + + \(queryPlan) + """, + sourceLocation: sourceLocation + ) case ..