diff --git a/Sources/App/Commands/Analyze.swift b/Sources/App/Commands/Analyze.swift index 4e3453968..28dcb7f25 100644 --- a/Sources/App/Commands/Analyze.swift +++ b/Sources/App/Commands/Analyze.swift @@ -152,10 +152,7 @@ extension Analyze { } } - try await updatePackages(client: client, - database: database, - results: packageResults, - stage: .analysis) + try await updatePackages(client: client, database: database, results: packageResults) try await RecentPackage.refresh(on: database) try await RecentRelease.refresh(on: database) diff --git a/Sources/App/Commands/Common.swift b/Sources/App/Commands/Common.swift index a8aabea88..41a2c81ea 100644 --- a/Sources/App/Commands/Common.swift +++ b/Sources/App/Commands/Common.swift @@ -17,111 +17,125 @@ import PostgresKit import Vapor -/// Update packages (in the `[Result, Error>]` array). -/// -/// - Parameters: -/// - client: `Client` object -/// - database: `Database` object -/// - results: `Joined` results to update -/// - stage: Processing stage -func updatePackages(client: Client, - database: Database, - results: [Result, Error>], - stage: Package.ProcessingStage) async throws { - do { - let total = results.count - let errors = results.filter(\.isError).count - let errorRate = total > 0 ? 100.0 * Double(errors) / Double(total) : 0.0 - switch errorRate { - case 0: - Current.logger().info("Updating \(total) packages for stage '\(stage)'") - case 0..<20: - Current.logger().info("Updating \(total) packages for stage '\(stage)' (errors: \(errors))") - default: - Current.logger().critical("updatePackages: unusually high error rate: \(errors)/\(total) = \(errorRate)%") - } - } - for result in results { - do { - try await updatePackage(client: client, database: database, result: result, stage: stage) - } catch { - Current.logger().critical("updatePackage failed: \(error)") - } - } - - Current.logger().debug("updateStatus ops: \(results.count)") +// TODO: Adopt ProcessingError also in Analysis and then factor out generic parts back into Common +protocol ProcessingError: Error, CustomStringConvertible { + associatedtype UnderlyingError: Error & CustomStringConvertible + var packageId: Package.Id { get } + var underlyingError: UnderlyingError { get } + var level: Logger.Level { get } + var status: Package.Status { get } } -func updatePackage(client: Client, - database: Database, - result: Result, Error>, - stage: Package.ProcessingStage) async throws { - switch result { - case .success(let res): - let pkg = res.package - if stage == .ingestion && pkg.status == .new { - // newly ingested package: leave status == .new for fast-track - // analysis - } else { - pkg.status = .ok +// TODO: Leaving this extension here for now in order to group the updating/error reporting in one place for both Ingestion and Analysis. Eventually these should either go to their respective files or move common parts into a Common namespace. +extension Analyze { + /// Update packages (in the `[Result, Error>]` array). + /// + /// - Parameters: + /// - client: `Client` object + /// - database: `Database` object + /// - results: `Joined` results to update + /// - stage: Processing stage + static func updatePackages(client: Client, + database: Database, + results: [Result, Error>]) async throws { + do { + let total = results.count + let errors = results.filter(\.isError).count + let errorRate = total > 0 ? 100.0 * Double(errors) / Double(total) : 0.0 + switch errorRate { + case 0: + Current.logger().info("Updating \(total) packages for stage 'analysis'") + case 0..<20: + Current.logger().info("Updating \(total) packages for stage 'analysis' (errors: \(errors))") + default: + Current.logger().critical("updatePackages: unusually high error rate: \(errors)/\(total) = \(errorRate)%") } - pkg.processingStage = stage + } + for result in results { do { - try await pkg.update(on: database) + try await updatePackage(client: client, database: database, result: result) } catch { - Current.logger().report(error: error) + Current.logger().critical("updatePackage failed: \(error)") } + } - // PSQLError also conforms to DatabaseError but we want to intercept it specifically, - // because it allows us to log more concise error messages via serverInfo[.message] - case let .failure(error) where error is PSQLError: - // Escalate database errors to critical - let error = error as! PSQLError - let msg = error.serverInfo?[.message] ?? String(reflecting: error) - Current.logger().critical("\(msg)") - try await recordError(database: database, error: error, stage: stage) + Current.logger().debug("updateStatus ops: \(results.count)") + } - case let .failure(error) where error is DatabaseError: - // Escalate database errors to critical - Current.logger().critical("\(String(reflecting: error))") - try await recordError(database: database, error: error, stage: stage) + static func updatePackage(client: Client, + database: Database, + result: Result, Error>) async throws { + switch result { + case .success(let res): + try await res.package.update(on: database, status: .ok, stage: .analysis) - case let .failure(error): - Current.logger().report(error: error) - try await recordError(database: database, error: error, stage: stage) + // PSQLError also conforms to DatabaseError but we want to intercept it specifically, + // because it allows us to log more concise error messages via serverInfo[.message] + case let .failure(error) where error is PSQLError: + // Escalate database errors to critical + let error = error as! PSQLError + let msg = error.serverInfo?[.message] ?? String(reflecting: error) + Current.logger().critical("\(msg)") + try await recordError(database: database, error: error) + + case let .failure(error) where error is DatabaseError: + // Escalate database errors to critical + Current.logger().critical("\(String(reflecting: error))") + try await recordError(database: database, error: error) + + case let .failure(error): + Current.logger().report(error: error) + try await recordError(database: database, error: error) + } } -} + static func recordError(database: Database, error: Error) async throws { + func setStatus(id: Package.Id?, status: Package.Status) async throws { + guard let id = id else { return } + try await Package.query(on: database) + .filter(\.$id == id) + .set(\.$processingStage, to: .analysis) + .set(\.$status, to: status) + .update() + } + + guard let error = error as? AppError else { return } -func recordError(database: Database, - error: Error, - stage: Package.ProcessingStage) async throws { - func setStatus(id: Package.Id?, status: Package.Status) async throws { - guard let id = id else { return } - try await Package.query(on: database) - .filter(\.$id == id) - .set(\.$processingStage, to: stage) - .set(\.$status, to: status) - .update() + switch error { + case let .analysisError(id, _): + try await setStatus(id: id, status: .analysisFailed) + case .envVariableNotSet, .shellCommandFailed: + break + case let .genericError(id, _): + try await setStatus(id: id, status: .ingestionFailed) + case let .invalidPackageCachePath(id, _): + try await setStatus(id: id, status: .invalidCachePath) + case let .cacheDirectoryDoesNotExist(id, _): + try await setStatus(id: id, status: .cacheDirectoryDoesNotExist) + case let .invalidRevision(id, _): + try await setStatus(id: id, status: .analysisFailed) + case let .noValidVersions(id, _): + try await setStatus(id: id, status: .noValidVersions) + } } +} - guard let error = error as? AppError else { return } - switch error { - case let .analysisError(id, _): - try await setStatus(id: id, status: .analysisFailed) - case .envVariableNotSet, .shellCommandFailed: - break - case let .genericError(id, _): - try await setStatus(id: id, status: .ingestionFailed) - case let .invalidPackageCachePath(id, _): - try await setStatus(id: id, status: .invalidCachePath) - case let .cacheDirectoryDoesNotExist(id, _): - try await setStatus(id: id, status: .cacheDirectoryDoesNotExist) - case let .invalidRevision(id, _): - try await setStatus(id: id, status: .analysisFailed) - case let .noValidVersions(id, _): - try await setStatus(id: id, status: .noValidVersions) +// TODO: Leaving this extension here for now in order to group the updating/error reporting in one place for both Ingestion and Analysis. Eventually these should either go to their respective files or move common parts into a Common namespace. +extension Ingestion { + static func updatePackage(client: Client, + database: Database, + result: Result, Ingestion.Error>, + stage: Package.ProcessingStage) async throws { + switch result { + case .success(let res): + // for newly ingested package leave status == .new in order to fast-track analysis + let updatedStatus: Package.Status = res.package.status == .new ? .new : .ok + try await res.package.update(on: database, status: updatedStatus, stage: stage) + case .failure(let failure): + Current.logger().log(level: failure.level, "\(failure)") + try await Package.update(for: failure.packageId, on: database, status: failure.status, stage: stage) + } } } diff --git a/Sources/App/Commands/Ingest.swift b/Sources/App/Commands/Ingest.swift index 7bf64744f..c19501bef 100644 --- a/Sources/App/Commands/Ingest.swift +++ b/Sources/App/Commands/Ingest.swift @@ -12,8 +12,68 @@ // See the License for the specific language governing permissions and // limitations under the License. -import Vapor +import Dependencies import Fluent +import PostgresKit +import Vapor + + +enum Ingestion { + struct Error: ProcessingError { + var packageId: Package.Id + var underlyingError: UnderlyingError + + var description: String { + "Ingestion.Error(\(packageId), \(underlyingError))" + } + + enum UnderlyingError: Swift.Error, CustomStringConvertible { + case fetchMetadataFailed(owner: String, name: String, details: String) + case findOrCreateRepositoryFailed(url: String, details: Swift.Error) + case invalidURL(String) + case noRepositoryMetadata(owner: String?, name: String?) + case repositorySaveFailed(owner: String?, name: String?, details: String) + case repositorySaveUniqueViolation(owner: String?, name: String?, details: String) + + var description: String { + switch self { + case let .fetchMetadataFailed(owner, name, details): + "fetchMetadataFailed(\(owner), \(name), \(details))" + case let .findOrCreateRepositoryFailed(url, details): + "findOrCreateRepositoryFailed(\(url), \(details))" + case let .invalidURL(url): + "invalidURL(\(url))" + case let .noRepositoryMetadata(owner, name): + "noRepositoryMetadata(\(owner), \(name))" + case let .repositorySaveFailed(owner, name, details): + "repositorySaveFailed(\(owner), \(name), \(details))" + case let .repositorySaveUniqueViolation(owner, name, details): + "repositorySaveUniqueViolation(\(owner), \(name), \(details))" + } + } + } + + var level: Logger.Level { + switch underlyingError { + case .fetchMetadataFailed, .invalidURL, .noRepositoryMetadata: + return .warning + case .findOrCreateRepositoryFailed, .repositorySaveFailed, .repositorySaveUniqueViolation: + return .critical + } + } + + var status: Package.Status { + switch underlyingError { + case .fetchMetadataFailed, .findOrCreateRepositoryFailed, .noRepositoryMetadata, .repositorySaveFailed: + return .ingestionFailed + case .invalidURL: + return .invalidUrl + case .repositorySaveUniqueViolation: + return .ingestionFailed + } + } + } +} struct IngestCommand: AsyncCommand { @@ -21,7 +81,7 @@ struct IngestCommand: AsyncCommand { var help: String { "Run package ingestion (fetching repository metadata)" } - func run(using context: CommandContext, signature: SPICommand.Signature) async throws { + func run(using context: CommandContext, signature: SPICommand.Signature) async { let client = context.application.client let db = context.application.db Current.setLogger(Logger(component: "ingest")) @@ -98,74 +158,133 @@ func ingest(client: Client, await withTaskGroup(of: Void.self) { group in for pkg in packages { - group.addTask { - let result = await Result { - Current.logger().info("Ingesting \(pkg.package.url)") - let (metadata, license, readme) = try await fetchMetadata(client: client, package: pkg) - let repo = try await Repository.findOrCreate(on: database, for: pkg.model) - - let s3Readme: S3Readme? - do { - if let upstreamEtag = readme?.etag, - repo.s3Readme?.needsUpdate(upstreamEtag: upstreamEtag) ?? true, - let owner = metadata.repositoryOwner, - let repository = metadata.repositoryName, - let html = readme?.html { - let objectUrl = try await Current.storeS3Readme(owner, repository, html) - if let imagesToCache = readme?.imagesToCache, imagesToCache.isEmpty == false { - try await Current.storeS3ReadmeImages(client, imagesToCache) - } - s3Readme = .cached(s3ObjectUrl: objectUrl, githubEtag: upstreamEtag) - } else { - s3Readme = repo.s3Readme - } - } catch { - // We don't want to fail ingestion in case storing the readme fails - warn and continue. - Current.logger().warning("storeS3Readme failed") - s3Readme = .error("\(error)") - } - - let fork = await getFork(on: database, parent: metadata.repository?.parent) - - try await updateRepository(on: database, - for: repo, - metadata: metadata, - licenseInfo: license, - readmeInfo: readme, - s3Readme: s3Readme, - fork: fork) - return pkg - } + group.addTask { + await Ingestion.ingest(client: client, database: database, package: pkg) + } + } + } +} - switch result { - case .success: - AppMetrics.ingestMetadataSuccessCount?.inc() - case .failure: - AppMetrics.ingestMetadataFailureCount?.inc() - } - do { - try await updatePackage(client: client, database: database, result: result, stage: .ingestion) - } catch { - Current.logger().report(error: error) +extension Ingestion { + static func ingest(client: Client, database: Database, package: Joined) async { + let result = await Result { () async throws(Ingestion.Error) -> Joined in + @Dependency(\.environment) var environment + Current.logger().info("Ingesting \(package.package.url)") + + // Even though we have a `Joined` as a parameter, we must not rely + // on `repository` for owner/name as it will be nil when a package is first ingested. + // The only way to get `owner` and `repository` here is by parsing them from the URL. + let (owner, repository) = try await run { + if environment.shouldFail(failureMode: .invalidURL) { + throw Github.Error.invalidURL(package.model.url) } + return try Github.parseOwnerName(url: package.model.url) + } rethrowing: { _ in + Ingestion.Error.invalidURL(packageId: package.model.id!, url: package.model.url) + } + + let (metadata, license, readme) = try await run { + try await fetchMetadata(client: client, package: package.model, owner: owner, repository: repository) + } rethrowing: { + Ingestion.Error(packageId: package.model.id!, + underlyingError: .fetchMetadataFailed(owner: owner, name: repository, details: "\($0)")) + } + let repo = try await findOrCreateRepository(on: database, for: package) + + let s3Readme: S3Readme? + do throws(S3Readme.Error) { + s3Readme = try await storeS3Readme(client: client, repository: repo, metadata: metadata, readme: readme) + } catch { + // We don't want to fail ingestion in case storing the readme fails - warn and continue. + Current.logger().warning("storeS3Readme failed: \(error)") + s3Readme = .error("\(error)") + } + + let fork = await getFork(on: database, parent: metadata.repository?.parent) + + try await run { () async throws(Ingestion.Error.UnderlyingError) in + try await updateRepository(on: database, for: repo, metadata: metadata, licenseInfo: license, readmeInfo: readme, s3Readme: s3Readme, fork: fork) + } rethrowing: { + Ingestion.Error(packageId: package.model.id!, underlyingError: $0) } + return package + } + + switch result { + case .success: + AppMetrics.ingestMetadataSuccessCount?.inc() + case .failure: + AppMetrics.ingestMetadataFailureCount?.inc() + } + + do { + try await updatePackage(client: client, database: database, result: result, stage: .ingestion) + } catch { + Current.logger().report(error: error) } } -} -func fetchMetadata(client: Client, package: Joined) async throws -> (Github.Metadata, Github.License?, Github.Readme?) { - // Even though we get through a `Joined` as a parameter, it's - // we must not rely on `repository` as it will be nil when a package is first ingested. - // The only way to get `owner` and `repository` here is by parsing them from the URL. - let (owner, repository) = try Github.parseOwnerName(url: package.model.url) + static func findOrCreateRepository(on database: Database, for package: Joined) async throws(Ingestion.Error) -> Repository { + try await run { + @Dependency(\.environment) var environment + if environment.shouldFail(failureMode: .findOrCreateRepositoryFailed) { + throw Abort(.internalServerError) + } + + return try await Repository.findOrCreate(on: database, for: package.model) + } rethrowing: { + Ingestion.Error( + packageId: package.model.id!, + underlyingError: .findOrCreateRepositoryFailed(url: package.model.url, details: $0) + ) + } + } + + + static func storeS3Readme(client: Client, repository: Repository, metadata: Github.Metadata, readme: Github.Readme?) async throws(S3Readme.Error) -> S3Readme? { + if let upstreamEtag = readme?.etag, + repository.s3Readme?.needsUpdate(upstreamEtag: upstreamEtag) ?? true, + let owner = metadata.repositoryOwner, + let repository = metadata.repositoryName, + let html = readme?.html { + let objectUrl = try await Current.storeS3Readme(owner, repository, html) + if let imagesToCache = readme?.imagesToCache, imagesToCache.isEmpty == false { + try await Current.storeS3ReadmeImages(client, imagesToCache) + } + return .cached(s3ObjectUrl: objectUrl, githubEtag: upstreamEtag) + } else { + return repository.s3Readme + } + } - async let metadata = try await Current.fetchMetadata(client, owner, repository) - async let license = await Current.fetchLicense(client, owner, repository) - async let readme = await Current.fetchReadme(client, owner, repository) - return try await (metadata, license, readme) + static func fetchMetadata(client: Client, package: Package, owner: String, repository: String) async throws(Github.Error) -> (Github.Metadata, Github.License?, Github.Readme?) { + @Dependency(\.environment) var environment + if environment.shouldFail(failureMode: .fetchMetadataFailed) { + throw Github.Error.requestFailed(.internalServerError) + } + + async let metadata = try await Current.fetchMetadata(client, owner, repository) + async let license = await Current.fetchLicense(client, owner, repository) + async let readme = await Current.fetchReadme(client, owner, repository) + + do { + return try await (metadata, license, readme) + } catch let error as Github.Error { + throw error + } catch { + // This whole do { ... } catch { ... } should be unnecessary - it's a workaround for + // https://github.com/swiftlang/swift/issues/76169 + assert(false, "Unexpected error type: \(type(of: error))") + // We need to throw _something_ here (we should never hit this codepath though) + throw Github.Error.requestFailed(.internalServerError) + // We could theoretically avoid this whole second catch and just do + // error as! GithubError + // but let's play it safe and not risk a server crash, unlikely as it may be. + } + } } @@ -181,13 +300,23 @@ func updateRepository(on database: Database, licenseInfo: Github.License?, readmeInfo: Github.Readme?, s3Readme: S3Readme?, - fork: Fork? = nil) async throws { + fork: Fork? = nil) async throws(Ingestion.Error.UnderlyingError) { + @Dependency(\.environment) var environment + if environment.shouldFail(failureMode: .noRepositoryMetadata) { + throw .noRepositoryMetadata(owner: repository.owner, name: repository.name) + } + if environment.shouldFail(failureMode: .repositorySaveFailed) { + throw .repositorySaveFailed(owner: repository.owner, + name: repository.name, + details: "TestError") + } + if environment.shouldFail(failureMode: .repositorySaveUniqueViolation) { + throw .repositorySaveUniqueViolation(owner: repository.owner, + name: repository.name, + details: "TestError") + } guard let repoMetadata = metadata.repository else { - if repository.$package.value == nil { - try await repository.$package.load(on: database) - } - throw AppError.genericError(repository.package.id, - "repository metadata is nil for package \(repository.name ?? "unknown")") + throw .noRepositoryMetadata(owner: repository.owner, name: repository.name) } repository.defaultBranch = repoMetadata.defaultBranch @@ -215,7 +344,23 @@ func updateRepository(on database: Database, repository.summary = repoMetadata.description repository.forkedFrom = fork - try await repository.save(on: database) + do { + try await repository.save(on: database) + } catch let error as PSQLError where error.isUniqueViolation { + let details = error.serverInfo?[.message] ?? "" + throw Ingestion.Error.UnderlyingError.repositorySaveUniqueViolation(owner: repository.owner, + name: repository.name, + details: details) + } catch let error as PSQLError { + let details = error.serverInfo?[.message] ?? "" + throw Ingestion.Error.UnderlyingError.repositorySaveFailed(owner: repository.owner, + name: repository.name, + details: details) + } catch { + throw Ingestion.Error.UnderlyingError.repositorySaveFailed(owner: repository.owner, + name: repository.name, + details: "\(error)") + } } func getFork(on database: Database, parent: Github.Metadata.Parent?) async -> Fork? { @@ -252,3 +397,9 @@ private extension Github.Metadata.Parent { return normalizedURL } } + +private extension Ingestion.Error { + static func invalidURL(packageId: Package.Id, url: String) -> Self { + Ingestion.Error(packageId: packageId, underlyingError: .invalidURL(url)) + } +} diff --git a/Sources/App/Core/AppEnvironment.swift b/Sources/App/Core/AppEnvironment.swift index f71d5e11b..196451658 100644 --- a/Sources/App/Core/AppEnvironment.swift +++ b/Sources/App/Core/AppEnvironment.swift @@ -25,7 +25,7 @@ import FoundationNetworking struct AppEnvironment: Sendable { var fetchHTTPStatusCode: @Sendable (_ url: String) async throws -> HTTPStatus var fetchLicense: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.License? - var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> Github.Metadata + var fetchMetadata: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws(Github.Error) -> Github.Metadata var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme? var fetchS3Readme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> String var fileManager: FileManager @@ -51,9 +51,9 @@ struct AppEnvironment: Sendable { var siteURL: @Sendable () -> String var storeS3Readme: @Sendable (_ owner: String, _ repository: String, - _ readme: String) async throws -> String + _ readme: String) async throws(S3Readme.Error) -> String var storeS3ReadmeImages: @Sendable (_ client: Client, - _ imagesToCache: [Github.Readme.ImageToCache]) async throws -> Void + _ imagesToCache: [Github.Readme.ImageToCache]) async throws(S3Readme.Error) -> Void var timeZone: @Sendable () -> TimeZone var triggerBuild: @Sendable (_ client: Client, _ buildId: Build.Id, @@ -86,9 +86,9 @@ extension AppEnvironment { static let live = AppEnvironment( fetchHTTPStatusCode: { url in try await Networking.fetchHTTPStatusCode(url) }, fetchLicense: { client, owner, repo in await Github.fetchLicense(client:client, owner: owner, repository: repo) }, - fetchMetadata: { client, owner, repo in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) }, + fetchMetadata: { client, owner, repo throws(Github.Error) in try await Github.fetchMetadata(client:client, owner: owner, repository: repo) }, fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) }, - fetchS3Readme: { client, owner, repo in try await S3Store.fetchReadme(client:client, owner: owner, repository: repo) }, + fetchS3Readme: { client, owner, repo in try await S3Readme.fetchReadme(client:client, owner: owner, repository: repo) }, fileManager: .live, getStatusCount: { client, status in try await Gitlab.Builder.getStatusCount(client: client, @@ -131,8 +131,12 @@ extension AppEnvironment { setLogger: { logger in Self.logger = logger }, shell: .live, siteURL: { Environment.get("SITE_URL") ?? "http://localhost:8080" }, - storeS3Readme: { owner, repo, readme in try await S3Store.storeReadme(owner: owner, repository: repo, readme: readme) }, - storeS3ReadmeImages: { client, images in try await S3Store.storeReadmeImages(client: client, imagesToCache: images) }, + storeS3Readme: { owner, repo, readme throws(S3Readme.Error) in + try await S3Readme.storeReadme(owner: owner, repository: repo, readme: readme) + }, + storeS3ReadmeImages: { client, images throws(S3Readme.Error) in + try await S3Readme.storeReadmeImages(client: client, imagesToCache: images) + }, timeZone: { .current }, triggerBuild: { client, buildId, cloneURL, isDocBuild, platform, ref, swiftVersion, versionID in try await Gitlab.Builder.triggerBuild(client: client, diff --git a/Sources/App/Core/Dependencies/EnvironmentClient.swift b/Sources/App/Core/Dependencies/EnvironmentClient.swift index b3da31ab7..166993620 100644 --- a/Sources/App/Core/Dependencies/EnvironmentClient.swift +++ b/Sources/App/Core/Dependencies/EnvironmentClient.swift @@ -44,6 +44,16 @@ struct EnvironmentClient { var mastodonCredentials: @Sendable () -> Mastodon.Credentials? var mastodonPost: @Sendable (_ client: Client, _ post: String) async throws -> Void var random: @Sendable (_ range: ClosedRange) -> Double = { XCTFail("random"); return Double.random(in: $0) } + + enum FailureMode: String { + case fetchMetadataFailed + case findOrCreateRepositoryFailed + case invalidURL + case noRepositoryMetadata + case repositorySaveFailed + case repositorySaveUniqueViolation + } + var shouldFail: @Sendable (_ failureMode: FailureMode) -> Bool = { _ in false } } @@ -100,7 +110,14 @@ extension EnvironmentClient: DependencyKey { .map(Mastodon.Credentials.init(accessToken:)) }, mastodonPost: { client, message in try await Mastodon.post(client: client, message: message) }, - random: { range in Double.random(in: range) } + random: { range in Double.random(in: range) }, + shouldFail: { failureMode in + let shouldFail = Environment.get("FAILURE_MODE") + .map { Data($0.utf8) } + .flatMap { try? JSONDecoder().decode([String: Double].self, from: $0) } ?? [:] + guard let rate = shouldFail[failureMode.rawValue] else { return false } + return Double.random(in: 0...1) <= rate + } ) } } diff --git a/Sources/App/Core/AsyncDefer.swift b/Sources/App/Core/ErrorHandlingHelpers.swift similarity index 69% rename from Sources/App/Core/AsyncDefer.swift rename to Sources/App/Core/ErrorHandlingHelpers.swift index 50bc1dedd..6436b0fbf 100644 --- a/Sources/App/Core/AsyncDefer.swift +++ b/Sources/App/Core/ErrorHandlingHelpers.swift @@ -15,7 +15,7 @@ @discardableResult func run(_ operation: () async throws -> T, - defer deferredOperation: () async throws -> Void) async throws -> T { + defer deferredOperation: () async throws -> Void) async throws -> T { do { let result = try await operation() try await deferredOperation() @@ -25,3 +25,15 @@ func run(_ operation: () async throws -> T, throw error } } + + +@discardableResult +func run(_ operation: () async throws(E1) -> T, + rethrowing transform: (E1) -> E2) async throws(E2) -> T { + do { + let result = try await operation() + return result + } catch { + throw transform(error) + } +} diff --git a/Sources/App/Core/Extensions/S3Readme+ext.swift b/Sources/App/Core/Extensions/S3Readme+ext.swift new file mode 100644 index 000000000..99aab319d --- /dev/null +++ b/Sources/App/Core/Extensions/S3Readme+ext.swift @@ -0,0 +1,78 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Dependencies +import S3Store +import Vapor + + +extension S3Readme { + enum Error: Swift.Error { + case envVariableNotSet(String) + case invalidURL(String) + case missingBody + case requestFailed(key: S3Store.Key, error: Swift.Error) + case storeReadmeFailed + case storeImagesFailed + } + + static func fetchReadme(client: Client, owner: String, repository: String) async throws(S3Readme.Error) -> String { + let key = try S3Store.Key.readme(owner: owner, repository: repository) + let response: ClientResponse + do { + response = try await client.get(URI(string: key.objectUrl)) + } catch { + throw .requestFailed(key: key, error: error) + } + guard let body = response.body else { throw .missingBody } + return body.asString() + } + + static func storeReadme(owner: String, repository: String, readme: String) async throws(S3Readme.Error) -> String { + @Dependency(\.environment) var environment + guard let accessKeyId = environment.awsAccessKeyId() else { throw .envVariableNotSet("AWS_ACCESS_KEY_ID") } + guard let secretAccessKey = environment.awsSecretAccessKey() else { throw .envVariableNotSet("AWS_SECRET_ACCESS_KEY")} + let store = S3Store(credentials: .init(keyId: accessKeyId, secret: secretAccessKey)) + let key = try S3Store.Key.readme(owner: owner, repository: repository) + + Current.logger().debug("Copying readme to \(key.s3Uri) ...") + do { + try await store.save(payload: readme, to: key) + } catch { + throw .requestFailed(key: key, error: error) + } + + return key.objectUrl + } + + static func storeReadmeImages(client: Client, imagesToCache: [Github.Readme.ImageToCache]) async throws(S3Readme.Error) { + @Dependency(\.environment) var environment + guard let accessKeyId = environment.awsAccessKeyId() else { throw .envVariableNotSet("AWS_ACCESS_KEY_ID") } + guard let secretAccessKey = environment.awsSecretAccessKey() else { throw .envVariableNotSet("AWS_SECRET_ACCESS_KEY")} + + let store = S3Store(credentials: .init(keyId: accessKeyId, secret: secretAccessKey)) + for imageToCache in imagesToCache { + Current.logger().debug("Copying readme image to \(imageToCache.s3Key.s3Uri) ...") + do { + let response = try await client.get(URI(stringLiteral: imageToCache.originalUrl)) + if var body = response.body, let imageData = body.readData(length: body.readableBytes) { + try await store.save(payload: imageData, to: imageToCache.s3Key) + } + } catch { + throw .requestFailed(key: imageToCache.s3Key, error: error) + } + } + } + +} diff --git a/Sources/App/Core/Extensions/S3Store+ext.swift b/Sources/App/Core/Extensions/S3Store+ext.swift index 83e35b87f..4f0ee6a60 100644 --- a/Sources/App/Core/Extensions/S3Store+ext.swift +++ b/Sources/App/Core/Extensions/S3Store+ext.swift @@ -12,68 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -import S3Store -import Vapor -import Dependencies - - -extension S3Store { - - static func fetchReadme(client: Client, owner: String, repository: String) async throws -> String { - let key = try Key.readme(owner: owner, repository: repository) - guard let body = try await client.get(URI(string: key.objectUrl)).body else { - throw Error.genericError("No body") - } - return body.asString() - } - - static func storeReadme(owner: String, repository: String, readme: String) async throws -> String { - @Dependency(\.environment) var environment - guard let accessKeyId = environment.awsAccessKeyId(), - let secretAccessKey = environment.awsSecretAccessKey() - else { - throw Error.genericError("missing AWS credentials") - } - let store = S3Store(credentials: .init(keyId: accessKeyId, secret: secretAccessKey)) - let key = try Key.readme(owner: owner, repository: repository) - - Current.logger().debug("Copying readme to \(key.s3Uri) ...") - try await store.save(payload: readme, to: key) +import Foundation - return key.objectUrl - } - - static func storeReadmeImages(client: Client, imagesToCache: [Github.Readme.ImageToCache]) async throws { - @Dependency(\.environment) var environment - guard let accessKeyId = environment.awsAccessKeyId(), - let secretAccessKey = environment.awsSecretAccessKey() - else { - throw Error.genericError("missing AWS credentials") - } - - let store = S3Store(credentials: .init(keyId: accessKeyId, secret: secretAccessKey)) - for imageToCache in imagesToCache { - Current.logger().debug("Copying readme image to \(imageToCache.s3Key.s3Uri) ...") - let response = try await client.get(URI(stringLiteral: imageToCache.originalUrl)) - if var body = response.body, let imageData = body.readData(length: body.readableBytes) { - try await store.save(payload: imageData, to: imageToCache.s3Key) - } - } - } - -} +import Dependencies +import S3Store extension S3Store.Key { - static func readme(owner: String, repository: String, imageUrl: String? = nil) throws -> Self { + static func readme(owner: String, repository: String, imageUrl: String? = nil) throws(S3Readme.Error) -> Self { @Dependency(\.environment) var environment - guard let bucket = environment.awsReadmeBucket() else { - throw S3Store.Error.genericError("AWS_README_BUCKET not set") - } + guard let bucket = environment.awsReadmeBucket() else { throw .envVariableNotSet("AWS_README_BUCKET") } if let imageUrl { - guard let url = URL(string: imageUrl) - else { throw S3Store.Error.genericError("Invalid imageUrl \(imageUrl)") } + guard let url = URL(string: imageUrl) else { throw .invalidURL(imageUrl) } let filename = url.lastPathComponent let path = "\(owner)/\(repository)/\(filename)".lowercased() return .init(bucket: bucket, path: path) diff --git a/Sources/App/Core/Github.swift b/Sources/App/Core/Github.swift index 8bccce4f2..d4f0c0cfc 100644 --- a/Sources/App/Core/Github.swift +++ b/Sources/App/Core/Github.swift @@ -19,24 +19,13 @@ import S3Store enum Github { - enum Error: LocalizedError { + enum Error: Swift.Error { + case decodeContentFailed(URI, Swift.Error) case missingToken case noBody - case invalidURI(Package.Id?, _ url: String) + case invalidURL(String) + case postRequestFailed(URI, Swift.Error) case requestFailed(HTTPStatus) - - var errorDescription: String? { - switch self { - case .missingToken: - return "missing Github API token" - case .noBody: - return "no body" - case let .invalidURI(id, url): - return "invalid URL: \(url) (id: \(id?.uuidString ?? "nil"))" - case .requestFailed(let statusCode): - return "request failed with status code: \(statusCode)" - } - } } static var decoder: JSONDecoder { @@ -60,13 +49,13 @@ enum Github { return response.status == .forbidden && limit == 0 } - static func parseOwnerName(url: String) throws -> (owner: String, name: String) { + static func parseOwnerName(url: String) throws(Github.Error) -> (owner: String, name: String) { let parts = url .droppingGithubComPrefix .droppingGitExtension .split(separator: "/") .map(String.init) - guard parts.count == 2 else { throw Error.invalidURI(nil, url) } + guard parts.count == 2 else { throw Error.invalidURL(url) } return (owner: parts[0], name: parts[1]) } @@ -181,13 +170,18 @@ extension Github { var query: String } - static func fetchResource(_ type: T.Type, client: Client, query: GraphQLQuery) async throws -> T { + static func fetchResource(_ type: T.Type, client: Client, query: GraphQLQuery) async throws(Github.Error) -> T { guard let token = Current.githubToken() else { throw Error.missingToken } - let response = try await client.post(Self.graphQLApiUri, headers: defaultHeaders(with: token)) { - try $0.content.encode(query) + let response: ClientResponse + do { + response = try await client.post(Self.graphQLApiUri, headers: defaultHeaders(with: token)) { + try $0.content.encode(query) + } + } catch { + throw .postRequestFailed(Self.graphQLApiUri, error) } guard !isRateLimited(response) else { @@ -200,10 +194,14 @@ extension Github { throw Error.requestFailed(response.status) } - return try response.content.decode(T.self, using: decoder) + do { + return try response.content.decode(T.self, using: decoder) + } catch { + throw .decodeContentFailed(Self.graphQLApiUri, error) + } } - static func fetchMetadata(client: Client, owner: String, repository: String) async throws -> Metadata { + static func fetchMetadata(client: Client, owner: String, repository: String) async throws(Github.Error) -> Metadata { struct Response: Decodable, Equatable { var data: T } diff --git a/Sources/App/Models/Package.swift b/Sources/App/Models/Package.swift index 76f87c22e..ff6a914a8 100644 --- a/Sources/App/Models/Package.swift +++ b/Sources/App/Models/Package.swift @@ -267,6 +267,21 @@ extension Package { """# ).run() } + + static func update(for id: Package.Id, on database: Database, + status: Status, stage: ProcessingStage) async throws { + try await Package.query(on: database) + .filter(\.$id == id) + .set(\.$processingStage, to: stage) + .set(\.$status, to: status) + .update() + } + + func update(on database: Database, status: Status, stage: ProcessingStage) async throws { + self.status = status + self.processingStage = stage + try await update(on: database) + } } diff --git a/Sources/S3Store/S3Store.swift b/Sources/S3Store/S3Store.swift index f101e19d7..3d743544c 100644 --- a/Sources/S3Store/S3Store.swift +++ b/Sources/S3Store/S3Store.swift @@ -61,7 +61,7 @@ extension S3Store { } } - public struct Key: Equatable { + public struct Key: Equatable, Sendable { public let bucket: String public let path: String diff --git a/Tests/AppTests/AnalyzerTests.swift b/Tests/AppTests/AnalyzerTests.swift index 6d2ced970..f72888e57 100644 --- a/Tests/AppTests/AnalyzerTests.swift +++ b/Tests/AppTests/AnalyzerTests.swift @@ -869,10 +869,7 @@ class AnalyzerTests: AppTestCase { ] // MUT - try await updatePackages(client: app.client, - database: app.db, - results: results, - stage: .analysis) + try await Analyze.updatePackages(client: app.client, database: app.db, results: results) // validate do { diff --git a/Tests/AppTests/ErrorReportingTests.swift b/Tests/AppTests/ErrorReportingTests.swift index 64e776b93..8bee183b5 100644 --- a/Tests/AppTests/ErrorReportingTests.swift +++ b/Tests/AppTests/ErrorReportingTests.swift @@ -20,22 +20,21 @@ import XCTVapor class ErrorReportingTests: AppTestCase { - func test_recordError() async throws { + func test_Analyze_recordError() async throws { let pkg = try await savePackage(on: app.db, "1") - try await recordError(database: app.db, - error: AppError.cacheDirectoryDoesNotExist(pkg.id, "path"), - stage: .ingestion) + try await Analyze.recordError(database: app.db, + error: AppError.cacheDirectoryDoesNotExist(pkg.id, "path")) do { let pkg = try await XCTUnwrapAsync(try await Package.find(pkg.id, on: app.db)) XCTAssertEqual(pkg.status, .cacheDirectoryDoesNotExist) - XCTAssertEqual(pkg.processingStage, .ingestion) + XCTAssertEqual(pkg.processingStage, .analysis) } } func test_Ingestor_error_reporting() async throws { // setup - try await Package(url: "1", processingStage: .reconciliation).save(on: app.db) - Current.fetchMetadata = { _, _, _ in throw Github.Error.invalidURI(nil, "1") } + try await Package(id: .id0, url: "1", processingStage: .reconciliation).save(on: app.db) + Current.fetchMetadata = { _, _, _ throws(Github.Error) in throw Github.Error.invalidURL("1") } try await withDependencies { $0.date.now = .now @@ -47,7 +46,7 @@ class ErrorReportingTests: AppTestCase { // validation logger.logs.withValue { XCTAssertEqual($0, [.init(level: .warning, - message: #"App.Github.Error.invalidURI(nil, "1")"#)]) + message: #"Ingestion.Error(\#(UUID.id0), invalidURL(1))"#)]) } } diff --git a/Tests/AppTests/GithubTests.swift b/Tests/AppTests/GithubTests.swift index 254f34842..87c6c54af 100644 --- a/Tests/AppTests/GithubTests.swift +++ b/Tests/AppTests/GithubTests.swift @@ -35,11 +35,13 @@ class GithubTests: AppTestCase { XCTAssertEqual(res.owner, "foo") XCTAssertEqual(res.name, "bar") } - XCTAssertThrowsError( - try Github.parseOwnerName(url: "https://github.com/foo/bar/baz") - ) { error in - XCTAssertEqual(error.localizedDescription, - "invalid URL: https://github.com/foo/bar/baz (id: nil)") + do { + _ = try Github.parseOwnerName(url: "https://github.com/foo/bar/baz") + XCTFail("Expected error") + } catch let Github.Error.invalidURL(url) { + XCTAssertEqual(url, "https://github.com/foo/bar/baz") + } catch { + XCTFail("Unexpected error: \(error)") } } @@ -203,7 +205,7 @@ class GithubTests: AppTestCase { _ = try await Github.fetchMetadata(client: client, packageUrl: pkg.url) XCTFail("expected error to be thrown") } catch { - guard case Github.Error.invalidURI = error else { + guard case Github.Error.invalidURL = error else { XCTFail("unexpected error: \(error.localizedDescription)") return } @@ -223,12 +225,15 @@ class GithubTests: AppTestCase { do { _ = try await Github.fetchMetadata(client: client, packageUrl: pkg.url) XCTFail("expected error to be thrown") - } catch { + } catch let Github.Error.decodeContentFailed(uri, error) { // validation + XCTAssertEqual(uri, "https://api.github.com/graphql") guard case DecodingError.dataCorrupted = error else { XCTFail("unexpected error: \(error.localizedDescription)") return } + } catch { + XCTFail("Unexpected error: \(error)") } } diff --git a/Tests/AppTests/IngestorTests.swift b/Tests/AppTests/IngestorTests.swift index e63d8a82b..a2374419c 100644 --- a/Tests/AppTests/IngestorTests.swift +++ b/Tests/AppTests/IngestorTests.swift @@ -65,16 +65,12 @@ class IngestorTests: AppTestCase { func test_ingest_continue_on_error() async throws { // Test completion of ingestion despite early error // setup - enum TestError: Error, Equatable { - case badRequest - } - let packages = try await savePackages(on: app.db, ["https://github.com/foo/1", - "https://github.com/foo/2"]) + "https://github.com/foo/2"], processingStage: .reconciliation) .map(Joined.init(model:)) - Current.fetchMetadata = { _, owner, repository in + Current.fetchMetadata = { _, owner, repository throws(Github.Error) in if owner == "foo" && repository == "1" { - throw TestError.badRequest + throw Github.Error.requestFailed(.badRequest) } return .mock(owner: owner, repository: repository) } @@ -83,12 +79,17 @@ class IngestorTests: AppTestCase { // MUT await ingest(client: app.client, database: app.db, packages: packages) - // validate the second package's license is updated - let repo = try await Repository.query(on: app.db) - .filter(\.$name == "2") - .first() - .unwrap() - XCTAssertEqual(repo.licenseUrl, "license") + do { + // validate the second package's license is updated + let repo = try await Repository.query(on: app.db) + .filter(\.$name == "2") + .first() + .unwrap() + XCTAssertEqual(repo.licenseUrl, "license") + for pkg in try await Package.query(on: app.db).all() { + XCTAssertEqual(pkg.processingStage, .ingestion, "\(pkg.url) must be in ingestion") + } + } } func test_updateRepository_insert() async throws { @@ -249,16 +250,19 @@ class IngestorTests: AppTestCase { let pkgs = try await savePackages(on: app.db, ["https://github.com/foo/1", "https://github.com/foo/2"]) .map(Joined.init(model:)) - let results: [Result, Error>] = [ - .failure(AppError.genericError(try pkgs[0].model.requireID(), "error 1")), + let pkgId0 = try pkgs[0].model.requireID() + let results: [Result, Ingestion.Error>] = [ + .failure(.init(packageId: pkgId0, underlyingError: .fetchMetadataFailed(owner: "", name: "", details: ""))), .success(pkgs[1]) ] // MUT - try await updatePackages(client: app.client, - database: app.db, - results: results, - stage: .ingestion) + for result in results { + try await Ingestion.updatePackage(client: app.client, + database: app.db, + result: result, + stage: .ingestion) + } // validate do { @@ -268,7 +272,7 @@ class IngestorTests: AppTestCase { } } - func test_updatePackages_new() async throws { + func test_updatePackage_new() async throws { // Ensure newly ingested packages are passed on with status = new to fast-track // them into analysis let pkgs = [ @@ -276,14 +280,16 @@ class IngestorTests: AppTestCase { Package(id: UUID(), url: "https://github.com/foo/2", status: .new, processingStage: .reconciliation) ] try await pkgs.save(on: app.db) - let results: [Result, Error>] = [ .success(.init(model: pkgs[0])), - .success(.init(model: pkgs[1]))] + let results: [Result, Ingestion.Error>] = [ .success(.init(model: pkgs[0])), + .success(.init(model: pkgs[1]))] // MUT - try await updatePackages(client: app.client, - database: app.db, - results: results, - stage: .ingestion) + for result in results { + try await Ingestion.updatePackage(client: app.client, + database: app.db, + result: result, + stage: .ingestion) + } // validate do { @@ -318,11 +324,10 @@ class IngestorTests: AppTestCase { let urls = ["https://github.com/foo/1", "https://github.com/foo/2", "https://github.com/foo/3"] - let packages = try await savePackages(on: app.db, urls.asURLs, - processingStage: .reconciliation) - Current.fetchMetadata = { _, owner, repository in + try await savePackages(on: app.db, urls.asURLs, processingStage: .reconciliation) + Current.fetchMetadata = { _, owner, repository throws(Github.Error) in if owner == "foo" && repository == "2" { - throw AppError.genericError(packages[1].id, "error 2") + throw Github.Error.requestFailed(.badRequest) } return .mock(owner: owner, repository: repository) } @@ -354,12 +359,12 @@ class IngestorTests: AppTestCase { func test_ingest_unique_owner_name_violation() async throws { // Test error behaviour when two packages resolving to the same owner/name are ingested: - // - don't update package // - don't create repository records // setup - for url in ["https://github.com/foo/1", "https://github.com/foo/2"].asURLs { - try await Package(url: url, processingStage: .reconciliation).save(on: app.db) - } + try await Package(id: .id0, url: "https://github.com/foo/0", status: .ok, processingStage: .reconciliation) + .save(on: app.db) + try await Package(id: .id1, url: "https://github.com/foo/1", status: .ok, processingStage: .reconciliation) + .save(on: app.db) // Return identical metadata for both packages, same as a for instance a redirected // package would after a rename / ownership change Current.fetchMetadata = { _, _, _ in @@ -379,7 +384,6 @@ class IngestorTests: AppTestCase { stars: 0, summary: "desc") } - let lastUpdate = Date() try await withDependencies { $0.date.now = .now @@ -390,31 +394,51 @@ class IngestorTests: AppTestCase { // validate repositories (single element pointing to the ingested package) let repos = try await Repository.query(on: app.db).all() - let ingested = try await Package.query(on: app.db) - .filter(\.$processingStage == .ingestion) + XCTAssertEqual(repos.count, 1) + + // validate packages - one should have succeeded, one should have failed + let succeeded = try await Package.query(on: app.db) + .filter(\.$status == .ok) .first() .unwrap() - XCTAssertEqual(repos.map(\.$package.id), [try ingested.requireID()]) - - // validate packages - let reconciled = try await Package.query(on: app.db) - .filter(\.$processingStage == .reconciliation) + let failed = try await Package.query(on: app.db) + .filter(\.$status == .ingestionFailed) .first() .unwrap() - // the ingested package has the update ... - XCTAssertEqual(ingested.status, .new) - XCTAssertEqual(ingested.processingStage, .ingestion) - XCTAssert(ingested.updatedAt! > lastUpdate) - // ... while the reconciled package remains unchanged ... - XCTAssertEqual(reconciled.status, .new) - XCTAssertEqual(reconciled.processingStage, .reconciliation) - XCTAssert(reconciled.updatedAt! < lastUpdate) - // ... and an error has been logged + XCTAssertEqual(succeeded.processingStage, .ingestion) + XCTAssertEqual(failed.processingStage, .ingestion) + // an error must have been logged try logger.logs.withValue { logs in XCTAssertEqual(logs.count, 1) let log = try XCTUnwrap(logs.first) XCTAssertEqual(log.level, .critical) - XCTAssertEqual(log.message, #"duplicate key value violates unique constraint "idx_repositories_owner_name""#) + XCTAssertEqual(log.message, #"Ingestion.Error(\#(try failed.requireID()), repositorySaveUniqueViolation(owner, name, duplicate key value violates unique constraint "idx_repositories_owner_name"))"#) + } + + // ensure analysis can process these packages + try await withDependencies { + $0.date.now = .now + $0.environment.allowSocialPosts = { false } + } operation: { [db = app.db] in + Current.fileManager.fileExists = { @Sendable _ in true } + Current.git.commitCount = { @Sendable _ in 1 } + Current.git.firstCommitDate = { @Sendable _ in .t0 } + Current.git.lastCommitDate = { @Sendable _ in .t0 } + Current.git.getTags = { @Sendable _ in [] } + Current.git.hasBranch = { @Sendable _, _ in true } + Current.git.revisionInfo = { @Sendable _, _ in .init(commit: "sha0", date: .t0) } + Current.git.shortlog = { @Sendable _ in "" } + Current.shell.run = { @Sendable cmd, _ in + if cmd.description.hasSuffix("package dump-package") { + return .packageDump(name: "foo") + } + return "" + } + + try await Analyze.analyze(client: app.client, database: db, mode: .id(.id0)) + try await Analyze.analyze(client: app.client, database: db, mode: .id(.id1)) + try await XCTAssertEqualAsync(try await Package.find(.id0, on: db)?.processingStage, .analysis) + try await XCTAssertEqualAsync(try await Package.find(.id1, on: db)?.processingStage, .analysis) } } @@ -569,10 +593,9 @@ class IngestorTests: AppTestCase { imagesToCache: []) } let storeCalls = QueueIsolated(0) - struct Error: Swift.Error { } - Current.storeS3Readme = { owner, repo, html in + Current.storeS3Readme = { owner, repo, html throws(S3Readme.Error) in storeCalls.increment() - throw Error() + throw .storeReadmeFailed } do { // first ingestion, no readme has been saved @@ -597,11 +620,8 @@ class IngestorTests: AppTestCase { func test_issue_761_no_license() async throws { // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/761 // setup - let pkg = try await { - let p = Package(url: "https://github.com/foo/1") - try await p.save(on: app.db) - return Joined(model: p) - }() + let pkg = Package(url: "https://github.com/foo/1") + try await pkg.save(on: app.db) // use mock for metadata request which we're not interested in ... Current.fetchMetadata = { _, _, _ in Github.Metadata() } // and live fetch request for fetchLicense, whose behaviour we want to test ... @@ -611,7 +631,7 @@ class IngestorTests: AppTestCase { let client = MockClient { _, resp in resp.status = .notFound } // MUT - let (_, license, _) = try await fetchMetadata(client: client, package: pkg) + let (_, license, _) = try await Ingestion.fetchMetadata(client: client, package: pkg, owner: "foo", repository: "1") // validate XCTAssertEqual(license, nil) @@ -660,3 +680,24 @@ class IngestorTests: AppTestCase { XCTAssertEqual(fork5, nil) } } + + +private extension String { + static func packageDump(name: String) -> Self { + #""" + { + "name": "\#(name)", + "products": [ + { + "name": "p1", + "targets": [], + "type": { + "executable": null + } + } + ], + "targets": [] + } + """# + } +} diff --git a/app.yml b/app.yml index 0245834ff..e2306bc2b 100644 --- a/app.yml +++ b/app.yml @@ -46,6 +46,7 @@ x-shared: &shared DATABASE_USERNAME: ${DATABASE_USERNAME} DATABASE_PASSWORD: ${DATABASE_PASSWORD} DATABASE_USE_TLS: ${DATABASE_USE_TLS} + FAILURE_MODE: ${FAILURE_MODE} GITHUB_TOKEN: ${GITHUB_TOKEN} GITLAB_API_TOKEN: ${GITLAB_API_TOKEN} GITLAB_PIPELINE_LIMIT: ${GITLAB_PIPELINE_LIMIT}