diff --git a/Package.resolved b/Package.resolved index c1d729ef6..be2aefffa 100644 --- a/Package.resolved +++ b/Package.resolved @@ -1,5 +1,5 @@ { - "originHash" : "522f8d8b891d1a65e94cabb612d629f37de0bf6025993255871335482d355051", + "originHash" : "c68e60d7c703a8fa9fb0a3a11024cf6b2304fb294ba9f124de1d41f024b11cc9", "pins" : [ { "identity" : "async-http-client", @@ -28,15 +28,6 @@ "version" : "2.0.2" } }, - { - "identity" : "cache", - "kind" : "remoteSourceControl", - "location" : "https://github.com/0xLeif/Cache.git", - "state" : { - "revision" : "a535c68aab7bf0b42bda7a5de66b04f86e48c6e6", - "version" : "2.1.0" - } - }, { "identity" : "canonicalpackageurl", "kind" : "remoteSourceControl", diff --git a/Package.swift b/Package.swift index bc6225c79..9adebaf27 100644 --- a/Package.swift +++ b/Package.swift @@ -27,7 +27,6 @@ let package = Package( .library(name: "S3Store", targets: ["S3Store"]), ], dependencies: [ - .package(url: "https://github.com/0xLeif/Cache.git", from: "2.1.0"), .package(url: "https://github.com/JohnSundell/Ink.git", from: "0.5.1"), .package(url: "https://github.com/swift-server/swift-prometheus.git", from: "1.0.0"), .package(url: "https://github.com/SwiftPackageIndex/Plot.git", branch: "main"), @@ -63,7 +62,6 @@ let package = Package( .product(name: "SPIManifest", package: "SPIManifest"), .product(name: "SemanticVersion", package: "SemanticVersion"), .product(name: "SwiftSoup", package: "SwiftSoup"), - .product(name: "Cache", package: "cache"), .product(name: "CanonicalPackageURL", package: "CanonicalPackageURL"), .product(name: "CustomDump", package: "swift-custom-dump"), .product(name: "Dependencies", package: "swift-dependencies"), diff --git a/Sources/App/Core/CurrentReferenceCache.swift b/Sources/App/Core/CurrentReferenceCache.swift deleted file mode 100644 index f7de4336d..000000000 --- a/Sources/App/Core/CurrentReferenceCache.swift +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -@preconcurrency import Cache - -typealias CurrentReferenceCache = ExpiringCache - -extension CurrentReferenceCache { - static let live = CurrentReferenceCache(duration: .minutes(5)) - - subscript(owner owner: String, repository repository: String) -> String? { - get { - let key = "\(owner)/\(repository)".lowercased() - return self[key] - } - set { - let key = "\(owner)/\(repository)".lowercased() - self[key] = newValue - } - } -} diff --git a/Sources/App/Core/Dependencies/CurrentReferenceCacheClient.swift b/Sources/App/Core/Dependencies/CurrentReferenceCacheClient.swift new file mode 100644 index 000000000..793a65f8b --- /dev/null +++ b/Sources/App/Core/Dependencies/CurrentReferenceCacheClient.swift @@ -0,0 +1,86 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Dependencies +import DependenciesMacros + + +@DependencyClient +struct CurrentReferenceCacheClient { + var set: @Sendable (_ owner: String, _ repository: String, _ reference: String?) async -> Void + var get: @Sendable (_ owner: String, _ repository: String) async -> String? +} + + +extension CurrentReferenceCacheClient: DependencyKey { + static let timeToLive: Duration = .seconds(5*60) + + static var liveValue: CurrentReferenceCacheClient { + .init( + set: { owner, repository, reference async in + @Dependency(\.redis) var redis + await redis.set(key: getKey(owner: owner, repository: repository), + value: reference, + expiresIn: timeToLive) + }, + get: { owner, repository in + @Dependency(\.redis) var redis + return await redis.get(key: getKey(owner: owner, repository: repository)) + } + ) + } + + static func getKey(owner: String, repository: String) -> String { + "\(owner)/\(repository)".lowercased() + } +} + + +extension CurrentReferenceCacheClient: TestDependencyKey { + static var testValue: Self { Self() } +} + + +extension DependencyValues { + var currentReferenceCache: CurrentReferenceCacheClient { + get { self[CurrentReferenceCacheClient.self] } + set { self[CurrentReferenceCacheClient.self] = newValue } + } +} + + +#if DEBUG +extension CurrentReferenceCacheClient { + static var disabled: Self { + .init(set: { _, _, _ in }, get: { _, _ in nil }) + } + + static var inMemory: Self { + .init( + set: { owner, repository, reference in + _cache.withValue { + $0[getKey(owner: owner, repository: repository)] = reference + } + }, + get: { owner, repository in + _cache.withValue { + $0[getKey(owner: owner, repository: repository)] + } + } + ) + } + + nonisolated(unsafe) static var _cache = LockIsolated<[String: String]>([:]) +} +#endif diff --git a/Sources/App/Core/Dependencies/EnvironmentClient.swift b/Sources/App/Core/Dependencies/EnvironmentClient.swift index 67ee40016..371b570bf 100644 --- a/Sources/App/Core/Dependencies/EnvironmentClient.swift +++ b/Sources/App/Core/Dependencies/EnvironmentClient.swift @@ -39,7 +39,6 @@ struct EnvironmentClient { var collectionSigningCertificateChain: @Sendable () -> [URL] = { XCTFail("collectionSigningCertificateChain"); return [] } var collectionSigningPrivateKey: @Sendable () -> Data? var current: @Sendable () -> Environment = { XCTFail("current"); return .development } - var currentReferenceCache: @Sendable () -> CurrentReferenceCache? var dbId: @Sendable () -> String? var mastodonCredentials: @Sendable () -> Mastodon.Credentials? var random: @Sendable (_ range: ClosedRange) -> Double = { XCTFail("random"); return Double.random(in: $0) } @@ -102,7 +101,6 @@ extension EnvironmentClient: DependencyKey { Environment.get("COLLECTION_SIGNING_PRIVATE_KEY").map { Data($0.utf8) } }, current: { (try? Environment.detect()) ?? .development }, - currentReferenceCache: { .live }, dbId: { Environment.get("DATABASE_ID") }, mastodonCredentials: { Environment.get("MASTODON_ACCESS_TOKEN") diff --git a/Sources/App/Core/Dependencies/RedisClient.swift b/Sources/App/Core/Dependencies/RedisClient.swift new file mode 100644 index 000000000..c4582b4f6 --- /dev/null +++ b/Sources/App/Core/Dependencies/RedisClient.swift @@ -0,0 +1,127 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import NIOCore +@preconcurrency import RediStack +import Dependencies +import DependenciesMacros + + +@DependencyClient +struct RedisClient { + var set: @Sendable (_ key: String, _ value: String?, Duration?) async -> Void + var get: @Sendable (_ key: String) async -> String? +} + + +extension RedisClient { + func set(key: String, value: String?, expiresIn: Duration? = nil) async { + await set(key: key, value: value, expiresIn) + } +} + + +extension RedisClient: DependencyKey { + static var liveValue: RedisClient { + .init( + set: { key, value, expiresIn in + await Redis.shared?.set(key: key, value: value, expiresIn: expiresIn) + }, + get: { key in await Redis.shared?.get(key: key) } + ) + } +} + + +extension RedisClient: TestDependencyKey { + static var testValue: Self { Self() } +} + + +extension DependencyValues { + var redis: RedisClient { + get { self[RedisClient.self] } + set { self[RedisClient.self] = newValue } + } +} + + +#if DEBUG +extension RedisClient { + static var disabled: Self { + .init(set: { _, _, _ in }, get: { _ in nil }) + } +} +#endif + + +private actor Redis { + var client: RediStack.RedisClient + static private var task: Task? + + static var shared: Redis? { + get async { + if let task { + return await task.value + } + let task = Task { + var attemptsLeft = maxConnectionAttempts + while attemptsLeft > 0 { + do { + return try await Redis() + } catch { + attemptsLeft -= 1 + Current.logger().warning("Redis connection failed, \(attemptsLeft) attempts left. Error: \(error)") + try? await Task.sleep(for: .milliseconds(500)) + } + } + return nil + } + self.task = task + return await task.value + } + } + + private init() async throws { + let connection = RedisConnection.make( + configuration: try .init(hostname: Redis.hostname), + boundEventLoop: NIOSingletons.posixEventLoopGroup.any() + ) + self.client = try await connection.get() + } + + // This hostname has to match the redir service name in app.yml. + static let hostname = "redis" + static let maxConnectionAttempts = 3 + + func set(key: String, value: String?, expiresIn: Duration?) async -> Void { + if let value { + let buffer = ByteBuffer(string: value) + let value = RESPValue.bulkString(buffer) + if let expiresIn { + let ttl = Int(expiresIn.components.seconds) + try? await client.setex(.init(key), to: value, expirationInSeconds: ttl).get() + } else { + try? await client.set(.init(key), to: value).get() + } + } else { + _ = try? await client.delete([.init(key)]).get() + } + } + + func get(key: String) async -> String? { + return try? await client.get(.init(key)).map(\.string).get() + } +} + diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index e1846a1b9..c8a65988a 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -14,7 +14,6 @@ import Fluent import FluentPostgresDriver -import Redis import Vapor @@ -78,13 +77,6 @@ public func configure(_ app: Application) async throws -> String { sqlLogLevel: .debug), as: .psql) - // Setup Redis connection - do { - app.redis.configuration = try RedisConfiguration(hostname: "redis") - } catch { - app.logger.warning("Failed to configure Redis, caching disabled. Error: \(error)") - } - do { // Migration 001 - schema 1.0 app.migrations.add(CreatePackage()) app.migrations.add(CreateRepository()) diff --git a/Sources/App/routes+documentation.swift b/Sources/App/routes+documentation.swift index aef2fa664..595d4171f 100644 --- a/Sources/App/routes+documentation.swift +++ b/Sources/App/routes+documentation.swift @@ -178,14 +178,15 @@ extension Request { let docVersion = try await { () -> DocVersion in if reference == String.current { - if let ref = environment.currentReferenceCache()?[owner: owner, repository: repository] { + @Dependency(\.currentReferenceCache) var currentReferenceCache + if let ref = await currentReferenceCache.get(owner: owner, repository: repository) { return .current(referencing: ref) } guard let params = try await DocumentationTarget.query(on: db, owner: owner, repository: repository)?.internal else { throw Abort(.notFound) } - environment.currentReferenceCache()?[owner: owner, repository: repository] = "\(params.docVersion)" + await currentReferenceCache.set(owner: owner, repository: repository, reference: "\(params.docVersion)") return .current(referencing: "\(params.docVersion)") } else { return .reference(reference) diff --git a/Tests/AppTests/PackageController+routesTests.swift b/Tests/AppTests/PackageController+routesTests.swift index eed792c9e..4d0d8c672 100644 --- a/Tests/AppTests/PackageController+routesTests.swift +++ b/Tests/AppTests/PackageController+routesTests.swift @@ -566,8 +566,8 @@ class PackageController_routesTests: SnapshotTestCase { // Test the current (~) documentation routes: // /owner/package/documentation/~ + various path elements try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = { @Sendable _ in .init(status: .ok, body: .mockIndexHTML()) } } operation: { // setup @@ -649,8 +649,8 @@ class PackageController_routesTests: SnapshotTestCase { // Test the current (~) documentation routes with baseURL rewriting: // /owner/package/documentation/~ + various path elements try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = { @Sendable _ in .init(status: .ok, body: .mockIndexHTML(baseURL: "/owner/package/1.0.0")) } } operation: { // setup @@ -941,8 +941,8 @@ class PackageController_routesTests: SnapshotTestCase { func test_documentation_current_css() async throws { try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = App.HTTPClient.echoURL() } operation: { // setup @@ -996,8 +996,8 @@ class PackageController_routesTests: SnapshotTestCase { func test_documentation_current_js() async throws { try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = App.HTTPClient.echoURL() } operation: { // setup @@ -1051,8 +1051,8 @@ class PackageController_routesTests: SnapshotTestCase { func test_documentation_current_data() async throws { try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = App.HTTPClient.echoURL() } operation: { // setup @@ -1156,8 +1156,8 @@ class PackageController_routesTests: SnapshotTestCase { // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/2287 // Ensure references are path encoded try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.httpClient.fetchDocumentation = { @Sendable _ in .init(status: .ok, body: .mockIndexHTML()) } } operation: { // setup @@ -1220,8 +1220,8 @@ class PackageController_routesTests: SnapshotTestCase { func test_documentation_routes_tutorials() async throws { try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { nil } $0.environment.dbId = { nil } $0.httpClient.fetchDocumentation = { @Sendable _ in .init(status: .ok, body: .mockIndexHTML()) } } operation: { @@ -1479,8 +1479,8 @@ class PackageController_routesTests: SnapshotTestCase { // Ensures default branch updates don't introduce a "documentation gap" // https://github.com/SwiftPackageIndex/SwiftPackageIndex-Server/issues/2288 try await withDependencies { + $0.currentReferenceCache = .disabled $0.environment.awsDocsBucket = { "docs-bucket" } - $0.environment.currentReferenceCache = { .live } $0.httpClient.fetchDocumentation = { @Sendable _ in .init(status: .ok, body: .mockIndexHTML()) } } operation: { // setup @@ -1556,9 +1556,8 @@ class PackageController_routesTests: SnapshotTestCase { } func test_getDocRoute_documentation_current() async throws { - nonisolated(unsafe) let cache = CurrentReferenceCache() try await withDependencies { - $0.environment.currentReferenceCache = { cache } + $0.currentReferenceCache = .inMemory } operation: { // owner/repo/~/documentation/archive let req = Request(application: app, url: "", on: app.eventLoopGroup.next()) @@ -1576,7 +1575,8 @@ class PackageController_routesTests: SnapshotTestCase { XCTFail("unexpected error: \(error)") } - cache[owner: "owner", repository: "repo"] = "1.2.3" + @Dependency(\.currentReferenceCache) var cache + await cache.set(owner: "owner", repository: "repo", reference: "1.2.3") do { // Now with the cache in place this resolves let route = try await req.getDocRoute(fragment: .documentation)