Merge branch 'main' into heckj/cfrayRequestLogging

Author: heckj

Package.resolved

@@ -27,7 +27,6 @@ let package = Package(
         .library(name: "S3Store", targets: ["S3Store"]),
     ],
     dependencies: [
-        .package(url: "https://github.com/0xLeif/Cache.git", from: "2.1.0"),
         .package(url: "https://github.com/JohnSundell/Ink.git", from: "0.5.1"),
         .package(url: "https://github.com/swift-server/swift-prometheus.git", from: "1.0.0"),
         .package(url: "https://github.com/SwiftPackageIndex/Plot.git", branch: "main"),
@@ -63,7 +62,6 @@ let package = Package(
                     .product(name: "SPIManifest", package: "SPIManifest"),
                     .product(name: "SemanticVersion", package: "SemanticVersion"),
                     .product(name: "SwiftSoup", package: "SwiftSoup"),
-                    .product(name: "Cache", package: "cache"),
                     .product(name: "CanonicalPackageURL", package: "CanonicalPackageURL"),
                     .product(name: "CustomDump", package: "swift-custom-dump"),
                     .product(name: "Dependencies", package: "swift-dependencies"),

Package.swift

@@ -267,10 +267,11 @@ enum Ingestion {
         //   Sending 'github' into async let risks causing data races between async let uses and local uses
         @Dependency(\.github.fetchMetadata) var fetchMetadata
         @Dependency(\.github.fetchLicense) var fetchLicense
+        @Dependency(\.github.fetchReadme) var fetchReadme
 
         async let metadata = try await fetchMetadata(owner, repository)
         async let license = await fetchLicense(owner, repository)
-        async let readme = await Current.fetchReadme(client, owner, repository)
+        async let readme = await fetchReadme(owner, repository)
 
         do {
             return try await (metadata, license, readme)

Sources/App/Commands/Ingestion.swift

@@ -23,7 +23,6 @@ import FoundationNetworking
 
 
 struct AppEnvironment: Sendable {
-    var fetchReadme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async -> Github.Readme?
     var fetchS3Readme: @Sendable (_ client: Client, _ owner: String, _ repository: String) async throws -> String
     var fileManager: FileManager
     var getStatusCount: @Sendable (_ client: Client, _ status: Gitlab.Builder.Status) async throws -> Int
@@ -64,7 +63,6 @@ extension AppEnvironment {
     nonisolated(unsafe) static var logger: Logger!
 
     static let live = AppEnvironment(
-        fetchReadme: { client, owner, repo in await Github.fetchReadme(client:client, owner: owner, repository: repo) },
         fetchS3Readme: { client, owner, repo in try await S3Readme.fetchReadme(client:client, owner: owner, repository: repo) },
         fileManager: .live,
         getStatusCount: { client, status in

Sources/App/Core/AppEnvironment.swift

@@ -0,0 +1,86 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Dependencies
+import DependenciesMacros
+
+
+@DependencyClient
+struct CurrentReferenceCacheClient {
+    var set: @Sendable (_ owner: String, _ repository: String, _ reference: String?) async -> Void
+    var get: @Sendable (_ owner: String, _ repository: String) async -> String?
+}
+
+
+extension CurrentReferenceCacheClient: DependencyKey {
+    static let timeToLive: Duration = .seconds(5*60)
+
+    static var liveValue: CurrentReferenceCacheClient {
+        .init(
+            set: { owner, repository, reference async in
+                @Dependency(\.redis) var redis
+                try? await redis.set(key: getKey(owner: owner, repository: repository),
+                                     value: reference,
+                                     expiresIn: timeToLive)
+            },
+            get: { owner, repository in
+                @Dependency(\.redis) var redis
+                return try? await redis.get(key: getKey(owner: owner, repository: repository))
+            }
+        )
+    }
+
+    static func getKey(owner: String, repository: String) -> String {
+        "\(owner)/\(repository)".lowercased()
+    }
+}
+
+
+extension CurrentReferenceCacheClient: TestDependencyKey {
+    static var testValue: Self { Self() }
+}
+
+
+extension DependencyValues {
+    var currentReferenceCache: CurrentReferenceCacheClient {
+        get { self[CurrentReferenceCacheClient.self] }
+        set { self[CurrentReferenceCacheClient.self] = newValue }
+    }
+}
+
+
+#if DEBUG
+extension CurrentReferenceCacheClient {
+    static var disabled: Self {
+        .init(set: { _, _, _ in }, get: { _, _ in nil })
+    }
+
+    static var inMemory: Self {
+        .init(
+            set: { owner, repository, reference in
+                _cache.withValue {
+                    $0[getKey(owner: owner, repository: repository)] = reference
+                }
+            },
+            get: { owner, repository in
+                _cache.withValue {
+                    $0[getKey(owner: owner, repository: repository)]
+                }
+            }
+        )
+    }
+
+    nonisolated(unsafe) static var _cache = LockIsolated<[String: String]>([:])
+}
+#endif

Sources/App/Core/CurrentReferenceCache.swift

@@ -39,7 +39,6 @@ struct EnvironmentClient {
     var collectionSigningCertificateChain: @Sendable () -> [URL] = { XCTFail("collectionSigningCertificateChain"); return [] }
     var collectionSigningPrivateKey: @Sendable () -> Data?
     var current: @Sendable () -> Environment = { XCTFail("current"); return .development }
-    var currentReferenceCache: @Sendable () -> CurrentReferenceCache?
     var dbId: @Sendable () -> String?
     var enableCFRayLogging: @Sendable () -> Bool = { XCTFail("enableCFRayLogging"); return true }
     var mastodonCredentials: @Sendable () -> Mastodon.Credentials?
@@ -103,7 +102,6 @@ extension EnvironmentClient: DependencyKey {
                 Environment.get("COLLECTION_SIGNING_PRIVATE_KEY").map { Data($0.utf8) }
             },
             current: { (try? Environment.detect()) ?? .development },
-            currentReferenceCache: { .live },
             dbId: { Environment.get("DATABASE_ID") },
             enableCFRayLogging: {
                 Environment.get("ENABLE_CF_RAY_LOGGING")

Sources/App/Core/Dependencies/CurrentReferenceCacheClient.swift

@@ -23,14 +23,16 @@ import IssueReporting
 struct GithubClient {
     var fetchLicense: @Sendable (_ owner: String, _ repository: String) async -> Github.License?
     var fetchMetadata: @Sendable (_ owner: String, _ repository: String) async throws(Github.Error) -> Github.Metadata = { _,_ in reportIssue("fetchMetadata"); return .init() }
+    var fetchReadme: @Sendable (_ owner: String, _ repository: String) async -> Github.Readme?
 }
 
 
 extension GithubClient: DependencyKey {
     static var liveValue: Self {
         .init(
             fetchLicense: { owner, repo in await Github.fetchLicense(owner: owner, repository: repo) },
-            fetchMetadata: { owner, repo throws(Github.Error) in try await Github.fetchMetadata(owner: owner, repository: repo) }
+            fetchMetadata: { owner, repo throws(Github.Error) in try await Github.fetchMetadata(owner: owner, repository: repo) },
+            fetchReadme: { owner, repo in await Github.fetchReadme(owner: owner, repository: repo) }
         )
     }
 }
@@ -40,7 +42,8 @@ extension GithubClient: TestDependencyKey {
     static var testValue: Self {
         .init(
             fetchLicense: { _, _ in unimplemented("fetchLicense"); return nil },
-            fetchMetadata: { _, _ in unimplemented("fetchMetadata"); return .init() }
+            fetchMetadata: { _, _ in unimplemented("fetchMetadata"); return .init() },
+            fetchReadme: { _, _ in unimplemented("fetchReadme"); return nil }
         )
     }
 }

Sources/App/Core/Dependencies/EnvironmentClient.swift

@@ -0,0 +1,154 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import NIOCore
+@preconcurrency import RediStack
+import Dependencies
+import DependenciesMacros
+
+
+@DependencyClient
+struct RedisClient {
+    var set: @Sendable (_ key: String, _ value: String?, Duration?) async throws -> Void
+    var get: @Sendable (_ key: String) async throws -> String?
+    var expire: @Sendable (_ key: String, _ after: Duration) async throws -> Bool
+    var increment: @Sendable (_ key: String, _ by: Int) async throws -> Int
+}
+
+
+extension RedisClient {
+    func set(key: String, value: String?, expiresIn: Duration? = nil) async throws {
+        try await set(key: key, value: value, expiresIn)
+    }
+
+    func increment(key: String) async throws -> Int {
+        try await increment(key: key, by: 1)
+    }
+}
+
+
+extension RedisClient: DependencyKey {
+    static var liveValue: RedisClient {
+        .init(
+            set: { key, value, expiresIn in
+                try await Redis.shared.set(key: key, value: value, expiresIn: expiresIn)
+            },
+            get: { key in try await Redis.shared.get(key: key) },
+            expire: { key, ttl in try await Redis.shared.expire(key: key, after: ttl) },
+            increment: { key, value in try await Redis.shared.increment(key: key, by: value) }
+        )
+    }
+}
+
+
+extension RedisClient: TestDependencyKey {
+    static var testValue: Self { Self() }
+}
+
+
+extension DependencyValues {
+    var redis: RedisClient {
+        get { self[RedisClient.self] }
+        set { self[RedisClient.self] = newValue }
+    }
+}
+
+
+#if DEBUG
+extension RedisClient {
+    static var disabled: Self {
+        .init(set: { _, _, _ in },
+              get: { _ in nil },
+              expire: { _, _ in true },
+              increment: { _, value in value })
+    }
+}
+#endif
+
+
+private actor Redis {
+    var client: RediStack.RedisClient
+    static private var task: Task<Redis, Swift.Error>?
+
+    static var shared: Redis {
+        get async throws {
+            if let task {
+                return try await task.value
+            }
+            let task = Task<Redis, Swift.Error> {
+                var attemptsLeft = maxConnectionAttempts
+                while attemptsLeft > 0 {
+                    do {
+                        return try await Redis()
+                    } catch {
+                        attemptsLeft -= 1
+                        Current.logger().warning("Redis connection failed, \(attemptsLeft) attempts left. Error: \(error)")
+                        try? await Task.sleep(for: .milliseconds(500))
+                    }
+                }
+                throw Error.unavailable
+            }
+            self.task = task
+            return try await task.value
+        }
+    }
+
+    enum Error: Swift.Error {
+        case unavailable
+    }
+
+    private init() async throws {
+        let connection = RedisConnection.make(
+            configuration: try .init(hostname: Redis.hostname),
+            boundEventLoop: NIOSingletons.posixEventLoopGroup.any()
+        )
+        self.client = try await connection.get()
+    }
+
+    // This hostname has to match the redir service name in app.yml.
+    static let hostname = "redis"
+    static let maxConnectionAttempts = 3
+
+    func set(key: String, value: String?, expiresIn: Duration?) async {
+        if let value {
+            let buffer = ByteBuffer(string: value)
+            let value = RESPValue.bulkString(buffer)
+            if let expiresIn {
+                let ttl = Int(expiresIn.components.seconds)
+                try? await client.setex(.init(key), to: value, expirationInSeconds: ttl).get()
+            } else {
+                try? await client.set(.init(key), to: value).get()
+            }
+        } else {
+            _ = try? await client.delete([.init(key)]).get()
+        }
+    }
+
+    func get(key: String) async -> String? {
+        return try? await client.get(.init(key)).map(\.string).get()
+    }
+
+    func expire(key: String, after ttl: Duration) async throws -> Bool {
+        try await client.expire(.init(key), after: .init(ttl)).get()
+    }
+
+    func increment(key: String) async throws -> Int {
+        try await client.increment(.init(key)).get()
+    }
+
+    func increment(key: String, by value: Int) async throws -> Int {
+        try await client.increment(.init(key), by: value).get()
+    }
+}
+