Merge pull request #3070 from SwiftPackageIndex/alerting-command

Alerting command

Author: finestructure

Sources/App/Commands/Alerting.swift

@@ -0,0 +1,240 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+import SQLKit
+import Vapor
+import NIOCore
+
+
+enum Alerting {
+    struct Command: AsyncCommand {
+        var help: String { "Application level alerting" }
+
+        struct Signature: CommandSignature {
+            @Option(name: "time-period", short: "t")
+            var timePeriod: Int?
+
+            @Option(name: "limit", short: "l")
+            var limit: Int?
+
+            static let defaultLimit = 2000
+            static let defaultTimePeriod = 2
+
+            var duration: TimeAmount {
+                .hours(Int64(timePeriod ?? Self.defaultTimePeriod))
+            }
+        }
+
+        func run(using context: CommandContext, signature: Signature) async throws {
+            Current.setLogger(Logger(component: "alerting"))
+
+            Current.logger().info("Running alerting...")
+
+            let timePeriod = signature.duration
+            let limit = signature.limit ?? Signature.defaultLimit
+
+            Current.logger().info("Validation time interval: \(timePeriod.hours)h, limit: \(limit)")
+
+            let builds = try await Alerting.fetchBuilds(on: context.application.db, timePeriod: timePeriod, limit: limit)
+            try await Alerting.runChecks(for: builds)
+        }
+    }
+}
+
+extension Alerting {
+    struct BuildInfo {
+        var createdAt: Date
+        var updatedAt: Date
+        var builderVersion: String?
+        var platform: Build.Platform
+        var runnerId: String?
+        var status: Build.Status
+        var swiftVersion: SwiftVersion
+
+        init(createdAt: Date, updatedAt: Date, builderVersion: String? = nil, platform: Build.Platform, runnerId: String? = nil, status: Build.Status, swiftVersion: SwiftVersion) {
+            self.createdAt = createdAt
+            self.updatedAt = updatedAt
+            self.builderVersion = builderVersion
+            self.platform = platform
+            self.runnerId = runnerId
+            self.status = status
+            self.swiftVersion = swiftVersion
+        }
+
+        init(_ build: Build) {
+            self.createdAt = build.createdAt!
+            self.updatedAt = build.updatedAt!
+            self.builderVersion = build.builderVersion
+            self.platform = build.platform
+            self.runnerId = build.runnerId
+            self.status = build.status
+            self.swiftVersion = build.swiftVersion
+        }
+    }
+
+    static func runChecks(for builds: [BuildInfo]) async throws {
+        // to do
+        // - [ ] doc gen is configured but it failed
+
+        Current.logger().info("Build records selected: \(builds.count)")
+        if let oldest = builds.last {
+            Current.logger().info("Oldest selected: \(oldest.createdAt)")
+        }
+        if let mostRecent = builds.first {
+            Current.logger().info("Most recent selected: \(mostRecent.createdAt)")
+        }
+        builds.validateBuildsPresent().log(check: "CHECK_BUILDS_PRESENT")
+        builds.validatePlatformsPresent().log(check: "CHECK_BUILDS_PLATFORMS_PRESENT")
+        builds.validateSwiftVersionsPresent().log(check: "CHECK_BUILDS_SWIFT_VERSIONS_PRESENT")
+        builds.validatePlatformsSuccessful().log(check: "CHECK_BUILDS_PLATFORMS_SUCCESSFUL")
+        builds.validateSwiftVersionsSuccessful().log(check: "CHECK_BUILDS_SWIFT_VERSIONS_SUCCESSFUL")
+        builds.validateRunnerIdsPresent().log(check: "CHECK_BUILDS_RUNNER_IDS_PRESENT")
+        builds.validateRunnerIdsSuccessful().log(check: "CHECK_BUILDS_RUNNER_IDS_SUCCESSFUL")
+        if builds.count >= 1000 {  // only run this test if we have a decent number of builds
+            builds.validateSuccessRateInRange().log(check: "CHECK_BUILDS_SUCCESS_RATE_IN_RANGE")
+        }
+    }
+
+    static func fetchBuilds(on database: Database, timePeriod: TimeAmount, limit: Int) async throws -> [Alerting.BuildInfo] {
+        let start = Date.now
+        defer {
+            Current.logger().debug("fetchBuilds elapsed: \(Date.now.timeIntervalSince(start).rounded(decimalPlaces: 2))s")
+        }
+        let cutoff = Current.date().addingTimeInterval(-timePeriod.timeInterval)
+        let builds = try await Build.query(on: database)
+            .field(\.$createdAt)
+            .field(\.$updatedAt)
+            .field(\.$builderVersion)
+            .field(\.$platform)
+            .field(\.$runnerId)
+            .field(\.$status)
+            .field(\.$swiftVersion)
+            .filter(Build.self, \.$createdAt >= cutoff)
+            .sort(\.$createdAt, .descending)
+            .limit(limit)
+            .all()
+            .map(BuildInfo.init)
+        return builds
+    }
+}
+
+
+extension Alerting {
+    enum Validation: Equatable {
+        case ok
+        case failed(reasons: [String])
+
+        func log(check: String) {
+            switch self {
+                case .ok:
+                    Current.logger().debug("\(check) passed")
+                case .failed(let reasons):
+                    for reason in reasons {
+                        Current.logger().critical("\(check) failed: \(reason)")
+                    }
+            }
+        }
+    }
+}
+
+
+extension [Alerting.BuildInfo] {
+    func validateBuildsPresent() -> Alerting.Validation {
+        isEmpty ? .failed(reasons: ["No builds"]) : .ok
+    }
+
+    func validatePlatformsPresent() -> Alerting.Validation {
+        var notSeen = Set(Build.Platform.allCases)
+        for build in self {
+            notSeen.remove(build.platform)
+            if notSeen.isEmpty { return .ok }
+        }
+        return .failed(reasons: notSeen.sorted().map { "Missing platform: \($0)" })
+    }
+
+    func validateSwiftVersionsPresent() -> Alerting.Validation {
+        var notSeen = Set(SwiftVersion.allActive)
+        for build in self {
+            notSeen.remove(build.swiftVersion)
+            if notSeen.isEmpty { return .ok }
+        }
+        return .failed(reasons: notSeen.sorted().map { "Missing Swift version: \($0)" })
+    }
+
+    func validatePlatformsSuccessful() -> Alerting.Validation {
+        var noSuccess = Set(Build.Platform.allCases)
+        for build in self where build.status == .ok {
+            noSuccess.remove(build.platform)
+            if noSuccess.isEmpty { return .ok }
+        }
+        return .failed(reasons: noSuccess.sorted().map { "Platform without successful builds: \($0)" })
+    }
+
+    func validateSwiftVersionsSuccessful() -> Alerting.Validation {
+        var noSuccess = Set(SwiftVersion.allActive)
+        for build in self where build.status == .ok {
+            noSuccess.remove(build.swiftVersion)
+            if noSuccess.isEmpty { return .ok }
+        }
+        return .failed(reasons: noSuccess.sorted().map { "Swift version without successful builds: \($0)" })
+    }
+
+    func validateRunnerIdsPresent() -> Alerting.Validation {
+        var notSeen = Set(Current.runnerIds())
+        for build in self where build.runnerId != nil {
+            notSeen.remove(build.runnerId!)
+            if notSeen.isEmpty { return .ok }
+        }
+        return .failed(reasons: notSeen.sorted().map { "Missing runner id: \($0)" })
+    }
+
+    func validateRunnerIdsSuccessful() -> Alerting.Validation {
+        var noSuccess = Set(Current.runnerIds())
+        for build in self where build.runnerId != nil && build.status == .ok {
+            noSuccess.remove(build.runnerId!)
+            if noSuccess.isEmpty { return .ok }
+        }
+        return .failed(reasons: noSuccess.sorted().map { "Runner id without successful builds: \($0)" })
+    }
+
+    func validateSuccessRateInRange() -> Alerting.Validation {
+        let successRate = Double(filter { $0.status == .ok }.count) / Double(count)
+        // Success rate has been around 30% generally
+        if 0.2 <= successRate && successRate <= 0.4 {
+            return .ok
+        } else {
+            let percentSuccessRate = (successRate * 1000).rounded() / 10
+            return .failed(reasons: ["Global success rate of \(percentSuccessRate)% out of bounds"])
+        }
+    }
+}
+
+
+private extension TimeAmount {
+    var timeInterval: TimeInterval {
+        Double(nanoseconds) * 1e-9
+    }
+    var hours: Int {
+        Int(timeInterval / 3600.0 + 0.5)
+    }
+}
+
+
+private extension TimeInterval {
+    func rounded(decimalPlaces: Int) -> Self {
+        let scale = (pow(10, decimalPlaces) as NSDecimalNumber).doubleValue
+        return (self * scale).rounded(.toNearestOrAwayFromZero) / scale
+    }
+}

Sources/App/Commands/Swift6Trigger.swift

@@ -33,7 +33,8 @@ struct Swift6TriggerCommand: AsyncCommand {
 
     var help: String { "Trigger Swift 6 builds" }
 
-    func run(using context: CommandContext, signature: Signature) async throws {        Current.setLogger(Logger(component: "swift-6-trigger"))
+    func run(using context: CommandContext, signature: Signature) async throws {
+        Current.setLogger(Logger(component: "swift-6-trigger"))
 
         do {
             if signature.dryRun {
@@ -48,11 +49,6 @@ struct Swift6TriggerCommand: AsyncCommand {
             Current.logger().critical("\(error)")
         }
     }
-
-    func printUsage(using context: CommandContext) {
-        var context = context
-        outputHelp(using: &context)
-    }
 }
 
 

Sources/App/Core/AppEnvironment.swift

@@ -68,6 +68,7 @@ struct AppEnvironment {
     var plausibleBackendReportingSiteID: () -> String?
     var postPlausibleEvent: (Client, Plausible.Event.Kind, Plausible.Path, User?) async throws -> Void
     var random: (_ range: ClosedRange<Double>) -> Double
+    var runnerIds: () -> [String]
     var setHTTPClient: (Client) -> Void
     var setLogger: (Logger) -> Void
     var shell: Shell
@@ -205,6 +206,12 @@ extension AppEnvironment {
         plausibleBackendReportingSiteID: { Environment.get("PLAUSIBLE_BACKEND_REPORTING_SITE_ID") },
         postPlausibleEvent: Plausible.postEvent,
         random: Double.random,
+        runnerIds: {
+            Environment.get("RUNNER_IDS")
+                .map { Data($0.utf8) }
+                .flatMap { try? JSONDecoder().decode([String].self, from: $0) }
+            ?? []
+        },
         setHTTPClient: { client in Self.httpClient = client },
         setLogger: { logger in Self.logger = logger },
         shell: .live,

Sources/App/configure.swift

@@ -339,6 +339,7 @@ public func configure(_ app: Application) async throws -> String {
     app.commands.use(ReconcileCommand(), as: "reconcile")
     app.commands.use(TriggerBuildsCommand(), as: "trigger-builds")
     app.commands.use(ReAnalyzeVersions.Command(), as: "re-analyze-versions")
+    app.commands.use(Alerting.Command(), as: "alerting")
     if Current.environment() == .development {
         app.commands.use(Swift6TriggerCommand(), as: "swift-6-trigger")
     }