Custom package collections

Package.swift

@@ -66,6 +66,7 @@ let package = Package(
                     .product(name: "CanonicalPackageURL", package: "CanonicalPackageURL"),
                     .product(name: "CustomDump", package: "swift-custom-dump"),
                     .product(name: "Dependencies", package: "swift-dependencies"),
+                    .product(name: "DependenciesMacros", package: "swift-dependencies"),
                     .product(name: "DependencyResolution", package: "DependencyResolution"),
                     .product(name: "Fluent", package: "fluent"),
                     .product(name: "FluentPostgresDriver", package: "fluent-postgres-driver"),

Sources/App/Commands/Reconcile.swift

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+import Dependencies
 import Fluent
 import Vapor
 
@@ -24,13 +25,13 @@ struct ReconcileCommand: AsyncCommand {
     func run(using context: CommandContext, signature: Signature) async throws {
         Current.setLogger(Logger(component: "reconcile"))
 
-        Current.logger().info("Reconciling ...")
+        Current.logger().info("Reconciling...")
 
         do {
             try await reconcile(client: context.application.client,
                                 database: context.application.db)
         } catch {
-            Current.logger().error("\(error.localizedDescription)")
+            Current.logger().error("\(error)")
         }
 
         Current.logger().info("done.")
@@ -39,7 +40,7 @@ struct ReconcileCommand: AsyncCommand {
             try await AppMetrics.push(client: context.application.client,
                                       jobName: "reconcile")
         } catch {
-            Current.logger().warning("\(error.localizedDescription)")
+            Current.logger().warning("\(error)")
         }
     }
 }
@@ -48,6 +49,24 @@ struct ReconcileCommand: AsyncCommand {
 func reconcile(client: Client, database: Database) async throws {
     let start = DispatchTime.now().uptimeNanoseconds
     defer { AppMetrics.reconcileDurationSeconds?.time(since: start) }
+
+    // reconcile main package list
+    Current.logger().info("Reconciling main list...")
+    let fullPackageList = try await reconcileMainPackageList(client: client, database: database)
+
+    do { // reconcile custom package collections
+        Current.logger().info("Reconciling custom collections...")
+        @Dependency(\.packageListRepository) var packageListRepository
+        let collections = try await packageListRepository.fetchCustomCollections(client: client)
+        for collection in collections {
+            Current.logger().info("Reconciling '\(collection.name)' collection...")
+            try await reconcileCustomCollection(client: client, database: database, fullPackageList: fullPackageList, collection)
+        }
+    }
+}
+
+
+func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] {
     async let sourcePackageList = try Current.fetchPackageList(client)
     async let sourcePackageDenyList = try Current.fetchPackageDenyList(client)
     async let currentList = try fetchCurrentPackageList(database)
@@ -58,6 +77,8 @@ func reconcile(client: Client, database: Database) async throws {
     try await reconcileLists(db: database,
                              source: packageList,
                              target: currentList)
+
+    return packageList
 }
 
 
@@ -115,6 +136,7 @@ func reconcileLists(db: Database, source: [URL], target: [URL]) async throws {
     }
 }
 
+
 func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] {
     // Note: If the implementation of this function ever changes, the `RemoveDenyList`
     // command in the Validator will also need updating to match.
@@ -140,3 +162,15 @@ func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] {
             .subtracting(Set(denyList.map(CaseInsensitiveURL.init)))
     ).map(\.url)
 }
+
+
+func reconcileCustomCollection(client: Client, database: Database, fullPackageList: [URL], _ dto: CustomCollection.DTO) async throws {
+    let collection = try await CustomCollection.findOrCreate(on: database, dto)
+
+    // Limit incoming URLs to 50 since this is input outside of our control
+    @Dependency(\.packageListRepository) var packageListRepository
+    let incomingURLs = try await packageListRepository.fetchCustomCollection(client: client, url: collection.url)
+        .prefix(Constants.maxCustomPackageCollectionSize)
+
+    try await collection.reconcile(on: database, packageURLs: incomingURLs)
+}

Sources/App/Core/Constants.swift

@@ -26,6 +26,9 @@ enum Constants {
 
     static let packageListUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/packages.json")
     static let packageDenyListUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/denylist.json")
+    static let customCollectionsUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/custom-package-collections.json")
+
+    static let maxCustomPackageCollectionSize = 50
 
     // NB: the underlying materialised views also have a limit, this is just an additional
     // limit to ensure we don't spill too many rows onto the home page

Sources/App/Core/Dependencies/PackageListRepositoryClient.swift

@@ -0,0 +1,59 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Dependencies
+import DependenciesMacros
+import Vapor
+
+
+@DependencyClient
+struct PackageListRepositoryClient {
+    var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL]
+    var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO]
+    // TODO: move other package list dependencies here
+}
+
+
+extension PackageListRepositoryClient: DependencyKey {
+    static var liveValue: PackageListRepositoryClient {
+        .init(
+            fetchCustomCollection: { client, url in
+                try await client
+                    .get(URI(string: url.absoluteString))
+                    .content
+                    .decode([URL].self, using: JSONDecoder())
+            },
+            fetchCustomCollections: { client in
+                try await client
+                    .get(Constants.customCollectionsUri)
+                    .content
+                    .decode([CustomCollection.DTO].self, using: JSONDecoder())
+            }
+        )
+    }
+}
+
+
+extension PackageListRepositoryClient: Sendable, TestDependencyKey {
+    static var testValue: Self { Self() }
+}
+
+
+extension DependencyValues {
+    var packageListRepository: PackageListRepositoryClient {
+        get { self[PackageListRepositoryClient.self] }
+        set { self[PackageListRepositoryClient.self] = newValue }
+    }
+}
+

Sources/App/Migrations/081/CreateCustomCollection.swift

@@ -0,0 +1,44 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+import SQLKit
+
+
+struct CreateCustomCollection: AsyncMigration {
+    func prepare(on database: Database) async throws {
+        try await database.schema("custom_collections")
+
+        // managed fields
+            .id()
+            .field("created_at", .datetime)
+            .field("updated_at", .datetime)
+
+        // data fields
+            .field("name", .string, .required)
+            .field("description", .string)
+            .field("badge", .string)
+            .field("url", .string, .required)
+
+        // constraints
+            .unique(on: "name")
+            .unique(on: "url")
+
+            .create()
+    }
+
+    func revert(on database: Database) async throws {
+        try await database.schema("custom_collections").delete()
+    }
+}

Sources/App/Migrations/081/CreateCustomCollectionPackage.swift

@@ -0,0 +1,43 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+import SQLKit
+
+
+struct CreateCustomCollectionPackage: AsyncMigration {
+    func prepare(on database: Database) async throws {
+        try await database.schema("custom_collections+packages")
+
+        // managed fields
+            .id()
+            .field("created_at", .datetime)
+            .field("updated_at", .datetime)
+
+        // reference fields
+            .field("custom_collection_id", .uuid,
+                   .references("custom_collections", "id", onDelete: .cascade), .required)
+            .field("package_id", .uuid,
+                   .references("packages", "id", onDelete: .cascade), .required)
+
+        // constraints
+            .unique(on: "custom_collection_id", "package_id")
+
+            .create()
+    }
+
+    func revert(on database: Database) async throws {
+        try await database.schema("custom_collections+packages").delete()
+    }
+}

Sources/App/Models/CustomCollection.swift

@@ -0,0 +1,115 @@
+// Copyright Dave Verwer, Sven A. Schmidt, and other contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import Fluent
+import Vapor
+
+
+final class CustomCollection: @unchecked Sendable, Model, Content {
+    static let schema = "custom_collections"
+
+    typealias Id = UUID
+
+    // managed fields
+
+    @ID(key: .id)
+    var id: Id?
+
+    @Timestamp(key: "created_at", on: .create)
+    var createdAt: Date?
+
+    // periphery:ignore
+    @Timestamp(key: "updated_at", on: .update)
+    var updatedAt: Date?
+
+    // data fields
+
+    @Field(key: "name")
+    var name: String
+
+    @Field(key: "description")
+    var description: String?
+
+    @Field(key: "badge")
+    var badge: String?
+
+    @Field(key: "url")
+    var url: URL
+
+    // reference fields
+    @Siblings(through: CustomCollectionPackage.self, from: \.$customCollection, to: \.$package)
+    var packages: [Package]
+
+    init() { }
+
+    init(id: Id? = nil, createdAt: Date? = nil, updatedAt: Date? = nil, _ dto: DTO) {
+        self.id = id
+        self.createdAt = createdAt
+        self.updatedAt = updatedAt
+        self.name = dto.name
+        self.description = dto.description
+        self.badge = dto.badge
+        self.url = dto.url
+    }
+}
+
+
+extension CustomCollection {
+    struct DTO: Codable {
+        var name: String
+        var description: String?
+        var badge: String?
+        var url: URL
+    }
+
+    static func findOrCreate(on database: Database, _ dto: DTO) async throws -> CustomCollection {
+        if let collection = try await CustomCollection.query(on: database)
+            .filter(\.$url == dto.url)
+            .first() {
+            return collection
+        } else {
+            let collection = CustomCollection(dto)
+            try await collection.save(on: database)
+            return collection
+        }
+    }
+
+    func reconcile(on database: Database, packageURLs: some Collection<URL>) async throws {
+        let incoming: [Package.Id: Package] = .init(
+            packages: try await Package.query(on: database)
+                .filter(by: packageURLs)
+                .all()
+        )
+        try await $packages.load(on: database)
+        let existing: [Package.Id: Package] = .init(packages: packages)
+        let newIDs = Set(incoming.keys).subtracting(Set(existing.keys))
+        try await $packages.attach(incoming[newIDs], on: database)
+        let removedIDs = Set(existing.keys).subtracting(Set(incoming.keys))
+        try await $packages.detach(existing[removedIDs], on: database)
+    }
+}
+
+
+private extension [Package.Id: Package] {
+    init(packages: [Package]) {
+        self.init(
+            packages.compactMap({ pkg in pkg.id.map({ ($0, pkg) }) }),
+            uniquingKeysWith: { (first, second) in first }
+        )
+    }
+
+    subscript(ids: some Collection<Package.Id>) -> [Package] {
+        Array(ids.compactMap { self[$0] })
+    }
+}