diff --git a/Package.swift b/Package.swift index 1b03ae382..2212f5951 100644 --- a/Package.swift +++ b/Package.swift @@ -66,6 +66,7 @@ let package = Package( .product(name: "CanonicalPackageURL", package: "CanonicalPackageURL"), .product(name: "CustomDump", package: "swift-custom-dump"), .product(name: "Dependencies", package: "swift-dependencies"), + .product(name: "DependenciesMacros", package: "swift-dependencies"), .product(name: "DependencyResolution", package: "DependencyResolution"), .product(name: "Fluent", package: "fluent"), .product(name: "FluentPostgresDriver", package: "fluent-postgres-driver"), diff --git a/Sources/App/Commands/Reconcile.swift b/Sources/App/Commands/Reconcile.swift index 931d1a769..10500fc9e 100644 --- a/Sources/App/Commands/Reconcile.swift +++ b/Sources/App/Commands/Reconcile.swift @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import Dependencies import Fluent import Vapor @@ -24,13 +25,13 @@ struct ReconcileCommand: AsyncCommand { func run(using context: CommandContext, signature: Signature) async throws { Current.setLogger(Logger(component: "reconcile")) - Current.logger().info("Reconciling ...") + Current.logger().info("Reconciling...") do { try await reconcile(client: context.application.client, database: context.application.db) } catch { - Current.logger().error("\(error.localizedDescription)") + Current.logger().error("\(error)") } Current.logger().info("done.") @@ -39,7 +40,7 @@ struct ReconcileCommand: AsyncCommand { try await AppMetrics.push(client: context.application.client, jobName: "reconcile") } catch { - Current.logger().warning("\(error.localizedDescription)") + Current.logger().warning("\(error)") } } } @@ -48,6 +49,24 @@ struct ReconcileCommand: AsyncCommand { func reconcile(client: Client, database: Database) async throws { let start = DispatchTime.now().uptimeNanoseconds defer { AppMetrics.reconcileDurationSeconds?.time(since: start) } + + // reconcile main package list + Current.logger().info("Reconciling main list...") + let fullPackageList = try await reconcileMainPackageList(client: client, database: database) + + do { // reconcile custom package collections + Current.logger().info("Reconciling custom collections...") + @Dependency(\.packageListRepository) var packageListRepository + let collections = try await packageListRepository.fetchCustomCollections(client: client) + for collection in collections { + Current.logger().info("Reconciling '\(collection.name)' collection...") + try await reconcileCustomCollection(client: client, database: database, fullPackageList: fullPackageList, collection) + } + } +} + + +func reconcileMainPackageList(client: Client, database: Database) async throws -> [URL] { async let sourcePackageList = try Current.fetchPackageList(client) async let sourcePackageDenyList = try Current.fetchPackageDenyList(client) async let currentList = try fetchCurrentPackageList(database) @@ -58,6 +77,8 @@ func reconcile(client: Client, database: Database) async throws { try await reconcileLists(db: database, source: packageList, target: currentList) + + return packageList } @@ -115,6 +136,7 @@ func reconcileLists(db: Database, source: [URL], target: [URL]) async throws { } } + func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] { // Note: If the implementation of this function ever changes, the `RemoveDenyList` // command in the Validator will also need updating to match. @@ -140,3 +162,15 @@ func processPackageDenyList(packageList: [URL], denyList: [URL]) -> [URL] { .subtracting(Set(denyList.map(CaseInsensitiveURL.init))) ).map(\.url) } + + +func reconcileCustomCollection(client: Client, database: Database, fullPackageList: [URL], _ dto: CustomCollection.DTO) async throws { + let collection = try await CustomCollection.findOrCreate(on: database, dto) + + // Limit incoming URLs to 50 since this is input outside of our control + @Dependency(\.packageListRepository) var packageListRepository + let incomingURLs = try await packageListRepository.fetchCustomCollection(client: client, url: collection.url) + .prefix(Constants.maxCustomPackageCollectionSize) + + try await collection.reconcile(on: database, packageURLs: incomingURLs) +} diff --git a/Sources/App/Core/Constants.swift b/Sources/App/Core/Constants.swift index 0905609bb..27bf1af3b 100644 --- a/Sources/App/Core/Constants.swift +++ b/Sources/App/Core/Constants.swift @@ -26,6 +26,9 @@ enum Constants { static let packageListUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/packages.json") static let packageDenyListUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/denylist.json") + static let customCollectionsUri = URI(string: "https://raw.githubusercontent.com/SwiftPackageIndex/PackageList/main/custom-package-collections.json") + + static let maxCustomPackageCollectionSize = 50 // NB: the underlying materialised views also have a limit, this is just an additional // limit to ensure we don't spill too many rows onto the home page diff --git a/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift new file mode 100644 index 000000000..8b68a3f4b --- /dev/null +++ b/Sources/App/Core/Dependencies/PackageListRepositoryClient.swift @@ -0,0 +1,59 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Dependencies +import DependenciesMacros +import Vapor + + +@DependencyClient +struct PackageListRepositoryClient { + var fetchCustomCollection: @Sendable (_ client: Client, _ url: URL) async throws -> [URL] + var fetchCustomCollections: @Sendable (_ client: Client) async throws -> [CustomCollection.DTO] + // TODO: move other package list dependencies here +} + + +extension PackageListRepositoryClient: DependencyKey { + static var liveValue: PackageListRepositoryClient { + .init( + fetchCustomCollection: { client, url in + try await client + .get(URI(string: url.absoluteString)) + .content + .decode([URL].self, using: JSONDecoder()) + }, + fetchCustomCollections: { client in + try await client + .get(Constants.customCollectionsUri) + .content + .decode([CustomCollection.DTO].self, using: JSONDecoder()) + } + ) + } +} + + +extension PackageListRepositoryClient: Sendable, TestDependencyKey { + static var testValue: Self { Self() } +} + + +extension DependencyValues { + var packageListRepository: PackageListRepositoryClient { + get { self[PackageListRepositoryClient.self] } + set { self[PackageListRepositoryClient.self] = newValue } + } +} + diff --git a/Sources/App/Migrations/081/CreateCustomCollection.swift b/Sources/App/Migrations/081/CreateCustomCollection.swift new file mode 100644 index 000000000..e2a844a5f --- /dev/null +++ b/Sources/App/Migrations/081/CreateCustomCollection.swift @@ -0,0 +1,44 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Fluent +import SQLKit + + +struct CreateCustomCollection: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("custom_collections") + + // managed fields + .id() + .field("created_at", .datetime) + .field("updated_at", .datetime) + + // data fields + .field("name", .string, .required) + .field("description", .string) + .field("badge", .string) + .field("url", .string, .required) + + // constraints + .unique(on: "name") + .unique(on: "url") + + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("custom_collections").delete() + } +} diff --git a/Sources/App/Migrations/081/CreateCustomCollectionPackage.swift b/Sources/App/Migrations/081/CreateCustomCollectionPackage.swift new file mode 100644 index 000000000..3fcae566c --- /dev/null +++ b/Sources/App/Migrations/081/CreateCustomCollectionPackage.swift @@ -0,0 +1,43 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Fluent +import SQLKit + + +struct CreateCustomCollectionPackage: AsyncMigration { + func prepare(on database: Database) async throws { + try await database.schema("custom_collections+packages") + + // managed fields + .id() + .field("created_at", .datetime) + .field("updated_at", .datetime) + + // reference fields + .field("custom_collection_id", .uuid, + .references("custom_collections", "id", onDelete: .cascade), .required) + .field("package_id", .uuid, + .references("packages", "id", onDelete: .cascade), .required) + + // constraints + .unique(on: "custom_collection_id", "package_id") + + .create() + } + + func revert(on database: Database) async throws { + try await database.schema("custom_collections+packages").delete() + } +} diff --git a/Sources/App/Models/CustomCollection.swift b/Sources/App/Models/CustomCollection.swift new file mode 100644 index 000000000..650acede0 --- /dev/null +++ b/Sources/App/Models/CustomCollection.swift @@ -0,0 +1,115 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Fluent +import Vapor + + +final class CustomCollection: @unchecked Sendable, Model, Content { + static let schema = "custom_collections" + + typealias Id = UUID + + // managed fields + + @ID(key: .id) + var id: Id? + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + // periphery:ignore + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + // data fields + + @Field(key: "name") + var name: String + + @Field(key: "description") + var description: String? + + @Field(key: "badge") + var badge: String? + + @Field(key: "url") + var url: URL + + // reference fields + @Siblings(through: CustomCollectionPackage.self, from: \.$customCollection, to: \.$package) + var packages: [Package] + + init() { } + + init(id: Id? = nil, createdAt: Date? = nil, updatedAt: Date? = nil, _ dto: DTO) { + self.id = id + self.createdAt = createdAt + self.updatedAt = updatedAt + self.name = dto.name + self.description = dto.description + self.badge = dto.badge + self.url = dto.url + } +} + + +extension CustomCollection { + struct DTO: Codable { + var name: String + var description: String? + var badge: String? + var url: URL + } + + static func findOrCreate(on database: Database, _ dto: DTO) async throws -> CustomCollection { + if let collection = try await CustomCollection.query(on: database) + .filter(\.$url == dto.url) + .first() { + return collection + } else { + let collection = CustomCollection(dto) + try await collection.save(on: database) + return collection + } + } + + func reconcile(on database: Database, packageURLs: some Collection) async throws { + let incoming: [Package.Id: Package] = .init( + packages: try await Package.query(on: database) + .filter(by: packageURLs) + .all() + ) + try await $packages.load(on: database) + let existing: [Package.Id: Package] = .init(packages: packages) + let newIDs = Set(incoming.keys).subtracting(Set(existing.keys)) + try await $packages.attach(incoming[newIDs], on: database) + let removedIDs = Set(existing.keys).subtracting(Set(incoming.keys)) + try await $packages.detach(existing[removedIDs], on: database) + } +} + + +private extension [Package.Id: Package] { + init(packages: [Package]) { + self.init( + packages.compactMap({ pkg in pkg.id.map({ ($0, pkg) }) }), + uniquingKeysWith: { (first, second) in first } + ) + } + + subscript(ids: some Collection) -> [Package] { + Array(ids.compactMap { self[$0] }) + } +} diff --git a/Sources/App/Models/CustomCollectionPackage.swift b/Sources/App/Models/CustomCollectionPackage.swift new file mode 100644 index 000000000..8272ed0b6 --- /dev/null +++ b/Sources/App/Models/CustomCollectionPackage.swift @@ -0,0 +1,53 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import Fluent +import Vapor + + +final class CustomCollectionPackage: @unchecked Sendable, Model, Content { + static let schema = "custom_collections+packages" + + typealias Id = UUID + + // managed fields + + @ID(key: .id) + var id: Id? + + @Timestamp(key: "created_at", on: .create) + var createdAt: Date? + + // periphery:ignore + @Timestamp(key: "updated_at", on: .update) + var updatedAt: Date? + + // reference fields + + @Parent(key: "custom_collection_id") + var customCollection: CustomCollection + + @Parent(key: "package_id") + var package: Package + + init() { } + + init(id: Id? = nil, createdAt: Date? = nil, updatedAt: Date? = nil, customCollection: CustomCollection, package: Package) { + self.id = id + self.createdAt = createdAt + self.updatedAt = updatedAt + self.customCollection = customCollection + self.package = package + } +} diff --git a/Sources/App/Models/Package.swift b/Sources/App/Models/Package.swift index 387dddcf0..7d9a0e5dd 100644 --- a/Sources/App/Models/Package.swift +++ b/Sources/App/Models/Package.swift @@ -271,6 +271,11 @@ extension QueryBuilder where Model == Package { func filter(by url: URL) -> Self { filter(\.$url == url.absoluteString) } + + func filter(by urls: some Collection) -> Self { + // TODO: make case-insensitive and canonicalise incoming URLs + filter(\.$url ~~ urls.map(\.absoluteString)) + } } diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 28cbd3fb6..98e15cd3d 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -340,6 +340,10 @@ public func configure(_ app: Application) async throws -> String { do { // Migration 080 - Set`forked_from` to NULL because of Fork model change in Repository app.migrations.add(UpdateRepositoryResetForkedFrom()) } + do { // Migration 081 - Create `custom_collections` + app.migrations.add(CreateCustomCollection()) + app.migrations.add(CreateCustomCollectionPackage()) + } app.asyncCommands.use(Analyze.Command(), as: "analyze") app.asyncCommands.use(CreateRestfileCommand(), as: "create-restfile") diff --git a/Tests/AppTests/CustomCollectionTests.swift b/Tests/AppTests/CustomCollectionTests.swift new file mode 100644 index 000000000..3dbd9c807 --- /dev/null +++ b/Tests/AppTests/CustomCollectionTests.swift @@ -0,0 +1,310 @@ +// Copyright Dave Verwer, Sven A. Schmidt, and other contributors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import XCTest + +@testable import App + +import Dependencies + + +class CustomCollectionTests: AppTestCase { + + func test_CustomCollection_save() async throws { + // MUT + try await CustomCollection(id: .id0, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + .save(on: app.db) + + do { // validate + let collection = try await CustomCollection.find(.id0, on: app.db).unwrap() + XCTAssertEqual(collection.name, "List") + XCTAssertEqual(collection.url, "https://github.com/foo/bar/list.json") + } + + do { // ensure name is unique + try await CustomCollection(.init(name: "List", url: "https://github.com/foo/bar/other-list.json")) + .save(on: app.db) + XCTFail("Expected failure") + } catch { + let msg = String(reflecting: error) + XCTAssert(msg.contains(#"duplicate key value violates unique constraint "uq:custom_collections.name""#), + "was: \(msg)") + } + + do { // ensure url is unique + try await CustomCollection(.init(name: "List 2", url: "https://github.com/foo/bar/list.json")) + .save(on: app.db) + XCTFail("Expected failure") + } catch { + let msg = String(reflecting: error) + XCTAssert(msg.contains(#"duplicate key value violates unique constraint "uq:custom_collections.url""#), + "was: \(msg)") + } + } + + func test_CustomCollection_findOrCreate() async throws { + do { // initial call creates collection + // MUT + let res = try await CustomCollection.findOrCreate(on: app.db, .init(name: "List", url: "url")) + + // validate + XCTAssertEqual(res.name, "List") + XCTAssertEqual(res.url, "url") + + let c = try await CustomCollection.query(on: app.db).all() + XCTAssertEqual(c.count, 1) + XCTAssertEqual(c.first?.name, "List") + XCTAssertEqual(c.first?.url, "url") + } + + do { // re-running is idempotent + // MUT + let res = try await CustomCollection.findOrCreate(on: app.db, .init(name: "List", url: "url")) + + // validate + XCTAssertEqual(res.name, "List") + XCTAssertEqual(res.url, "url") + + let c = try await CustomCollection.query(on: app.db).all() + XCTAssertEqual(c.count, 1) + XCTAssertEqual(c.first?.name, "List") + XCTAssertEqual(c.first?.url, "url") + } + } + + func test_CustomCollectionPackage_attach() async throws { + // setup + let pkg = Package(id: .id0, url: "1".asGithubUrl.url) + try await pkg.save(on: app.db) + let collection = CustomCollection(id: .id1, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + + // MUT + try await collection.$packages.attach(pkg, on: app.db) + + do { // validate + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 1) + let pivot = try await CustomCollectionPackage.query(on: app.db).first().unwrap() + try await pivot.$package.load(on: app.db) + XCTAssertEqual(pivot.package.id, .id0) + XCTAssertEqual(pivot.package.url, "1".asGithubUrl) + try await pivot.$customCollection.load(on: app.db) + XCTAssertEqual(pivot.customCollection.id, .id1) + XCTAssertEqual(pivot.customCollection.name, "List") + } + + do { // ensure package is unique per list + try await collection.$packages.attach(pkg, on: app.db) + } catch { + let msg = String(reflecting: error) + XCTAssert(msg.contains(#"duplicate key value violates unique constraint "uq:custom_collections+packages.custom_collection_id+custom_coll""#), + "was: \(msg)") + } + } + + func test_CustomCollectionPackage_detach() async throws { + // setup + let pkg = Package(id: .id0, url: "1".asGithubUrl.url) + try await pkg.save(on: app.db) + let collection = CustomCollection(id: .id1, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await collection.$packages.attach(pkg, on: app.db) + + // MUT + try await collection.$packages.detach(pkg, on: app.db) + + do { // validate + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 0) + } + + do { // ensure packag and collection are untouched + _ = try await Package.find(.id0, on: app.db).unwrap() + _ = try await CustomCollection.find(.id1, on: app.db).unwrap() + } + } + + func test_CustomCollection_packages() async throws { + // setup + let p1 = Package(id: .id0, url: "1".asGithubUrl.url) + try await p1.save(on: app.db) + let p2 = Package(id: .id1, url: "2".asGithubUrl.url) + try await p2.save(on: app.db) + let collection = CustomCollection(id: .id2, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await collection.$packages.attach([p1, p2], on: app.db) + + do { // MUT + let collection = try await CustomCollection.find(.id2, on: app.db).unwrap() + try await collection.$packages.load(on: app.db) + let packages = collection.packages + XCTAssertEqual(Set(packages.map(\.id)) , Set([.id0, .id1])) + } + } + + func test_CustomCollection_cascade() async throws { + // setup + let pkg = Package(id: .id0, url: "1".asGithubUrl.url) + try await pkg.save(on: app.db) + let collection = CustomCollection(id: .id1, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await collection.$packages.attach(pkg, on: app.db) + do { + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + do { + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + do { + let count = try await Package.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + + // MUT + try await collection.delete(on: app.db) + + // validation + do { + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 0) + } + do { + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 0) + } + do { + let count = try await Package.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + } + + func test_Package_cascade() async throws { + // setup + let pkg = Package(id: .id0, url: "1".asGithubUrl.url) + try await pkg.save(on: app.db) + let collection = CustomCollection(id: .id1, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await collection.$packages.attach(pkg, on: app.db) + do { + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + do { + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + do { + let count = try await Package.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + + // MUT + try await pkg.delete(on: app.db) + + // validation + do { + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + } + do { + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 0) + } + do { + let count = try await Package.query(on: app.db).count() + XCTAssertEqual(count, 0) + } + } + + func test_CustomCollection_reconcile() async throws { + // Test reconciliation of a custom collection against a list of package URLs + let collection = CustomCollection(id: .id0, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await Package(id: .id1, url: URL("a")).save(on: app.db) + try await Package(id: .id2, url: URL("b")).save(on: app.db) + + do { // Initial set of URLs + // MUT + try await collection.reconcile(on: app.db, packageURLs: [URL("a")]) + + do { // validate + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.find(.id0, on: app.db).unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), ["a"]) + } + } + + do { // Add more URLs + // MUT + try await collection.reconcile(on: app.db, packageURLs: [ + URL("a"), + URL("b") + ]) + + do { // validate + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 2) + let collection = try await CustomCollection.find(.id0, on: app.db).unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url).sorted(), [ + "a", + "b" + ]) + } + } + + do { // Remove URLs + // MUT + try await collection.reconcile(on: app.db, packageURLs: [ + URL("b") + ]) + + do { // validate + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.find(.id0, on: app.db).unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), ["b"]) + } + } + } + + func test_CustomCollection_reconcile_caseSensitive() async throws { + // Test reconciliation with a case-insensitive matching URL + let collection = CustomCollection(id: .id0, .init(name: "List", url: "https://github.com/foo/bar/list.json")) + try await collection.save(on: app.db) + try await Package(id: .id1, url: URL("a")).save(on: app.db) + + // MUT + try await collection.reconcile(on: app.db, packageURLs: [URL("A")]) + + do { // validate + // The package is not added to the custom collection, because it is not an + // exact match for the package URL. + // This is currently a limiting of the Fluent ~~ operator in the query + // filter(\.$url ~~ urls.map(\.absoluteString)) + let count = try await CustomCollectionPackage.query(on: app.db).count() + XCTAssertEqual(count, 0) + let collection = try await CustomCollection.find(.id0, on: app.db).unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), []) + } + } + +} diff --git a/Tests/AppTests/MastodonTests.swift b/Tests/AppTests/MastodonTests.swift index 2708464dd..1f5b0ccf7 100644 --- a/Tests/AppTests/MastodonTests.swift +++ b/Tests/AppTests/MastodonTests.swift @@ -58,6 +58,8 @@ final class MastodonTests: AppTestCase { try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } } operation: { // run first two processing steps try await reconcile(client: app.client, database: app.db) diff --git a/Tests/AppTests/MetricsTests.swift b/Tests/AppTests/MetricsTests.swift index 0ecc64c2e..49fe99ac7 100644 --- a/Tests/AppTests/MetricsTests.swift +++ b/Tests/AppTests/MetricsTests.swift @@ -14,6 +14,7 @@ @testable import App +import Dependencies import Prometheus import XCTest @@ -98,14 +99,18 @@ class MetricsTests: AppTestCase { } func test_reconcileDurationSeconds() async throws { - // setup - Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs } + try await withDependencies { + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } + } operation: { + // setup + Current.fetchPackageList = { _ in ["1", "2", "3"].asURLs } - // MUT - try await reconcile(client: app.client, database: app.db) + // MUT + try await reconcile(client: app.client, database: app.db) - // validation - XCTAssert((AppMetrics.reconcileDurationSeconds?.get()) ?? 0 > 0) + // validation + XCTAssert((AppMetrics.reconcileDurationSeconds?.get()) ?? 0 > 0) + } } func test_ingestDurationSeconds() async throws { diff --git a/Tests/AppTests/PackageTests.swift b/Tests/AppTests/PackageTests.swift index f7cadab63..518a28cfa 100644 --- a/Tests/AppTests/PackageTests.swift +++ b/Tests/AppTests/PackageTests.swift @@ -136,6 +136,14 @@ final class PackageTests: AppTestCase { XCTAssertEqual(res.map(\.url), ["https://foo.com/1"]) } + func test_filter_by_urls() async throws { + for url in ["https://foo.com/1", "https://foo.com/2", "https://foo.com/a", "https://foo.com/A"] { + try await Package(url: url.url).save(on: app.db) + } + let res = try await Package.query(on: app.db).filter(by: ["https://foo.com/2", "https://foo.com/a"]).all() + XCTAssertEqual(res.map(\.url), ["https://foo.com/2", "https://foo.com/a"]) + } + func test_repository() async throws { let pkg = try await savePackage(on: app.db, "1") do { @@ -289,6 +297,7 @@ final class PackageTests: AppTestCase { func test_isNew() async throws { try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } } operation: { // setup let url = "1".asGithubUrl diff --git a/Tests/AppTests/PipelineTests.swift b/Tests/AppTests/PipelineTests.swift index 80efbb85b..c66c3884e 100644 --- a/Tests/AppTests/PipelineTests.swift +++ b/Tests/AppTests/PipelineTests.swift @@ -160,6 +160,8 @@ class PipelineTests: AppTestCase { func test_processing_pipeline() async throws { try await withDependencies { $0.date.now = .now + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in [] } + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [] } } operation: { // Test pipeline pick-up end to end // setup diff --git a/Tests/AppTests/ReconcilerTests.swift b/Tests/AppTests/ReconcilerTests.swift index 3d25e2d01..0eb97a18d 100644 --- a/Tests/AppTests/ReconcilerTests.swift +++ b/Tests/AppTests/ReconcilerTests.swift @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +import XCTest + @testable import App +import Dependencies import Vapor -import XCTest class ReconcilerTests: AppTestCase { @@ -33,13 +35,13 @@ class ReconcilerTests: AppTestCase { XCTAssertEqual(urls.map(\.absoluteString).sorted(), ["1", "2", "3"]) } - func test_basic_reconciliation() async throws { + func test_reconcileMainPackageList() async throws { // setup let urls = ["1", "2", "3"] Current.fetchPackageList = { _ in urls.asURLs } // MUT - try await reconcile(client: app.client, database: app.db) + _ = try await reconcileMainPackageList(client: app.client, database: app.db) // validate let packages = try await Package.query(on: app.db).all() @@ -53,7 +55,7 @@ class ReconcilerTests: AppTestCase { } } - func test_adds_and_deletes() async throws { + func test_reconcileMainPackageList_adds_and_deletes() async throws { // save intial set of packages 1, 2, 3 for url in ["1", "2", "3"].asURLs { try await Package(url: url).save(on: app.db) @@ -64,14 +66,14 @@ class ReconcilerTests: AppTestCase { Current.fetchPackageList = { _ in urls.asURLs } // MUT - try await reconcile(client: app.client, database: app.db) + _ = try await reconcileMainPackageList(client: app.client, database: app.db) // validate let packages = try await Package.query(on: app.db).all() XCTAssertEqual(packages.map(\.url).sorted(), urls.sorted()) } - func test_packageDenyList() async throws { + func test_reconcileMainPackageList_packageDenyList() async throws { // Save the intial set of packages for url in ["1", "2", "3"].asURLs { try await Package(url: url).save(on: app.db) @@ -86,14 +88,14 @@ class ReconcilerTests: AppTestCase { Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } // MUT - try await reconcile(client: app.client, database: app.db) + _ = try await reconcileMainPackageList(client: app.client, database: app.db) // validate let packages = try await Package.query(on: app.db).all() XCTAssertEqual(packages.map(\.url).sorted(), ["1", "3", "5"]) } - func test_packageDenyList_caseSensitivity() async throws { + func test_reconcileMainPackageList_packageDenyList_caseSensitivity() async throws { // Save the intial set of packages for url in ["https://example.com/one/one", "https://example.com/two/two"].asURLs { try await Package(url: url).save(on: app.db) @@ -108,10 +110,136 @@ class ReconcilerTests: AppTestCase { Current.fetchPackageDenyList = { _ in packageDenyList.asURLs } // MUT - try await reconcile(client: app.client, database: app.db) + _ = try await reconcileMainPackageList(client: app.client, database: app.db) // validate let packages = try await Package.query(on: app.db).all() XCTAssertEqual(packages.map(\.url).sorted(), ["https://example.com/two/two"]) } + + func test_reconcileCustomCollections() async throws { + // Test single custom collection reconciliation + // setup + var fullPackageList = [URL("a"), URL("b"), URL("c")] + for url in fullPackageList { try await Package(url: url).save(on: app.db) } + + // Initial run + try await withDependencies { + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [URL("b")] } + } operation: { + // MUT + try await reconcileCustomCollection(client: app.client, + database: app.db, + fullPackageList: fullPackageList, + .init(name: "List", url: "url")) + + // validate + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.query(on: app.db).first().unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), ["b"]) + } + + // Reconcile again with an updated list of packages in the collection + try await withDependencies { + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [URL("c")] } + } operation: { + // MUT + try await reconcileCustomCollection(client: app.client, + database: app.db, + fullPackageList: fullPackageList, + .init(name: "List", url: "url")) + + // validate + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.query(on: app.db).first().unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), ["c"]) + } + + // Re-run after the single package in the list has been deleted in the full package list + fullPackageList = [URL("a"), URL("b")] + try await Package.query(on: app.db).filter(by: URL("c")).first()?.delete(on: app.db) + try await withDependencies { + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in [URL("c")] } + } operation: { + // MUT + try await reconcileCustomCollection(client: app.client, + database: app.db, + fullPackageList: fullPackageList, + .init(name: "List", url: "url")) + + // validate + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.query(on: app.db).first().unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), []) + } + } + + func test_reconcileCustomCollections_limit() async throws { + // Test custom collection reconciliation size limit + // setup + let fullPackageList = (1...60).map { URL(string: "\($0)")! } + for url in fullPackageList { try await Package(url: url).save(on: app.db) } + + try await withDependencies { + $0.packageListRepository.fetchCustomCollection = { @Sendable _, _ in + fullPackageList + } + } operation: { + // MUT + try await reconcileCustomCollection(client: app.client, + database: app.db, + fullPackageList: fullPackageList, + .init(name: "List", url: "url")) + + // validate + let collection = try await CustomCollection.query(on: app.db).first().unwrap() + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.count, 50) + XCTAssertEqual(collection.packages.first?.url, "1") + XCTAssertEqual(collection.packages.last?.url, "50") + } + } + + func test_reconcile() async throws { + let fullPackageList = (1...3).map { URL(string: "\($0)")! } + struct TestError: Error { var message: String } + + try await withDependencies { + $0.packageListRepository.fetchCustomCollection = { @Sendable _, url in + if url == "collectionURL" { + return [URL("2")] + } else { + throw TestError(message: "collection not found: \(url)") + } + } + $0.packageListRepository.fetchCustomCollections = { @Sendable _ in + [.init(name: "List", url: "collectionURL")] + } + } operation: { + // setup + Current.fetchPackageList = { _ in fullPackageList } + + // MUT + _ = try await reconcile(client: app.client, database: app.db) + + // validate + let packages = try await Package.query(on: app.db).all() + XCTAssertEqual(packages.map(\.url).sorted(), + fullPackageList.map(\.absoluteString).sorted()) + let count = try await CustomCollection.query(on: app.db).count() + XCTAssertEqual(count, 1) + let collection = try await CustomCollection.query(on: app.db).first().unwrap() + XCTAssertEqual(collection.name, "List") + XCTAssertEqual(collection.url, "collectionURL") + try await collection.$packages.load(on: app.db) + XCTAssertEqual(collection.packages.map(\.url), ["2"]) + } + } + }