Skip to content

Commit 2d216b4

Browse files
finestructurefinestructure
authored
Merge pull request #3557 from SwiftPackageIndex/ingestion-namespace
Ingestion namespace
2 parents 8f8c2f0 + 2c386dc commit 2d216b4

File tree

8 files changed

+211
-211
lines changed

8 files changed

+211
-211
lines changed

‎Sources/App/Commands/Ingest.swift‎ renamed to ‎Sources/App/Commands/Ingestion.swift‎

Lines changed: 157 additions & 157 deletions
Large diffs are not rendered by default.

‎Sources/App/configure.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ public func configure(_ app: Application) async throws -> String {
354354
app.asyncCommands.use(Analyze.Command(), as: "analyze")
355355
app.asyncCommands.use(CreateRestfileCommand(), as: "create-restfile")
356356
app.asyncCommands.use(DeleteBuildsCommand(), as: "delete-builds")
357-
app.asyncCommands.use(IngestCommand(), as: "ingest")
357+
app.asyncCommands.use(Ingestion.Command(), as: "ingest")
358358
app.asyncCommands.use(ReconcileCommand(), as: "reconcile")
359359
app.asyncCommands.use(TriggerBuildsCommand(), as: "trigger-builds")
360360
app.asyncCommands.use(ReAnalyzeVersions.Command(), as: "re-analyze-versions")

‎Tests/AppTests/ErrorReportingTests.swift‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ class ErrorReportingTests: AppTestCase {
3131
}
3232
}
3333

34-
func test_Ingestor_error_reporting() async throws {
34+
func test_Ingestion_error_reporting() async throws {
3535
// setup
3636
try await Package(id: .id0, url: "1", processingStage: .reconciliation).save(on: app.db)
3737
Current.fetchMetadata = { _, _, _ throws(Github.Error) in throw Github.Error.invalidURL("1") }
@@ -40,7 +40,7 @@ class ErrorReportingTests: AppTestCase {
4040
$0.date.now = .now
4141
} operation: {
4242
// MUT
43-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
43+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
4444
}
4545

4646
// validation

‎Tests/AppTests/IngestorTests.swift‎ renamed to ‎Tests/AppTests/IngestionTests.swift‎

Lines changed: 40 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import S3Store
2222
import Vapor
2323

2424

25-
class IngestorTests: AppTestCase {
25+
class IngestionTests: AppTestCase {
2626

2727
func test_ingest_basic() async throws {
2828
// setup
@@ -38,7 +38,7 @@ class IngestorTests: AppTestCase {
3838
$0.date.now = .now
3939
} operation: {
4040
// MUT
41-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
41+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
4242
}
4343

4444
// validate
@@ -77,7 +77,7 @@ class IngestorTests: AppTestCase {
7777
Current.fetchLicense = { _, _, _ in Github.License(htmlUrl: "license") }
7878

7979
// MUT
80-
await ingest(client: app.client, database: app.db, packages: packages)
80+
await Ingestion.ingest(client: app.client, database: app.db, packages: packages)
8181

8282
do {
8383
// validate the second package's license is updated
@@ -97,12 +97,12 @@ class IngestorTests: AppTestCase {
9797
let repo = Repository(packageId: try pkg.requireID())
9898

9999
// MUT
100-
try await updateRepository(on: app.db,
101-
for: repo,
102-
metadata: .mock(owner: "foo", repository: "bar"),
103-
licenseInfo: .init(htmlUrl: ""),
104-
readmeInfo: .init(html: "", htmlUrl: "", imagesToCache: []),
105-
s3Readme: nil)
100+
try await Ingestion.updateRepository(on: app.db,
101+
for: repo,
102+
metadata: .mock(owner: "foo", repository: "bar"),
103+
licenseInfo: .init(htmlUrl: ""),
104+
readmeInfo: .init(html: "", htmlUrl: "", imagesToCache: []),
105+
s3Readme: nil)
106106

107107
// validate
108108
do {
@@ -154,16 +154,16 @@ class IngestorTests: AppTestCase {
154154
summary: "package desc")
155155

156156
// MUT
157-
try await updateRepository(on: app.db,
158-
for: repo,
159-
metadata: md,
160-
licenseInfo: .init(htmlUrl: "license url"),
161-
readmeInfo: .init(etag: "etag",
162-
html: "readme html https://img.shields.io/endpoint?url=https%3A%2F%2Fswiftpackageindex.com",
163-
htmlUrl: "readme html url",
164-
imagesToCache: []),
165-
s3Readme: .cached(s3ObjectUrl: "url", githubEtag: "etag"),
166-
fork: .parentURL("https://github.com/foo/bar.git"))
157+
try await Ingestion.updateRepository(on: app.db,
158+
for: repo,
159+
metadata: md,
160+
licenseInfo: .init(htmlUrl: "license url"),
161+
readmeInfo: .init(etag: "etag",
162+
html: "readme html https://img.shields.io/endpoint?url=https%3A%2F%2Fswiftpackageindex.com",
163+
htmlUrl: "readme html url",
164+
imagesToCache: []),
165+
s3Readme: .cached(s3ObjectUrl: "url", githubEtag: "etag"),
166+
fork: .parentURL("https://github.com/foo/bar.git"))
167167

168168
// validate
169169
do {
@@ -229,14 +229,14 @@ class IngestorTests: AppTestCase {
229229
summary: "package desc")
230230

231231
// MUT
232-
try await updateRepository(on: app.db,
233-
for: repo,
234-
metadata: md,
235-
licenseInfo: .init(htmlUrl: "license url"),
236-
readmeInfo: .init(html: "readme html",
237-
htmlUrl: "readme html url",
238-
imagesToCache: []),
239-
s3Readme: nil)
232+
try await Ingestion.updateRepository(on: app.db,
233+
for: repo,
234+
metadata: md,
235+
licenseInfo: .init(htmlUrl: "license url"),
236+
readmeInfo: .init(html: "readme html",
237+
htmlUrl: "readme html url",
238+
imagesToCache: []),
239+
s3Readme: nil)
240240

241241
// validate
242242
do {
@@ -310,7 +310,7 @@ class IngestorTests: AppTestCase {
310310
$0.date.now = .now
311311
} operation: {
312312
// MUT
313-
try await ingest(client: app.client, database: app.db, mode: .limit(testUrls.count))
313+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(testUrls.count))
314314
}
315315

316316
// validate
@@ -337,7 +337,7 @@ class IngestorTests: AppTestCase {
337337
$0.date.now = .now
338338
} operation: {
339339
// MUT
340-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
340+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
341341
}
342342

343343
// validate
@@ -389,7 +389,7 @@ class IngestorTests: AppTestCase {
389389
$0.date.now = .now
390390
} operation: {
391391
// MUT
392-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
392+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
393393
}
394394

395395
// validate repositories (single element pointing to the ingested package)
@@ -490,7 +490,7 @@ class IngestorTests: AppTestCase {
490490

491491
do { // first ingestion, no readme has been saved
492492
// MUT
493-
try await ingest(client: app.client, database: app.db, mode: .limit(1))
493+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(1))
494494

495495
// validate
496496
try await XCTAssertEqualAsync(await Repository.query(on: app.db).count(), 1)
@@ -506,7 +506,7 @@ class IngestorTests: AppTestCase {
506506
try await pkg.save(on: app.db)
507507

508508
// MUT
509-
try await ingest(client: app.client, database: app.db, mode: .limit(1))
509+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(1))
510510

511511
// validate
512512
try await XCTAssertEqualAsync(await Repository.query(on: app.db).count(), 1)
@@ -522,7 +522,7 @@ class IngestorTests: AppTestCase {
522522
try await pkg.save(on: app.db)
523523

524524
// MUT
525-
try await ingest(client: app.client, database: app.db, mode: .limit(1))
525+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(1))
526526

527527
// validate
528528
try await XCTAssertEqualAsync(await Repository.query(on: app.db).count(), 1)
@@ -573,7 +573,7 @@ class IngestorTests: AppTestCase {
573573
$0.date.now = .now
574574
} operation: {
575575
// MUT
576-
try await ingest(client: app.client, database: app.db, mode: .limit(1))
576+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(1))
577577
}
578578

579579
// There should only be one call as `storeS3ReadmeImages` takes the array of images.
@@ -604,7 +604,7 @@ class IngestorTests: AppTestCase {
604604
} operation: {
605605
// MUT
606606
let app = self.app!
607-
try await ingest(client: app.client, database: app.db, mode: .limit(1))
607+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(1))
608608
}
609609

610610
// validate
@@ -660,23 +660,23 @@ class IngestorTests: AppTestCase {
660660
try await Package(url: "https://github.com/bar/forked.git", processingStage: .analysis).save(on: app.db)
661661

662662
// test lookup when package is in the index
663-
let fork = await getFork(on: app.db, parent: .init(url: "https://github.com/foo/parent.git"))
663+
let fork = await Ingestion.getFork(on: app.db, parent: .init(url: "https://github.com/foo/parent.git"))
664664
XCTAssertEqual(fork, .parentId(id: .id0, fallbackURL: "https://github.com/foo/parent.git"))
665665

666666
// test lookup when package is in the index but with different case in URL
667-
let fork2 = await getFork(on: app.db, parent: .init(url: "https://github.com/Foo/Parent.git"))
667+
let fork2 = await Ingestion.getFork(on: app.db, parent: .init(url: "https://github.com/Foo/Parent.git"))
668668
XCTAssertEqual(fork2, .parentId(id: .id0, fallbackURL: "https://github.com/Foo/Parent.git"))
669669

670670
// test whem metadata repo url doesn't have `.git` at end
671-
let fork3 = await getFork(on: app.db, parent: .init(url: "https://github.com/Foo/Parent"))
671+
let fork3 = await Ingestion.getFork(on: app.db, parent: .init(url: "https://github.com/Foo/Parent"))
672672
XCTAssertEqual(fork3, .parentId(id: .id0, fallbackURL: "https://github.com/Foo/Parent.git"))
673673

674674
// test lookup when package is not in the index
675-
let fork4 = await getFork(on: app.db, parent: .init(url: "https://github.com/some/other.git"))
675+
let fork4 = await Ingestion.getFork(on: app.db, parent: .init(url: "https://github.com/some/other.git"))
676676
XCTAssertEqual(fork4, .parentURL("https://github.com/some/other.git"))
677677

678678
// test lookup when parent url is nil
679-
let fork5 = await getFork(on: app.db, parent: nil)
679+
let fork5 = await Ingestion.getFork(on: app.db, parent: nil)
680680
XCTAssertEqual(fork5, nil)
681681
}
682682
}

‎Tests/AppTests/MastodonTests.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,7 @@ final class MastodonTests: AppTestCase {
6666
} operation: {
6767
// run first two processing steps
6868
try await reconcile(client: app.client, database: app.db)
69-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
69+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
7070

7171
// MUT - analyze, triggering the post
7272
try await Analyze.analyze(client: app.client,
@@ -86,7 +86,7 @@ final class MastodonTests: AppTestCase {
8686
try await withDependencies {
8787
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
8888
} operation: {
89-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
89+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
9090

9191
// MUT - analyze, triggering posts if any
9292
try await Analyze.analyze(client: app.client,
@@ -104,7 +104,7 @@ final class MastodonTests: AppTestCase {
104104
// fast forward our clock by the deadtime interval again (*2) and re-ingest
105105
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime * 2)
106106
} operation: {
107-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
107+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
108108
// MUT - analyze again
109109
try await Analyze.analyze(client: app.client,
110110
database: app.db,

‎Tests/AppTests/MetricsTests.swift‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ class MetricsTests: AppTestCase {
120120
let pkg = try await savePackage(on: app.db, "1")
121121

122122
// MUT
123-
try await ingest(client: app.client, database: app.db, mode: .id(pkg.id!))
123+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .id(pkg.id!))
124124

125125
// validation
126126
XCTAssert((AppMetrics.ingestDurationSeconds?.get()) ?? 0 > 0)

‎Tests/AppTests/PackageTests.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -350,8 +350,8 @@ final class PackageTests: AppTestCase {
350350
}
351351

352352
// run ingestion to progress package through pipeline
353-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
354-
353+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
354+
355355
// MUT & validate
356356
do {
357357
let pkg = try await XCTUnwrapAsync(try await Package.query(on: app.db).first())
@@ -378,7 +378,7 @@ final class PackageTests: AppTestCase {
378378
try await withDependencies {
379379
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
380380
} operation: {
381-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
381+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
382382

383383
do {
384384
let pkg = try await XCTUnwrapAsync(try await Package.query(on: app.db).first())

‎Tests/AppTests/PipelineTests.swift‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -201,8 +201,8 @@ class PipelineTests: AppTestCase {
201201
}
202202

203203
// MUT - second stage
204-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
205-
204+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
205+
206206
do { // validate
207207
let packages = try await Package.query(on: app.db).sort(\.$url).all()
208208
XCTAssertEqual(packages.map(\.url), ["1", "2", "3"].asGithubUrls)
@@ -240,7 +240,7 @@ class PipelineTests: AppTestCase {
240240
}
241241

242242
// MUT - ingest again
243-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
243+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
244244

245245
do { // validate - only new package moves to .ingestion stage
246246
let packages = try await Package.query(on: app.db).sort(\.$url).all()
@@ -270,7 +270,7 @@ class PipelineTests: AppTestCase {
270270
$0.date.now = .now.addingTimeInterval(Constants.reIngestionDeadtime)
271271
} operation: {
272272
// MUT - ingest yet again
273-
try await ingest(client: app.client, database: app.db, mode: .limit(10))
273+
try await Ingestion.ingest(client: app.client, database: app.db, mode: .limit(10))
274274

275275
do { // validate - now all three packages should have been updated
276276
let packages = try await Package.query(on: app.db).sort(\.$url).all()

0 commit comments

Comments
 (0)