Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 5 additions & 3 deletions Sources/App/Controllers/PackageController+routes.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,11 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import Dependencies
import Fluent
import Plot
import Vapor
import SemanticVersion
import Vapor


enum PackageController {
Expand Down Expand Up @@ -222,7 +223,7 @@ enum PackageController {
// r.owner,
// v.reference,
// d.linkable_paths_count
// '(owner: "' || r.owner || '", repository: "' || r.name || '"), // ' || d.linkable_paths_count || ' urls'
// '(owner: "' || r.owner || '", repository: "' || r.name || '"), // ' || d.linkable_paths_count || ' urls'
// FROM
// packages p
// INNER JOIN repositories r ON p.id = r.package_id
Expand Down Expand Up @@ -437,7 +438,8 @@ extension PackageController {

extension PackageController {
static func awsDocumentationURL(route: DocRoute) throws -> URI {
guard let bucket = Current.awsDocsBucket() else {
@Dependency(\.environment) var environment
guard let bucket = environment.awsDocsBucket() else {
throw AppError.envVariableNotSet("AWS_DOCS_BUCKET")
}

Expand Down
8 changes: 0 additions & 8 deletions Sources/App/Core/AppEnvironment.swift
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,6 @@ import FoundationNetworking
struct AppEnvironment: Sendable {
var apiSigningKey: @Sendable () -> String?
var appVersion: @Sendable () -> String?
var awsAccessKeyId: @Sendable () -> String?
var awsDocsBucket: @Sendable () -> String?
var awsReadmeBucket: @Sendable () -> String?
var awsSecretAccessKey: @Sendable () -> String?
var buildTriggerAllowList: @Sendable () -> [Package.Id]
var buildTriggerDownscaling: @Sendable () -> Double
var buildTriggerLatestSwiftVersionDownscaling: @Sendable () -> Double
Expand Down Expand Up @@ -106,10 +102,6 @@ extension AppEnvironment {
static let live = AppEnvironment(
apiSigningKey: { Environment.get("API_SIGNING_KEY") },
appVersion: { App.appVersion },
awsAccessKeyId: { Environment.get("AWS_ACCESS_KEY_ID") },
awsDocsBucket: { Environment.get("AWS_DOCS_BUCKET") },
awsReadmeBucket: { Environment.get("AWS_README_BUCKET") },
awsSecretAccessKey: { Environment.get("AWS_SECRET_ACCESS_KEY") },
buildTriggerAllowList: {
Environment.get("BUILD_TRIGGER_ALLOW_LIST")
.map { Data($0.utf8) }
Expand Down
8 changes: 8 additions & 0 deletions Sources/App/Core/Dependencies/EnvironmentClient.swift
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ struct EnvironmentClient {
// regarding the use of XCTFail here.
var allowBuildTriggers: @Sendable () -> Bool = { XCTFail(#function); return true }
var allowSocialPosts: @Sendable () -> Bool = { XCTFail(#function); return true }
var awsAccessKeyId: @Sendable () -> String?
var awsDocsBucket: @Sendable () -> String?
var awsReadmeBucket: @Sendable () -> String?
var awsSecretAccessKey: @Sendable () -> String?
var builderToken: @Sendable () -> String?
var buildTimeout: @Sendable () -> Int = { XCTFail(#function); return 10 }
// We're not defaulting current to XCTFail, because its use is too pervasive and would require the vast
Expand All @@ -47,6 +51,10 @@ extension EnvironmentClient: DependencyKey {
.flatMap(\.asBool)
?? Constants.defaultAllowSocialPosts
},
awsAccessKeyId: { Environment.get("AWS_ACCESS_KEY_ID") },
awsDocsBucket: { Environment.get("AWS_DOCS_BUCKET") },
awsReadmeBucket: { Environment.get("AWS_README_BUCKET") },
awsSecretAccessKey: { Environment.get("AWS_SECRET_ACCESS_KEY") },
builderToken: { Environment.get("BUILDER_TOKEN") },
buildTimeout: { Environment.get("BUILD_TIMEOUT").flatMap(Int.init) ?? 10 },
current: { (try? Environment.detect()) ?? .development },
Expand Down
14 changes: 9 additions & 5 deletions Sources/App/Core/Extensions/S3Store+ext.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

import S3Store
import Vapor
import Dependencies


extension S3Store {
Expand All @@ -27,8 +28,9 @@ extension S3Store {
}

static func storeReadme(owner: String, repository: String, readme: String) async throws -> String {
guard let accessKeyId = Current.awsAccessKeyId(),
let secretAccessKey = Current.awsSecretAccessKey()
@Dependency(\.environment) var environment
guard let accessKeyId = environment.awsAccessKeyId(),
let secretAccessKey = environment.awsSecretAccessKey()
else {
throw Error.genericError("missing AWS credentials")
}
Expand All @@ -42,8 +44,9 @@ extension S3Store {
}

static func storeReadmeImages(client: Client, imagesToCache: [Github.Readme.ImageToCache]) async throws {
guard let accessKeyId = Current.awsAccessKeyId(),
let secretAccessKey = Current.awsSecretAccessKey()
@Dependency(\.environment) var environment
guard let accessKeyId = environment.awsAccessKeyId(),
let secretAccessKey = environment.awsSecretAccessKey()
else {
throw Error.genericError("missing AWS credentials")
}
Expand All @@ -63,7 +66,8 @@ extension S3Store {

extension S3Store.Key {
static func readme(owner: String, repository: String, imageUrl: String? = nil) throws -> Self {
guard let bucket = Current.awsReadmeBucket() else {
@Dependency(\.environment) var environment
guard let bucket = environment.awsReadmeBucket() else {
throw S3Store.Error.genericError("AWS_README_BUCKET not set")
}

Expand Down
2 changes: 1 addition & 1 deletion Sources/App/Core/Gitlab.swift
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ extension Gitlab.Builder {
guard let pipelineToken = Current.gitlabPipelineToken(),
let builderToken = environment.builderToken()
else { throw Gitlab.Error.missingToken }
guard let awsDocsBucket = Current.awsDocsBucket() else {
guard let awsDocsBucket = environment.awsDocsBucket() else {
throw Gitlab.Error.missingConfiguration("AWS_DOCS_BUCKET")
}
let timeout = environment.buildTimeout() + (isDocBuild ? 5 : 0)
Expand Down
2 changes: 2 additions & 0 deletions Tests/AppTests/BuildTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,7 @@ class BuildTests: AppTestCase {

func test_trigger() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -197,6 +198,7 @@ class BuildTests: AppTestCase {

func test_trigger_isDocBuild() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down
10 changes: 10 additions & 0 deletions Tests/AppTests/BuildTriggerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,7 @@ class BuildTriggerTests: AppTestCase {

func test_triggerBuildsUnchecked() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -390,6 +391,7 @@ class BuildTriggerTests: AppTestCase {

func test_triggerBuildsUnchecked_supported() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -470,6 +472,7 @@ class BuildTriggerTests: AppTestCase {

func test_triggerBuildsUnchecked_build_exists() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -550,6 +553,7 @@ class BuildTriggerTests: AppTestCase {
func test_triggerBuilds_checked() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -662,6 +666,7 @@ class BuildTriggerTests: AppTestCase {
func test_triggerBuilds_multiplePackages() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -712,6 +717,7 @@ class BuildTriggerTests: AppTestCase {
func test_triggerBuilds_trimming() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
} operation: {
// Ensure we trim builds as part of triggering
Expand Down Expand Up @@ -747,6 +753,7 @@ class BuildTriggerTests: AppTestCase {
func test_triggerBuilds_error() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -843,6 +850,7 @@ class BuildTriggerTests: AppTestCase {

func test_override_switch() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -917,6 +925,7 @@ class BuildTriggerTests: AppTestCase {
func test_downscaling() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down Expand Up @@ -990,6 +999,7 @@ class BuildTriggerTests: AppTestCase {
func test_downscaling_allow_list_override() async throws {
try await withDependencies {
$0.environment.allowBuildTriggers = { true }
$0.environment.awsDocsBucket = { "awsDocsBucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Expand Down
82 changes: 44 additions & 38 deletions Tests/AppTests/GithubTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.

import XCTest

@testable import App

import Vapor
import SwiftSoup
import Dependencies
import S3Store
import XCTest
import SwiftSoup
import Vapor


class GithubTests: AppTestCase {
Expand Down Expand Up @@ -396,9 +398,10 @@
}

func test_extractImagesRequiringCaching() async throws {
Current.awsReadmeBucket = { "awsReadmeBucket" }

var readme = """
try withDependencies {
$0.environment.awsReadmeBucket = { "awsReadmeBucket" }
} operation: {
var readme = """
<html>
<head></head>
<body>
Expand All @@ -411,35 +414,37 @@
</html>
"""

// MUT
let images = Github.replaceImagesRequiringCaching(owner: "owner", repository: "repo", readme: &readme)

XCTAssertEqual(images, [
.init(originalUrl: "https://private-user-images.githubusercontent.com/with-jwt.jpg?jwt=some-jwt",
s3Key: S3Store.Key.init(bucket: "awsReadmeBucket", path: "owner/repo/with-jwt.jpg"))
])

let document = try SwiftSoup.parse(readme)
let imageElements = try document.select("img").array()

XCTAssertEqual(try imageElements.map { try $0.attr("src") }, [
"https://awsReadmeBucket.s3.us-east-2.amazonaws.com/owner/repo/with-jwt.jpg",
"https://private-user-images.githubusercontent.com/without-jwt.jpg",
"https://raw.githubusercontent.com/raw-image.png",
"https://github.com/example/repo/branch/assets/example.png",
"https://example.com/other-domain.jpg"
])

XCTAssertEqual(try imageElements.map { try $0.attr("data-original-src") }, [
"https://private-user-images.githubusercontent.com/with-jwt.jpg?jwt=some-jwt",
"", "", "", "" // This attribute only gets added to images that will be cached.
])
// MUT
let images = Github.replaceImagesRequiringCaching(owner: "owner", repository: "repo", readme: &readme)

XCTAssertEqual(images, [
.init(originalUrl: "https://private-user-images.githubusercontent.com/with-jwt.jpg?jwt=some-jwt",
s3Key: S3Store.Key.init(bucket: "awsReadmeBucket", path: "owner/repo/with-jwt.jpg"))
])

let document = try SwiftSoup.parse(readme)
let imageElements = try document.select("img").array()

XCTAssertEqual(try imageElements.map { try $0.attr("src") }, [
"https://awsReadmeBucket.s3.us-east-2.amazonaws.com/owner/repo/with-jwt.jpg",
"https://private-user-images.githubusercontent.com/without-jwt.jpg",
"https://raw.githubusercontent.com/raw-image.png",
"https://github.com/example/repo/branch/assets/example.png",
"https://example.com/other-domain.jpg"
])

XCTAssertEqual(try imageElements.map { try $0.attr("data-original-src") }, [
"https://private-user-images.githubusercontent.com/with-jwt.jpg?jwt=some-jwt",
"", "", "", "" // This attribute only gets added to images that will be cached.
])
}
}

func test_extractImagesRequiringCaching_noUnnecessaryChanges() async throws {
Current.awsReadmeBucket = { "awsReadmeBucket" }

var readme = """
try withDependencies {

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

no calls to throwing functions occur within 'try' expression

Check warning on line 444 in Tests/AppTests/GithubTests.swift

View workflow job for this annotation

GitHub Actions / Query Performance Test

no calls to throwing functions occur within 'try' expression
$0.environment.awsReadmeBucket = { "awsReadmeBucket" }
} operation: {
var readme = """
<html>
<head></head>
<body>
Expand All @@ -450,14 +455,15 @@
</html>
"""

let originalReadme = readme
let originalReadme = readme

// MUT
let images = Github.replaceImagesRequiringCaching(owner: "owner", repository: "repo", readme: &readme)
// MUT
let images = Github.replaceImagesRequiringCaching(owner: "owner", repository: "repo", readme: &readme)

// Checks
XCTAssertEqual(originalReadme, readme)
XCTAssertEqual(images, [])
// Checks
XCTAssertEqual(originalReadme, readme)
XCTAssertEqual(images, [])
}
}

func test_Readme_containsSPIBadge() throws {
Expand Down
7 changes: 4 additions & 3 deletions Tests/AppTests/GitlabBuilderTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,10 @@ class GitlabBuilderTests: AppTestCase {

func test_triggerBuild() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "docs-bucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Current.awsDocsBucket = { "docs-bucket" }
Current.gitlabPipelineToken = { "pipeline token" }
Current.siteURL = { "http://example.com" }
let buildId = UUID()
Expand Down Expand Up @@ -104,10 +104,10 @@ class GitlabBuilderTests: AppTestCase {

func test_issue_588() async throws {
try await withDependencies {
$0.environment.awsDocsBucket = { "docs-bucket" }
$0.environment.builderToken = { "builder token" }
$0.environment.buildTimeout = { 10 }
} operation: {
Current.awsDocsBucket = { "docs-bucket" }
Current.gitlabPipelineToken = { "pipeline token" }
Current.siteURL = { "http://example.com" }

Expand Down Expand Up @@ -181,12 +181,13 @@ class LiveGitlabBuilderTests: AppTestCase {
// Set this to a valid value if you want to report build results back to the server
ProcessInfo.processInfo.environment["LIVE_BUILDER_TOKEN"]
}
// make sure environment variables are configured for live access
$0.environment.awsDocsBucket = { "spi-dev-docs" }
} operation: {
// set build branch to trigger on
Gitlab.Builder.branch = "main"

// make sure environment variables are configured for live access
Current.awsDocsBucket = { "spi-dev-docs" }
Current.gitlabPipelineToken = {
// This Gitlab token is required in order to trigger the pipeline
ProcessInfo.processInfo.environment["LIVE_GITLAB_PIPELINE_TOKEN"]
Expand Down
8 changes: 6 additions & 2 deletions Tests/AppTests/IngestorTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -419,8 +419,12 @@ class IngestorTests: AppTestCase {
}

func test_S3Store_Key_readme() throws {
XCTAssertEqual(try S3Store.Key.readme(owner: "foo", repository: "bar").path, "foo/bar/readme.html")
XCTAssertEqual(try S3Store.Key.readme(owner: "FOO", repository: "bar").path, "foo/bar/readme.html")
try withDependencies {
$0.environment.awsReadmeBucket = { "readme-bucket" }
} operation: {
XCTAssertEqual(try S3Store.Key.readme(owner: "foo", repository: "bar").path, "foo/bar/readme.html")
XCTAssertEqual(try S3Store.Key.readme(owner: "FOO", repository: "bar").path, "foo/bar/readme.html")
}
}

func test_ingest_storeS3Readme() async throws {
Expand Down
Loading
Loading