From 06f82be48b2b652d1c2b2211bede714b5e7a0cbc Mon Sep 17 00:00:00 2001 From: Piotr Krzeminski Date: Mon, 15 Dec 2025 23:27:35 +0100 Subject: [PATCH] chore(server): count calls to GitHub's API as metric Part of https://github.com/typesafegithub/github-workflows-kt/issues/2160. Thanks to this, we'll be able to assess the load on GitHub, and better understand if some server's instabilities are related to e.g. being throttled by GitHub. --- .../workflows/jitbindingserver/Main.kt | 18 ++++++++-- .../jitbindingserver/ArtifactRoutesTest.kt | 8 ++--- .../jitbindingserver/MetadataRoutesTest.kt | 12 ++++--- .../mavenbinding/MavenMetadataBuilding.kt | 5 ++- .../mavenbinding/PackageArtifactsBuilding.kt | 3 ++ .../mavenbinding/MavenMetadataBuildingTest.kt | 13 ++++--- shared-internal/build.gradle.kts | 1 + .../workflows/shared/internal/GithubApi.kt | 34 ++++++++++++++++--- 8 files changed, 73 insertions(+), 21 deletions(-) diff --git a/jit-binding-server/src/main/kotlin/io/github/typesafegithub/workflows/jitbindingserver/Main.kt b/jit-binding-server/src/main/kotlin/io/github/typesafegithub/workflows/jitbindingserver/Main.kt index b50513a050..bb51985e6e 100644 --- a/jit-binding-server/src/main/kotlin/io/github/typesafegithub/workflows/jitbindingserver/Main.kt +++ b/jit-binding-server/src/main/kotlin/io/github/typesafegithub/workflows/jitbindingserver/Main.kt @@ -21,6 +21,8 @@ import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty import io.ktor.server.response.respondText import io.ktor.server.routing.routing +import io.micrometer.core.instrument.MeterRegistry +import io.micrometer.core.instrument.Tag import io.micrometer.prometheusmetrics.PrometheusConfig import io.micrometer.prometheusmetrics.PrometheusMeterRegistry import java.time.Duration @@ -65,7 +67,12 @@ fun main() { fun Application.appModule( buildVersionArtifacts: suspend (ActionCoords, HttpClient) -> VersionArtifacts?, - buildPackageArtifacts: suspend (ActionCoords, String, (Collection) -> Unit) -> Map, + buildPackageArtifacts: suspend ( + ActionCoords, + String, + (Collection) -> Unit, + MeterRegistry, + ) -> Map, getGithubAuthToken: () -> String, ) { val httpClient = @@ -75,6 +82,7 @@ fun Application.appModule( val counter = prometheusRegistry.counter( "calls_to_github", + listOf(Tag.of("type", "static")), ) counter.increment() } @@ -106,7 +114,12 @@ private fun buildBindingsCache( @Suppress("ktlint:standard:function-signature") // Conflict with detekt. private fun buildMetadataCache( bindingsCache: LoadingCache, - buildPackageArtifacts: suspend (ActionCoords, String, (Collection) -> Unit) -> Map, + buildPackageArtifacts: suspend ( + ActionCoords, + String, + (Collection) -> Unit, + MeterRegistry, + ) -> Map, getGithubAuthToken: () -> String, ): LoadingCache = Caffeine @@ -118,6 +131,7 @@ private fun buildMetadataCache( it, getGithubAuthToken(), { coords -> prefetchBindingArtifacts(coords, bindingsCache) }, + prometheusRegistry, ) } diff --git a/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/ArtifactRoutesTest.kt b/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/ArtifactRoutesTest.kt index 7c60281051..3c2493c85f 100644 --- a/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/ArtifactRoutesTest.kt +++ b/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/ArtifactRoutesTest.kt @@ -30,7 +30,7 @@ class ArtifactRoutesTest : ) }, // Irrelevant for these tests. - buildPackageArtifacts = { _, _, _ -> emptyMap() }, + buildPackageArtifacts = { _, _, _, _ -> emptyMap() }, getGithubAuthToken = { "" }, ) } @@ -51,7 +51,7 @@ class ArtifactRoutesTest : appModule( buildVersionArtifacts = { _, _ -> null }, // Irrelevant for these tests. - buildPackageArtifacts = { _, _, _ -> emptyMap() }, + buildPackageArtifacts = { _, _, _, _ -> emptyMap() }, getGithubAuthToken = { "" }, ) } @@ -71,7 +71,7 @@ class ArtifactRoutesTest : appModule( buildVersionArtifacts = { _, _ -> error("An internal error occurred!") }, // Irrelevant for these tests. - buildPackageArtifacts = { _, _, _ -> emptyMap() }, + buildPackageArtifacts = { _, _, _, _ -> emptyMap() }, getGithubAuthToken = { "" }, ) } @@ -98,7 +98,7 @@ class ArtifactRoutesTest : appModule( buildVersionArtifacts = mockBuildVersionArtifacts, // Irrelevant for these tests. - buildPackageArtifacts = { _, _, _ -> emptyMap() }, + buildPackageArtifacts = { _, _, _, _ -> emptyMap() }, getGithubAuthToken = { "" }, ) } diff --git a/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/MetadataRoutesTest.kt b/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/MetadataRoutesTest.kt index 3f50a281e9..abc0a5c04d 100644 --- a/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/MetadataRoutesTest.kt +++ b/jit-binding-server/src/test/kotlin/io/github/typesafegithub/workflows/jitbindingserver/MetadataRoutesTest.kt @@ -8,6 +8,7 @@ import io.ktor.client.request.get import io.ktor.client.statement.bodyAsText import io.ktor.http.HttpStatusCode import io.ktor.server.testing.testApplication +import io.micrometer.core.instrument.MeterRegistry import io.mockk.every import io.mockk.mockk import io.mockk.verify @@ -20,7 +21,7 @@ class MetadataRoutesTest : // Given application { appModule( - buildPackageArtifacts = { _, _, _ -> + buildPackageArtifacts = { _, _, _, _ -> mapOf("maven-metadata.xml" to "Some XML contents") }, getGithubAuthToken = { "some-token" }, @@ -48,7 +49,7 @@ class MetadataRoutesTest : // Given application { appModule( - buildPackageArtifacts = { _, _, _ -> + buildPackageArtifacts = { _, _, _, _ -> emptyMap() }, getGithubAuthToken = { "some-token" }, @@ -75,7 +76,7 @@ class MetadataRoutesTest : // Given application { appModule( - buildPackageArtifacts = { _, _, _ -> + buildPackageArtifacts = { _, _, _, _ -> error("An internal error occurred!") }, getGithubAuthToken = { "some-token" }, @@ -106,9 +107,10 @@ class MetadataRoutesTest : ActionCoords, String, (Collection) -> Unit, + MeterRegistry?, ) -> Map, >() - every { mockBuildPackageArtifacts(any(), any(), any()) } throws + every { mockBuildPackageArtifacts(any(), any(), any(), any()) } throws Exception("An internal error occurred!") andThen mapOf("maven-metadata.xml" to "Some XML contents") application { @@ -135,7 +137,7 @@ class MetadataRoutesTest : // Then response2.status shouldBe HttpStatusCode.OK - verify(exactly = 2) { mockBuildPackageArtifacts(any(), any(), any()) } + verify(exactly = 2) { mockBuildPackageArtifacts(any(), any(), any(), any()) } } } } diff --git a/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuilding.kt b/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuilding.kt index ef1fe14994..d99f6ea7e4 100644 --- a/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuilding.kt +++ b/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuilding.kt @@ -7,21 +7,24 @@ import io.github.typesafegithub.workflows.actionbindinggenerator.domain.ActionCo import io.github.typesafegithub.workflows.actionbindinggenerator.domain.SignificantVersion.FULL import io.github.typesafegithub.workflows.shared.internal.fetchAvailableVersions import io.github.typesafegithub.workflows.shared.internal.model.Version +import io.micrometer.core.instrument.MeterRegistry import java.time.format.DateTimeFormatter private val logger = logger { } internal suspend fun ActionCoords.buildMavenMetadataFile( githubAuthToken: String, + meterRegistry: MeterRegistry? = null, fetchAvailableVersions: suspend ( owner: String, name: String, githubAuthToken: String?, + meterRegistry: MeterRegistry?, ) -> Either> = ::fetchAvailableVersions, prefetchBindingArtifacts: (Collection) -> Unit = {}, ): String? { val availableVersions = - fetchAvailableVersions(owner, name, githubAuthToken) + fetchAvailableVersions(owner, name, githubAuthToken, meterRegistry) .getOrElse { logger.error { it } emptyList() diff --git a/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/PackageArtifactsBuilding.kt b/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/PackageArtifactsBuilding.kt index 55e58817b1..ec241e62c5 100644 --- a/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/PackageArtifactsBuilding.kt +++ b/maven-binding-builder/src/main/kotlin/io/github/typesafegithub/workflows/mavenbinding/PackageArtifactsBuilding.kt @@ -1,16 +1,19 @@ package io.github.typesafegithub.workflows.mavenbinding import io.github.typesafegithub.workflows.actionbindinggenerator.domain.ActionCoords +import io.micrometer.core.instrument.MeterRegistry suspend fun buildPackageArtifacts( actionCoords: ActionCoords, githubAuthToken: String, prefetchBindingArtifacts: (Collection) -> Unit, + meterRegistry: MeterRegistry, ): Map { val mavenMetadata = actionCoords.buildMavenMetadataFile( githubAuthToken = githubAuthToken, prefetchBindingArtifacts = prefetchBindingArtifacts, + meterRegistry = meterRegistry, ) ?: return emptyMap() return mapOf( "maven-metadata.xml" to mavenMetadata, diff --git a/maven-binding-builder/src/test/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuildingTest.kt b/maven-binding-builder/src/test/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuildingTest.kt index 76adcdfb6c..7d97eeb10e 100644 --- a/maven-binding-builder/src/test/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuildingTest.kt +++ b/maven-binding-builder/src/test/kotlin/io/github/typesafegithub/workflows/mavenbinding/MavenMetadataBuildingTest.kt @@ -9,6 +9,7 @@ import io.github.typesafegithub.workflows.shared.internal.model.Version import io.kotest.core.spec.style.FunSpec import io.kotest.matchers.nulls.shouldBeNull import io.kotest.matchers.shouldBe +import io.micrometer.core.instrument.MeterRegistry import java.time.ZonedDateTime class MavenMetadataBuildingTest : @@ -26,7 +27,8 @@ class MavenMetadataBuildingTest : String, String, String?, - ) -> Either> = { _, _, _ -> + MeterRegistry?, + ) -> Either> = { _, _, _, _ -> listOf( Version(version = "v3-beta", dateProvider = { ZonedDateTime.parse("2024-07-01T00:00:00Z") }), Version(version = "v2", dateProvider = { ZonedDateTime.parse("2024-05-01T00:00:00Z") }), @@ -70,7 +72,8 @@ class MavenMetadataBuildingTest : String, String, String?, - ) -> Either> = { _, _, _ -> + MeterRegistry?, + ) -> Either> = { _, _, _, _ -> listOf( Version(version = "v1.1", dateProvider = { ZonedDateTime.parse("2024-03-07T00:00:00Z") }), Version(version = "v1.1.0", dateProvider = { ZonedDateTime.parse("2024-03-07T00:00:00Z") }), @@ -95,7 +98,8 @@ class MavenMetadataBuildingTest : String, String, String?, - ) -> Either> = { _, _, _ -> + MeterRegistry?, + ) -> Either> = { _, _, _, _ -> emptyList().right() } @@ -115,7 +119,8 @@ class MavenMetadataBuildingTest : String, String, String?, - ) -> Either> = { owner, name, _ -> + MeterRegistry?, + ) -> Either> = { owner, name, _, _ -> listOf( Version(version = "v3-beta", dateProvider = { ZonedDateTime.parse("2024-07-01T00:00:00Z") }), Version(version = "v2", dateProvider = { ZonedDateTime.parse("2024-05-01T00:00:00Z") }), diff --git a/shared-internal/build.gradle.kts b/shared-internal/build.gradle.kts index ce2cdfaadd..4d09f2fb67 100644 --- a/shared-internal/build.gradle.kts +++ b/shared-internal/build.gradle.kts @@ -14,6 +14,7 @@ dependencies { // we cannot use a BOM due to limitation in kotlin scripting when resolving the transitive KMM variant dependencies // note: see https://youtrack.jetbrains.com/issue/KT-67618 api("io.ktor:ktor-client-core:3.3.3") + api("io.micrometer:micrometer-core:1.15.5") implementation("io.ktor:ktor-client-cio:3.3.3") implementation("io.ktor:ktor-client-content-negotiation:3.3.3") implementation("io.ktor:ktor-client-logging:3.3.3") diff --git a/shared-internal/src/main/kotlin/io/github/typesafegithub/workflows/shared/internal/GithubApi.kt b/shared-internal/src/main/kotlin/io/github/typesafegithub/workflows/shared/internal/GithubApi.kt index c2d8912cfd..54a3eadc7a 100644 --- a/shared-internal/src/main/kotlin/io/github/typesafegithub/workflows/shared/internal/GithubApi.kt +++ b/shared-internal/src/main/kotlin/io/github/typesafegithub/workflows/shared/internal/GithubApi.kt @@ -7,15 +7,18 @@ import io.github.oshai.kotlinlogging.KotlinLogging.logger import io.github.typesafegithub.workflows.shared.internal.model.Version import io.ktor.client.HttpClient import io.ktor.client.call.body +import io.ktor.client.plugins.HttpSend import io.ktor.client.plugins.contentnegotiation.ContentNegotiation import io.ktor.client.plugins.logging.LogLevel.ALL import io.ktor.client.plugins.logging.Logger import io.ktor.client.plugins.logging.Logging +import io.ktor.client.plugins.plugin import io.ktor.client.request.bearerAuth import io.ktor.client.request.get import io.ktor.client.statement.bodyAsText import io.ktor.http.isSuccess import io.ktor.serialization.kotlinx.json.json +import io.micrometer.core.instrument.MeterRegistry import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json import java.time.ZonedDateTime @@ -26,25 +29,29 @@ suspend fun fetchAvailableVersions( owner: String, name: String, githubAuthToken: String?, + meterRegistry: MeterRegistry? = null, githubEndpoint: String = "https://api.github.com", ): Either> = either { - buildHttpClient().use { httpClient -> + buildHttpClient(meterRegistry = meterRegistry).use { httpClient -> return listOf( apiTagsUrl(githubEndpoint = githubEndpoint, owner = owner, name = name), apiBranchesUrl(githubEndpoint = githubEndpoint, owner = owner, name = name), ).flatMap { url -> fetchGithubRefs(url, githubAuthToken, httpClient).bind() } - .versions(githubAuthToken) + .versions(githubAuthToken, meterRegistry = meterRegistry) } } -private fun List.versions(githubAuthToken: String?): Either> = +private fun List.versions( + githubAuthToken: String?, + meterRegistry: MeterRegistry?, +): Either> = either { this@versions.map { githubRef -> val version = githubRef.ref.substringAfterLast("/") Version(version) { val response = - buildHttpClient().use { httpClient -> + buildHttpClient(meterRegistry = meterRegistry).use { httpClient -> httpClient .get(urlString = githubRef.`object`.url) { if (githubAuthToken != null) { @@ -122,7 +129,7 @@ private data class Person( val date: String, ) -private fun buildHttpClient() = +private fun buildHttpClient(meterRegistry: MeterRegistry?) = HttpClient { val klogger = logger install(Logging) { @@ -141,4 +148,21 @@ private fun buildHttpClient() = }, ) } + }.apply { + if (meterRegistry != null) { + plugin(HttpSend).intercept { request -> + if (request.url.host == "api.github.com") { + val counter = + meterRegistry.counter( + "calls_to_github", + listOf( + io.micrometer.core.instrument.Tag + .of("type", "api"), + ), + ) + counter.increment() + } + execute(request) + } + } }