CID-2911: merge main

Author: mohamedlajmileanix

src/main/kotlin/net/leanix/githubagent/client/GitHubClient.kt

@@ -28,7 +28,17 @@ interface GitHubClient {
     ): GitHubAppResponse
 
     @GetMapping("/api/v3/app/installations")
-    fun getInstallations(@RequestHeader("Authorization") jwt: String): List<Installation>
+    fun getInstallations(
+        @RequestHeader("Authorization") jwt: String,
+        @RequestParam("per_page", defaultValue = "30") perPage: Int,
+        @RequestParam("page", defaultValue = "1") page: Int
+    ): List<Installation>
+
+    @GetMapping("/api/v3/app/installations/{installationId}")
+    fun getInstallation(
+        @PathVariable("installationId") installationId: Long,
+        @RequestHeader("Authorization") jwt: String
+    ): Installation
 
     @PostMapping("/api/v3/app/installations/{installationId}/access_tokens")
     fun createInstallationToken(
@@ -37,7 +47,11 @@ interface GitHubClient {
     ): InstallationTokenResponse
 
     @GetMapping("/api/v3/organizations")
-    fun getOrganizations(@RequestHeader("Authorization") token: String): List<Organization>
+    fun getOrganizations(
+        @RequestHeader("Authorization") jwt: String,
+        @RequestParam("per_page", defaultValue = "30") perPage: Int,
+        @RequestParam("since", defaultValue = "1") since: Int
+    ): List<Organization>
 
     @GetMapping("/api/v3/orgs/{org}/repos")
     fun getRepositories(

src/main/kotlin/net/leanix/githubagent/dto/GitHubResponsesDto.kt

@@ -14,7 +14,9 @@ data class InstallationTokenResponse(
 @JsonIgnoreProperties(ignoreUnknown = true)
 data class Installation(
     @JsonProperty("id") val id: Long,
-    @JsonProperty("account") val account: Account
+    @JsonProperty("account") val account: Account,
+    @JsonProperty("permissions") val permissions: Map<String, String>,
+    @JsonProperty("events") val events: List<String>
 )
 
 @JsonIgnoreProperties(ignoreUnknown = true)

src/main/kotlin/net/leanix/githubagent/services/GitHubAPIService.kt

@@ -0,0 +1,42 @@
+package net.leanix.githubagent.services
+
+import net.leanix.githubagent.client.GitHubClient
+import net.leanix.githubagent.dto.Installation
+import net.leanix.githubagent.dto.Organization
+import org.springframework.stereotype.Service
+
+@Service
+class GitHubAPIService(
+    private val gitHubClient: GitHubClient,
+) {
+
+    companion object {
+        private const val PAGE_SIZE = 30 // Maximum allowed by GitHub API is 100
+    }
+
+    fun getPaginatedInstallations(jwtToken: String): List<Installation> {
+        val installations = mutableListOf<Installation>()
+        var page = 1
+        var currentInstallations: List<Installation>
+
+        do {
+            currentInstallations = gitHubClient.getInstallations("Bearer $jwtToken", PAGE_SIZE, page)
+            if (currentInstallations.isNotEmpty()) installations.addAll(currentInstallations) else break
+            page++
+        } while (currentInstallations.size == PAGE_SIZE)
+        return installations
+    }
+
+    fun getPaginatedOrganizations(installationToken: String): List<Organization> {
+        val organizations = mutableListOf<Organization>()
+        var since = 1
+        var currentOrganizations: List<Organization>
+
+        do {
+            currentOrganizations = gitHubClient.getOrganizations("Bearer $installationToken", PAGE_SIZE, since)
+            if (currentOrganizations.isNotEmpty()) organizations.addAll(currentOrganizations) else break
+            since = currentOrganizations.last().id
+        } while (currentOrganizations.size == PAGE_SIZE)
+        return organizations
+    }
+}

src/main/kotlin/net/leanix/githubagent/services/GitHubAuthenticationService.kt

@@ -6,6 +6,7 @@ import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.config.GitHubEnterpriseProperties
 import net.leanix.githubagent.dto.Installation
 import net.leanix.githubagent.exceptions.FailedToCreateJWTException
+import net.leanix.githubagent.exceptions.JwtTokenNotFound
 import org.bouncycastle.jce.provider.BouncyCastleProvider
 import org.slf4j.LoggerFactory
 import org.springframework.core.io.ResourceLoader
@@ -26,7 +27,8 @@ class GitHubAuthenticationService(
     private val githubEnterpriseProperties: GitHubEnterpriseProperties,
     private val resourceLoader: ResourceLoader,
     private val gitHubEnterpriseService: GitHubEnterpriseService,
-    private val gitHubClient: GitHubClient
+    private val gitHubClient: GitHubClient,
+    private val gitHubAPIService: GitHubAPIService,
 ) {
 
     companion object {
@@ -38,11 +40,9 @@ class GitHubAuthenticationService(
 
     fun refreshTokens() {
         generateAndCacheJwtToken()
-        val jwtToken = cachingService.get("jwtToken")
-        generateAndCacheInstallationTokens(
-            gitHubClient.getInstallations("Bearer $jwtToken"),
-            jwtToken.toString()
-        )
+        val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
+        val installations = gitHubAPIService.getPaginatedInstallations(jwtToken.toString())
+        generateAndCacheInstallationTokens(installations, jwtToken.toString())
     }
 
     fun generateAndCacheJwtToken() {

src/main/kotlin/net/leanix/githubagent/services/GitHubEnterpriseService.kt

@@ -1,14 +1,17 @@
 package net.leanix.githubagent.services
 
 import net.leanix.githubagent.client.GitHubClient
-import net.leanix.githubagent.dto.GitHubAppResponse
 import net.leanix.githubagent.exceptions.GitHubAppInsufficientPermissionsException
 import net.leanix.githubagent.exceptions.UnableToConnectToGitHubEnterpriseException
+import net.leanix.githubagent.shared.GITHUB_APP_LABEL
 import org.slf4j.LoggerFactory
 import org.springframework.stereotype.Service
 
 @Service
-class GitHubEnterpriseService(private val githubClient: GitHubClient) {
+class GitHubEnterpriseService(
+    private val githubClient: GitHubClient,
+    private val syncLogService: SyncLogService,
+) {
 
     companion object {
         val expectedPermissions = listOf("administration", "contents", "metadata")
@@ -19,29 +22,34 @@ class GitHubEnterpriseService(private val githubClient: GitHubClient) {
     fun verifyJwt(jwt: String) {
         runCatching {
             val githubApp = getGitHubApp(jwt)
-            validateGithubAppResponse(githubApp)
+            validateEnabledPermissionsAndEvents(GITHUB_APP_LABEL, githubApp.permissions, githubApp.events)
             logger.info("Authenticated as GitHub App: '${githubApp.slug}'")
         }.onFailure {
-            logger.error("Failed to verify JWT token", it)
             when (it) {
-                is GitHubAppInsufficientPermissionsException -> throw it
-                else -> throw UnableToConnectToGitHubEnterpriseException("Failed to verify JWT token")
+                is GitHubAppInsufficientPermissionsException -> {
+                    logger.error(it.message)
+                    syncLogService.sendErrorLog(it.message!!)
+                }
+                else -> {
+                    logger.error("Failed to verify JWT token", it)
+                    throw UnableToConnectToGitHubEnterpriseException("Failed to verify JWT token")
+                }
             }
         }
     }
 
-    fun validateGithubAppResponse(response: GitHubAppResponse) {
-        val missingPermissions = expectedPermissions.filterNot { response.permissions.containsKey(it) }
-        val missingEvents = expectedEvents.filterNot { response.events.contains(it) }
+    fun validateEnabledPermissionsAndEvents(type: String, permissions: Map<String, String>, events: List<String>) {
+        val missingPermissions = expectedPermissions.filterNot { permissions.containsKey(it) }
+        val missingEvents = expectedEvents.filterNot { events.contains(it) }
 
         if (missingPermissions.isNotEmpty() || missingEvents.isNotEmpty()) {
-            var message = "GitHub App is missing the following "
+            var message = "$type missing the following "
             if (missingPermissions.isNotEmpty()) {
                 message = message.plus("permissions: $missingPermissions")
             }
             if (missingEvents.isNotEmpty()) {
                 if (missingPermissions.isNotEmpty()) {
-                    message = message.plus(", and the following")
+                    message = message.plus(", and the following ")
                 }
                 message = message.plus("events: $missingEvents")
             }

src/main/kotlin/net/leanix/githubagent/services/GitHubScanningService.kt

@@ -9,9 +9,11 @@ import net.leanix.githubagent.dto.Organization
 import net.leanix.githubagent.dto.OrganizationDto
 import net.leanix.githubagent.dto.RateLimitType
 import net.leanix.githubagent.dto.RepositoryDto
+import net.leanix.githubagent.exceptions.GitHubAppInsufficientPermissionsException
 import net.leanix.githubagent.exceptions.JwtTokenNotFound
 import net.leanix.githubagent.exceptions.ManifestFileNotFoundException
 import net.leanix.githubagent.handler.RateLimitHandler
+import net.leanix.githubagent.shared.INSTALLATION_LABEL
 import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
 import net.leanix.githubagent.shared.fileNameMatchRegex
 import net.leanix.githubagent.shared.generateFullPath
@@ -27,6 +29,8 @@ class GitHubScanningService(
     private val gitHubAuthenticationService: GitHubAuthenticationService,
     private val syncLogService: SyncLogService,
     private val rateLimitHandler: RateLimitHandler,
+    private val gitHubEnterpriseService: GitHubEnterpriseService,
+    private val gitHubAPIService: GitHubAPIService,
 ) {
 
     private val logger = LoggerFactory.getLogger(GitHubScanningService::class.java)
@@ -36,17 +40,36 @@ class GitHubScanningService(
         val installations = getInstallations(jwtToken.toString())
         fetchAndSendOrganisationsData(installations)
         installations.forEach { installation ->
-            fetchAndSendRepositoriesData(installation)
-                .forEach { repository ->
-                    fetchManifestFilesAndSend(installation, repository)
+            kotlin.runCatching {
+                gitHubEnterpriseService.validateEnabledPermissionsAndEvents(
+                    INSTALLATION_LABEL,
+                    installation.permissions,
+                    installation.events
+                )
+                fetchAndSendRepositoriesData(installation)
+                    .forEach { repository ->
+                        fetchManifestFilesAndSend(installation, repository)
+                    }
+                syncLogService.sendInfoLog("Finished initial full scan for organization ${installation.account.login}.")
+            }.onFailure {
+                val message = "Failed to scan organization ${installation.account.login}."
+                when (it) {
+                    is GitHubAppInsufficientPermissionsException -> {
+                        syncLogService.sendErrorLog("$message ${it.message}")
+                        logger.error("$message ${it.message}")
+                    }
+                    else -> {
+                        syncLogService.sendErrorLog(message)
+                        logger.error(message, it)
+                    }
                 }
-            syncLogService.sendInfoLog("Finished initial full scan for organization ${installation.account.login}.")
+            }
         }
         syncLogService.sendInfoLog("Finished full scan for all available organizations.")
     }
 
     private fun getInstallations(jwtToken: String): List<Installation> {
-        val installations = gitHubClient.getInstallations("Bearer $jwtToken")
+        val installations = gitHubAPIService.getPaginatedInstallations(jwtToken)
         gitHubAuthenticationService.generateAndCacheInstallationTokens(installations, jwtToken)
         return installations
     }
@@ -61,7 +84,7 @@ class GitHubScanningService(
         }
         val installationToken = cachingService.get("installationToken:${installations.first().id}")
         val organizations = rateLimitHandler.executeWithRateLimitHandler(RateLimitType.REST) {
-            gitHubClient.getOrganizations("Bearer $installationToken")
+            gitHubAPIService.getPaginatedOrganizations(installationToken.toString())
                 .map { organization ->
                     if (installations.find { it.account.login == organization.login } != null) {
                         OrganizationDto(organization.id, organization.login, true)
@@ -105,6 +128,7 @@ class GitHubScanningService(
     }
 
     fun fetchManifestFilesAndSend(installation: Installation, repository: RepositoryDto) {
+        if (repository.archived) return
         val manifestFiles = fetchManifestFiles(installation, repository.name).getOrThrow().items
         val manifestFilesContents = fetchManifestContents(
             installation,

src/main/kotlin/net/leanix/githubagent/services/WebhookEventService.kt

@@ -2,13 +2,14 @@ package net.leanix.githubagent.services
 
 import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import com.fasterxml.jackson.module.kotlin.readValue
-import net.leanix.githubagent.dto.Account
-import net.leanix.githubagent.dto.Installation
+import net.leanix.githubagent.client.GitHubClient
 import net.leanix.githubagent.dto.InstallationEventPayload
 import net.leanix.githubagent.dto.ManifestFileAction
 import net.leanix.githubagent.dto.ManifestFileUpdateDto
 import net.leanix.githubagent.dto.PushEventCommit
 import net.leanix.githubagent.dto.PushEventPayload
+import net.leanix.githubagent.exceptions.JwtTokenNotFound
+import net.leanix.githubagent.shared.INSTALLATION_LABEL
 import net.leanix.githubagent.shared.MANIFEST_FILE_NAME
 import net.leanix.githubagent.shared.fileNameMatchRegex
 import net.leanix.githubagent.shared.generateFullPath
@@ -24,7 +25,9 @@ class WebhookEventService(
     private val gitHubAuthenticationService: GitHubAuthenticationService,
     private val gitHubScanningService: GitHubScanningService,
     private val syncLogService: SyncLogService,
-    @Value("\${webhookEventService.waitingTime}") private val waitingTime: Long
+    @Value("\${webhookEventService.waitingTime}") private val waitingTime: Long,
+    private val gitHubClient: GitHubClient,
+    private val gitHubEnterpriseService: GitHubEnterpriseService
 ) {
 
     private val logger = LoggerFactory.getLogger(WebhookEventService::class.java)
@@ -77,9 +80,15 @@ class WebhookEventService(
         }
         syncLogService.sendFullScanStart(installationEventPayload.installation.account.login)
         kotlin.runCatching {
-            val installation = Installation(
+            val jwtToken = cachingService.get("jwtToken") ?: throw JwtTokenNotFound()
+            val installation = gitHubClient.getInstallation(
                 installationEventPayload.installation.id.toLong(),
-                Account(installationEventPayload.installation.account.login)
+                "Bearer $jwtToken"
+            )
+            gitHubEnterpriseService.validateEnabledPermissionsAndEvents(
+                INSTALLATION_LABEL,
+                installation.permissions,
+                installation.events
             )
             gitHubAuthenticationService.refreshTokens()
             gitHubScanningService.fetchAndSendOrganisationsData(listOf(installation))

src/main/kotlin/net/leanix/githubagent/shared/Constants.kt

@@ -13,3 +13,6 @@ val SUPPORTED_EVENT_TYPES = listOf(
 )
 
 val fileNameMatchRegex = Regex("/?$MANIFEST_FILE_NAME\$")
+
+const val GITHUB_APP_LABEL = "GitHub App"
+const val INSTALLATION_LABEL = "Installation"

src/test/kotlin/net/leanix/githubagent/services/GitHubAPIServiceTest.kt

@@ -0,0 +1,81 @@
+package net.leanix.githubagent.services
+
+import io.mockk.every
+import io.mockk.mockk
+import net.leanix.githubagent.client.GitHubClient
+import net.leanix.githubagent.dto.Account
+import net.leanix.githubagent.dto.Installation
+import net.leanix.githubagent.dto.Organization
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+
+class GitHubAPIServiceTest {
+
+    private val gitHubClient = mockk<GitHubClient>()
+    private val gitHubAPIService = GitHubAPIService(gitHubClient)
+
+    private val permissions = mapOf("administration" to "read", "contents" to "read", "metadata" to "read")
+    private val events = listOf("label", "public", "repository", "push")
+
+    @Test
+    fun `test getPaginatedInstallations with one page`() {
+        val jwtToken = "test-jwt-token"
+        val installationsPage1 = listOf(
+            Installation(1, Account("test-account"), permissions, events),
+            Installation(2, Account("test-account"), permissions, events)
+        )
+
+        every { gitHubClient.getInstallations(any(), any(), any()) } returns installationsPage1
+
+        val installations = gitHubAPIService.getPaginatedInstallations(jwtToken)
+        assertEquals(2, installations.size)
+        assertEquals(installationsPage1, installations)
+    }
+
+    @Test
+    fun `test getPaginatedInstallations with multiple pages`() {
+        val jwtToken = "test-jwt-token"
+        val perPage = 30
+        val totalInstallations = 100
+        val installations = (1..totalInstallations).map {
+            Installation(it.toLong(), Account("test-account-$it"), permissions, events)
+        }
+        val pages = installations.chunked(perPage)
+
+        every { gitHubClient.getInstallations(any(), any(), any()) } returnsMany pages + listOf(emptyList())
+
+        val result = gitHubAPIService.getPaginatedInstallations(jwtToken)
+        assertEquals(totalInstallations, result.size)
+        assertEquals(installations, result)
+    }
+
+    @Test
+    fun `test getPaginatedOrganizations with one page`() {
+        val installationToken = "test-installation-token"
+        val organizationsPage1 = listOf(
+            Organization("org-1", 1),
+            Organization("org-2", 2)
+        )
+
+        every { gitHubClient.getOrganizations(any(), any(), any()) } returns organizationsPage1
+
+        val organizations = gitHubAPIService.getPaginatedOrganizations(installationToken)
+        assertEquals(2, organizations.size)
+        assertEquals(organizationsPage1, organizations)
+    }
+
+    @Test
+    fun `test getPaginatedOrganizations with multiple pages`() {
+        val installationToken = "test-installation-token"
+        val perPage = 30
+        val totalOrganizations = 100
+        val organizations = (1..totalOrganizations).map { Organization("org-$it", it) }
+        val pages = organizations.chunked(perPage)
+
+        every { gitHubClient.getOrganizations(any(), any(), any()) } returnsMany pages + listOf(emptyList())
+
+        val result = gitHubAPIService.getPaginatedOrganizations(installationToken)
+        assertEquals(totalOrganizations, result.size)
+        assertEquals(organizations, result)
+    }
+}