diff --git a/adoptium-api-v3-persistence/pom.xml b/adoptium-api-v3-persistence/pom.xml index 776a3eabe..ff52120e9 100644 --- a/adoptium-api-v3-persistence/pom.xml +++ b/adoptium-api-v3-persistence/pom.xml @@ -32,6 +32,10 @@ com.fasterxml.jackson.datatype jackson-datatype-jsr310 + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + org.glassfish jakarta.json diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/XmlMapper.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/XmlMapper.kt new file mode 100644 index 000000000..987009a75 --- /dev/null +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/XmlMapper.kt @@ -0,0 +1,17 @@ +package net.adoptium.api.v3 + +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.databind.DeserializationFeature +import com.fasterxml.jackson.databind.MapperFeature +import com.fasterxml.jackson.dataformat.xml.JacksonXmlModule +import com.fasterxml.jackson.module.kotlin.registerKotlinModule + +object XmlMapper { + + val mapper: ObjectMapper = XmlMapper(JacksonXmlModule().apply { + setDefaultUseWrapper(false) + }).registerKotlinModule() + .configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true) + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) +} diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStore.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStore.kt index 89d32ba08..807b021ff 100644 --- a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStore.kt +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStore.kt @@ -1,6 +1,7 @@ package net.adoptium.api.v3.dataSources import net.adoptium.api.v3.dataSources.models.AdoptRepos +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo import net.adoptium.api.v3.dataSources.persitence.mongo.UpdatedInfo import net.adoptium.api.v3.models.ReleaseInfo @@ -8,8 +9,11 @@ interface APIDataStore { fun schedulePeriodicUpdates() fun getAdoptRepos(): AdoptRepos fun setAdoptRepos(binaryRepos: AdoptRepos) + fun getAdoptAttestationRepo(): AdoptAttestationRepo + fun setAdoptAttestationRepo(attestationRepo: AdoptAttestationRepo) fun getReleaseInfo(): ReleaseInfo fun loadDataFromDb(forceUpdate: Boolean, logEntries: Boolean = true): AdoptRepos + fun loadAttestationDataFromDb(forceUpdate: Boolean, logEntries: Boolean = true): AdoptAttestationRepo fun getUpdateInfo(): UpdatedInfo suspend fun isConnectedToDb(): Boolean } diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStoreImpl.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStoreImpl.kt index 1858b513d..0cfaa857a 100644 --- a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStoreImpl.kt +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/APIDataStoreImpl.kt @@ -4,6 +4,7 @@ import jakarta.enterprise.context.ApplicationScoped import jakarta.inject.Inject import kotlinx.coroutines.runBlocking import net.adoptium.api.v3.dataSources.models.AdoptRepos +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo import net.adoptium.api.v3.dataSources.models.FeatureRelease import net.adoptium.api.v3.dataSources.models.Releases import net.adoptium.api.v3.dataSources.persitence.ApiPersistence @@ -23,6 +24,7 @@ open class APIDataStoreImpl : APIDataStore { private var dataStore: ApiPersistence private var updatedAt: UpdatedInfo private var binaryRepos: AdoptRepos + private var attestationRepo: AdoptAttestationRepo private var releaseInfo: ReleaseInfo private var schedule: ScheduledFuture<*>? @@ -67,6 +69,32 @@ open class APIDataStoreImpl : APIDataStore { } } + fun loadAttestationDataFromDb( + dataStore: ApiPersistence, + previousUpdateInfo: UpdatedInfo, + forceUpdate: Boolean, + previousRepo: AdoptAttestationRepo?, + logEntries: Boolean = true): Pair { + + return runBlocking { + val updated = dataStore.getUpdatedAt() + + if (previousRepo == null || forceUpdate || updated != previousUpdateInfo) { + val data = dataStore.readAttestationData() + val updatedAt = dataStore.getUpdatedAt() + + val newData = AdoptAttestationRepo(data) + + if (logEntries) { + LOGGER.info("Loaded Attestations: $updatedAt") + } + Pair(newData, updatedAt) + } else { + Pair(previousRepo, previousUpdateInfo) + } + } + } + private fun filterValidAssets(data: List): AdoptRepos { // Ensure that we filter out valid releases/binaries for this ecosystem val filtered = AdoptRepos(data) @@ -131,6 +159,20 @@ open class APIDataStoreImpl : APIDataStore { AdoptRepos(listOf()) } + attestationRepo = try { + val update = loadAttestationDataFromDb( + dataStore, + updatedAt, + true, + null + ) + updatedAt = update.second + update.first + } catch (e: Exception) { + LOGGER.error("Failed to read attestation db", e) + AdoptAttestationRepo(listOf()) + } + releaseInfo = loadReleaseInfo() } @@ -194,6 +236,25 @@ open class APIDataStoreImpl : APIDataStore { } + override fun loadAttestationDataFromDb( + forceUpdate: Boolean, + logEntries: Boolean + ): AdoptAttestationRepo { + val update = loadAttestationDataFromDb( + dataStore, + updatedAt, + forceUpdate, + attestationRepo, + logEntries + ) + + this.updatedAt = update.second + this.attestationRepo = update.first + + return attestationRepo + + } + override fun getUpdateInfo(): UpdatedInfo { return updatedAt } @@ -211,6 +272,14 @@ open class APIDataStoreImpl : APIDataStore { this.binaryRepos = binaryRepos } + override fun getAdoptAttestationRepo(): AdoptAttestationRepo { + return attestationRepo + } + + override fun setAdoptAttestationRepo(attestationRepo: AdoptAttestationRepo) { + this.attestationRepo = attestationRepo + } + private fun periodicUpdate() { // Must catch errors or may kill the scheduler try { diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/ApiPersistence.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/ApiPersistence.kt index 568190f92..a71fe3394 100644 --- a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/ApiPersistence.kt +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/ApiPersistence.kt @@ -1,6 +1,7 @@ package net.adoptium.api.v3.dataSources.persitence import net.adoptium.api.v3.dataSources.models.AdoptRepos +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo import net.adoptium.api.v3.dataSources.models.FeatureRelease import net.adoptium.api.v3.dataSources.models.GitHubId import net.adoptium.api.v3.dataSources.models.ReleaseNotes @@ -10,11 +11,14 @@ import net.adoptium.api.v3.models.GHReleaseMetadata import net.adoptium.api.v3.models.GitHubDownloadStatsDbEntry import net.adoptium.api.v3.models.ReleaseInfo import net.adoptium.api.v3.models.Vendor +import net.adoptium.api.v3.models.Attestation import java.time.ZonedDateTime interface ApiPersistence { suspend fun updateAllRepos(repos: AdoptRepos, checksum: String) + //suspend fun updateAttestationRepo(repos: AdoptAttestationRepo, checksum: String) suspend fun readReleaseData(featureVersion: Int): FeatureRelease + suspend fun readAttestationData(): List suspend fun addGithubDownloadStatsEntries(stats: List) suspend fun getStatsForFeatureVersion(featureVersion: Int): List diff --git a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/mongo/MongoApiPersistence.kt b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/mongo/MongoApiPersistence.kt index c27296649..9e7c9a8ec 100644 --- a/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/mongo/MongoApiPersistence.kt +++ b/adoptium-api-v3-persistence/src/main/kotlin/net/adoptium/api/v3/dataSources/persitence/mongo/MongoApiPersistence.kt @@ -10,6 +10,8 @@ import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.flow.toList import net.adoptium.api.v3.TimeSource import net.adoptium.api.v3.dataSources.models.AdoptRepos +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo +import net.adoptium.api.v3.dataSources.models.AttestationRepoSummary import net.adoptium.api.v3.dataSources.models.FeatureRelease import net.adoptium.api.v3.dataSources.models.GitHubId import net.adoptium.api.v3.dataSources.models.ReleaseNotes @@ -21,6 +23,7 @@ import net.adoptium.api.v3.models.GitHubDownloadStatsDbEntry import net.adoptium.api.v3.models.Release import net.adoptium.api.v3.models.ReleaseInfo import net.adoptium.api.v3.models.Vendor +import net.adoptium.api.v3.models.Attestation import org.bson.BsonArray import org.bson.BsonBoolean import org.bson.BsonDateTime @@ -34,6 +37,7 @@ import java.time.ZonedDateTime open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : MongoInterface(), ApiPersistence { private val githubReleaseMetadataCollection: MongoCollection = createCollection(mongoClient.getDatabase(), GH_RELEASE_METADATA) private val releasesCollection: MongoCollection = createCollection(mongoClient.getDatabase(), RELEASE_DB) + private var attestationsCollection: MongoCollection = createCollection(mongoClient.getDatabase(), ATTESTATIONS_DB) private val gitHubStatsCollection: MongoCollection = createCollection(mongoClient.getDatabase(), GITHUB_STATS_DB) private val dockerStatsCollection: MongoCollection = createCollection(mongoClient.getDatabase(), DOCKER_STATS_DB) private val releaseInfoCollection: MongoCollection = createCollection(mongoClient.getDatabase(), RELEASE_INFO_DB) @@ -46,6 +50,7 @@ open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : M private val LOGGER = LoggerFactory.getLogger(this::class.java) const val GH_RELEASE_METADATA = "githubReleaseMetadata" const val RELEASE_DB = "release" + const val ATTESTATIONS_DB = "attestations" const val GITHUB_STATS_DB = "githubStats" const val DOCKER_STATS_DB = "dockerStats" const val RELEASE_INFO_DB = "releaseInfo" @@ -66,6 +71,15 @@ open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : M } } +// override suspend fun updateAttestationRepo(repo: AdoptAttestationRepo, checksum: String) { +// +// try { +// writeAttestations(repo.attestations) +// } finally { +// updateUpdatedTime(TimeSource.now(), checksum, repo.hashCode()) +// } +// } + private suspend fun writeReleases(featureVersion: Int, value: FeatureRelease) { val toAdd = value.releases.getReleases().toList() if (toAdd.isNotEmpty()) { @@ -74,6 +88,16 @@ open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : M } } + private suspend fun writeAttestations(attestations: List) { + if (attestations.isNotEmpty()) { + // Delete all existing + attestationsCollection.drop() + attestationsCollection = createCollection(client.getDatabase(), ATTESTATIONS_DB) + + attestationsCollection.insertMany(attestations, InsertManyOptions()) + } + } + override suspend fun readReleaseData(featureVersion: Int): FeatureRelease { val releases = releasesCollection .find(majorVersionMatcher(featureVersion)) @@ -82,6 +106,12 @@ open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : M return FeatureRelease(featureVersion, Releases(releases)) } + override suspend fun readAttestationData(): List { + return attestationsCollection + .find(Document()) + .toList() + } + override suspend fun addGithubDownloadStatsEntries(stats: List) { gitHubStatsCollection.insertMany(stats) } @@ -182,6 +212,7 @@ open class MongoApiPersistence @Inject constructor(mongoClient: MongoClient) : M } private fun majorVersionMatcher(featureVersion: Int) = Document("version_data.major", featureVersion) + private fun attestationMajorVersionMatcher(featureVersion: Int) = Document("featureVersion", featureVersion) override suspend fun getGhReleaseMetadata(gitHubId: GitHubId): GHReleaseMetadata? { return githubReleaseMetadataCollection.find(matchGithubId(gitHubId)).firstOrNull() diff --git a/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AdoptAttestationRepo.kt b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AdoptAttestationRepo.kt new file mode 100644 index 000000000..4284ddc2d --- /dev/null +++ b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AdoptAttestationRepo.kt @@ -0,0 +1,5 @@ +package net.adoptium.api.v3.dataSources.models + +import net.adoptium.api.v3.models.Attestation + +class AdoptAttestationRepo(val attestations: List) diff --git a/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AttestationRepoSummary.kt b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AttestationRepoSummary.kt new file mode 100644 index 000000000..4335fb2d6 --- /dev/null +++ b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/dataSources/models/AttestationRepoSummary.kt @@ -0,0 +1,114 @@ +package net.adoptium.api.v3.dataSources.models + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonProperty + +/* Format example: +Query: +query RepoFiles($owner: String!, $name: String!, $expr: String!) { + repository(owner: $owner, name: $name) { + object(expression: $expr) { + ... on Tree { + entries { + name + type + object { + ... on Tree { + entries { + name + type + object { + ... on Blob { + commitResourcePath + } + } + } + } + } + } + } + } + } +} +Response: +{ + "data": { + "repository": { + "object": { + "entries": [ + { + "name": ".github", + "type": "tree", + "object": { + "entries": [ + { + "name": "workflows", + "type": "tree", + "object": {} + } + ] + } + }, + { + "name": "21", + "type": "tree", + "object": { + "entries": [ + { + "name": "jdk_21_0_5_11_x64_linux_Adoptium.xml", + "type": "blob", + "object": { + "commitResourcePath": "/andrew-m-leonard/temurin-attestations/commit/97d84d270a34f66749e5e66b0a42996828bda4e1" + } + }, + { + "name": "jdk_21_0_5_11_x64_linux_Adoptium.xml.sign.pub", + "type": "blob", + "object": { + "commitResourcePath": "/andrew-m-leonard/temurin-attestations/commit/d65966533a06d90147e8f4ad963dcf7bb52e645e" + } + } + ] + } + }, + { + "name": "README.md", + "type": "blob", + "object": {} + } + ] + } + } + } +} + +*/ + +data class AttestationRepoSummary @JsonCreator constructor( + @JsonProperty("data") val data: AttestationRepoSummaryData? +) { +} + +data class AttestationRepoSummaryData @JsonCreator constructor( + @JsonProperty("repository") val repository: AttestationRepoSummaryRepository? +) { +} + +data class AttestationRepoSummaryRepository @JsonCreator constructor( + @JsonProperty("object") val att_object: AttestationRepoSummaryObject? +) { +} + +data class AttestationRepoSummaryObject @JsonCreator constructor( + @JsonProperty("commitResourcePath") val commitResourcePath: String?, + @JsonProperty("entries") val entries: List? +) { +} + +data class AttestationRepoSummaryEntry @JsonCreator constructor( + @JsonProperty("name") val name: String, + @JsonProperty("type") val type: String, + @JsonProperty("object") val att_object: AttestationRepoSummaryObject? +) { +} + diff --git a/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/models/Attestation.kt b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/models/Attestation.kt new file mode 100644 index 000000000..4310d2fef --- /dev/null +++ b/adoptium-models-parent/adoptium-api-v3-models/src/main/kotlin/net/adoptium/api/v3/models/Attestation.kt @@ -0,0 +1,130 @@ +package net.adoptium.api.v3.models + +import com.fasterxml.jackson.annotation.JsonCreator +import org.eclipse.microprofile.openapi.annotations.media.Schema + +class Attestation { + + val id: String + + val commitResourcePath: String + + val featureVersion: Int + + @Schema(example = "jdk-21.0.5+11") + val release_name: String + + val os: OperatingSystem + + val architecture: Architecture + + val image_type: ImageType + + val jvm_impl: JvmImpl + + val vendor: Vendor + + @Schema(description = "Assessor checksum of attested target") + val target_checksum: String? + + @Schema(example = "Acme Ltd") + val assessor_org: String + + @Schema(example = "We claim a verified reproducible build.") + val assessor_affirmation: String + + @Schema(example = "VERIFIED_REPRODUCIBLE_BUILD") + val assessor_claim_predicate: String + + @Schema(example = "https://github.com/adoptium/temurin-attestations/21/jdk_21_0_6_7_x64-linux_MyOrgLtd.xml") + val attestation_link: String + + @Schema(example = "https://github.com/adoptium/temurin-attestations/21/jdk_21_0_6_7_x64-linux_MyOrgLtd.xml.sign.pub") + val attestation_public_signing_key_link: String + + @JsonCreator + constructor( + id: String, + commitResourcePath: String, + featureVersion: Int, + release_name: String, + os: OperatingSystem, + architecture: Architecture, + image_type: ImageType, + jvm_impl: JvmImpl, + vendor: Vendor, + target_checksum: String?, + assessor_org: String, + assessor_affirmation: String, + assessor_claim_predicate: String, + attestation_link: String, + attestation_public_signing_key_link: String + ) { + this.id = id + this.commitResourcePath = commitResourcePath + this.featureVersion = featureVersion + this.release_name = release_name + this.os = os + this.architecture = architecture + this.image_type = image_type + this.jvm_impl = jvm_impl + this.vendor = vendor + this.target_checksum = target_checksum + this.assessor_org = assessor_org + this.assessor_affirmation = assessor_affirmation + this.assessor_claim_predicate = assessor_claim_predicate + this.attestation_link = attestation_link + this.attestation_public_signing_key_link = attestation_public_signing_key_link + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Attestation + + if (id != other.id) return false + if (commitResourcePath != other.commitResourcePath) return false + if (featureVersion != other.featureVersion) return false + if (release_name != other.release_name) return false + if (os != other.os) return false + if (architecture != other.architecture) return false + if (image_type != other.image_type) return false + if (jvm_impl != other.jvm_impl) return false + if (vendor != other.vendor) return false + if (target_checksum != other.target_checksum) return false + if (assessor_org != other.assessor_org) return false + if (assessor_affirmation != other.assessor_affirmation) return false + if (assessor_claim_predicate != other.assessor_claim_predicate) return false + if (attestation_link != other.attestation_link) return false + if (attestation_public_signing_key_link != other.attestation_public_signing_key_link) return false + + return true + } + + override fun hashCode(): Int { + var result = featureVersion.hashCode() + result = 31 * result + id.hashCode() + result = 31 * result + commitResourcePath.hashCode() + result = 31 * result + release_name.hashCode() + result = 31 * result + os.hashCode() + result = 31 * result + architecture.hashCode() + result = 31 * result + image_type.hashCode() + result = 31 * result + jvm_impl.hashCode() + result = 31 * result + vendor.hashCode() + result = 31 * result + target_checksum.hashCode() + result = 31 * result + assessor_org.hashCode() + result = 31 * result + assessor_affirmation.hashCode() + result = 31 * result + assessor_claim_predicate.hashCode() + result = 31 * result + attestation_link.hashCode() + result = 31 * result + attestation_public_signing_key_link.hashCode() + return result + } + + override fun toString(): String { + return "Attestation(id='$id', commitResourcePath='$commitResourcePath', featureVersion='$featureVersion', release_name='$release_name', os='$os', architecture='$architecture', image_type='$image_type', jvm_impl='$jvm_impl', vendor='$vendor'" + + "assessor_org='$assessor_org', assessor_affirmation='$assessor_affirmation', assessor_claim_predicate.hashCode='$assessor_claim_predicate.hashCode', " + + "target_checksum='$target_checksum', "+ + "attestation_link='$attestation_link', attestation_public_signing_key_link='$attestation_public_signing_key_link')" + } +} diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepoBuilder.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepoBuilder.kt new file mode 100644 index 000000000..55a093fd0 --- /dev/null +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepoBuilder.kt @@ -0,0 +1,40 @@ +package net.adoptium.api.v3 + +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import net.adoptium.api.v3.dataSources.VersionSupplier +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo +import net.adoptium.api.v3.dataSources.models.FeatureRelease +import net.adoptium.api.v3.dataSources.models.GitHubId +import net.adoptium.api.v3.dataSources.models.Releases +import net.adoptium.api.v3.mapping.ReleaseMapper +import net.adoptium.api.v3.models.GHReleaseMetadata +import net.adoptium.api.v3.models.Release +import org.slf4j.LoggerFactory +import java.time.temporal.ChronoUnit +import kotlin.math.absoluteValue + +@ApplicationScoped +class AdoptAttestationRepoBuilder @Inject constructor( + private var adoptAttestationRepository: AdoptAttestationRepository, + private var versionSupplier: VersionSupplier + ) { + + companion object { + @JvmStatic + private val LOGGER = LoggerFactory.getLogger(this::class.java) + } + + suspend fun build(): AdoptAttestationRepo { + // Fetch attestations in parallel + val attestationMap = versionSupplier + .getAllVersions() + .reversed() + .mapNotNull { version -> + adoptAttestationRepository.getAttestations(version) + } + .associateBy { it.featureVersion } + LOGGER.info("DONE") + return AdoptAttestationRepo(attestationMap) + } +} diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepository.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepository.kt new file mode 100644 index 000000000..fa776b51c --- /dev/null +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/AdoptAttestationRepository.kt @@ -0,0 +1,97 @@ +package net.adoptium.api.v3 + +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import net.adoptium.api.v3.dataSources.github.GitHubApi +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAsset +import net.adoptium.api.v3.dataSources.github.graphql.models.PageInfo +import net.adoptium.api.v3.dataSources.models.AdoptAttestationRepo +import net.adoptium.api.v3.dataSources.models.GitHubId +import net.adoptium.api.v3.mapping.AttestationMapper +import net.adoptium.api.v3.mapping.adopt.AdoptAttestationMapperFactory +import net.adoptium.api.v3.models.Vendor +import org.slf4j.LoggerFactory + +interface AdoptAttestationRepository { + suspend fun getAttestationsSummary(): AttestationRepoSummary? + suspend fun getAttestationByName() : GHAttestation? +} + +@ApplicationScoped +open class AdoptAttestationRepositoryImpl @Inject constructor( + val client: GitHubApi, + adoptAttestationMapperFactory: AdoptAttestationMapperFactory +) : AdoptAttestationRepository { + + companion object { + const val ADOPT_ORG = "AdoptOpenJDK" + const val ADOPTIUM_ORG = "adoptium" + + @JvmStatic + private val LOGGER = LoggerFactory.getLogger(this::class.java) + } + + private val mappers = mapOf( + ".*/temurin-attestations/.*".toRegex() to adoptAttestationMapperFactory.get(Vendor.eclipse), + ) + + private fun getMapperForRepo(url: String): ReleaseMapper { + val mapper = mappers + .entries + .firstOrNull { url.matches(it.key) } + + if (mapper == null) { + throw IllegalStateException("No mapper found for repo $url") + } + + return mapper.value + } + + override suspend fun getAttestationByName(owner: String, repoName: String, name: String): GHAttestation? { + val attestation = client.getAttestationByName(owner, repoName, name) + + return attestation + } + + private fun getAttestationRepoSummary(): suspend (Vendor, String, String) -> AttestationRepoSummary? { + return { vendor: Vendor, owner: String, repoName: String -> getAttestationRepoSummary(vendor, owner, repoName) } + } + + private suspend fun getAttestationRepoSummary(vendor: Vendor, owner: String, repoName: String): AttestationRepoSummary? { + return client.getAttestationSummary(owner, repoName) + } + + private fun getDataForEachRepo( + version: Int, + getFun: suspend (Vendor, String, String) -> E + ): Deferred> { + LOGGER.info("getting $version") + return GlobalScope.async { + + return@async listOf( + getRepoDataAsync(ADOPTIUM_ORG, Vendor.eclipse, "temurin-attestations", getFun), + ) + .map { repo -> repo.await() } + } + } + + private fun getRepoDataAsync( + owner: String, + vendor: Vendor, + repoName: String, + getFun: suspend (Vendor, String, String) -> E + ): Deferred { + return GlobalScope.async { + if (!Vendor.validVendor(vendor)) { + return@async null + } + LOGGER.info("getting attestations for $owner $repoName") + val attestations = getFun(vendor, owner, repoName) + LOGGER.info("Done getting attestations for $owner $repoName") + return@async attestations + } + } +} diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/V3Updater.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/V3Updater.kt index 1911af0dd..9d621effa 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/V3Updater.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/V3Updater.kt @@ -42,6 +42,7 @@ class V3UpdaterApp : Application() @ApplicationScoped class V3Updater @Inject constructor( private val adoptReposBuilder: AdoptReposBuilder, + private val adoptAttestationRepoBuilder: AdoptAttestationRepoBuilder, private val apiDataStore: APIDataStore, private val database: ApiPersistence, private val statsInterface: StatsInterface, @@ -69,6 +70,18 @@ class V3Updater @Inject constructor( return String(Base64.getEncoder().encode(md.digest())) } + fun calculateAttestationChecksum(repo: AdoptAttestationRepo): String { + val md = MessageDigest.getInstance("SHA256") + val outputStream = object : OutputStream() { + override fun write(b: Int) { + md.update(b.toByte()) + } + } + UpdaterJsonMapper.mapper.writeValue(outputStream, repo) + + return String(Base64.getEncoder().encode(md.digest())) + } + fun copyOldReleasesIntoNewRepo(currentRepo: AdoptRepos, newRepoData: AdoptRepos, filter: ReleaseIncludeFilter) = currentRepo .removeReleases { vendor, startTime, isPrerelease -> filter.filter(vendor, startTime, isPrerelease) } .addAll(newRepoData @@ -121,6 +134,23 @@ class V3Updater @Inject constructor( } } + private fun incrementalAttestationUpdate(oldRepo: AdoptAttestationRepo): AdoptAttestationRepo? { + return runBlocking { + // Must catch errors or may kill the scheduler + try { + LOGGER.info("Starting Incremental attestations update") + + // Just do a full update for Attestations repo + val after = fullAttestationUpdate(oldRepo) + printAttestationRepoDebugInfo(oldRepo, after, null) + return@runBlocking after + } catch (e: Exception) { + LOGGER.error("Failed to perform incremental attestations update", e) + } + return@runBlocking null + } + } + private fun printRepoDebugInfo( oldRepo: AdoptRepos, afterInMemory: AdoptRepos, @@ -139,6 +169,16 @@ class V3Updater @Inject constructor( } } + private fun printAttestationRepoDebugInfo( + oldRepo: AdoptAttestationRepo, + afterInMemory: AdoptAttestationRepo, + afterInDb: AdoptAttestationRepo?) { + + if (APIConfig.DEBUG) { + LOGGER.debug("Attestation updated and db version comparison {} {} {} {}", calculateAttestationChecksum(oldRepo), oldRepo.hashCode(), calculateAttestationChecksum(afterInMemory), afterInMemory.hashCode()) + } + } + private fun deepDiffDebugPrint(repoA: AdoptRepos, repoB: AdoptRepos) { repoA .allReleases @@ -230,12 +270,25 @@ class V3Updater @Inject constructor( AdoptRepos(emptyList()) } + var attestationRepo: AdoptAttestationRepo = try { + apiDataStore.loadAttestationDataFromDb(true) + } catch (e: java.lang.Exception) { + LOGGER.error("Failed to load attestation db", e) + if (e is MongoException) { + LOGGER.error("Failed to connect to attestation db, exiting") + Quarkus.asyncExit(2) + Quarkus.waitForExit() + } + AdoptAttestationRepo(emptyList()) + } + val incrementalUpdateScheduled = AtomicBoolean(false) executor.scheduleWithFixedDelay( timerTask { try { runUpdate(repo, incrementalUpdateScheduled, executor) + runAttestationUpdate(attestationRepo, incrementalUpdateScheduled, executor) } catch (e: InvalidUpdateException) { LOGGER.error("Failed to perform update", e) } @@ -264,6 +317,25 @@ class V3Updater @Inject constructor( return repo1 } + fun runAttestationUpdate( + repo: AdoptAttestationRepo, + incrementalUpdateScheduled: AtomicBoolean, + executor: ScheduledExecutorService + ): AdoptRepos { + var repo1 = repo + repo1 = fullAttestationUpdate(repo1) ?: repo1 + repo1 = incrementalAttestationUpdate(repo1) ?: repo1 + if (!incrementalUpdateScheduled.getAndSet(true)) { + executor.scheduleWithFixedDelay( + timerTask { + repo1 = incrementalAttestationUpdate(repo1) ?: repo1 + }, + 1, 6, TimeUnit.MINUTES + ) + } + return repo1 + } + private fun assertConnectedToDb() { val connected = try { runBlocking { @@ -339,6 +411,47 @@ class V3Updater @Inject constructor( return null } + @Throws(InvalidUpdateException::class) + private fun fullAttestationUpdate(currentRepo: AdoptAttestationRepo): AdoptAttestationRepo? { + // Must catch errors or may kill the scheduler + try { + return runBlocking { + LOGGER.info("Starting Full Attestation update") + + updatableVersionSupplier.updateVersions() + + val repo = adoptAttestationRepoBuilder.build() + + printAttestationRepoDebugInfo(currentRepo, repo, null) + + val checksum = calculateAttestationChecksum(repo) + + val dataInDb = mutex.withLock { + runBlocking { + database.updateAttestationRepo(repo, checksum) + + apiDataStore.loadAttestationDataFromDb(forceUpdate = true, logEntries = false) + } + } + + LOGGER.info("Updating Release Notes") + adoptReleaseNotes.updateReleaseNotes(repo) + + printAttestationRepoDebugInfo(currentRepo, repo, dataInDb) + + LOGGER.info("Full update done") + return@runBlocking repo + } + } catch (e: Exception) { + LOGGER.error("Failed to perform full update", e) + } catch (e: Throwable) { + // Log and rethrow, may be unrecoverable error such as OutOfMemoryError + LOGGER.error("Error during full update", e) + throw e + } + return null + } + class InvalidUpdateException(message: String) : Exception(message) private suspend fun validateStats(repo: AdoptRepos): Boolean { diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/dataSources/UpdatableVersionSupplierImpl.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/dataSources/UpdatableVersionSupplierImpl.kt index 6d51b22f9..2d693dbba 100644 --- a/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/dataSources/UpdatableVersionSupplierImpl.kt +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/main/kotlin/net/adoptium/api/v3/dataSources/UpdatableVersionSupplierImpl.kt @@ -12,7 +12,7 @@ class UpdatableVersionSupplierImpl @Inject constructor(val updaterHtmlClient: Up private val LOGGER: Logger = LoggerFactory.getLogger(UpdatableVersionSupplierImpl::class.java) } - private val DEFAULT_LATEST_JAVA_VERSION = 24 + private val DEFAULT_LATEST_JAVA_VERSION = 21 private val LATEST_JAVA_VERSION_PROPERTY = "LATEST_JAVA_VERSION" private val VERSION_FILE_URL = "https://raw.githubusercontent.com/openjdk/jdk/master/make/conf/version-numbers.conf" @@ -20,11 +20,11 @@ class UpdatableVersionSupplierImpl @Inject constructor(val updaterHtmlClient: Up private var tipVersion: Int? = null private var latestJavaVersion: Int private var versions: Array - private var ltsVersions: Array = arrayOf(8, 11, 17, 21) + private var ltsVersions: Array = arrayOf(21) init { latestJavaVersion = Integer.parseInt(System.getProperty(LATEST_JAVA_VERSION_PROPERTY, DEFAULT_LATEST_JAVA_VERSION.toString())) - versions = (8..latestJavaVersion).toList().toTypedArray() + versions = (21..latestJavaVersion).toList().toTypedArray() runBlocking { updateVersions() } diff --git a/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/AttestationMapperTest.kt b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/AttestationMapperTest.kt new file mode 100644 index 000000000..499935b58 --- /dev/null +++ b/adoptium-updater-parent/adoptium-api-v3-updater/src/test/kotlin/net/adoptium/api/AttestationMapperTest.kt @@ -0,0 +1,114 @@ +package net.adoptium.api + +import io.mockk.clearMocks +import io.mockk.mockk +import kotlinx.coroutines.runBlocking +import net.adoptium.api.v3.XmlMapper +import net.adoptium.api.v3.dataSources.github.GitHubHtmlClient +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation +import net.adoptium.api.v3.mapping.adopt.AdoptAttestationMapper +import net.adoptium.api.v3.models.Architecture +import net.adoptium.api.v3.models.ImageType +import net.adoptium.api.v3.models.JvmImpl +import net.adoptium.api.v3.models.OperatingSystem +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInstance +import org.junit.jupiter.api.TestInstance.Lifecycle +import org.junit.jupiter.api.Assertions.assertEquals +import org.slf4j.LoggerFactory + +@TestInstance(Lifecycle.PER_CLASS) +class AttestationMapperTest { + + private val LOGGER = LoggerFactory.getLogger(AttestationMapperTest::class.java) + + private val fakeGithubHtmlClient = mockk() + private val adoptAttestationMapper = AdoptAttestationMapper(fakeGithubHtmlClient) + + companion object { + val ghAttestation = XmlMapper.mapper.readValue( + """ + + + + + + true + + Acme Inc + + + + + + Eclipse Temurin Attestation + assessor-1 + + + claim-1 + + + + + + + target-jdk-1 + VERIFIED_REPRODUCIBLE_BUILD + + + + + + Temurin jdk-21.0.5+11 aarch64 Linux + jdk-21.0.5+11 + + + https://api.adoptium.net/v3/binary/version/jdk-21.0.5+11/linux/aarch64/jdk/hotspot/normal/eclipse + + 1234567890123456789012345678901234567890123456789012345678901234 + + + + + aarch64_linux + + + + + + Acme confirms a verified reproducible build + + + + + """.trimIndent(), + GHAttestation::class.java + ) + } + + @BeforeEach + fun beforeEach() { + clearMocks(fakeGithubHtmlClient) + } + + @Test + fun `Test Attestation xml mapper parsing`() { + + runBlocking { + val parsed = adoptAttestationMapper.toAttestationList(listOf(ghAttestation)) + LOGGER.info("Parsed Attestation: ${parsed[0]}") + assertEquals("jdk-21.0.5+11", parsed[0].release_name) + assertEquals(OperatingSystem.linux, parsed[0].os) + assertEquals(Architecture.aarch64, parsed[0].architecture) + assertEquals(ImageType.jdk, parsed[0].image_type) + assertEquals(JvmImpl.hotspot, parsed[0].jvm_impl) + assertEquals("Acme Inc", parsed[0].assessor_org) + assertEquals("Acme confirms a verified reproducible build", parsed[0].assessor_affirmation) + assertEquals("VERIFIED_REPRODUCIBLE_BUILD", parsed[0].assessor_claim_predicate) + assertEquals("1234567890123456789012345678901234567890123456789012345678901234", parsed[0].target_checksum) + assertEquals("attestation_link", parsed[0].attestation_link) + assertEquals("attestation_public_signing_key_link", parsed[0].attestation_public_signing_key_link) + } + } +} diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubApi.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubApi.kt index f27d1f49f..84d9b3946 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubApi.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/GitHubApi.kt @@ -2,11 +2,15 @@ package net.adoptium.api.v3.dataSources.github import net.adoptium.api.v3.dataSources.github.graphql.models.GHRelease import net.adoptium.api.v3.dataSources.github.graphql.models.GHRepository +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation import net.adoptium.api.v3.dataSources.github.graphql.models.summary.GHRepositorySummary import net.adoptium.api.v3.dataSources.models.GitHubId +import net.adoptium.api.v3.dataSources.models.AttestationRepoSummary interface GitHubApi { suspend fun getRepository(owner: String, repoName: String, filter: (updatedAt: String, isPrerelease: Boolean) -> Boolean): GHRepository suspend fun getRepositorySummary(owner: String, repoName: String): GHRepositorySummary suspend fun getReleaseById(id: GitHubId): GHRelease? + suspend fun getAttestationSummary(org: String, repo: String): AttestationRepoSummary? + suspend fun getAttestationByName(org: String, repo: String, name: String): GHAttestation? } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/GraphQLGitHubClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/GraphQLGitHubClient.kt index 15166684c..0ece9e41f 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/GraphQLGitHubClient.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/GraphQLGitHubClient.kt @@ -6,16 +6,22 @@ import net.adoptium.api.v3.dataSources.github.GitHubApi import net.adoptium.api.v3.dataSources.github.graphql.clients.GraphQLGitHubReleaseClient import net.adoptium.api.v3.dataSources.github.graphql.clients.GraphQLGitHubRepositoryClient import net.adoptium.api.v3.dataSources.github.graphql.clients.GraphQLGitHubSummaryClient +import net.adoptium.api.v3.dataSources.github.graphql.clients.GraphQLGitHubAttestationSummaryClient +import net.adoptium.api.v3.dataSources.github.graphql.clients.GraphQLGitHubAttestationClient import net.adoptium.api.v3.dataSources.github.graphql.models.GHRelease import net.adoptium.api.v3.dataSources.github.graphql.models.GHRepository +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation import net.adoptium.api.v3.dataSources.github.graphql.models.summary.GHRepositorySummary import net.adoptium.api.v3.dataSources.models.GitHubId +import net.adoptium.api.v3.dataSources.models.AttestationRepoSummary @ApplicationScoped open class GraphQLGitHubClient @Inject constructor( private val summaryClient: GraphQLGitHubSummaryClient, private val releaseClient: GraphQLGitHubReleaseClient, - private val repositoryClientClient: GraphQLGitHubRepositoryClient + private val repositoryClientClient: GraphQLGitHubRepositoryClient, + private val attestationSummaryClient: GraphQLGitHubAttestationSummaryClient, + private val attestationClient: GraphQLGitHubAttestationClient ) : GitHubApi { override suspend fun getRepositorySummary(owner: String, repoName: String): GHRepositorySummary { @@ -29,4 +35,12 @@ open class GraphQLGitHubClient @Inject constructor( override suspend fun getRepository(owner: String, repoName: String, filter: (updatedAt: String, isPrerelease: Boolean) -> Boolean): GHRepository { return repositoryClientClient.getRepository(owner, repoName, filter) } + + override suspend fun getAttestationSummary(org: String, repo: String): AttestationRepoSummary? { + return attestationSummaryClient.getAttestationSummary(org, repo) + } + + override suspend fun getAttestationByName(org: String, repo: String, name: String): GHAttestation? { + return attestationClient.getAttestationByName(org, repo, name) + } } diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationClient.kt new file mode 100644 index 000000000..62a28c3e6 --- /dev/null +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationClient.kt @@ -0,0 +1,69 @@ +package net.adoptium.api.v3.dataSources.github.graphql.clients + +import com.expediagroup.graphql.client.types.GraphQLClientRequest +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import net.adoptium.api.v3.XmlMapper +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestationResponse +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation +import net.adoptium.api.v3.dataSources.models.GitHubId +import org.slf4j.LoggerFactory +import kotlin.reflect.KClass + +@ApplicationScoped +open class GraphQLGitHubAttestationClient @Inject constructor( + private val graphQLGitHubInterface: GraphQLGitHubInterface +) { + + companion object { + @JvmStatic + private val LOGGER = LoggerFactory.getLogger(this::class.java) + } + + open suspend fun getAttestationByName(org: String, repo: String, name: String): GHAttestation? { + + LOGGER.debug("Getting attestation $org/$repo/$name") + + val query = RequestAttestationByName(org, repo, name) + + val result: GHAttestationResponse = graphQLGitHubInterface.queryApi(query::withCursor, null) + if (result == null) { + return result + } + + val ghAttestation = XmlMapper.mapper.readValue(result.text, GHAttestation::class.java) + + return ghAttestation + } + + class RequestAttestationByName(org: String, repo: String, name: String) : GraphQLClientRequest { + fun withCursor(cursor: String?): RequestAttestationByName { + this + } + + override val query: String + get() = + """ + query { + repository(owner: $org, name: $repo) { + object(expression: HEAD:$name) { + ... on Blob { + id + commitResourcePath + text + } + } + } + rateLimit { + cost, + remaining + } + } + """ + + override fun responseType(): KClass { + return GHAttestationResponse::class + } + } +} + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationSummaryClient.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationSummaryClient.kt new file mode 100644 index 000000000..9dad59532 --- /dev/null +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/clients/GraphQLGitHubAttestationSummaryClient.kt @@ -0,0 +1,67 @@ +package net.adoptium.api.v3.dataSources.github.graphql.clients + +import com.expediagroup.graphql.client.types.GraphQLClientRequest +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import net.adoptium.api.v3.dataSources.models.AttestationRepoSummary +import net.adoptium.api.v3.dataSources.models.GitHubId +import org.slf4j.LoggerFactory +import kotlin.reflect.KClass + +@ApplicationScoped +open class GraphQLGitHubAttestationSummaryClient @Inject constructor( + private val graphQLGitHubInterface: GraphQLGitHubInterface +) { + + companion object { + @JvmStatic + private val LOGGER = LoggerFactory.getLogger(this::class.java) + } + + open suspend fun getAttestationSummary(org: String, repo: String): AttestationRepoSummary { + + LOGGER.debug("Getting tree file summary of attestations github repository $org/$repo") + + val query = RequestAttestationRepoSummary(org, repo) + + val result: AttestationRepoSummary = graphQLGitHubInterface.queryApi(query, null) + + return result + } + + class RequestAttestationRepoSummary(org: String, repo: String) : GraphQLClientRequest { + override val query: String + get() = + """ + query { + repository(owner: "${org}", name: "${repo}") { + object(expression: HEAD:) { + ... on Tree { + entries { + name + type + object { + ... on Tree { + entries { + name + type + } + } + } + } + } + } + } + rateLimit { + cost, + remaining + } + } + """ + + override fun responseType(): KClass { + return AttestationRepoSummary::class + } + } +} + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestation.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestation.kt new file mode 100644 index 000000000..66d35c065 --- /dev/null +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestation.kt @@ -0,0 +1,169 @@ +package net.adoptium.api.v3.dataSources.github.graphql.models + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonIgnoreProperties +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlText +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper +import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlRootElement + +data class Organization( + @JacksonXmlProperty(localName = "name") + var name: String +) + +data class Assessor( + @JacksonXmlProperty(localName = "thirdParty") + var thirdParty: Boolean, + + @JacksonXmlProperty(localName = "organization") + var organization: Organization +) + +data class Assessors( + @get:JacksonXmlElementWrapper(useWrapping = false) + var assessor: List +) + +data class Claim( + @JacksonXmlProperty(localName = "target") + var target: String, + + @JacksonXmlProperty(localName = "predicate") + var predicate: String +) + +data class Claims( + @get:JacksonXmlElementWrapper(useWrapping = false) + var claim: List +) + +data class Affirmation( + @JacksonXmlProperty(localName = "statement") + var statement: String, +) + +// Cannot use "data class" due to known bug: https://github.com/FasterXML/jackson-module-kotlin/issues/138 +@JacksonXmlRootElement(localName = "claim") +class ClaimRef() { + @JacksonXmlText() + var claimRef: String? = null +} + +data class ClaimRefs( + @get:JacksonXmlElementWrapper(useWrapping = false) + var claim: List +) + +data class ClaimMap( + @JacksonXmlProperty(localName = "claims") + var claims: ClaimRefs +) + +data class Attestation( + @JacksonXmlProperty(localName = "summary") + var summary: String, + + @JacksonXmlProperty(localName = "assessor") + var assessor: String, + + @JacksonXmlProperty(localName = "map") + var map: ClaimMap +) + +data class Attestations( + @get:JacksonXmlElementWrapper(useWrapping = false) + var attestation: List +) + +// Cannot use "data class" due to known bug: https://github.com/FasterXML/jackson-module-kotlin/issues/138 +@JacksonXmlRootElement(localName = "hash") +class Hash() { + @JacksonXmlText() + var sha256: String? = null +} + +data class Hashes( + @get:JacksonXmlElementWrapper(useWrapping = false) + var hash: List +) + +data class Reference( + @JacksonXmlProperty(localName = "url") + var url: String, + + @JacksonXmlProperty(localName = "hashes") + var hashes: Hashes +) + +data class ExternalReferences( + @get:JacksonXmlElementWrapper(useWrapping = false) + var reference: List +) + +// Cannot use "data class" due to known bug: https://github.com/FasterXML/jackson-module-kotlin/issues/138 +@JacksonXmlRootElement(localName = "property") +class Property() { + @JacksonXmlProperty(isAttribute = true) + var name: String? = null + + @JacksonXmlText() + var value: String? = null +} + +data class Properties( + @get:JacksonXmlElementWrapper(useWrapping = false) + var property: List +) + +data class Component( + @JacksonXmlProperty(localName = "name") + var name: String, + + @JacksonXmlProperty(localName = "version") + var version: String, + + @JacksonXmlProperty(localName = "externalReferences") + var externalReferences: ExternalReferences, + + @JacksonXmlProperty(localName = "properties") + var properties: Properties +) + +data class Components( + @get:JacksonXmlElementWrapper(useWrapping = false) + var component: List +) + +data class Targets( + @JacksonXmlProperty(localName = "components") + var components: Components +) + +data class Declarations( + @JacksonXmlProperty(localName = "assessors") + var assessors: Assessors, + + @JacksonXmlProperty(localName = "claims") + var claims: Claims, + + @JacksonXmlProperty(localName = "attestations") + var attestations: Attestations, + + @JacksonXmlProperty(localName = "targets") + var targets: Targets, + + @JacksonXmlProperty(localName = "affirmation") + var affirmation: Affirmation +) + +@JsonIgnoreProperties(ignoreUnknown = true) +@JacksonXmlRootElement(localName = "bom") +data class GHAttestation @JsonCreator constructor( + @JacksonXmlProperty(localName = "declarations") + var declarations: Declarations, + + @JacksonXmlProperty(localName = "signature") + var signature: String? = null +) + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestationResponse.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestationResponse.kt new file mode 100644 index 000000000..79f122eb2 --- /dev/null +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GHAttestationResponse.kt @@ -0,0 +1,57 @@ +package net.adoptium.api.v3.dataSources.github.graphql.models + +import com.fasterxml.jackson.annotation.JsonCreator +import com.fasterxml.jackson.annotation.JsonProperty +import com.fasterxml.jackson.databind.annotation.JsonDeserialize + +/* Format example: +Query: +query RepoFiles($owner: String!, $name: String!, $expr: String!) { + repository(owner: $owner, name: $name) { + object(expression: HEAD:21/jdk_21_0_5_11_x64_linux_Adoptium.xml) { + ... on Blob { + id + commitResourcePath + text + } + } + } +} +Response: +{ + "data": { + "repository": { + "object": { + "id": "B_kwDONeZDk9oAKDk3ZDg0ZDI3MGEzNGY2Njc0OWU1ZTY2YjBhNDI5OTY4MjhiZGE0ZTE", + "commitResourcePath": "/andrew-m-leonard/temurin-attestations/commit/97d84d270a34f66749e5e66b0a42996828bda4e1", + "text": "\n\n" + } + } + } +} +*/ + +data class GHAttestationResponse @JsonCreator constructor( + @JsonProperty("data") val data: GHAttestationResponseData +) : HasRateLimit(rateLimit) { +} + +data class GHAttestationResponseData @JsonCreator constructor( + @JsonProperty("repository") val repository: GHAttestationResponseRepository +) { +} + +data class GHAttestationResponseRepository @JsonCreator constructor( + @JsonProperty("object") val repository: GHAttestationResponseObject +) { +} + +data class GHAttestationResponseObject @JsonCreator constructor( + @JsonProperty("id") + @JsonDeserialize(using = GitHubIdDeserializer::class) + val id: GitHubId, + @JsonProperty("commitResourcePath") val type: String, + @JsonProperty("text") val text: String +) { +} + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GraphQLQueries.kt b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GraphQLQueries.kt index e9868daca..b9c79bd6f 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GraphQLQueries.kt +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-github-datasource/src/main/kotlin/net/adoptium/api/v3/dataSources/github/graphql/models/GraphQLQueries.kt @@ -3,6 +3,7 @@ package net.adoptium.api.v3.dataSources.github.graphql.models import com.fasterxml.jackson.annotation.JsonCreator import com.fasterxml.jackson.annotation.JsonProperty import net.adoptium.api.v3.dataSources.github.graphql.models.summary.GHRepositorySummary +import net.adoptium.api.v3.dataSources.github.graphql.models.GitHubIdDeserializer /* Models that encapsulate how GitHub represents its release data @@ -34,3 +35,48 @@ class ReleaseQueryData @JsonCreator constructor( @JsonProperty("node") val assetNode: GHAssetNode?, @JsonProperty("rateLimit") rateLimit: RateLimit ) : HasRateLimit(rateLimit) + +/* + Attestation file query example: +Query: +query RepoFiles($owner: String!, $name: String!, $expr: String!) { + repository(owner: $owner, name: $name) { + object(expression: HEAD:21/jdk_21_0_5_11_x64_linux_Adoptium.xml) { + ... on Blob { + id + commitResourcePath + text + } + } + } +} +Response: +{ + "data": { + "repository": { + "object": { + "id": "B_kwDONeZDk9oAKDk3ZDg0ZDI3MGEzNGY2Njc0OWU1ZTY2YjBhNDI5OTY4MjhiZGE0ZTE", + "commitResourcePath": "/andrew-m-leonard/temurin-attestations/commit/97d84d270a34f66749e5e66b0a42996828bda4e1", + "text": "\n\n" + } + } + } +} +*/ +class AttestationQueryData @JsonCreator constructor( + @JsonProperty("repository") val repository: GHAttestationRepository?, + @JsonProperty("rateLimit") rateLimit: RateLimit +) : HasRateLimit(rateLimit) + +class GHAttestationRepository @JsonCreator constructor( + @JsonProperty("object") val object: GHAttestationObject? +) + +class GHAttestationObject @JsonCreator constructor( + @JsonProperty("id") + @JsonDeserialize(using = GitHubIdDeserializer::class) + val id: GitHubId, + @JsonProperty("commitResourcePath") val commitResourcePath: String, + @JsonProperty("text") val text: String +) + diff --git a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml index 537b2da80..4b8c03494 100644 --- a/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml +++ b/adoptium-updater-parent/adoptium-datasources-parent/adoptium-http-client-datasource/pom.xml @@ -36,6 +36,10 @@ com.fasterxml.jackson.datatype jackson-datatype-jakarta-jsonp + + com.fasterxml.jackson.dataformat + jackson-dataformat-xml + org.jetbrains.kotlinx kotlinx-coroutines-core diff --git a/adoptium-updater-parent/adoptium-mappers-parent/adopt-mappers/src/main/kotlin/net/adoptium/api/v3/mapping/adopt/AdoptAttestationMapper.kt b/adoptium-updater-parent/adoptium-mappers-parent/adopt-mappers/src/main/kotlin/net/adoptium/api/v3/mapping/adopt/AdoptAttestationMapper.kt new file mode 100644 index 000000000..04f326a96 --- /dev/null +++ b/adoptium-updater-parent/adoptium-mappers-parent/adopt-mappers/src/main/kotlin/net/adoptium/api/v3/mapping/adopt/AdoptAttestationMapper.kt @@ -0,0 +1,98 @@ +package net.adoptium.api.v3.mapping.adopt + +import jakarta.enterprise.context.ApplicationScoped +import jakarta.inject.Inject +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.GlobalScope +import kotlinx.coroutines.async +import net.adoptium.api.v3.dataSources.github.GitHubHtmlClient +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation +import net.adoptium.api.v3.models.Attestation +import net.adoptium.api.v3.models.Architecture +import net.adoptium.api.v3.models.ImageType +import net.adoptium.api.v3.models.JvmImpl +import net.adoptium.api.v3.models.OperatingSystem +import net.adoptium.api.v3.models.Vendor +import net.adoptium.api.v3.mapping.AttestationMapper +import org.slf4j.LoggerFactory +import java.util.EnumMap + + +@ApplicationScoped +open class AdoptAttestationMapperFactory @Inject constructor( + val htmlClient: GitHubHtmlClient +) { + private val mappers: MutableMap = EnumMap(Vendor::class.java) + + open fun get(vendor: Vendor): AttestationMapper { + return if (mappers.containsKey(vendor)) { + mappers[vendor]!! + } else { + val mapper = AdoptAttestationMapper(htmlClient) + mappers[vendor] = mapper + mapper + } + } +} + +private class AdoptAttestationMapper( + val htmlClient: GitHubHtmlClient +) : AttestationMapper() { + companion object { + @JvmStatic + private val LOGGER = LoggerFactory.getLogger(this::class.java) + } + + override suspend fun toAttestationList(ghAttestationAssets: List): List { + return ghAttestationAssets + .map { asset -> assetToAttestationAsync(asset) } + .mapNotNull { it.await() } + } + + private fun assetToAttestationAsync( + ghAttestationAsset: GHAttestation + ): Deferred { + return GlobalScope.async { + try { + // Temurin Attestations (https://github.com/adoptium/temurin-attestations/blob/main/.github/workflows/validate-cdxa.yml) have: + // ONE attestation + // ONE target component + // ONE assessor + // ONE claim + // ONE target component externalReferences reference with a single hash + // target component version is of format jdk-$MAJOR_VERSION+$BUILD_NUM or jdk-$MAJOR_VERSION.0.$UPDATE_VERSION+$BUILD_NUM + + val releaseName: String = ghAttestationAsset.declarations.targets.components.component[0].version + // featureVersion derived from releaseName + val featureVersion: Int = releaseName.split("[-\\+\\.]")[1].toInt() + + val assessor_org: String = ghAttestationAsset.declarations.assessors.assessor[0].organization.name + val assessor_affirmation: String = ghAttestationAsset.declarations.affirmation.statement + val assessor_claim_predicate: String = ghAttestationAsset.declarations.claims.claim[0].predicate + val target_checksum: String? = ghAttestationAsset.declarations.targets.components.component[0].externalReferences.reference[0].hashes.hash[0].sha256 + + var archStr: String = "" + var osStr: String = "" + for (property in ghAttestationAsset.declarations.targets.components.component[0].properties.property) { + if (property.name == "platform") { + val split_platform: List? = property.value?.split("_") + if (split_platform != null) { + archStr = split_platform[0] + osStr = split_platform[1] + } + } + } + + val arch: Architecture = Architecture.valueOf(archStr) //by lazy { Architecture.valueOf(archStr) } + val os: OperatingSystem = OperatingSystem.valueOf(osStr) //by lazy { OperatingSystem.valueOf(osStr) } + + return@async Attestation(featureVersion, releaseName, os, arch, ImageType.jdk, JvmImpl.hotspot, + target_checksum, assessor_org, assessor_affirmation, assessor_claim_predicate, + "attestation_link", "attestation_public_signing_key_link") + } catch (e: Exception) { + LOGGER.error("Failed to fetch attestation ${ghAttestationAsset}", e) + return@async null + } + } + } +} diff --git a/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/AttestationResult.kt b/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/AttestationResult.kt new file mode 100644 index 000000000..14bf47bb3 --- /dev/null +++ b/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/AttestationResult.kt @@ -0,0 +1,7 @@ +package net.adoptium.api.v3 + +import net.adoptium.api.v3.models.Attestation + +class AttestationResult(val result: List? = null, val error: String? = null) { + fun succeeded() = error == null && result != null +} diff --git a/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/mapping/AttestationMapper.kt b/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/mapping/AttestationMapper.kt new file mode 100644 index 000000000..2f6824ec4 --- /dev/null +++ b/adoptium-updater-parent/adoptium-mappers-parent/mappers-common/src/main/kotlin/net/adoptium/api/v3/mapping/AttestationMapper.kt @@ -0,0 +1,19 @@ +package net.adoptium.api.v3.mapping + +import net.adoptium.api.v3.models.Attestation +import net.adoptium.api.v3.TimeSource +import net.adoptium.api.v3.dataSources.github.graphql.models.GHAttestation +import java.time.Instant +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter + +abstract class AttestationMapper { + abstract suspend fun toAttestationList(ghAttestationAssets: List): List + + companion object { + fun parseDate(date: String): ZonedDateTime { + return Instant.from(DateTimeFormatter.ISO_INSTANT.parse(date)) + .atZone(TimeSource.ZONE) + } + } +}