Skip to content
This repository was archived by the owner on Jul 8, 2025. It is now read-only.

Commit 1fa7a7d

Browse files
slissonslisson
committed
feat: draft branch can be merged with changes from git
1 parent 1a604e4 commit 1fa7a7d

File tree

9 files changed

+378
-242
lines changed

9 files changed

+378
-242
lines changed

‎gradle/libs.versions.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ ktor = "3.2.0"
88
logback = "1.5.18"
99
modelix-core = "15.4.1"
1010
modelix-mps-plugins = "0.11.1"
11+
modelix-openapi = "1.2.0-pr35-1"
1112

1213
[libraries]
1314
auth0-jwt = { group = "com.auth0", name = "java-jwt", version = "4.5.0" }
@@ -56,7 +57,7 @@ zt-zip = { group = "org.zeroturnaround", name = "zt-zip", version = "1.17" }
5657
modelix-syncPlugin3 = { group = "org.modelix.mps", name = "mps-sync-plugin3", version.ref = "modelix-core" }
5758
modelix-mpsPlugins-generator = { group = "org.modelix.mps", name = "generator-execution-plugin", version.ref = "modelix-mps-plugins" }
5859
modelix-mpsPlugins-diff = { group = "org.modelix.mps", name = "diff-plugin", version.ref = "modelix-mps-plugins" }
59-
modelix-api-server-stubs = { group = "org.modelix", name = "api-server-stubs-ktor", version = "1.2.0" }
60+
modelix-api-server-stubs = { group = "org.modelix", name = "api-server-stubs-ktor", version.ref = "modelix-openapi" }
6061

6162
[bundles]
6263
ktor-client = [

‎workspace-client-plugin/build.gradle.kts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ plugins {
2020
group = "org.modelix.mps"
2121

2222
kotlin {
23-
jvmToolchain(17)
23+
jvmToolchain(11)
2424
}
2525

2626
dependencies {

‎workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitConnectorManager.kt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,11 +54,11 @@ class GitConnectorManager(
5454

5555
fun getOrCreateImportTask(taskKey: GitImportTask.Key): GitImportTask {
5656
return importTasks.getOrCreateTask(taskKey) {
57-
GitImportTask(
57+
GitImportTaskUsingKubernetesJob(
5858
key = taskKey,
5959
scope = scope,
60-
kestraClient = kestraClient,
6160
modelClient = modelClient,
61+
jwtUtil = kestraClient.jwtUtil,
6262
)
6363
}
6464
}

‎workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitImportTask.kt‎

Lines changed: 92 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,110 @@
11
package org.modelix.services.gitconnector
22

3+
import io.kubernetes.client.openapi.models.V1Container
4+
import io.kubernetes.client.openapi.models.V1Job
35
import kotlinx.coroutines.CoroutineScope
46
import kotlinx.coroutines.delay
7+
import org.modelix.authorization.ModelixJWTUtil
58
import org.modelix.model.IVersion
69
import org.modelix.model.client2.IModelClientV2
710
import org.modelix.model.lazy.RepositoryId
11+
import org.modelix.model.server.ModelServerPermissionSchema
812
import org.modelix.services.gitconnector.stubs.models.GitRepositoryConfig
13+
import org.modelix.services.workspaces.metadata
14+
import org.modelix.services.workspaces.spec
15+
import org.modelix.services.workspaces.template
16+
import org.modelix.workspace.manager.ITaskInstance
917
import org.modelix.workspace.manager.KestraClient
18+
import org.modelix.workspace.manager.KubernetesJobTask
1019
import org.modelix.workspace.manager.TaskInstance
1120
import kotlin.time.Duration.Companion.seconds
1221

13-
class GitImportTask(
14-
val key: Key,
22+
interface GitImportTask : ITaskInstance<IVersion> {
23+
data class Key(
24+
val repo: GitRepositoryConfig,
25+
val gitBranchName: String,
26+
val gitRevision: String,
27+
val modelixBranchName: String,
28+
)
29+
}
30+
31+
class GitImportTaskUsingKubernetesJob(
32+
val key: GitImportTask.Key,
33+
scope: CoroutineScope,
34+
val modelClient: IModelClientV2,
35+
val jwtUtil: ModelixJWTUtil,
36+
) : GitImportTask, KubernetesJobTask<IVersion>(scope) {
37+
38+
private val repoId = requireNotNull(key.repo.modelixRepository?.let { RepositoryId(it) }) { "Repository ID missing" }
39+
private val branchRef = repoId.getBranchReference(key.modelixBranchName)
40+
private suspend fun modelixBranchExists() = modelClient.listBranches(repoId).contains(branchRef)
41+
42+
override suspend fun tryGetResult(): IVersion? {
43+
return if (modelixBranchExists()) {
44+
return modelClient.lazyLoadVersion(branchRef).takeIf { it.gitCommit == key.gitRevision }
45+
} else {
46+
return null
47+
}
48+
}
49+
50+
@Suppress("ktlint")
51+
override fun generateJobYaml(): V1Job {
52+
val remote = requireNotNull(key.repo.remotes?.firstOrNull()) { "No remotes specified" }
53+
val modelixBranch =
54+
RepositoryId((key.repo.modelixRepository ?: key.repo.id)).getBranchReference("git-import/${key.gitBranchName}")
55+
val token = jwtUtil.createAccessToken(
56+
"git-import@modelix.org",
57+
listOf(
58+
ModelServerPermissionSchema.repository(modelixBranch.repositoryId).create.fullId,
59+
ModelServerPermissionSchema.repository(modelixBranch.repositoryId).read.fullId,
60+
ModelServerPermissionSchema.branch(modelixBranch).rewrite.fullId,
61+
),
62+
)
63+
64+
return V1Job().apply {
65+
metadata {
66+
name = "gitimportjob-$id"
67+
}
68+
spec {
69+
template {
70+
spec {
71+
addContainersItem(V1Container().apply {
72+
name = "importer"
73+
image = System.getenv("GIT_IMPORT_IMAGE")
74+
args = listOf(
75+
"git-import-remote",
76+
remote.url,
77+
"--git-user",
78+
remote.credentials?.username,
79+
"--git-password",
80+
remote.credentials?.password,
81+
"--limit",
82+
"50",
83+
"--model-server",
84+
System.getenv("model_server_url"),
85+
"--token",
86+
token,
87+
"--repository",
88+
modelixBranch.repositoryId.id,
89+
"--branch",
90+
modelixBranch.branchName,
91+
"--rev",
92+
key.gitRevision,
93+
)
94+
})
95+
}
96+
}
97+
}
98+
}
99+
}
100+
}
101+
102+
class GitImportTaskUsingKestra(
103+
val key: GitImportTask.Key,
15104
scope: CoroutineScope,
16105
val kestraClient: KestraClient,
17106
val modelClient: IModelClientV2,
18-
) : TaskInstance<IVersion>(scope) {
107+
) : GitImportTask, TaskInstance<IVersion>(scope) {
19108

20109
private val repoId = requireNotNull(key.repo.modelixRepository?.let { RepositoryId(it) }) { "Repository ID missing" }
21110
private val branchRef = repoId.getBranchReference(key.modelixBranchName)
@@ -51,13 +140,6 @@ class GitImportTask(
51140
delay(3.seconds)
52141
}
53142
}
54-
55-
data class Key(
56-
val repo: GitRepositoryConfig,
57-
val gitBranchName: String,
58-
val gitRevision: String,
59-
val modelixBranchName: String,
60-
)
61143
}
62144

63145
val IVersion.gitCommit: String? get() = getAttributes()["git-commit"]

‎workspace-manager/src/main/kotlin/org/modelix/services/workspaces/KubernetesApiExtensions.kt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ import kotlin.coroutines.suspendCoroutine
2020
fun KubernetesObject.metadata(body: V1ObjectMeta.() -> Unit): V1ObjectMeta {
2121
return (metadata ?: V1ObjectMeta().also { setMetadata(it) }).apply(body)
2222
}
23+
fun V1PodTemplateSpec.metadata(body: V1ObjectMeta.() -> Unit): V1ObjectMeta {
24+
return (metadata ?: V1ObjectMeta().also { setMetadata(it) }).apply(body)
25+
}
2326

2427
fun KubernetesObject.setMetadata(data: V1ObjectMeta) {
2528
when (this) {

‎workspace-manager/src/main/kotlin/org/modelix/workspace/manager/KubernetesJobTask.kt‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package org.modelix.workspace.manager
2+
3+
import io.kubernetes.client.openapi.apis.BatchV1Api
4+
import io.kubernetes.client.openapi.models.V1Job
5+
import io.kubernetes.client.openapi.models.V1Toleration
6+
import kotlinx.coroutines.CoroutineScope
7+
import kotlinx.coroutines.delay
8+
import kotlinx.coroutines.withTimeout
9+
import org.modelix.services.workspaces.ContinuingCallback
10+
import org.modelix.services.workspaces.metadata
11+
import org.modelix.services.workspaces.spec
12+
import org.modelix.services.workspaces.template
13+
import org.modelix.workspace.manager.WorkspaceJobQueue.Companion.KUBERNETES_NAMESPACE
14+
import kotlin.coroutines.suspendCoroutine
15+
import kotlin.time.Duration.Companion.minutes
16+
17+
abstract class KubernetesJobTask<Out : Any>(scope: CoroutineScope) : TaskInstance<Out>(scope) {
18+
companion object {
19+
const val JOB_ID_LABEL = "modelix.workspace.job.id"
20+
}
21+
22+
abstract suspend fun tryGetResult(): Out?
23+
abstract fun generateJobYaml(): V1Job
24+
25+
override suspend fun process() = withTimeout(30.minutes) {
26+
tryGetResult()?.let { return@withTimeout it }
27+
28+
findJob()?.let { deleteJob(it) }
29+
createJob()
30+
31+
var jobCreationFailedConfirmations = 0
32+
while (true) {
33+
delay(1000)
34+
35+
tryGetResult()?.let { return@withTimeout it }
36+
37+
val job = findJob()
38+
39+
// https://kubernetes.io/docs/concepts/workloads/controllers/job/#terminal-job-conditions
40+
val jobFailed = job?.status?.conditions.orEmpty().any { it.type == "Failed" }
41+
val jobSucceeded = job?.status?.conditions.orEmpty().any { it.type == "Complete" }
42+
if (jobFailed || jobSucceeded) break
43+
44+
if (job == null) {
45+
jobCreationFailedConfirmations++
46+
} else {
47+
jobCreationFailedConfirmations = 0
48+
}
49+
if (jobCreationFailedConfirmations > 10) {
50+
break
51+
}
52+
}
53+
checkNotNull(tryGetResult()) { "Job finished without producing the expected result" }
54+
}
55+
56+
private suspend fun createJob() {
57+
suspendCoroutine {
58+
val job = generateJobYaml().apply {
59+
apiVersion = "batch/v1"
60+
kind = "Job"
61+
metadata {
62+
putLabelsItem(JOB_ID_LABEL, id.toString())
63+
}
64+
spec {
65+
ttlSecondsAfterFinished = 300
66+
activeDeadlineSeconds = 3600
67+
backoffLimit = 0
68+
template {
69+
spec {
70+
addTolerationsItem(
71+
V1Toleration().apply {
72+
key = "workspace-client"
73+
operator = "Exists"
74+
effect = "NoExecute"
75+
},
76+
)
77+
activeDeadlineSeconds = 3600
78+
restartPolicy = "Never"
79+
}
80+
metadata {
81+
putLabelsItem(JOB_ID_LABEL, id.toString())
82+
}
83+
}
84+
}
85+
}
86+
BatchV1Api().createNamespacedJob(KUBERNETES_NAMESPACE, job).executeAsync(ContinuingCallback(it))
87+
}
88+
}
89+
90+
private suspend fun findJob(): V1Job? {
91+
val jobs = suspendCoroutine {
92+
BatchV1Api().listNamespacedJob(KUBERNETES_NAMESPACE)
93+
.executeAsync(ContinuingCallback(it))
94+
}
95+
return jobs.items.firstOrNull { it.metadata.labels?.get(JOB_ID_LABEL) == id.toString() }
96+
}
97+
98+
private suspend fun deleteJob(job: V1Job) {
99+
suspendCoroutine {
100+
BatchV1Api().deleteNamespacedJob(job.metadata!!.name, job.metadata!!.namespace)
101+
.executeAsync(ContinuingCallback(it))
102+
}
103+
}
104+
}

‎workspace-manager/src/main/kotlin/org/modelix/workspace/manager/TaskInstance.kt‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,26 +5,33 @@ import kotlinx.coroutines.Deferred
55
import kotlinx.coroutines.async
66
import java.util.UUID
77

8-
abstract class TaskInstance<R>(val scope: CoroutineScope) {
8+
interface ITaskInstance<R> {
9+
fun getOutput(): Result<R>?
10+
suspend fun waitForOutput(): R
11+
fun getState(): TaskState
12+
fun launch(): Deferred<R>
13+
}
14+
15+
abstract class TaskInstance<R>(val scope: CoroutineScope) : ITaskInstance<R> {
916
val id: UUID = UUID.randomUUID()
1017
private var job: Deferred<R>? = null
1118
private var result: Result<R>? = null
1219
protected abstract suspend fun process(): R
1320

1421
@Synchronized
15-
fun launch(): Deferred<R> {
22+
override fun launch(): Deferred<R> {
1623
return job ?: scope.async {
1724
runCatching { process() }.also { result = it }.getOrThrow()
1825
}.also { job = it }
1926
}
2027

21-
fun getOutput(): Result<R>? = result
28+
override fun getOutput(): Result<R>? = result
2229

23-
suspend fun waitForOutput(): R {
30+
override suspend fun waitForOutput(): R {
2431
return launch().await()
2532
}
2633

27-
fun getState() = job.let {
34+
override fun getState(): TaskState = job.let {
2835
when {
2936
it == null -> TaskState.CREATED
3037
it.isCompleted -> TaskState.COMPLETED
@@ -35,7 +42,7 @@ abstract class TaskInstance<R>(val scope: CoroutineScope) {
3542
}
3643
}
3744

38-
class ReusableTasks<K, V : TaskInstance<*>> {
45+
class ReusableTasks<K, V : ITaskInstance<*>> {
3946
private val tasks = LinkedHashMap<K, V>()
4047

4148
fun getOrCreateTask(key: K, factory: (K) -> V): V {

0 commit comments

Comments
 (0)