feat: export of draft changes back to git

gradle.properties

@@ -2,11 +2,10 @@ org.gradle.caching=true
 
 # Supported MPS versions by Modelix workspaces.
 # Docker images are built for each of the versions.
-mpsMajorVersions=2020.3,2021.1,2021.2,2021.3,2022.2,2022.3,2023.2,2023.3,2024.1
+mpsMajorVersions=2021.1,2021.2,2021.3,2022.2,2022.3,2023.2,2023.3,2024.1
 
 # For each supported major MPS version define:
 # * The exact MPS version including the minor version.
-mpsVersion2020.3=2020.3.6
 mpsVersion2021.1=2021.1.4
 mpsVersion2021.2=2021.2.6
 mpsVersion2021.3=2021.3.5

gradle/libs.versions.toml

@@ -10,9 +10,9 @@ kotlinSerialization="1.9.0"
 kotlinx-coroutines = "1.10.2"
 ktor = "3.3.1"
 logback = "1.5.20"
-modelix-core = "16.2.3"
+modelix-core = "16.5.0"
 modelix-mps-plugins = "0.12.0"
-modelix-openapi = "1.3.0"
+modelix-openapi = "1.4.0"
 
 [libraries]
 auth0-jwt = { group = "com.auth0", name = "java-jwt", version = "4.5.0" }
@@ -68,7 +68,7 @@ maven-invoker = { group = "org.apache.maven.shared", name = "maven-invoker", ver
 modelix-api-client-ktor = { group = "org.modelix", name = "api-client-ktor", version.ref = "modelix-openapi" }
 modelix-api-server-stubs = { group = "org.modelix", name = "api-server-stubs-ktor", version.ref = "modelix-openapi" }
 modelix-authorization = { group = "org.modelix", name = "authorization", version.ref = "modelix-core" }
-modelix-dashboard = { group = "org.modelix", name = "dashboard-spa", version = "1.1.1" }
+modelix-dashboard = { group = "org.modelix", name = "dashboard-spa", version = "1.2.0" }
 modelix-model-client = { group = "org.modelix", name = "model-client", version.ref = "modelix-core" }
 modelix-model-server = { group = "org.modelix", name = "model-server", version.ref = "modelix-core" }
 modelix-mps-build-tools = { group = "org.modelix.mps", name="build-tools-lib", version = "2.0.1"}

helm/modelix/values.yaml

@@ -17,13 +17,13 @@ oauthProxy:
 modelServer:
   image:
     repository: modelix/model-server
-    tag: "16.2.3"
+    tag: "16.5.0"
     pullPolicy: IfNotPresent
 
 gitImport:
   image:
     repository: modelix/mps-git-import
-    tag: "16.2.3"
+    tag: "16.5.0"
     pullPolicy: IfNotPresent
 
 proxy:

workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitConnectorController.kt

@@ -8,6 +8,8 @@ import io.ktor.server.routing.Route
 import kotlinx.serialization.Serializable
 import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsController
 import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsController.Companion.modelixGitConnectorDraftsRoutes
+import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsExportJobController
+import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsExportJobController.Companion.modelixGitConnectorDraftsExportJobRoutes
 import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsPreparationJobController
 import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsPreparationJobController.Companion.modelixGitConnectorDraftsPreparationJobRoutes
 import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorDraftsRebaseJobController
@@ -25,6 +27,7 @@ import org.modelix.services.gitconnector.stubs.controllers.ModelixGitConnectorRe
 import org.modelix.services.gitconnector.stubs.controllers.TypedApplicationCall
 import org.modelix.services.gitconnector.stubs.models.DraftConfig
 import org.modelix.services.gitconnector.stubs.models.DraftConfigList
+import org.modelix.services.gitconnector.stubs.models.DraftExportJob
 import org.modelix.services.gitconnector.stubs.models.DraftPreparationJob
 import org.modelix.services.gitconnector.stubs.models.DraftRebaseJob
 import org.modelix.services.gitconnector.stubs.models.GitBranchList
@@ -304,6 +307,43 @@ class GitConnectorController(val manager: GitConnectorManager) {
                 call.respondJob(task)
             }
         })
+
+        modelixGitConnectorDraftsExportJobRoutes(object : ModelixGitConnectorDraftsExportJobController {
+            suspend fun TypedApplicationCall<DraftExportJob>.respondJob(task: GitExportTask) {
+                respondTyped(
+                    DraftExportJob(
+                        active = when (task.getState()) {
+                            TaskState.CREATED, TaskState.ACTIVE -> true
+                            TaskState.CANCELLED, TaskState.COMPLETED, TaskState.UNKNOWN -> false
+                        },
+                        errorMessage = task.getOutput()?.exceptionOrNull()?.stackTraceToString(),
+                        gitBranchName = task.gitBranchName,
+                        modelixVersionHash = task.key.modelixVersionHash,
+                    ),
+                )
+            }
+
+            override suspend fun getDraftExportJob(
+                draftId: String,
+                call: TypedApplicationCall<DraftExportJob>,
+            ) {
+                val draft = manager.getDraft(draftId)
+                    ?: return call.respondText("Draft not found: $draftId", status = HttpStatusCode.NotFound)
+                val task = manager.exportTasks.getAll().lastOrNull { it.key.modelixBranchName == draft.modelixBranchName }
+                    ?: return call.respondText("No export job found for draft $draftId", status = HttpStatusCode.NotFound)
+                call.respondJob(task)
+            }
+
+            override suspend fun exportDraft(
+                draftId: String,
+                draftExportJob: DraftExportJob,
+                call: TypedApplicationCall<DraftExportJob>,
+            ) {
+                val task = manager.getOrCreateExportTask(draftId)
+                task.launch()
+                call.respondJob(task)
+            }
+        })
     }
 }
 

workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitConnectorManager.kt

@@ -27,6 +27,7 @@ class GitConnectorManager(
 ) {
 
     private val importTasks = ReusableTasks<GitImportTask.Key, GitImportTask>()
+    val exportTasks = ReusableTasks<GitExportTask.Key, GitExportTask>()
     val draftPreparationTasks = ReusableTasks<DraftPreparationTask.Key, DraftPreparationTask>()
     private val draftRebaseTasks = ReusableTasks<DraftRebaseTask.Key, DraftRebaseTask>()
 
@@ -63,6 +64,32 @@ class GitConnectorManager(
         }
     }
 
+    suspend fun getOrCreateExportTask(draftId: String): GitExportTask {
+        val data = connectorData.getValue()
+        val draft = requireNotNull(data.drafts[draftId]) { "Draft not found: $draftId" }
+        val gitRepoConfig = requireNotNull(data.repositories[draft.gitRepositoryId]) { "Repository not found: ${draft.gitRepositoryId}" }
+        val modelixBranch = gitRepoConfig.getModelixRepositoryId().getBranchReference(draft.modelixBranchName)
+        val versionHash = modelClient.pullHash(modelixBranch)
+        val key = GitExportTask.Key(
+            repo = gitRepoConfig,
+            modelixVersionHash = versionHash,
+            modelixBranchName = draft.modelixBranchName,
+            gitBaseBranch = draft.gitBranchName,
+        )
+        return getOrCreateExportTask(key)
+    }
+
+    fun getOrCreateExportTask(taskKey: GitExportTask.Key): GitExportTask {
+        return exportTasks.getOrCreateTask(taskKey) {
+            GitExportTask(
+                key = taskKey,
+                scope = scope,
+                modelClient = modelClient,
+                jwtUtil = kestraClient.jwtUtil,
+            )
+        }
+    }
+
     fun getOrCreateDraftPreparationTask(draftId: String): DraftPreparationTask {
         val key = DraftPreparationTask.Key(
             draftId = draftId,

workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitExportTask.kt

@@ -0,0 +1,139 @@
+package org.modelix.services.gitconnector
+
+import io.kubernetes.client.openapi.models.V1Container
+import io.kubernetes.client.openapi.models.V1EnvVar
+import io.kubernetes.client.openapi.models.V1Job
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.withContext
+import org.eclipse.jgit.api.Git
+import org.modelix.authorization.ModelixJWTUtil
+import org.modelix.model.client2.IModelClientV2
+import org.modelix.model.lazy.RepositoryId
+import org.modelix.model.server.ModelServerPermissionSchema
+import org.modelix.services.gitconnector.stubs.models.GitRepositoryConfig
+import org.modelix.services.workspaces.metadata
+import org.modelix.services.workspaces.spec
+import org.modelix.services.workspaces.template
+import org.modelix.workspace.manager.ITaskInstance
+import org.modelix.workspace.manager.KubernetesJobTask
+import java.time.ZoneId
+import java.time.format.DateTimeFormatter
+import kotlin.time.Clock
+import kotlin.time.Duration
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.ExperimentalTime
+import kotlin.time.toJavaInstant
+
+class GitExportTask(
+    val key: Key,
+    scope: CoroutineScope,
+    val modelClient: IModelClientV2,
+    val jwtUtil: ModelixJWTUtil,
+) : ITaskInstance<FetchedBranch>, KubernetesJobTask<FetchedBranch>(scope) {
+
+    companion object {
+        @OptIn(ExperimentalTime::class)
+        fun timeForBranchName() = DateTimeFormatter.ofPattern("yyyyMMdd-HHmmss")
+            .withZone(ZoneId.systemDefault())
+            .format(Clock.System.now().toJavaInstant())
+    }
+
+    data class Key(
+        val repo: GitRepositoryConfig,
+        val modelixVersionHash: String,
+        val modelixBranchName: String,
+        val gitBaseBranch: String,
+    )
+
+    private val repoId = requireNotNull(key.repo.modelixRepository?.let { RepositoryId(it) }) { "Repository ID missing" }
+    private val modelixBranch = repoId.getBranchReference(key.modelixBranchName)
+    val gitBranchName = "modelix-export/${key.gitBaseBranch}/${timeForBranchName()}-${key.modelixVersionHash.replace("*", "")}"
+
+    private fun chooseRemote() = requireNotNull(key.repo.remotes?.firstOrNull()) { "No remotes specified" }
+
+    override fun getResultCheckingInterval(): Duration = 30.seconds
+
+    override suspend fun tryGetResult(): FetchedBranch? {
+        val remote = chooseRemote()
+        val cmd = Git.lsRemoteRepository()
+        cmd.setRemote(remote.url)
+        cmd.setHeads(true)
+        cmd.setTags(false)
+
+        val username = remote.credentials?.username.orEmpty()
+        val password = remote.credentials?.password.orEmpty()
+        if (password.isNotEmpty()) {
+            cmd.applyCredentials(username, password)
+        }
+        cmd.configureHttpProxy()
+
+        val refs = withContext(Dispatchers.IO) {
+            cmd.call()
+        }
+
+        for (ref in refs) {
+            if (!ref.name.startsWith("refs/heads/")) continue
+            val branchName = ref.name.removePrefix("refs/heads/")
+            if (branchName == this.gitBranchName) {
+                return FetchedBranch(
+                    remoteName = remote.name,
+                    branchName = branchName,
+                    commitHash = ref.objectId.name,
+                )
+            }
+        }
+        return null
+    }
+
+    @Suppress("ktlint")
+    override fun generateJobYaml(): V1Job {
+        val remote = chooseRemote()
+        val token = jwtUtil.createAccessToken(
+            "git-sync@modelix.org",
+            listOf(
+                ModelServerPermissionSchema.repository(modelixBranch.repositoryId).read.fullId,
+                ModelServerPermissionSchema.branch(modelixBranch).read.fullId,
+            ),
+        )
+
+        return V1Job().apply {
+            metadata {
+                name = "gitexportjob-$id"
+            }
+            spec {
+                template {
+                    spec {
+                        addContainersItem(V1Container().apply {
+                            name = "importer"
+                            image = System.getenv("GIT_IMPORT_IMAGE")
+                            System.getenv("MODELIX_HTTP_PROXY")?.let {
+                                addEnvItem(V1EnvVar().name("MODELIX_HTTP_PROXY").value(it))
+                            }
+                            args = listOf(
+                                "git-export-remote",
+                                remote.url,
+                                "--git-user",
+                                remote.credentials?.username,
+                                "--git-password",
+                                remote.credentials?.password,
+                                "--model-server",
+                                System.getenv("model_server_url"),
+                                "--token",
+                                token,
+                                "--modelix-repository",
+                                modelixBranch.repositoryId.id,
+                                "--modelix-branch",
+                                modelixBranch.branchName,
+                                "--version",
+                                key.modelixVersionHash,
+                                "--git-branch",
+                                gitBranchName,
+                            )
+                        })
+                    }
+                }
+            }
+        }
+    }
+}

workspace-manager/src/main/kotlin/org/modelix/services/gitconnector/GitImportTask.kt

@@ -42,9 +42,9 @@ class GitImportTaskUsingKubernetesJob(
 
     override suspend fun tryGetResult(): IVersion? {
         return if (modelixBranchExists()) {
-            return modelClient.lazyLoadVersion(branchRef).takeIf { it.gitCommit == key.gitRevision }
+            modelClient.lazyLoadVersion(branchRef).takeIf { it.gitCommit == key.gitRevision }
         } else {
-            return null
+            null
         }
     }
 

workspace-manager/src/main/kotlin/org/modelix/workspace/manager/KubernetesJobTask.kt

@@ -14,19 +14,35 @@ import org.modelix.services.workspaces.spec
 import org.modelix.services.workspaces.template
 import org.modelix.workspace.manager.WorkspaceJobQueue.Companion.KUBERNETES_NAMESPACE
 import kotlin.coroutines.suspendCoroutine
+import kotlin.time.Clock
+import kotlin.time.Duration
 import kotlin.time.Duration.Companion.minutes
+import kotlin.time.Duration.Companion.seconds
+import kotlin.time.ExperimentalTime
+import kotlin.time.Instant
 
+@OptIn(ExperimentalTime::class)
 abstract class KubernetesJobTask<Out : Any>(scope: CoroutineScope) : TaskInstance<Out>(scope) {
     companion object {
         const val JOB_ID_LABEL = "modelix.workspace.job.id"
         private val LOG = mu.KotlinLogging.logger {}
     }
 
+    private var lastResultCheck: Instant = Instant.fromEpochSeconds(0L)
+
     abstract suspend fun tryGetResult(): Out?
     abstract fun generateJobYaml(): V1Job
+    open fun getResultCheckingInterval(): Duration = 5.seconds
+
+    suspend fun checkForResult(): Out? {
+        val now = Clock.System.now()
+        if (now - lastResultCheck < getResultCheckingInterval()) return null
+        lastResultCheck = Clock.System.now()
+        return tryGetResult()
+    }
 
     override suspend fun process() = withTimeout(30.minutes) {
-        tryGetResult()?.let { return@withTimeout it }
+        checkForResult()?.let { return@withTimeout it }
 
         findJob()?.let { deleteJob(it) }
         createJob()
@@ -35,7 +51,7 @@ abstract class KubernetesJobTask<Out : Any>(scope: CoroutineScope) : TaskInstanc
         while (true) {
             delay(1000)
 
-            tryGetResult()?.let { return@withTimeout it }
+            checkForResult()?.let { return@withTimeout it }
 
             val job = findJob()
 

workspace-manager/src/test/kotlin/org/modelix/workspace/manager/GitExportTaskTest.kt

@@ -0,0 +1,13 @@
+package org.modelix.workspace.manager
+
+import org.modelix.services.gitconnector.GitExportTask
+import kotlin.test.Test
+import kotlin.test.assertTrue
+
+class GitExportTaskTest {
+
+    @Test
+    fun `timestamp format`() {
+        assertTrue(GitExportTask.timeForBranchName().matches(Regex("""\d{8}-\d{6}""")))
+    }
+}