From 3ed9812a8b5af4d707a174e89c7ae67e2e3c70a1 Mon Sep 17 00:00:00 2001 From: vsuraj Date: Mon, 15 Apr 2024 14:25:42 +0530 Subject: [PATCH 1/8] Temp to test credentials for aws code commit --- .../taurus/service/credentials/AWSCredentialsService.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java index 47eaff97a1..5ab3a4c556 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java @@ -88,7 +88,10 @@ public AWSCredentialsDTO createTemporaryCredentials() { var accessKeyId = serviceAccountCredentials.getAWSAccessKeyId(); var secretAccessKey = serviceAccountCredentials.getAWSSecretKey(); var sessionToken = serviceAccountCredentials.getSessionToken(); - + System.out.println("Test gitlab code commit access"); + System.out.println(accessKeyId); + System.out.println(secretAccessKey); + System.out.println(sessionToken); return new AWSCredentialsDTO( secretAccessKey, accessKeyId, sessionToken, awsCredentialsServiceConfig.getRegion()); } From 15d4c95ffa4645006184e65a7a3bb5c7f0b893d7 Mon Sep 17 00:00:00 2001 From: vsuraj Date: Thu, 18 Apr 2024 18:47:43 +0530 Subject: [PATCH 2/8] Code commit integration --- .../pipelines_control_service/build.gradle | 7 + .../credentials/AWSCredentialsService.java | 8 +- .../upload/CodeCommitCredentialProvider.java | 21 + .../taurus/service/upload/JobUpload.java | 15 +- .../src/main/resources/application.properties | 2 + .../taurus/service/upload/JobUploadTest.java | 668 +++++++++--------- 6 files changed, 378 insertions(+), 343 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java diff --git a/projects/control-service/projects/pipelines_control_service/build.gradle b/projects/control-service/projects/pipelines_control_service/build.gradle index fedca80a35..1fde7aca69 100644 --- a/projects/control-service/projects/pipelines_control_service/build.gradle +++ b/projects/control-service/projects/pipelines_control_service/build.gradle @@ -28,6 +28,12 @@ configurations { testImplementation.exclude group: 'com.vaadin.external.google', module: 'android-json' } +dependencyManagement { + imports { + mavenBom 'org.springframework.cloud:spring-cloud-dependencies:2021.0.9' + } +} + dependencies { // Implementation dependencies are found on compile classpath of this component and consumers. implementation project(':base') implementation 'com.vmware.taurus:model:3.1.+' @@ -37,6 +43,7 @@ dependencies { // Implementation dependencies are found on compile classpath of // for authorization implementation 'org.springframework.security:spring-security-oauth2-resource-server' + implementation 'org.springframework.cloud:spring-cloud-config-server' implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-data-jpa' diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java index 5ab3a4c556..149db43f20 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/credentials/AWSCredentialsService.java @@ -10,6 +10,8 @@ import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder; import com.amazonaws.services.securitytoken.model.AssumeRoleRequest; import java.util.UUID; + +import lombok.Getter; import org.springframework.stereotype.Service; /** @@ -26,6 +28,7 @@ public class AWSCredentialsService { public record AWSCredentialsDTO( String awsSecretAccessKey, String awsAccessKeyId, String awsSessionToken, String region) {} + @Getter private STSAssumeRoleSessionCredentialsProvider credentialsProvider; private AWSCredentialsServiceConfig awsCredentialsServiceConfig; @@ -88,10 +91,7 @@ public AWSCredentialsDTO createTemporaryCredentials() { var accessKeyId = serviceAccountCredentials.getAWSAccessKeyId(); var secretAccessKey = serviceAccountCredentials.getAWSSecretKey(); var sessionToken = serviceAccountCredentials.getSessionToken(); - System.out.println("Test gitlab code commit access"); - System.out.println(accessKeyId); - System.out.println(secretAccessKey); - System.out.println(sessionToken); + return new AWSCredentialsDTO( secretAccessKey, accessKeyId, sessionToken, awsCredentialsServiceConfig.getRegion()); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java new file mode 100644 index 0000000000..c7d7b93d1c --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java @@ -0,0 +1,21 @@ +package com.vmware.taurus.service.upload; + +import com.vmware.taurus.service.credentials.AWSCredentialsService; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.springframework.cloud.config.server.support.AwsCodeCommitCredentialProvider; +import org.springframework.stereotype.Component; + +@Component +public class CodeCommitCredentialProvider { + private final AWSCredentialsService awsCredentialsService; + + public CodeCommitCredentialProvider(AWSCredentialsService awsCredentialsService) { + this.awsCredentialsService = awsCredentialsService; + } + + public CredentialsProvider getProvider() { + AwsCodeCommitCredentialProvider codeCommitCredentialProvider = new AwsCodeCommitCredentialProvider(); + codeCommitCredentialProvider.setAwsCredentialProvider(awsCredentialsService.getCredentialsProvider()); + return codeCommitCredentialProvider; + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java index 8fbf5b586b..c1edb8e03c 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java @@ -35,7 +35,7 @@ public class JobUpload { private static final Logger log = LoggerFactory.getLogger(JobUpload.class); private final String datajobsTempStorageFolder; - private final GitCredentialsProvider gitCredentialsProvider; + private final CredentialsProvider credentialsProvider; private final GitWrapper gitWrapper; private final FeatureFlags featureFlags; private final AuthorizationProvider authorizationProvider; @@ -45,19 +45,27 @@ public class JobUpload { @Autowired public JobUpload( @Value("${datajobs.temp.storage.folder:}") String datajobsTempStorageFolder, + @Value("${datajobs.git.assumeIAMRole}") boolean assumeCodeCommitIAMRole, GitCredentialsProvider gitCredentialsProvider, + CodeCommitCredentialProvider codeCommitProvider, GitWrapper gitWrapper, FeatureFlags featureFlags, AuthorizationProvider authorizationProvider, JobUploadAllowListValidator jobUploadAllowListValidator, JobUploadFilterListValidator jobUploadFilterListValidator) { this.datajobsTempStorageFolder = datajobsTempStorageFolder; - this.gitCredentialsProvider = gitCredentialsProvider; this.gitWrapper = gitWrapper; this.featureFlags = featureFlags; this.authorizationProvider = authorizationProvider; this.jobUploadAllowListValidator = jobUploadAllowListValidator; this.jobUploadFilterListValidator = jobUploadFilterListValidator; + + if(assumeCodeCommitIAMRole){ + this.credentialsProvider = codeCommitProvider.getProvider(); + } + else{ + this.credentialsProvider = gitCredentialsProvider.getProvider(); + } } /** @@ -67,7 +75,6 @@ public JobUpload( * @return resource containing data job content in a zip format. */ public Optional getDataJob(String jobName) { - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "get data job source")) { Git git = @@ -115,7 +122,6 @@ public Optional getDataJob(String jobName) { public String publishDataJob(String jobName, Resource resource, String reason) { log.debug("Publish datajob to git {}", jobName); String jobVersion; - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "deploy")) { File jobFolder = FileUtils.unzipDataJob(resource, new File(tempDirPath.toFile(), "job"), jobName); @@ -155,7 +161,6 @@ public String publishDataJob(String jobName, Resource resource, String reason) { * @param reason reason specified by user for deleting the data job */ public void deleteDataJob(String jobName, String reason) { - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "delete")) { Git git = gitWrapper.cloneJobRepository( diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties index bf79ae1b39..46912f2431 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties @@ -119,6 +119,8 @@ datajobs.notification.owner.name=Versatile Data Kit # The gitlab repository and credentials for pulling data jobs code when building their images. datajobs.git.url=${GIT_URL} +# datajobs.git.assumeIAMRole tells the control-service if the Service Account pattern should be used for AWS CodeCommit. +datajobs.git.assumeIAMRole=${DATAJOBS_AWS_ASSUME_IAM_ROLE:true} datajobs.git.username=${GIT_USERNAME} datajobs.git.password=${GIT_PASSWORD} datajobs.git.branch=${GIT_BRANCH:master} diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java index d197bad704..50ecbb5d89 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java @@ -1,334 +1,334 @@ -/* - * Copyright 2023-2024 Broadcom - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.vmware.taurus.service.upload; - -import static com.vmware.taurus.service.upload.FileUtils.createTempDir; - -import com.google.common.collect.Iterables; -import com.vmware.taurus.ControlplaneApplication; -import com.vmware.taurus.TestIOUtils; -import com.vmware.taurus.authorization.provider.AuthorizationProvider; -import com.vmware.taurus.base.FeatureFlags; -import java.io.File; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Arrays; -import java.util.Optional; -import net.lingala.zip4j.ZipFile; -import net.lingala.zip4j.exception.ZipException; -import net.lingala.zip4j.model.ZipParameters; -import org.apache.commons.io.FileUtils; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.api.ResetCommand; -import org.eclipse.jgit.api.errors.GitAPIException; -import org.eclipse.jgit.revwalk.RevCommit; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.api.io.TempDir; -import org.mockito.Mock; -import org.mockito.Mockito; -import org.mockito.junit.jupiter.MockitoExtension; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.FileSystemResource; -import org.springframework.core.io.Resource; - -@ExtendWith(MockitoExtension.class) -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -@SpringBootTest(classes = ControlplaneApplication.class) -public class JobUploadTest { - - File remoteRepositoryDir; - private Git remoteGit; - - @Mock private GitCredentialsProvider gitCredentialsProvider; - - private GitWrapper gitWrapper; - - @Mock private FeatureFlags featureFlags; - - @Mock private AuthorizationProvider authorizationProvider; - - @Mock private JobUploadAllowListValidator jobUploadAllowListValidator; - - @Mock private JobUploadFilterListValidator jobUploadFilterListValidator; - - private JobUpload jobUpload; - - @Value("${datajobs.git.branch}") - public String gitDataJobsBranch; - - @Value("${datajobs.git.remote}") - public String gitDataJobsRemote; - - @BeforeEach - public void setup() throws GitAPIException, IOException { - remoteRepositoryDir = Files.createTempDirectory("remote_repo").toFile(); - remoteGit = Git.init().setDirectory(remoteRepositoryDir).call(); - gitWrapper = - new GitWrapper( - "file://" + remoteRepositoryDir.getAbsolutePath(), - gitDataJobsBranch, - gitDataJobsRemote, - true); - - jobUpload = - new JobUpload( - null, - gitCredentialsProvider, - gitWrapper, - featureFlags, - authorizationProvider, - jobUploadAllowListValidator, - jobUploadFilterListValidator); - } - - @AfterEach - public void cleanup() { - remoteRepositoryDir.delete(); - } - - @Test - public void testPushNoAuthentication() throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(false); - - Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); - - jobUpload.publishDataJob("example", jobResource, "example-reason"); - - refreshRemoteGitDirectoryWithLatestChanges(); - Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); - } - - @Test - public void testPushAuthentication() throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); - - jobUpload.publishDataJob("example", jobResource, "example-reason"); - - refreshRemoteGitDirectoryWithLatestChanges(); - Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); - - var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); - Assertions.assertEquals(1, commits.length); - Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); - } - - private void refreshRemoteGitDirectoryWithLatestChanges() throws GitAPIException { - remoteGit.reset().setMode(ResetCommand.ResetType.HARD).call(); - } - - @Test - public void testWithSecondUploadNoChanges(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - File jobDir = mkdir(tempDir.toFile(), "job_dir"); - writeToFile(jobDir, "file.txt", "11"); - - File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "some.zip")); - String firstUploadVersion = - jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); - // put unrelated commit between the two updates to verify commit return is of original job - jobUpload.publishDataJob( - "other-unrelated-job", new FileSystemResource(zipFile), "commit unrelated"); - String secondUploadVersion = - jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 2"); - // second version should be same as first as no changes are present in the job content - Assertions.assertEquals(firstUploadVersion, secondUploadVersion); - } - - @Test - public void testWithEmptyJobDirectory(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - File jobDir = mkdir(tempDir.toFile(), "job_dir"); - File jobFile = writeToFile(jobDir, "file.txt", "11"); - - File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "not_empty.zip")); - jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); - - jobFile.delete(); - zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "empty.zip")); - jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); - - Assertions.assertFalse(new File(remoteRepositoryDir, "job-name").exists()); - } - - /** - * If zip is foo/job-name/file.txt foo/job-name/nested_dir/file2.txt then data job dir uploaded - * will look like job-name/job-name/file.txt job-name/job-name/nested_dir/file2.txt - */ - @Test - public void testWithRootFolderNameInZip(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - File parentDir = mkdir(tempDir.toFile(), "foo"); - File jobDir = mkdir(parentDir, "job-name"); - writeToFile(jobDir, "file.txt", "11"); - - File zipFile = - createZipFromDirWithRootFolderNameInZip(jobDir, new File(tempDir.toFile(), "whatever.zip")); - jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); - - refreshRemoteGitDirectoryWithLatestChanges(); - TestIOUtils.compareDirectories(parentDir, new File(remoteRepositoryDir, "job-name")); - } - - @Test - public void testPushMultipleCommits(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - File jobDir = mkdir(tempDir.toFile(), "data-jobs-example"); - File nestedDir = mkdir(jobDir, "nested_dir"); - - File nestedDirFile = writeToFile(nestedDir, "file_nested_dir.txt", "11"); - File nestedDirFileToDelete = writeToFile(nestedDir, "file_nested_dir_delete.txt", "11"); - File fileToRemain = writeToFile(jobDir, "file_to_remain.txt", "11"); - File fileToDelete = writeToFile(jobDir, "file_to_delete.txt", "11"); - File fileToChange = writeToFile(jobDir, "file_to_change.txt", "11"); - - File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); - jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 1"); - - refreshRemoteGitDirectoryWithLatestChanges(); - TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); - - nestedDirFileToDelete.delete(); - fileToDelete.delete(); - writeToFile(jobDir, fileToChange.getName(), "22"); - - zipFile.delete(); - zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); - String jobCommit = - jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 2"); - - refreshRemoteGitDirectoryWithLatestChanges(); - Assertions.assertEquals(remoteGit.log().call().iterator().next().getName(), jobCommit); - TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); - } - - @Test - public void testDeleteDataJob(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - Resource jobResource = - new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); - - jobUpload.publishDataJob("example", jobResource, "example-reason"); - - refreshRemoteGitDirectoryWithLatestChanges(); - Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); - - jobUpload.deleteDataJob("example", "example-reason"); - refreshRemoteGitDirectoryWithLatestChanges(); - - var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); - Assertions.assertEquals(2, commits.length); - Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); - Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); - } - - @Test - public void testICanOverrideTheDefaultTempDirectoryAndUploadAndDeleteStillWork() - throws IOException, GitAPIException { - jobUpload = - new JobUpload( - createTempDir("DIFFERENT_DIRECTORY_TEST").toFile().toString(), - gitCredentialsProvider, - gitWrapper, - featureFlags, - authorizationProvider, - jobUploadAllowListValidator, - jobUploadFilterListValidator); - - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - Resource jobResource = - new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); - - jobUpload.publishDataJob("example", jobResource, "example-reason"); - - refreshRemoteGitDirectoryWithLatestChanges(); - Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); - - jobUpload.deleteDataJob("example", "example-reason"); - refreshRemoteGitDirectoryWithLatestChanges(); - - var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); - Assertions.assertEquals(2, commits.length); - Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); - Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); - } - - @Test - public void testGetDataJob(@TempDir Path tempDir) throws Exception { - Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); - Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); - - Resource jobResource = - new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); - jobUpload.publishDataJob("example", jobResource, "example-reason"); - refreshRemoteGitDirectoryWithLatestChanges(); - - Optional zippedDataJob = jobUpload.getDataJob("example"); - - Assertions.assertTrue(zippedDataJob.isPresent()); - Assertions.assertTrue(zippedDataJob.get().exists()); - - byte[] zippedDataJobBytes = zippedDataJob.get().getInputStream().readAllBytes(); - Assertions.assertTrue(zippedDataJobBytes.length > 0); - // check that valid zip is expected - first 4 chars of every zip match those - byte[] zipSignature = {0x50, 0x4b, 0x03, 0x04}; - Assertions.assertArrayEquals(zipSignature, Arrays.copyOfRange(zippedDataJobBytes, 0, 4)); - } - - private File createZipFromDir(File directory, File zipFile) throws ZipException { - ZipFile zip = new ZipFile(zipFile); - zip.createSplitZipFileFromFolder(directory, new ZipParameters(), false, 0); - return zipFile; - } - - private File createZipFromDirWithRootFolderNameInZip(File directory, File zipFile) - throws ZipException { - ZipFile zip = new ZipFile(zipFile); - ZipParameters params = new ZipParameters(); - params.setRootFolderNameInZip("foo"); - zip.createSplitZipFileFromFolder(directory, params, false, 0); - return zipFile; - } - - private static File writeToFile(File directory, String fileName, String content) - throws IOException { - FileUtils.writeStringToFile(new File(directory, fileName), content, Charset.defaultCharset()); - return new File(directory, fileName); - } - - // @NotNull - private static File mkdir(File file, String dirName) { - File dir = new File(file, dirName); - dir.mkdir(); - return dir; - } -} +///* +// * Copyright 2023-2024 Broadcom +// * SPDX-License-Identifier: Apache-2.0 +// */ +// +//package com.vmware.taurus.service.upload; +// +//import static com.vmware.taurus.service.upload.FileUtils.createTempDir; +// +//import com.google.common.collect.Iterables; +//import com.vmware.taurus.ControlplaneApplication; +//import com.vmware.taurus.TestIOUtils; +//import com.vmware.taurus.authorization.provider.AuthorizationProvider; +//import com.vmware.taurus.base.FeatureFlags; +//import java.io.File; +//import java.io.IOException; +//import java.nio.charset.Charset; +//import java.nio.file.Files; +//import java.nio.file.Path; +//import java.util.Arrays; +//import java.util.Optional; +//import net.lingala.zip4j.ZipFile; +//import net.lingala.zip4j.exception.ZipException; +//import net.lingala.zip4j.model.ZipParameters; +//import org.apache.commons.io.FileUtils; +//import org.eclipse.jgit.api.Git; +//import org.eclipse.jgit.api.ResetCommand; +//import org.eclipse.jgit.api.errors.GitAPIException; +//import org.eclipse.jgit.revwalk.RevCommit; +//import org.junit.jupiter.api.AfterEach; +//import org.junit.jupiter.api.Assertions; +//import org.junit.jupiter.api.BeforeEach; +//import org.junit.jupiter.api.Test; +//import org.junit.jupiter.api.TestInstance; +//import org.junit.jupiter.api.extension.ExtendWith; +//import org.junit.jupiter.api.io.TempDir; +//import org.mockito.Mock; +//import org.mockito.Mockito; +//import org.mockito.junit.jupiter.MockitoExtension; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.boot.test.context.SpringBootTest; +//import org.springframework.core.io.ClassPathResource; +//import org.springframework.core.io.FileSystemResource; +//import org.springframework.core.io.Resource; +// +//@ExtendWith(MockitoExtension.class) +//@TestInstance(TestInstance.Lifecycle.PER_CLASS) +//@SpringBootTest(classes = ControlplaneApplication.class) +//public class JobUploadTest { +// +// File remoteRepositoryDir; +// private Git remoteGit; +// +// @Mock private GitCredentialsProvider gitCredentialsProvider; +// +// private GitWrapper gitWrapper; +// +// @Mock private FeatureFlags featureFlags; +// +// @Mock private AuthorizationProvider authorizationProvider; +// +// @Mock private JobUploadAllowListValidator jobUploadAllowListValidator; +// +// @Mock private JobUploadFilterListValidator jobUploadFilterListValidator; +// +// private JobUpload jobUpload; +// +// @Value("${datajobs.git.branch}") +// public String gitDataJobsBranch; +// +// @Value("${datajobs.git.remote}") +// public String gitDataJobsRemote; +// +// @BeforeEach +// public void setup() throws GitAPIException, IOException { +// remoteRepositoryDir = Files.createTempDirectory("remote_repo").toFile(); +// remoteGit = Git.init().setDirectory(remoteRepositoryDir).call(); +// gitWrapper = +// new GitWrapper( +// "file://" + remoteRepositoryDir.getAbsolutePath(), +// gitDataJobsBranch, +// gitDataJobsRemote, +// true); +// +// jobUpload = +// new JobUpload( +// null, +// gitCredentialsProvider, +// gitWrapper, +// featureFlags, +// authorizationProvider, +// jobUploadAllowListValidator, +// jobUploadFilterListValidator); +// } +// +// @AfterEach +// public void cleanup() { +// remoteRepositoryDir.delete(); +// } +// +// @Test +// public void testPushNoAuthentication() throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(false); +// +// Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); +// +// jobUpload.publishDataJob("example", jobResource, "example-reason"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); +// } +// +// @Test +// public void testPushAuthentication() throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); +// +// jobUpload.publishDataJob("example", jobResource, "example-reason"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); +// +// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); +// Assertions.assertEquals(1, commits.length); +// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); +// } +// +// private void refreshRemoteGitDirectoryWithLatestChanges() throws GitAPIException { +// remoteGit.reset().setMode(ResetCommand.ResetType.HARD).call(); +// } +// +// @Test +// public void testWithSecondUploadNoChanges(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// File jobDir = mkdir(tempDir.toFile(), "job_dir"); +// writeToFile(jobDir, "file.txt", "11"); +// +// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "some.zip")); +// String firstUploadVersion = +// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); +// // put unrelated commit between the two updates to verify commit return is of original job +// jobUpload.publishDataJob( +// "other-unrelated-job", new FileSystemResource(zipFile), "commit unrelated"); +// String secondUploadVersion = +// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 2"); +// // second version should be same as first as no changes are present in the job content +// Assertions.assertEquals(firstUploadVersion, secondUploadVersion); +// } +// +// @Test +// public void testWithEmptyJobDirectory(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// File jobDir = mkdir(tempDir.toFile(), "job_dir"); +// File jobFile = writeToFile(jobDir, "file.txt", "11"); +// +// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "not_empty.zip")); +// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); +// +// jobFile.delete(); +// zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "empty.zip")); +// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); +// +// Assertions.assertFalse(new File(remoteRepositoryDir, "job-name").exists()); +// } +// +// /** +// * If zip is foo/job-name/file.txt foo/job-name/nested_dir/file2.txt then data job dir uploaded +// * will look like job-name/job-name/file.txt job-name/job-name/nested_dir/file2.txt +// */ +// @Test +// public void testWithRootFolderNameInZip(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// File parentDir = mkdir(tempDir.toFile(), "foo"); +// File jobDir = mkdir(parentDir, "job-name"); +// writeToFile(jobDir, "file.txt", "11"); +// +// File zipFile = +// createZipFromDirWithRootFolderNameInZip(jobDir, new File(tempDir.toFile(), "whatever.zip")); +// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// TestIOUtils.compareDirectories(parentDir, new File(remoteRepositoryDir, "job-name")); +// } +// +// @Test +// public void testPushMultipleCommits(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// File jobDir = mkdir(tempDir.toFile(), "data-jobs-example"); +// File nestedDir = mkdir(jobDir, "nested_dir"); +// +// File nestedDirFile = writeToFile(nestedDir, "file_nested_dir.txt", "11"); +// File nestedDirFileToDelete = writeToFile(nestedDir, "file_nested_dir_delete.txt", "11"); +// File fileToRemain = writeToFile(jobDir, "file_to_remain.txt", "11"); +// File fileToDelete = writeToFile(jobDir, "file_to_delete.txt", "11"); +// File fileToChange = writeToFile(jobDir, "file_to_change.txt", "11"); +// +// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); +// jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 1"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); +// +// nestedDirFileToDelete.delete(); +// fileToDelete.delete(); +// writeToFile(jobDir, fileToChange.getName(), "22"); +// +// zipFile.delete(); +// zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); +// String jobCommit = +// jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 2"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// Assertions.assertEquals(remoteGit.log().call().iterator().next().getName(), jobCommit); +// TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); +// } +// +// @Test +// public void testDeleteDataJob(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// Resource jobResource = +// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); +// +// jobUpload.publishDataJob("example", jobResource, "example-reason"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); +// +// jobUpload.deleteDataJob("example", "example-reason"); +// refreshRemoteGitDirectoryWithLatestChanges(); +// +// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); +// Assertions.assertEquals(2, commits.length); +// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); +// Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); +// } +// +// @Test +// public void testICanOverrideTheDefaultTempDirectoryAndUploadAndDeleteStillWork() +// throws IOException, GitAPIException { +// jobUpload = +// new JobUpload( +// createTempDir("DIFFERENT_DIRECTORY_TEST").toFile().toString(), +// gitCredentialsProvider, +// gitWrapper, +// featureFlags, +// authorizationProvider, +// jobUploadAllowListValidator, +// jobUploadFilterListValidator); +// +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// Resource jobResource = +// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); +// +// jobUpload.publishDataJob("example", jobResource, "example-reason"); +// +// refreshRemoteGitDirectoryWithLatestChanges(); +// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); +// +// jobUpload.deleteDataJob("example", "example-reason"); +// refreshRemoteGitDirectoryWithLatestChanges(); +// +// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); +// Assertions.assertEquals(2, commits.length); +// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); +// Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); +// } +// +// @Test +// public void testGetDataJob(@TempDir Path tempDir) throws Exception { +// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); +// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); +// +// Resource jobResource = +// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); +// jobUpload.publishDataJob("example", jobResource, "example-reason"); +// refreshRemoteGitDirectoryWithLatestChanges(); +// +// Optional zippedDataJob = jobUpload.getDataJob("example"); +// +// Assertions.assertTrue(zippedDataJob.isPresent()); +// Assertions.assertTrue(zippedDataJob.get().exists()); +// +// byte[] zippedDataJobBytes = zippedDataJob.get().getInputStream().readAllBytes(); +// Assertions.assertTrue(zippedDataJobBytes.length > 0); +// // check that valid zip is expected - first 4 chars of every zip match those +// byte[] zipSignature = {0x50, 0x4b, 0x03, 0x04}; +// Assertions.assertArrayEquals(zipSignature, Arrays.copyOfRange(zippedDataJobBytes, 0, 4)); +// } +// +// private File createZipFromDir(File directory, File zipFile) throws ZipException { +// ZipFile zip = new ZipFile(zipFile); +// zip.createSplitZipFileFromFolder(directory, new ZipParameters(), false, 0); +// return zipFile; +// } +// +// private File createZipFromDirWithRootFolderNameInZip(File directory, File zipFile) +// throws ZipException { +// ZipFile zip = new ZipFile(zipFile); +// ZipParameters params = new ZipParameters(); +// params.setRootFolderNameInZip("foo"); +// zip.createSplitZipFileFromFolder(directory, params, false, 0); +// return zipFile; +// } +// +// private static File writeToFile(File directory, String fileName, String content) +// throws IOException { +// FileUtils.writeStringToFile(new File(directory, fileName), content, Charset.defaultCharset()); +// return new File(directory, fileName); +// } +// +// // @NotNull +// private static File mkdir(File file, String dirName) { +// File dir = new File(file, dirName); +// dir.mkdir(); +// return dir; +// } +//} From eb1d5df38a95e611be0ec244a1b07b9de61b0ab3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 18 Apr 2024 13:18:29 +0000 Subject: [PATCH 3/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../taurus/service/upload/CodeCommitCredentialProvider.java | 5 +++++ .../java/com/vmware/taurus/service/upload/JobUploadTest.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java index c7d7b93d1c..309ef9859a 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java @@ -1,3 +1,8 @@ +/* + * Copyright 2023-2024 Broadcom + * SPDX-License-Identifier: Apache-2.0 + */ + package com.vmware.taurus.service.upload; import com.vmware.taurus.service.credentials.AWSCredentialsService; diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java index 50ecbb5d89..2d005b632e 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java @@ -1,3 +1,8 @@ +/* + * Copyright 2023-2024 Broadcom + * SPDX-License-Identifier: Apache-2.0 + */ + ///* // * Copyright 2023-2024 Broadcom // * SPDX-License-Identifier: Apache-2.0 From 22d3f778d091e5af62cee2c7cab82ba9ab75b57b Mon Sep 17 00:00:00 2001 From: vsuraj Date: Sun, 21 Apr 2024 17:41:04 +0530 Subject: [PATCH 4/8] Code commit message builder integration --- projects/control-service/cicd/.gitlab-ci.yml | 15 + .../templates/deployment.yaml | 4 + .../pipelines-control-service/values.yaml | 4 +- .../job-builder-code-commit/Dockerfile | 33 + .../Dockerfile.python.vdk | 30 + .../job-builder-code-commit/README.md | 1 + .../job-builder-code-commit/build_image.sh | 81 +++ .../publish-vdk-job-builder.sh | 23 + .../job-builder-code-commit/version.txt | 1 + .../service/deploy/JobImageBuilder.java | 20 + .../src/main/resources/application.properties | 4 +- .../taurus/service/upload/JobUploadTest.java | 674 +++++++++--------- 12 files changed, 554 insertions(+), 336 deletions(-) create mode 100644 projects/control-service/projects/job-builder-code-commit/Dockerfile create mode 100644 projects/control-service/projects/job-builder-code-commit/Dockerfile.python.vdk create mode 100644 projects/control-service/projects/job-builder-code-commit/README.md create mode 100644 projects/control-service/projects/job-builder-code-commit/build_image.sh create mode 100755 projects/control-service/projects/job-builder-code-commit/publish-vdk-job-builder.sh create mode 100644 projects/control-service/projects/job-builder-code-commit/version.txt diff --git a/projects/control-service/cicd/.gitlab-ci.yml b/projects/control-service/cicd/.gitlab-ci.yml index 30e8e42bb7..bd23774de2 100644 --- a/projects/control-service/cicd/.gitlab-ci.yml +++ b/projects/control-service/cicd/.gitlab-ci.yml @@ -216,6 +216,21 @@ control_service_publish_job_builder_image: changes: - projects/control-service/projects/job-builder/version.txt +control_service_publish_job_builder_codecommit_image: + extends: .images:dind:docker-push-to-vdk-repos + stage: publish_artifacts + script: + - apk add --no-cache bash + - docker login --username "${VDK_DOCKER_REGISTRY_USERNAME}" --password "${VDK_DOCKER_REGISTRY_PASSWORD}" "${VDK_DOCKER_REGISTRY_URL}" + - cd projects/control-service/projects/job-builder-code-commit + - bash -ex ./publish-vdk-job-builder.sh + retry: !reference [.control_service_retry, retry_options] + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - if: '$CI_COMMIT_BRANCH == "main"' + changes: + - projects/control-service/projects/job-builder/version.txt control_service_publish_job_builder_secure_image: extends: control_service_publish_job_builder_image diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml index 53c6c3c48d..a3b37e95dd 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/templates/deployment.yaml @@ -95,6 +95,10 @@ spec: value: "{{ .Values.deploymentEcrAwsServiceAccountSecretAccessKey }}" - name: DATAJOBS_AWS_ROLE_ARN value: "{{ .Values.deploymentEcrAwsRoleArn }}" + - name: DATAJOBS_CC_AWS_ASSUME_IAM_ROLE + value: "{{ .Values.deploymentCodeCommitAwsAssumeIamRole}}" + - name: GIT_GRC_URL + value: "{{ .Values.deploymentGitRemoteCodeCommitUrl}}" - name: DATAJOBS_AWS_DEFAULT_SESSION_DURATION_SECONDS value: "{{ .Values.deploymentEcrAwsDefaultSessionDurationSeconds }}" - name: DOCKER_REGISTRY_TYPE diff --git a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml index 8e8567a8fb..76c07ddb40 100644 --- a/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml +++ b/projects/control-service/projects/helm_charts/pipelines-control-service/values.yaml @@ -188,7 +188,9 @@ deploymentGitPassword: "" # Credentials with read and write access to the Git repository. uploadGitReadWriteUsername: "" uploadGitReadWritePassword: "" - +# Code commit properties +deploymentCodeCommitAwsAssumeIamRole: false +deploymentGitRemoteCodeCommitUrl: "" # List of file types that are allowed to be uploaded. # It is comma separated list with file types. For example "image/png,text/plain" # Only base type can be specified as well, then all files with that base type are allowed. diff --git a/projects/control-service/projects/job-builder-code-commit/Dockerfile b/projects/control-service/projects/job-builder-code-commit/Dockerfile new file mode 100644 index 0000000000..99ee9bf52a --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/Dockerfile @@ -0,0 +1,33 @@ +# Used to trigger a build for a data job image. + +FROM gcr.io/kaniko-project/executor + +FROM alpine + +COPY --from=0 /kaniko /kaniko + + +ENV PATH $PATH:/kaniko +ENV SSL_CERT_DIR=/kaniko/ssl/certs +ENV DOCKER_CONFIG /kaniko/.docker/ + +WORKDIR /workspace + +COPY Dockerfile.python.vdk /workspace/Dockerfile +COPY build_image.sh /build_image.sh +RUN chmod +x /build_image.sh + + +# Setup Python and Git +## Update & Install dependencies +RUN apk add --no-cache --update \ + git \ + bash + +RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.10/main python3=3.7.10-r0 py3-pip \ + && pip3 install awscli \ + && pip3 install git-remote-codecommit \ + && apk --purge -v del py3-pip \ + && rm -rf /var/cache/apk/* + +ENTRYPOINT ["/build_image.sh"] diff --git a/projects/control-service/projects/job-builder-code-commit/Dockerfile.python.vdk b/projects/control-service/projects/job-builder-code-commit/Dockerfile.python.vdk new file mode 100644 index 0000000000..f4a076df5a --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/Dockerfile.python.vdk @@ -0,0 +1,30 @@ +# https://docs.docker.com/develop/develop-images/dockerfile_best-practices + +ARG base_image=python:3.9-slim + +FROM $base_image + +ARG UID=1000 +ARG GID=1000 + +# Set the working directory +WORKDIR /job + +# Create necessary users and set home directory to /job +RUN groupadd -r -g $GID group && useradd -u $UID -g $GID -r user && chown -R $UID:$GID /job +ENV HOME=/job + +# Copy the actual job that has to be executed +ARG job_name +COPY --chown=$UID:$GID $job_name $job_name/ + +# TODO: this would trigger for any change in job even if requirements.txt does not change +# but there's no COPY_IF_EXISTS command in docker to try copy it. +ARG requirements_file=requirements.txt +RUN if [ -f "$job_name/$requirements_file" ]; then pip3 install --no-cache-dir --disable-pip-version-check -q -r "$job_name/$requirements_file" || ( echo ">requirements_failed<" && exit 1 ) ; fi + +ARG job_githash +ENV JOB_NAME $job_name +ENV VDK_JOB_GITHASH $job_githash + +USER $UID diff --git a/projects/control-service/projects/job-builder-code-commit/README.md b/projects/control-service/projects/job-builder-code-commit/README.md new file mode 100644 index 0000000000..923053b2f3 --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/README.md @@ -0,0 +1 @@ +This package provides a way to configure and build your own Data Job images. diff --git a/projects/control-service/projects/job-builder-code-commit/build_image.sh b/projects/control-service/projects/job-builder-code-commit/build_image.sh new file mode 100644 index 0000000000..79099d7f1c --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/build_image.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# Copyright 2021-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +# TODO: replace those as env variables + +aws_access_key_id=$1 +aws_secret_access_key=$2 +aws_region=$3 +docker_registry=$4 +git_repository=$7 +registry_type=$8 +registry_username=$9 +registry_password=${10} +aws_session_token=${11} + +# Within this property docker config should be included to connect to the registry used to pull the image from. +# it should be prefixed with a comma +# example: ,"ghcr.io/versatile-data-kit-dev/dp/versatiledatakit":{"auth":"dmVyc2F0aWxlLWRhdGEta2l0LWRldjo8bXlUb2tlbj4="}} +extra_auth=${extra_auth:-""} +# Echo selected data to be logged +echo "AWS_REGION=$aws_region" +echo "DOCKER_REGISTRY=$docker_registry" +echo "GIT_REPOSITORY=$git_repository" +echo "REGISTRY_TYPE=$registry_type" +# We default to generic repo. +# We have special support for ECR because +# even though Kaniko supports building and pushing images to ECR +# it doesn't create repository nor do they think they should support it - +# https://github.com/GoogleContainerTools/kaniko/pull/1537 +# And ECR requires for each image to create separate repository +# And ECR will not create new image repository on docker push +# So we need to do it manually. +if [ "$registry_type" = "ecr" ] || [ "$registry_type" = "ECR" ] ; then + # Setup credentials to connect to AWS - same creds will be used by kaniko as well. + aws configure set aws_access_key_id $aws_access_key_id + aws configure set aws_secret_access_key $aws_secret_access_key + + # Check if aws_session_token is set and not empty. + if [ -n "$aws_session_token" ] ; then + aws configure set aws_session_token "$aws_session_token" + fi + # https://stackoverflow.com/questions/1199613/extract-filename-and-path-from-url-in-bash-script + repository_prefix=${docker_registry#*/} + # Create docker repository if it does not exist + aws ecr describe-repositories --region $aws_region --repository-names $repository_prefix/${DATA_JOB_NAME} || + aws ecr create-repository --region $aws_region --repository-name $repository_prefix/${DATA_JOB_NAME} + echo '{ "credsStore": "ecr-login" }' > /kaniko/.docker/config.json +elif [ "$registry_type" = "generic" ] || [ "$registry_type" = "GENERIC" ]; then + export auth=$(echo -n $registry_username:$registry_password | base64 -w 0) +cat > /kaniko/.docker/config.json <<- EOM +{ + "auths": { + "$IMAGE_REGISTRY_PATH": { + "username":"$registry_username", + "password":"$registry_password", + "auth": "$auth" + } + $extra_auth + } +} +EOM +#cat /kaniko/.docker/config.json +fi +# Clone repo into /data-jobs dir to get job's source +git clone $git_repository ./data-jobs +cd ./data-jobs +git reset --hard $GIT_COMMIT || ( echo ">data-job-not-found<" && exit 1 ) +if [ ! -d ${DATA_JOB_NAME} ]; then + echo ">data-job-not-found<" + exit 1 +fi +cd .. +# kaniko supports building directly from git repository but as we are using codecommit +# and using aws session credentials, we need to build it beforehand +/kaniko/executor \ + --dockerfile=/workspace/Dockerfile \ + --destination="${IMAGE_REGISTRY_PATH}/${DATA_JOB_NAME}:${GIT_COMMIT}" \ + --build-arg=job_githash="$JOB_GITHASH" \ + --build-arg=base_image="$BASE_IMAGE" \ + --build-arg=job_name="$JOB_NAME" \ + --context=./data-jobs $EXTRA_ARGUMENTS diff --git a/projects/control-service/projects/job-builder-code-commit/publish-vdk-job-builder.sh b/projects/control-service/projects/job-builder-code-commit/publish-vdk-job-builder.sh new file mode 100755 index 0000000000..20db530e0f --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/publish-vdk-job-builder.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Copyright 2023-2024 Broadcom +# SPDX-License-Identifier: Apache-2.0 + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +VERSION_TAG=$(cat "$SCRIPT_DIR/version.txt") +VDK_DOCKER_REGISTRY_URL=${VDK_DOCKER_REGISTRY_URL:-"registry.hub.docker.com/versatiledatakit"} + +function build_and_push_image() { + name="$1" + docker_file="$2" + arguments="$3" + + image_repo="$VDK_DOCKER_REGISTRY_URL/$name" + image_tag="$image_repo:$VERSION_TAG" + + docker build -t $image_tag -t $image_repo:latest -f "$SCRIPT_DIR/$docker_file" $arguments "$SCRIPT_DIR" + docker_push_vdk.sh $image_tag + docker_push_vdk.sh $image_repo:latest +} + +build_and_push_image "job-builder" Dockerfile diff --git a/projects/control-service/projects/job-builder-code-commit/version.txt b/projects/control-service/projects/job-builder-code-commit/version.txt new file mode 100644 index 0000000000..3eefcb9dd5 --- /dev/null +++ b/projects/control-service/projects/job-builder-code-commit/version.txt @@ -0,0 +1 @@ +1.0.0 diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java index f71fafda22..a818f9cbf5 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageBuilder.java @@ -38,6 +38,12 @@ public class JobImageBuilder { @Value("${datajobs.git.url}") private String gitRepo; + @Value("${datajobs.git.cc.grc}") + private String gitCCRepo; + + @Value("${datajobs.git.assumeIAMRole}") + boolean assumeCodeCommitIAMRole; + @Value("${datajobs.git.username}") private String gitUsername; @@ -197,6 +203,20 @@ public boolean buildImage( registryUsername, registryPassword, builderAwsSessionToken); + if(assumeCodeCommitIAMRole){ + args = Arrays.asList( + builderAwsAccessKeyId, + builderAwsSecretAccessKey, + awsRegion, + dockerRepositoryUrl, + "", + "", + gitCCRepo, + registryType, + registryUsername, + registryPassword, + builderAwsSessionToken); + } var envs = getBuildParameters(dataJob, desiredDataJobDeployment); String builderImage = supportedPythonVersions.getBuilderImage(desiredDataJobDeployment.getPythonVersion()); diff --git a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties index 46912f2431..e81bc63567 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties +++ b/projects/control-service/projects/pipelines_control_service/src/main/resources/application.properties @@ -119,8 +119,10 @@ datajobs.notification.owner.name=Versatile Data Kit # The gitlab repository and credentials for pulling data jobs code when building their images. datajobs.git.url=${GIT_URL} +datajobs.git.cc.grc=${GIT_GRC_URL} + # datajobs.git.assumeIAMRole tells the control-service if the Service Account pattern should be used for AWS CodeCommit. -datajobs.git.assumeIAMRole=${DATAJOBS_AWS_ASSUME_IAM_ROLE:true} +datajobs.git.assumeIAMRole=${DATAJOBS_CC_AWS_ASSUME_IAM_ROLE:false} datajobs.git.username=${GIT_USERNAME} datajobs.git.password=${GIT_PASSWORD} datajobs.git.branch=${GIT_BRANCH:master} diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java index 2d005b632e..547f6bf7a8 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java @@ -3,337 +3,343 @@ * SPDX-License-Identifier: Apache-2.0 */ -///* -// * Copyright 2023-2024 Broadcom -// * SPDX-License-Identifier: Apache-2.0 -// */ -// -//package com.vmware.taurus.service.upload; -// -//import static com.vmware.taurus.service.upload.FileUtils.createTempDir; -// -//import com.google.common.collect.Iterables; -//import com.vmware.taurus.ControlplaneApplication; -//import com.vmware.taurus.TestIOUtils; -//import com.vmware.taurus.authorization.provider.AuthorizationProvider; -//import com.vmware.taurus.base.FeatureFlags; -//import java.io.File; -//import java.io.IOException; -//import java.nio.charset.Charset; -//import java.nio.file.Files; -//import java.nio.file.Path; -//import java.util.Arrays; -//import java.util.Optional; -//import net.lingala.zip4j.ZipFile; -//import net.lingala.zip4j.exception.ZipException; -//import net.lingala.zip4j.model.ZipParameters; -//import org.apache.commons.io.FileUtils; -//import org.eclipse.jgit.api.Git; -//import org.eclipse.jgit.api.ResetCommand; -//import org.eclipse.jgit.api.errors.GitAPIException; -//import org.eclipse.jgit.revwalk.RevCommit; -//import org.junit.jupiter.api.AfterEach; -//import org.junit.jupiter.api.Assertions; -//import org.junit.jupiter.api.BeforeEach; -//import org.junit.jupiter.api.Test; -//import org.junit.jupiter.api.TestInstance; -//import org.junit.jupiter.api.extension.ExtendWith; -//import org.junit.jupiter.api.io.TempDir; -//import org.mockito.Mock; -//import org.mockito.Mockito; -//import org.mockito.junit.jupiter.MockitoExtension; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.test.context.SpringBootTest; -//import org.springframework.core.io.ClassPathResource; -//import org.springframework.core.io.FileSystemResource; -//import org.springframework.core.io.Resource; -// -//@ExtendWith(MockitoExtension.class) -//@TestInstance(TestInstance.Lifecycle.PER_CLASS) -//@SpringBootTest(classes = ControlplaneApplication.class) -//public class JobUploadTest { -// -// File remoteRepositoryDir; -// private Git remoteGit; -// -// @Mock private GitCredentialsProvider gitCredentialsProvider; -// -// private GitWrapper gitWrapper; -// -// @Mock private FeatureFlags featureFlags; -// -// @Mock private AuthorizationProvider authorizationProvider; -// -// @Mock private JobUploadAllowListValidator jobUploadAllowListValidator; -// -// @Mock private JobUploadFilterListValidator jobUploadFilterListValidator; -// -// private JobUpload jobUpload; -// -// @Value("${datajobs.git.branch}") -// public String gitDataJobsBranch; -// -// @Value("${datajobs.git.remote}") -// public String gitDataJobsRemote; -// -// @BeforeEach -// public void setup() throws GitAPIException, IOException { -// remoteRepositoryDir = Files.createTempDirectory("remote_repo").toFile(); -// remoteGit = Git.init().setDirectory(remoteRepositoryDir).call(); -// gitWrapper = -// new GitWrapper( -// "file://" + remoteRepositoryDir.getAbsolutePath(), -// gitDataJobsBranch, -// gitDataJobsRemote, -// true); -// -// jobUpload = -// new JobUpload( -// null, -// gitCredentialsProvider, -// gitWrapper, -// featureFlags, -// authorizationProvider, -// jobUploadAllowListValidator, -// jobUploadFilterListValidator); -// } -// -// @AfterEach -// public void cleanup() { -// remoteRepositoryDir.delete(); -// } -// -// @Test -// public void testPushNoAuthentication() throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(false); -// -// Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); -// -// jobUpload.publishDataJob("example", jobResource, "example-reason"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); -// } -// -// @Test -// public void testPushAuthentication() throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); -// -// jobUpload.publishDataJob("example", jobResource, "example-reason"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); -// -// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); -// Assertions.assertEquals(1, commits.length); -// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); -// } -// -// private void refreshRemoteGitDirectoryWithLatestChanges() throws GitAPIException { -// remoteGit.reset().setMode(ResetCommand.ResetType.HARD).call(); -// } -// -// @Test -// public void testWithSecondUploadNoChanges(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// File jobDir = mkdir(tempDir.toFile(), "job_dir"); -// writeToFile(jobDir, "file.txt", "11"); -// -// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "some.zip")); -// String firstUploadVersion = -// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); -// // put unrelated commit between the two updates to verify commit return is of original job -// jobUpload.publishDataJob( -// "other-unrelated-job", new FileSystemResource(zipFile), "commit unrelated"); -// String secondUploadVersion = -// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 2"); -// // second version should be same as first as no changes are present in the job content -// Assertions.assertEquals(firstUploadVersion, secondUploadVersion); -// } -// -// @Test -// public void testWithEmptyJobDirectory(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// File jobDir = mkdir(tempDir.toFile(), "job_dir"); -// File jobFile = writeToFile(jobDir, "file.txt", "11"); -// -// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "not_empty.zip")); -// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); -// -// jobFile.delete(); -// zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "empty.zip")); -// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); -// -// Assertions.assertFalse(new File(remoteRepositoryDir, "job-name").exists()); -// } -// -// /** -// * If zip is foo/job-name/file.txt foo/job-name/nested_dir/file2.txt then data job dir uploaded -// * will look like job-name/job-name/file.txt job-name/job-name/nested_dir/file2.txt -// */ -// @Test -// public void testWithRootFolderNameInZip(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// File parentDir = mkdir(tempDir.toFile(), "foo"); -// File jobDir = mkdir(parentDir, "job-name"); -// writeToFile(jobDir, "file.txt", "11"); -// -// File zipFile = -// createZipFromDirWithRootFolderNameInZip(jobDir, new File(tempDir.toFile(), "whatever.zip")); -// jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// TestIOUtils.compareDirectories(parentDir, new File(remoteRepositoryDir, "job-name")); -// } -// -// @Test -// public void testPushMultipleCommits(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// File jobDir = mkdir(tempDir.toFile(), "data-jobs-example"); -// File nestedDir = mkdir(jobDir, "nested_dir"); -// -// File nestedDirFile = writeToFile(nestedDir, "file_nested_dir.txt", "11"); -// File nestedDirFileToDelete = writeToFile(nestedDir, "file_nested_dir_delete.txt", "11"); -// File fileToRemain = writeToFile(jobDir, "file_to_remain.txt", "11"); -// File fileToDelete = writeToFile(jobDir, "file_to_delete.txt", "11"); -// File fileToChange = writeToFile(jobDir, "file_to_change.txt", "11"); -// -// File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); -// jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 1"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); -// -// nestedDirFileToDelete.delete(); -// fileToDelete.delete(); -// writeToFile(jobDir, fileToChange.getName(), "22"); -// -// zipFile.delete(); -// zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); -// String jobCommit = -// jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 2"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// Assertions.assertEquals(remoteGit.log().call().iterator().next().getName(), jobCommit); -// TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); -// } -// -// @Test -// public void testDeleteDataJob(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// Resource jobResource = -// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); -// -// jobUpload.publishDataJob("example", jobResource, "example-reason"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); -// -// jobUpload.deleteDataJob("example", "example-reason"); -// refreshRemoteGitDirectoryWithLatestChanges(); -// -// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); -// Assertions.assertEquals(2, commits.length); -// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); -// Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); -// } -// -// @Test -// public void testICanOverrideTheDefaultTempDirectoryAndUploadAndDeleteStillWork() -// throws IOException, GitAPIException { -// jobUpload = -// new JobUpload( -// createTempDir("DIFFERENT_DIRECTORY_TEST").toFile().toString(), -// gitCredentialsProvider, -// gitWrapper, -// featureFlags, -// authorizationProvider, -// jobUploadAllowListValidator, -// jobUploadFilterListValidator); -// -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// Resource jobResource = -// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); -// -// jobUpload.publishDataJob("example", jobResource, "example-reason"); -// -// refreshRemoteGitDirectoryWithLatestChanges(); -// Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); -// -// jobUpload.deleteDataJob("example", "example-reason"); -// refreshRemoteGitDirectoryWithLatestChanges(); -// -// var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); -// Assertions.assertEquals(2, commits.length); -// Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); -// Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); -// } -// -// @Test -// public void testGetDataJob(@TempDir Path tempDir) throws Exception { -// Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); -// Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); -// -// Resource jobResource = -// new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); -// jobUpload.publishDataJob("example", jobResource, "example-reason"); -// refreshRemoteGitDirectoryWithLatestChanges(); -// -// Optional zippedDataJob = jobUpload.getDataJob("example"); -// -// Assertions.assertTrue(zippedDataJob.isPresent()); -// Assertions.assertTrue(zippedDataJob.get().exists()); -// -// byte[] zippedDataJobBytes = zippedDataJob.get().getInputStream().readAllBytes(); -// Assertions.assertTrue(zippedDataJobBytes.length > 0); -// // check that valid zip is expected - first 4 chars of every zip match those -// byte[] zipSignature = {0x50, 0x4b, 0x03, 0x04}; -// Assertions.assertArrayEquals(zipSignature, Arrays.copyOfRange(zippedDataJobBytes, 0, 4)); -// } -// -// private File createZipFromDir(File directory, File zipFile) throws ZipException { -// ZipFile zip = new ZipFile(zipFile); -// zip.createSplitZipFileFromFolder(directory, new ZipParameters(), false, 0); -// return zipFile; -// } -// -// private File createZipFromDirWithRootFolderNameInZip(File directory, File zipFile) -// throws ZipException { -// ZipFile zip = new ZipFile(zipFile); -// ZipParameters params = new ZipParameters(); -// params.setRootFolderNameInZip("foo"); -// zip.createSplitZipFileFromFolder(directory, params, false, 0); -// return zipFile; -// } -// -// private static File writeToFile(File directory, String fileName, String content) -// throws IOException { -// FileUtils.writeStringToFile(new File(directory, fileName), content, Charset.defaultCharset()); -// return new File(directory, fileName); -// } -// -// // @NotNull -// private static File mkdir(File file, String dirName) { -// File dir = new File(file, dirName); -// dir.mkdir(); -// return dir; -// } -//} +/* + * Copyright 2023-2024 Broadcom + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.vmware.taurus.service.upload; + +import static com.vmware.taurus.service.upload.FileUtils.createTempDir; + +import com.google.common.collect.Iterables; +import com.vmware.taurus.ControlplaneApplication; +import com.vmware.taurus.TestIOUtils; +import com.vmware.taurus.authorization.provider.AuthorizationProvider; +import com.vmware.taurus.base.FeatureFlags; +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Optional; +import net.lingala.zip4j.ZipFile; +import net.lingala.zip4j.exception.ZipException; +import net.lingala.zip4j.model.ZipParameters; +import org.apache.commons.io.FileUtils; +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.ResetCommand; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.FileSystemResource; +import org.springframework.core.io.Resource; + +@ExtendWith(MockitoExtension.class) +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@SpringBootTest(classes = ControlplaneApplication.class) +public class JobUploadTest { + + File remoteRepositoryDir; + private Git remoteGit; + + @Mock private GitCredentialsProvider gitCredentialsProvider; + + @Mock private CodeCommitCredentialProvider codeCommitCredentialProvider; + + private GitWrapper gitWrapper; + + @Mock private FeatureFlags featureFlags; + + @Mock private AuthorizationProvider authorizationProvider; + + @Mock private JobUploadAllowListValidator jobUploadAllowListValidator; + + @Mock private JobUploadFilterListValidator jobUploadFilterListValidator; + + private JobUpload jobUpload; + + @Value("${datajobs.git.branch}") + public String gitDataJobsBranch; + + @Value("${datajobs.git.remote}") + public String gitDataJobsRemote; + + @BeforeEach + public void setup() throws GitAPIException, IOException { + remoteRepositoryDir = Files.createTempDirectory("remote_repo").toFile(); + remoteGit = Git.init().setDirectory(remoteRepositoryDir).call(); + gitWrapper = + new GitWrapper( + "file://" + remoteRepositoryDir.getAbsolutePath(), + gitDataJobsBranch, + gitDataJobsRemote, + true); + + jobUpload = + new JobUpload( + null, + false, + gitCredentialsProvider, + codeCommitCredentialProvider, + gitWrapper, + featureFlags, + authorizationProvider, + jobUploadAllowListValidator, + jobUploadFilterListValidator); + } + + @AfterEach + public void cleanup() { + remoteRepositoryDir.delete(); + } + + @Test + public void testPushNoAuthentication() throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(false); + + Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); + + jobUpload.publishDataJob("example", jobResource, "example-reason"); + + refreshRemoteGitDirectoryWithLatestChanges(); + Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); + } + + @Test + public void testPushAuthentication() throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + Resource jobResource = new ClassPathResource("/file_test/test_job.zip"); + + jobUpload.publishDataJob("example", jobResource, "example-reason"); + + refreshRemoteGitDirectoryWithLatestChanges(); + Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").isDirectory()); + + var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); + Assertions.assertEquals(1, commits.length); + Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); + } + + private void refreshRemoteGitDirectoryWithLatestChanges() throws GitAPIException { + remoteGit.reset().setMode(ResetCommand.ResetType.HARD).call(); + } + + @Test + public void testWithSecondUploadNoChanges(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + File jobDir = mkdir(tempDir.toFile(), "job_dir"); + writeToFile(jobDir, "file.txt", "11"); + + File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "some.zip")); + String firstUploadVersion = + jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); + // put unrelated commit between the two updates to verify commit return is of original job + jobUpload.publishDataJob( + "other-unrelated-job", new FileSystemResource(zipFile), "commit unrelated"); + String secondUploadVersion = + jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 2"); + // second version should be same as first as no changes are present in the job content + Assertions.assertEquals(firstUploadVersion, secondUploadVersion); + } + + @Test + public void testWithEmptyJobDirectory(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + File jobDir = mkdir(tempDir.toFile(), "job_dir"); + File jobFile = writeToFile(jobDir, "file.txt", "11"); + + File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "not_empty.zip")); + jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); + + jobFile.delete(); + zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "empty.zip")); + jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); + + Assertions.assertFalse(new File(remoteRepositoryDir, "job-name").exists()); + } + + /** + * If zip is foo/job-name/file.txt foo/job-name/nested_dir/file2.txt then data job dir uploaded + * will look like job-name/job-name/file.txt job-name/job-name/nested_dir/file2.txt + */ + @Test + public void testWithRootFolderNameInZip(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + File parentDir = mkdir(tempDir.toFile(), "foo"); + File jobDir = mkdir(parentDir, "job-name"); + writeToFile(jobDir, "file.txt", "11"); + + File zipFile = + createZipFromDirWithRootFolderNameInZip(jobDir, new File(tempDir.toFile(), "whatever.zip")); + jobUpload.publishDataJob("job-name", new FileSystemResource(zipFile), "commit 1"); + + refreshRemoteGitDirectoryWithLatestChanges(); + TestIOUtils.compareDirectories(parentDir, new File(remoteRepositoryDir, "job-name")); + } + + @Test + public void testPushMultipleCommits(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + File jobDir = mkdir(tempDir.toFile(), "data-jobs-example"); + File nestedDir = mkdir(jobDir, "nested_dir"); + + File nestedDirFile = writeToFile(nestedDir, "file_nested_dir.txt", "11"); + File nestedDirFileToDelete = writeToFile(nestedDir, "file_nested_dir_delete.txt", "11"); + File fileToRemain = writeToFile(jobDir, "file_to_remain.txt", "11"); + File fileToDelete = writeToFile(jobDir, "file_to_delete.txt", "11"); + File fileToChange = writeToFile(jobDir, "file_to_change.txt", "11"); + + File zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); + jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 1"); + + refreshRemoteGitDirectoryWithLatestChanges(); + TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); + + nestedDirFileToDelete.delete(); + fileToDelete.delete(); + writeToFile(jobDir, fileToChange.getName(), "22"); + + zipFile.delete(); + zipFile = createZipFromDir(jobDir, new File(tempDir.toFile(), "example.zip")); + String jobCommit = + jobUpload.publishDataJob("example", new FileSystemResource(zipFile), "commit 2"); + + refreshRemoteGitDirectoryWithLatestChanges(); + Assertions.assertEquals(remoteGit.log().call().iterator().next().getName(), jobCommit); + TestIOUtils.compareDirectories(jobDir, new File(remoteRepositoryDir, "example")); + } + + @Test + public void testDeleteDataJob(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + Resource jobResource = + new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); + + jobUpload.publishDataJob("example", jobResource, "example-reason"); + + refreshRemoteGitDirectoryWithLatestChanges(); + Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); + + jobUpload.deleteDataJob("example", "example-reason"); + refreshRemoteGitDirectoryWithLatestChanges(); + + var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); + Assertions.assertEquals(2, commits.length); + Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); + Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); + } + + @Test + public void testICanOverrideTheDefaultTempDirectoryAndUploadAndDeleteStillWork() + throws IOException, GitAPIException { + jobUpload = + new JobUpload( + createTempDir("DIFFERENT_DIRECTORY_TEST").toFile().toString(), + false, + gitCredentialsProvider, + codeCommitCredentialProvider, + gitWrapper, + featureFlags, + authorizationProvider, + jobUploadAllowListValidator, + jobUploadFilterListValidator); + + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + Resource jobResource = + new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); + + jobUpload.publishDataJob("example", jobResource, "example-reason"); + + refreshRemoteGitDirectoryWithLatestChanges(); + Assertions.assertTrue(new File(this.remoteRepositoryDir, "example").exists()); + + jobUpload.deleteDataJob("example", "example-reason"); + refreshRemoteGitDirectoryWithLatestChanges(); + + var commits = Iterables.toArray(remoteGit.log().call(), RevCommit.class); + Assertions.assertEquals(2, commits.length); + Assertions.assertEquals("user", commits[0].getAuthorIdent().getName()); + Assertions.assertFalse(new File(this.remoteRepositoryDir, "example").exists()); + } + + @Test + public void testGetDataJob(@TempDir Path tempDir) throws Exception { + Mockito.when(featureFlags.isSecurityEnabled()).thenReturn(true); + Mockito.when(authorizationProvider.getUserId(Mockito.any())).thenReturn("user"); + + Resource jobResource = + new ClassPathResource("/file_test/test_job.zip", this.getClass().getClassLoader()); + jobUpload.publishDataJob("example", jobResource, "example-reason"); + refreshRemoteGitDirectoryWithLatestChanges(); + + Optional zippedDataJob = jobUpload.getDataJob("example"); + + Assertions.assertTrue(zippedDataJob.isPresent()); + Assertions.assertTrue(zippedDataJob.get().exists()); + + byte[] zippedDataJobBytes = zippedDataJob.get().getInputStream().readAllBytes(); + Assertions.assertTrue(zippedDataJobBytes.length > 0); + // check that valid zip is expected - first 4 chars of every zip match those + byte[] zipSignature = {0x50, 0x4b, 0x03, 0x04}; + Assertions.assertArrayEquals(zipSignature, Arrays.copyOfRange(zippedDataJobBytes, 0, 4)); + } + + private File createZipFromDir(File directory, File zipFile) throws ZipException { + ZipFile zip = new ZipFile(zipFile); + zip.createSplitZipFileFromFolder(directory, new ZipParameters(), false, 0); + return zipFile; + } + + private File createZipFromDirWithRootFolderNameInZip(File directory, File zipFile) + throws ZipException { + ZipFile zip = new ZipFile(zipFile); + ZipParameters params = new ZipParameters(); + params.setRootFolderNameInZip("foo"); + zip.createSplitZipFileFromFolder(directory, params, false, 0); + return zipFile; + } + + private static File writeToFile(File directory, String fileName, String content) + throws IOException { + FileUtils.writeStringToFile(new File(directory, fileName), content, Charset.defaultCharset()); + return new File(directory, fileName); + } + + // @NotNull + private static File mkdir(File file, String dirName) { + File dir = new File(file, dirName); + dir.mkdir(); + return dir; + } +} From a0d13f468ed9a9c1b93c57751e6160bc526a4386 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 21 Apr 2024 12:13:05 +0000 Subject: [PATCH 5/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../projects/job-builder-code-commit/build_image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/control-service/projects/job-builder-code-commit/build_image.sh b/projects/control-service/projects/job-builder-code-commit/build_image.sh index 79099d7f1c..bb4706cbb0 100644 --- a/projects/control-service/projects/job-builder-code-commit/build_image.sh +++ b/projects/control-service/projects/job-builder-code-commit/build_image.sh @@ -1,4 +1,7 @@ #!/bin/sh +# Copyright 2023-2024 Broadcom +# SPDX-License-Identifier: Apache-2.0 + # Copyright 2021-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 # TODO: replace those as env variables From e0740752e0bc0bb76703829eb74d8b2a19e464bd Mon Sep 17 00:00:00 2001 From: vsuraj Date: Tue, 23 Apr 2024 23:24:21 +0530 Subject: [PATCH 6/8] review comment --- .../upload/CodeCommitCredentialProvider.java | 2 +- .../upload/CredentialProviderConfig.java | 30 +++++++++++++++++++ .../upload/GitCredentialsProvider.java | 3 +- .../taurus/service/upload/JobUpload.java | 17 ++++------- .../upload/VCSCredentialsProvider.java | 12 ++++++++ .../taurus/service/upload/JobUploadTest.java | 13 +------- 6 files changed, 52 insertions(+), 25 deletions(-) create mode 100644 projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java create mode 100644 projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java index 309ef9859a..941909ad44 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java @@ -11,7 +11,7 @@ import org.springframework.stereotype.Component; @Component -public class CodeCommitCredentialProvider { +public class CodeCommitCredentialProvider implements VCSCredentialsProvider { private final AWSCredentialsService awsCredentialsService; public CodeCommitCredentialProvider(AWSCredentialsService awsCredentialsService) { diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java new file mode 100644 index 0000000000..66e564470a --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java @@ -0,0 +1,30 @@ +package com.vmware.taurus.service.upload; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class CredentialProviderConfig { + + + private final VCSCredentialsProvider credentialsProvider; + + @Autowired + public CredentialProviderConfig( + @Value("${datajobs.git.assumeIAMRole}") boolean assumeCodeCommitIAMRole, + GitCredentialsProvider gitCredentialsProvider, + CodeCommitCredentialProvider codeCommitProvider) { + if (assumeCodeCommitIAMRole) { + this.credentialsProvider = codeCommitProvider; + } else { + this.credentialsProvider = gitCredentialsProvider; + } + } + + @Bean + public VCSCredentialsProvider credentialsProvider() { + return credentialsProvider; + } +} diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/GitCredentialsProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/GitCredentialsProvider.java index 74f306654e..dd49c1dbce 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/GitCredentialsProvider.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/GitCredentialsProvider.java @@ -17,7 +17,7 @@ * Other providers are explained: https://www.codeaffine.com/2014/12/09/jgit-authentication/ */ @Component -public class GitCredentialsProvider { +public class GitCredentialsProvider implements VCSCredentialsProvider { @Value("${datajobs.git.read.write.username:}") private String gitReadWriteUsername; @@ -25,6 +25,7 @@ public class GitCredentialsProvider { @Value("${datajobs.git.read.write.password:}") private String gitReadWritePassword; + @Override public CredentialsProvider getProvider() { return new UsernamePasswordCredentialsProvider(gitReadWriteUsername, gitReadWritePassword); } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java index c1edb8e03c..0ec0bdc992 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java @@ -35,7 +35,7 @@ public class JobUpload { private static final Logger log = LoggerFactory.getLogger(JobUpload.class); private final String datajobsTempStorageFolder; - private final CredentialsProvider credentialsProvider; + private final VCSCredentialsProvider gitCredentialsProvider; private final GitWrapper gitWrapper; private final FeatureFlags featureFlags; private final AuthorizationProvider authorizationProvider; @@ -45,27 +45,19 @@ public class JobUpload { @Autowired public JobUpload( @Value("${datajobs.temp.storage.folder:}") String datajobsTempStorageFolder, - @Value("${datajobs.git.assumeIAMRole}") boolean assumeCodeCommitIAMRole, - GitCredentialsProvider gitCredentialsProvider, - CodeCommitCredentialProvider codeCommitProvider, + VCSCredentialsProvider gitCredentialsProvider, GitWrapper gitWrapper, FeatureFlags featureFlags, AuthorizationProvider authorizationProvider, JobUploadAllowListValidator jobUploadAllowListValidator, JobUploadFilterListValidator jobUploadFilterListValidator) { this.datajobsTempStorageFolder = datajobsTempStorageFolder; + this.gitCredentialsProvider = gitCredentialsProvider; this.gitWrapper = gitWrapper; this.featureFlags = featureFlags; this.authorizationProvider = authorizationProvider; this.jobUploadAllowListValidator = jobUploadAllowListValidator; this.jobUploadFilterListValidator = jobUploadFilterListValidator; - - if(assumeCodeCommitIAMRole){ - this.credentialsProvider = codeCommitProvider.getProvider(); - } - else{ - this.credentialsProvider = gitCredentialsProvider.getProvider(); - } } /** @@ -75,6 +67,7 @@ public JobUpload( * @return resource containing data job content in a zip format. */ public Optional getDataJob(String jobName) { + CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "get data job source")) { Git git = @@ -122,6 +115,7 @@ public Optional getDataJob(String jobName) { public String publishDataJob(String jobName, Resource resource, String reason) { log.debug("Publish datajob to git {}", jobName); String jobVersion; + CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "deploy")) { File jobFolder = FileUtils.unzipDataJob(resource, new File(tempDirPath.toFile(), "job"), jobName); @@ -161,6 +155,7 @@ public String publishDataJob(String jobName, Resource resource, String reason) { * @param reason reason specified by user for deleting the data job */ public void deleteDataJob(String jobName, String reason) { + CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "delete")) { Git git = gitWrapper.cloneJobRepository( diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java new file mode 100644 index 0000000000..3aa00867ba --- /dev/null +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java @@ -0,0 +1,12 @@ +package com.vmware.taurus.service.upload; + +import org.eclipse.jgit.transport.CredentialsProvider; + +/** + * Class responsible for handling different credential providers. + * + */ +public interface VCSCredentialsProvider { + + CredentialsProvider getProvider(); +} diff --git a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java index 547f6bf7a8..c968ef4524 100644 --- a/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java +++ b/projects/control-service/projects/pipelines_control_service/src/test/java/com/vmware/taurus/service/upload/JobUploadTest.java @@ -3,11 +3,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -/* - * Copyright 2023-2024 Broadcom - * SPDX-License-Identifier: Apache-2.0 - */ - package com.vmware.taurus.service.upload; import static com.vmware.taurus.service.upload.FileUtils.createTempDir; @@ -56,9 +51,7 @@ public class JobUploadTest { File remoteRepositoryDir; private Git remoteGit; - @Mock private GitCredentialsProvider gitCredentialsProvider; - - @Mock private CodeCommitCredentialProvider codeCommitCredentialProvider; + @Mock private VCSCredentialsProvider gitCredentialsProvider; private GitWrapper gitWrapper; @@ -92,9 +85,7 @@ public void setup() throws GitAPIException, IOException { jobUpload = new JobUpload( null, - false, gitCredentialsProvider, - codeCommitCredentialProvider, gitWrapper, featureFlags, authorizationProvider, @@ -263,9 +254,7 @@ public void testICanOverrideTheDefaultTempDirectoryAndUploadAndDeleteStillWork() jobUpload = new JobUpload( createTempDir("DIFFERENT_DIRECTORY_TEST").toFile().toString(), - false, gitCredentialsProvider, - codeCommitCredentialProvider, gitWrapper, featureFlags, authorizationProvider, From 8bef39567abd4fe7e4af2470a6db0d3f63aea14f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:56:01 +0000 Subject: [PATCH 7/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../taurus/service/upload/CredentialProviderConfig.java | 5 +++++ .../vmware/taurus/service/upload/VCSCredentialsProvider.java | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java index 66e564470a..38902e0259 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java @@ -1,3 +1,8 @@ +/* + * Copyright 2023-2024 Broadcom + * SPDX-License-Identifier: Apache-2.0 + */ + package com.vmware.taurus.service.upload; import org.springframework.beans.factory.annotation.Autowired; diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java index 3aa00867ba..4186029312 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/VCSCredentialsProvider.java @@ -1,3 +1,8 @@ +/* + * Copyright 2023-2024 Broadcom + * SPDX-License-Identifier: Apache-2.0 + */ + package com.vmware.taurus.service.upload; import org.eclipse.jgit.transport.CredentialsProvider; From 505fb09f702709bfba7d3626e02ca59ea518de74 Mon Sep 17 00:00:00 2001 From: vsuraj Date: Thu, 25 Apr 2024 10:44:55 +0530 Subject: [PATCH 8/8] bean fix --- .../upload/CodeCommitCredentialProvider.java | 1 + .../service/upload/CredentialProviderConfig.java | 2 +- .../com/vmware/taurus/service/upload/JobUpload.java | 13 +++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java index 941909ad44..6f22b7ae37 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CodeCommitCredentialProvider.java @@ -18,6 +18,7 @@ public CodeCommitCredentialProvider(AWSCredentialsService awsCredentialsService) this.awsCredentialsService = awsCredentialsService; } + @Override public CredentialsProvider getProvider() { AwsCodeCommitCredentialProvider codeCommitCredentialProvider = new AwsCodeCommitCredentialProvider(); codeCommitCredentialProvider.setAwsCredentialProvider(awsCredentialsService.getCredentialsProvider()); diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java index 38902e0259..8a5cd799cf 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/CredentialProviderConfig.java @@ -28,7 +28,7 @@ public CredentialProviderConfig( } } - @Bean + @Bean(name="credentialsProvider") public VCSCredentialsProvider credentialsProvider() { return credentialsProvider; } diff --git a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java index 0ec0bdc992..3b06e21770 100644 --- a/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java +++ b/projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/upload/JobUpload.java @@ -14,6 +14,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.Resource; import org.springframework.security.core.Authentication; @@ -35,7 +36,7 @@ public class JobUpload { private static final Logger log = LoggerFactory.getLogger(JobUpload.class); private final String datajobsTempStorageFolder; - private final VCSCredentialsProvider gitCredentialsProvider; + private final VCSCredentialsProvider vcsCredentialsProvider; private final GitWrapper gitWrapper; private final FeatureFlags featureFlags; private final AuthorizationProvider authorizationProvider; @@ -45,14 +46,14 @@ public class JobUpload { @Autowired public JobUpload( @Value("${datajobs.temp.storage.folder:}") String datajobsTempStorageFolder, - VCSCredentialsProvider gitCredentialsProvider, + @Qualifier("credentialsProvider") VCSCredentialsProvider vcsCredentialsProvider, GitWrapper gitWrapper, FeatureFlags featureFlags, AuthorizationProvider authorizationProvider, JobUploadAllowListValidator jobUploadAllowListValidator, JobUploadFilterListValidator jobUploadFilterListValidator) { this.datajobsTempStorageFolder = datajobsTempStorageFolder; - this.gitCredentialsProvider = gitCredentialsProvider; + this.vcsCredentialsProvider = vcsCredentialsProvider; this.gitWrapper = gitWrapper; this.featureFlags = featureFlags; this.authorizationProvider = authorizationProvider; @@ -67,7 +68,7 @@ public JobUpload( * @return resource containing data job content in a zip format. */ public Optional getDataJob(String jobName) { - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); + CredentialsProvider credentialsProvider = vcsCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "get data job source")) { Git git = @@ -115,7 +116,7 @@ public Optional getDataJob(String jobName) { public String publishDataJob(String jobName, Resource resource, String reason) { log.debug("Publish datajob to git {}", jobName); String jobVersion; - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); + CredentialsProvider credentialsProvider = vcsCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "deploy")) { File jobFolder = FileUtils.unzipDataJob(resource, new File(tempDirPath.toFile(), "job"), jobName); @@ -155,7 +156,7 @@ public String publishDataJob(String jobName, Resource resource, String reason) { * @param reason reason specified by user for deleting the data job */ public void deleteDataJob(String jobName, String reason) { - CredentialsProvider credentialsProvider = gitCredentialsProvider.getProvider(); + CredentialsProvider credentialsProvider = vcsCredentialsProvider.getProvider(); try (var tempDirPath = new EphemeralFile(datajobsTempStorageFolder, jobName, "delete")) { Git git = gitWrapper.cloneJobRepository(