Merge pull request #137 from mcanoy/migrate-artifacts

migration functionality for artifacts. now with API

Author: mcanoy

src/main/java/com/redhat/labs/lodestar/resource/MigrationResource.java

@@ -0,0 +1,35 @@
+package com.redhat.labs.lodestar.resource;
+
+import javax.inject.Inject;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.core.Response;
+
+import org.eclipse.microprofile.metrics.MetricUnits;
+import org.eclipse.microprofile.metrics.annotation.Counted;
+import org.eclipse.microprofile.metrics.annotation.Timed;
+import org.eclipse.microprofile.openapi.annotations.tags.Tag;
+
+import com.redhat.labs.lodestar.service.MigrationService;
+
+@Path("/api/migrate")
+@Tag(name = "Migration", description = "Migratiion services")
+public class MigrationResource {
+
+    @Inject
+    MigrationService migrationService;
+
+    @PUT
+    @Counted(name = "migration", description = "How many migation requests have been invoked")
+    @Timed(name = "performedMigration", description = "How much time it takes to migrate", unit = MetricUnits.MILLISECONDS)
+    public Response migrate(@QueryParam(value = "participants") boolean migrateParticipants,
+            @QueryParam(value = "artifacts") boolean migrateArtifacts,
+            @QueryParam(value = "uuids") boolean migrateUuids) {
+        
+        migrationService.migrate(migrateUuids, migrateParticipants, migrateArtifacts);
+        
+        return Response.ok().build();
+
+    }
+}

src/main/java/com/redhat/labs/lodestar/rest/client/GitLabService.java

@@ -12,8 +12,10 @@
 import javax.ws.rs.Path;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 import javax.ws.rs.core.Response;
 
+import org.apache.http.NoHttpResponseException;
 import org.eclipse.microprofile.faulttolerance.Retry;
 import org.eclipse.microprofile.rest.client.annotation.ClientHeaderParam;
 import org.eclipse.microprofile.rest.client.annotation.RegisterProvider;
@@ -30,7 +32,7 @@
 import com.redhat.labs.lodestar.models.gitlab.ProjectTransfer;
 import com.redhat.labs.lodestar.resources.filter.Logged;
 
-@Retry(maxRetries = 5, delay = 2000)
+@Retry(maxRetries = 5, delay = 1200, retryOn = NoHttpResponseException.class, abortOn = WebApplicationException.class)
 @Path("/api/v4")
 @RegisterRestClient(configKey = "gitlab.api")
 @RegisterProvider(value = GitLabServiceResponseMapper.class, priority = 50)

src/main/java/com/redhat/labs/lodestar/service/MigrationService.java

@@ -15,6 +15,7 @@
 import org.slf4j.LoggerFactory;
 
 import com.redhat.labs.lodestar.config.JsonMarshaller;
+import com.redhat.labs.lodestar.models.Artifact;
 import com.redhat.labs.lodestar.models.Engagement;
 import com.redhat.labs.lodestar.models.EngagementUser;
 import com.redhat.labs.lodestar.models.gitlab.File;
@@ -26,7 +27,8 @@
 public class MigrationService {
     private static final Logger LOGGER = LoggerFactory.getLogger(MigrationService.class);
 
-    private static final String userJson = "users.json";
+    private static final String PARTICIPANT_JSON = "participants.json";
+    private static final String ARTIFACT_JSON = "artifacts.json";
 
     @Inject
     EngagementService engagementService;
@@ -43,42 +45,48 @@ public class MigrationService {
     @ConfigProperty(name = "engagements.repository.id")
     int engagementRepositoryId;
 
-    @ConfigProperty(name = "migrate.users")
-    boolean migrateUsers;
+    @ConfigProperty(name = "commit.default.email")
+    String commitEmail;
 
-    @ConfigProperty(name = "migrate.uuid")
-    boolean migrateUuids;
+    @ConfigProperty(name = "commit.default.author")
+    String commitAuthor;
+    
+    @ConfigProperty(name = "commit.default.branch")
+    String commitBranch;
 
     private Map<Integer, Engagement> allEngagements = new HashMap<>();
 
     /**
-     * Currently the migration will only occur if config properties are true.
      * The migration is idempotent so no harm in rerunning. It will only update
-     * engagements that haven't been migrated. As we get closer to migration time
-     * we should evaluate whether this is the right approach.
-     * Once the start up is complete this service can not be called.
-     * @param ev
+     * engagements that haven't been migrated. 
      */
-    void onStart(@Observes StartupEvent ev) {
-        
-        if(migrateUsers) {
-            LOGGER.info("Migrate users: {}", migrateUsers);
-            migrateUsers();
-            LOGGER.info("End Migrate users");
-        }
-        
+    public void migrate(boolean migrateUuids, boolean migrateParticipants, boolean migrateArtifacts) {
         if(migrateUuids) {
-            LOGGER.info("Migrate uuids: {}", migrateUuids);
+            LOGGER.info("Start Migrate uuids: {}", migrateUuids);
             migrateUuids();
             LOGGER.info("End Migrate uuids");
-        }  
+        } 
+        
+        if(migrateParticipants) {
+            LOGGER.info("Start Migrate participants: {}", migrateParticipants);
+            migrateParticipants();
+            LOGGER.info("End Migrate participants");
+        } 
 
+        if(migrateArtifacts) {
+            LOGGER.info("Start Migrate artifacts");
+            migrateArtifacts();
+            LOGGER.info("End Migrate artifacts");
+        }
     }
+    
     /**
-     * Get all projects and split for individual update
+     * Get all projects and split for individual update. This will add a description to any 
+     * project that doesn't have one. The description will included the uuid of the engagement
      */
     private void migrateUuids() {
         List<Project> allProjects = projectService.getProjectsByGroup(engagementRepositoryId, true);
+        getAllEngagements(); //hydrate before stream
         allProjects.parallelStream().forEach(this::updateProjectWithUuid);
     }
 
@@ -94,21 +102,35 @@ private void updateProjectWithUuid(Project project) {
             projectService.updateProject(project);
 
             LOGGER.info("Added uuid {} to project {} {}", uuid, project.getId(), project);
+        } else {
+            LOGGER.info("Skipped uuid update because description is already set or the project {} is not in the engagement map", project.getId());
         }
     }
 
     /**
      * Get all engagements (engagement.json) and split for individual update
      */
-    private void migrateUsers() {     
-        getAllEngagements().values().parallelStream().forEach(this::migrateUsersToGitlab);   
+    private void migrateParticipants() {     
+        getAllEngagements().values().parallelStream().forEach(this::migrateParticipantsToGitlab);   
+    }
+    
+    private void migrateArtifacts() {
+        getAllEngagements().values().parallelStream().forEach(this::migrateArtifactsToGitlab);
+    }
+    
+    private void migrateArtifactsToGitlab(Engagement engagement) {
+        List<Artifact> artifacts = engagement.getArtifacts() == null ? Collections.emptyList() : engagement.getArtifacts();
+        String content = json.toJson(artifacts);
+        migrateToGitlab(engagement, content, ARTIFACT_JSON, artifacts.size());
+        
     }
 
     /**
      * Get All engagements. This is run by migrate users and uuids so only run once if both are active
      * @return
      */
     private Map<Integer, Engagement> getAllEngagements() {
+        LOGGER.debug("Engagement count (pre-fetch) {}", allEngagements.size());
         if (allEngagements.isEmpty()) {
             List<Engagement> engagements = engagementService.getAllEngagements(Optional.of(false), Optional.of(false));
             engagements.parallelStream().forEach(this::addToMap);
@@ -131,20 +153,24 @@ private void addToMap(Engagement engagement) {
      * fileService.deleteFile(engagement.getProjectId(), userJson);
      * @param engagement
      */
-    private void migrateUsersToGitlab(Engagement engagement) {
-        
-        List<EngagementUser> users = engagement.getEngagementUsers();
+    private void migrateParticipantsToGitlab(Engagement engagement) {
 
-        if(users == null) {
-            users = Collections.emptyList();
-        }
+        List<EngagementUser> participants = engagement.getEngagementUsers() == null ? Collections.emptyList() : engagement.getEngagementUsers();
+        String content = json.toJson(participants);
+        migrateToGitlab(engagement, content, PARTICIPANT_JSON, participants.size());
+    }
+    
+    /**
+     * This will write the user json to gitlab. Should you wish to rollback or redo you could add this code
+     * fileService.deleteFile(engagement.getProjectId(), userJson);
+     * @param engagement
+     */
+    private void migrateToGitlab(Engagement engagement, String content, String fileName, int size) {
 
-        if(fileService.getFile(engagement.getProjectId(), userJson).isEmpty()) {
-            String content = json.toJson(users);
-            File file = File.builder().content(content).authorEmail("[email protected]").authorName("Jim Bot").branch("master").commitMessage("migrating users").build();
-            
-            fileService.createFile(engagement.getProjectId(), userJson, file);
-            LOGGER.info("Migrated {} users for engagement {}", users.size(), engagement.getUuid());
+        if(fileService.getFile(engagement.getProjectId(), fileName).isEmpty()) {
+            File file = File.builder().content(content).authorEmail(commitEmail).authorName(commitAuthor).branch(commitBranch).commitMessage(String.format("migrating %s", fileName)).build();
+            fileService.createFile(engagement.getProjectId(), fileName, file);
+            LOGGER.info("Migrated {} {} for engagement {}", size, fileName, engagement.getUuid());
         }
     }
 }

src/main/resources/application.properties

@@ -32,8 +32,12 @@ config.gitlab.ref=${CONFIG_GITLAB_REF:master}
 commit.page.size=100
 commit.msg.filter.list=${COMMIT_FILTERED_MESSAGE_LIST:manual_refresh}
 commit.filter.list=${COMMIT_FILTERED_EMAIL_LIST:[email protected]}
+commit.default.email=[email protected]
+commit.default.author=Otto Mayshin
+commit.default.branch=master
 config.reload=${CONFIG_RELOAD:false}
 
+
 # engagements
 engagements.repository.id=${ENGAGEMENTS_REPOSITORY_ID:2}
 engagements.do.not.delete=${ENGAGEMENTS_PRESERVE:false}
@@ -44,10 +48,4 @@ git.tag=${GIT_API_GIT_TAG:not.set}
 
 environment.id=${ENV_ID:local}
 
-migrate.users=${MIGRATE_USERS:false}
-migrate.uuid=${MIGRATE_UUID:false}
-
-%test.migrate.users=true
-%test.migrate.uuid=true
-
-seed.file.list=users.json,artifacts.json
+seed.file.list=participants.json,artifacts.json

src/test/java/com/redhat/labs/lodestar/resource/MigrationResourceTest.java

@@ -0,0 +1,18 @@
+package com.redhat.labs.lodestar.resource;
+
+import static io.restassured.RestAssured.given;
+
+import org.junit.jupiter.api.Test;
+
+import io.quarkus.test.junit.QuarkusTest;
+
+@QuarkusTest
+public class MigrationResourceTest {
+    @Test
+    void testGetHookFileSuccess() {
+
+        given().when().put("/api/migrate").then().statusCode(200);
+        given().queryParam("participants", true).queryParam("uuid", true).queryParam("artifacts", true).when()
+                .put("/api/migrate").then().statusCode(200);
+    }
+}

src/test/java/com/redhat/labs/lodestar/service/MigrationServiceTest.java

@@ -10,6 +10,7 @@
 import org.junit.jupiter.api.Test;
 import org.mockito.Mockito;
 
+import com.redhat.labs.lodestar.models.Artifact;
 import com.redhat.labs.lodestar.models.Engagement;
 import com.redhat.labs.lodestar.models.EngagementUser;
 import com.redhat.labs.lodestar.models.gitlab.File;
@@ -45,6 +46,9 @@ public static void setup() {
 
         List<EngagementUser> engagementUsers = new ArrayList<>();
         engagementUsers.add(EngagementUser.builder().uuid("b2").build());
+        
+        List<Artifact> artifacts = new ArrayList<>();
+        artifacts.add(Artifact.builder().uuid("1").build());
 
         List<Engagement> allEngagements = new ArrayList<>();
 
@@ -54,29 +58,35 @@ public static void setup() {
         e = Engagement.builder().uuid("c3").projectId(3).engagementUsers(engagementUsers).build();
         allEngagements.add(e);
 
-        e = Engagement.builder().uuid("d4").projectId(4).engagementUsers(engagementUsers).build();
+        e = Engagement.builder().uuid("d4").projectId(4).engagementUsers(engagementUsers).artifacts(artifacts).build();
         allEngagements.add(e);
 
         mockPS = Mockito.mock(ProjectService.class);
         Mockito.when(mockPS.getProjectsByGroup(2, true)).thenReturn(projects);
         QuarkusMock.installMockForType(mockPS, ProjectService.class);
 
         fileServiceMock = Mockito.mock(FileService.class);
-        Mockito.when(fileServiceMock.getFile(4, "users.json")).thenReturn(Optional.of(File.builder().build()));
+        Mockito.when(fileServiceMock.getFile(4, "participants.json")).thenReturn(Optional.of(File.builder().build()));
         QuarkusMock.installMockForType(fileServiceMock, FileService.class);
 
         EngagementService mockES = Mockito.mock(EngagementService.class);
         Mockito.when(mockES.getAllEngagements(Optional.of(false), Optional.of(false))).thenReturn(allEngagements);
         QuarkusMock.installMockForType(mockES, EngagementService.class);
     }
+    
+    @Test 
+    void migrate() {
+        migrationService.migrate(false, false, false);
+        
+        Mockito.verify(mockPS, Mockito.never()).updateProject(Mockito.any());
+        Mockito.verify(fileServiceMock, Mockito.never()).createFile(Mockito.anyInt(), Mockito.eq("participants.json"), Mockito.any(File.class));
 
-    @Test
-    void startOk() {
-
-        migrationService.onStart(new StartupEvent());
+        migrationService.migrate(true, true, true);
 
         Mockito.verify(mockPS, Mockito.times(2)).updateProject(Mockito.any());
-        Mockito.verify(fileServiceMock, Mockito.times(2)).createFile(Mockito.anyInt(), Mockito.eq("users.json"), Mockito.any(File.class));
+        Mockito.verify(fileServiceMock, Mockito.times(2)).createFile(Mockito.anyInt(), Mockito.eq("participants.json"), Mockito.any(File.class));
     }
+    
+    
 
 }