Skip to content

Commit 99a0938

Browse files
mcanoymcanoy
committed
add migration framework for new services. add project look up by uuid
1 parent 5642b0a commit 99a0938

File tree

16 files changed

+389
-91
lines changed

16 files changed

+389
-91
lines changed

‎src/main/java/com/redhat/labs/lodestar/models/gitlab/Project.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,15 +36,15 @@ public class Project {
3636
@JsonbProperty("description")
3737
private String description;
3838
@JsonbProperty("issues_enabled")
39-
private Boolean issuesEnabled;
39+
private boolean issuesEnabled;
4040
@JsonbProperty("merge_requests_enabled")
41-
private Boolean mergeRequestsEnabled;
41+
private boolean mergeRequestsEnabled;
4242
@JsonbProperty("jobs_enabled")
43-
private Boolean jobsEnabled;
43+
private boolean jobsEnabled;
4444
@JsonbProperty("wiki_enabled")
45-
private Boolean wikiEnabled;
45+
private boolean wikiEnabled;
4646
@JsonbProperty("snippets_enabled")
47-
private Boolean snippetsEnabled;
47+
private boolean snippetsEnabled;
4848
@JsonbProperty("issues_access_level")
4949
private String issuesAccessLevel;
5050
@JsonbProperty("repository_access_level")
@@ -66,11 +66,11 @@ public class Project {
6666
@JsonbProperty("resolve_outdated_diff_discussions")
6767
private Boolean resolveOutdatedDiffDiscussions;
6868
@JsonbProperty("container_registry_enabled")
69-
private Boolean containerRegistryEnabled;
69+
private boolean containerRegistryEnabled;
7070
@JsonbProperty("container_expiration_policy_attributes")
7171
private Map<String, String> containerExpirationPolicyAttributes;
7272
@JsonbProperty("shared_runners_enabled")
73-
private Boolean sharedRunnersEnabled;
73+
private boolean sharedRunnersEnabled;
7474
@JsonbProperty("visibility")
7575
private String visibility;
7676
@JsonbProperty("import_url")
@@ -106,7 +106,7 @@ public class Project {
106106
@JsonbProperty("ci_config_path")
107107
private String ciConfigPath;
108108
@JsonbProperty("auto_devops_enabled")
109-
private Boolean autoDevopsEnabled;
109+
private boolean autoDevopsEnabled;
110110
@JsonbProperty("auto_devops_deploy_strategy")
111111
private String autoDevopsDeployStrategy;
112112
@JsonbProperty("repository_storage")

‎src/main/java/com/redhat/labs/lodestar/resource/EngagementResource.java‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import javax.ws.rs.GET;
1111
import javax.ws.rs.POST;
1212
import javax.ws.rs.Path;
13+
import javax.ws.rs.PathParam;
1314
import javax.ws.rs.Produces;
1415
import javax.ws.rs.QueryParam;
1516
import javax.ws.rs.core.Context;
@@ -22,7 +23,7 @@
2223
import org.eclipse.microprofile.metrics.MetricUnits;
2324
import org.eclipse.microprofile.metrics.annotation.Counted;
2425
import org.eclipse.microprofile.metrics.annotation.Timed;
25-
import org.jboss.resteasy.annotations.jaxrs.PathParam;
26+
import org.eclipse.microprofile.openapi.annotations.tags.Tag;
2627
import org.slf4j.Logger;
2728
import org.slf4j.LoggerFactory;
2829

@@ -37,6 +38,7 @@
3738
@Path("/api/v1/engagements")
3839
@Produces(MediaType.APPLICATION_JSON)
3940
@Consumes(MediaType.APPLICATION_JSON)
41+
@Tag(name = "Engagements", description = "Engagement data")
4042
public class EngagementResource {
4143

4244
public static final Logger LOGGER = LoggerFactory.getLogger(EngagementResource.class);
@@ -122,6 +124,7 @@ public Response deleteEngagement(@PathParam("customer") String customer,
122124
@Path("customer/{customer}/{engagement}/hooks")
123125
@Counted(name = "create-engagement-hook", description = "Count of create-hook requestst")
124126
@Timed(name = "performedHookCreate", description = "Time to create hook", unit = MetricUnits.MILLISECONDS)
127+
@Tag(name = "Hooks")
125128
public Response createProjectHook(Hook hook, @PathParam("customer") String customer,
126129
@PathParam("engagement") String engagement) {
127130

@@ -144,6 +147,7 @@ public Response getEngagementCommits(@PathParam("customer") String customer,
144147
@Path("customer/{customer}/{engagement}/hooks")
145148
@Counted(name = "get-hook", description = "Count of get-hook requests")
146149
@Timed(name = "performedHookGetAll", description = "Time to get all hooks", unit = MetricUnits.MILLISECONDS)
150+
@Tag(name = "Hooks")
147151
public Response findAllProjectHooks(@PathParam("customer") String customer,
148152
@PathParam("engagement") String engagement) {
149153

@@ -167,6 +171,7 @@ public Response getStatus(@PathParam("customer") String customer, @PathParam("en
167171

168172
@DELETE
169173
@Path("/hooks")
174+
@Tag(name = "Hooks")
170175
@Counted(name = "delete-hooks", description = "Count of delete-hooks requests")
171176
@Timed(name = "performedHooksDelete", description = "Time to delete hooks", unit = MetricUnits.MILLISECONDS)
172177
public Response deleteAllHooks() {
@@ -175,5 +180,20 @@ public Response deleteAllHooks() {
175180
return Response.ok().build();
176181

177182
}
183+
184+
@GET
185+
@Path("project/{uuid}")
186+
@Counted(name = "project-by-uuid", description = "Count of project-by-uuid requests")
187+
@Timed(name = "performedProjectByUuidGet", description = "Time to get project", unit = MetricUnits.MILLISECONDS)
188+
@Tag(name = "Projects", description = "Project retrieval")
189+
public Response getProjectByUuid(@PathParam("uuid") String uuid) {
190+
Optional<Project> p = engagementService.getProjectByUuid(uuid);
191+
192+
if(p.isEmpty()) {
193+
return Response.status(404).build();
194+
}
195+
196+
return Response.ok(p.get()).build();
197+
}
178198

179199
}

‎src/main/java/com/redhat/labs/lodestar/rest/client/GitLabService.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@ Response getSubGroups(@PathParam("id") @Encoded Integer groupId, @QueryParam("pe
7373
@Produces("application/json")
7474
Response getGroupByName(@QueryParam("search") @Encoded String name, @QueryParam("per_page") int perPage,
7575
@QueryParam("page") int page);
76+
77+
@GET
78+
@Path("/groups/{id}/search")
79+
List<Project> findProjectByEngagementId(@PathParam("id") @Encoded Integer groupId,
80+
@QueryParam("scope") String scope, @QueryParam("search") String search);
7681

7782
@GET
7883
@Logged

‎src/main/java/com/redhat/labs/lodestar/service/EngagementService.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ public class EngagementService {
6464

6565
@ConfigProperty(name = "orchestration.queue.directory", defaultValue = "queue")
6666
String orchestrationQueueDirectory;
67+
68+
@ConfigProperty(name = "seed.file.list", defaultValue = "queue")
69+
List<String> seedFileList;
6770

6871
@Inject
6972
ProjectService projectService;
@@ -117,6 +120,10 @@ public Project createEngagement(Engagement engagement, String author, String aut
117120

118121
// create user reset file(s) if required
119122
repoFiles.addAll(createUserManagementFiles(engagement));
123+
124+
if(project.isFirst()) {
125+
repoFiles.addAll(createMicroFiles(project));
126+
}
120127

121128
// create actions for multiple commit
122129
CommitMultiple commit = createCommitMultiple(repoFiles, project.getId(), DEFAULT_BRANCH, author, authorEmail,
@@ -213,6 +220,10 @@ public Optional<Project> getProject(String customerName, String engagementName)
213220
LOGGER.debug("Full path {}", fullPath);
214221
return projectService.getProjectByIdOrPath(fullPath);
215222
}
223+
224+
public Optional<Project> getProjectByUuid(String engagementUuid) {
225+
return projectService.getProjectByEngagementUuid(engagementRepositoryId, engagementUuid);
226+
}
216227

217228
/**
218229
* Gets all engagements from the base group Structure is BaseGroup - customer
@@ -312,6 +323,19 @@ private File createEngagmentFile(Engagement engagement) {
312323
String fileContent = json.toJson(engagement);
313324
return File.builder().content(fileContent).filePath(ENGAGEMENT_FILE).build();
314325
}
326+
327+
/**
328+
* Creates seed files for other services. Never updates.
329+
* @return
330+
*/
331+
private List<File> createMicroFiles(Project project) {
332+
List<File> files = new ArrayList<>();
333+
if(project.isFirst()) {
334+
seedFileList.forEach(f -> files.add(File.builder().content("[]").filePath(f).build()));
335+
}
336+
337+
return files;
338+
}
315339

316340
private List<File> createUserManagementFiles(Engagement engagement) {
317341

‎src/main/java/com/redhat/labs/lodestar/service/FileService.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -105,7 +105,7 @@ public Optional<File> deleteFile(Integer projectId, String filePath, String ref)
105105
// set branch
106106
file.setBranch(ref);
107107
// add commit message
108-
file.setCommitMessage("git api deleted file.");
108+
file.setCommitMessage(String.format("git api deleted file. {}", filePath));
109109

110110
gitLabService.deleteFile(projectId, filePath, file);
111111
}
@@ -147,11 +147,10 @@ public Optional<File> getFile(String projectIdOrPath, String filePath, String re
147147
optional = Optional.of(file);
148148
}
149149
} catch(WebApplicationException wae) {
150+
150151
if(wae.getResponse().getStatus() != 404) {
151152
LOGGER.error("Get file {} for project {} failed with code {}", filePath, projectIdOrPath, wae.getResponse().getStatus());
152153
throw wae;
153-
} else if(LOGGER.isDebugEnabled()) {
154-
LOGGER.debug("Get file {} for project {} failed with code {}", filePath, projectIdOrPath, wae.getResponse().getStatus());
155154
}
156155
}
157156

‎src/main/java/com/redhat/labs/lodestar/service/GroupService.java‎

Lines changed: 0 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@
1313
import org.slf4j.Logger;
1414
import org.slf4j.LoggerFactory;
1515

16-
import com.redhat.labs.lodestar.exception.UnexpectedGitLabResponseException;
1716
import com.redhat.labs.lodestar.models.gitlab.Group;
1817
import com.redhat.labs.lodestar.models.pagination.PagedResults;
1918
import com.redhat.labs.lodestar.rest.client.GitLabService;
@@ -29,30 +28,6 @@ public class GroupService {
2928
@ConfigProperty(name = "commit.page.size")
3029
int commitPageSize;
3130

32-
// get a group
33-
public Optional<Group> getGitLabGroupByName(String name, Integer parentId)
34-
throws UnexpectedGitLabResponseException {
35-
36-
Optional<Group> optional = Optional.empty();
37-
38-
PagedResults<Group> page = new PagedResults<>();
39-
40-
while(page.hasMore()) {
41-
Response response = gitLabService.getGroupByName(name, commitPageSize, page.getNumber());
42-
page.update(response, new GenericType<List<Group>>() {});
43-
}
44-
45-
// look for a match between returned name and provided path
46-
for (Group group : page.getResults()) {
47-
if (name.equals(group.getName()) && parentId.equals(group.getParentId())) {
48-
return Optional.of(group);
49-
}
50-
}
51-
52-
return optional;
53-
54-
}
55-
5631
public List<Group> getSubgroups(Integer groupId) {
5732

5833
PagedResults<Group> page = new PagedResults<>();

‎src/main/java/com/redhat/labs/lodestar/service/MigrationService.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
package com.redhat.labs.lodestar.service;
2+
3+
import java.util.Collections;
4+
import java.util.HashMap;
5+
import java.util.List;
6+
import java.util.Map;
7+
import java.util.Optional;
8+
9+
import javax.enterprise.context.ApplicationScoped;
10+
import javax.enterprise.event.Observes;
11+
import javax.inject.Inject;
12+
13+
import org.eclipse.microprofile.config.inject.ConfigProperty;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
16+
17+
import com.redhat.labs.lodestar.config.JsonMarshaller;
18+
import com.redhat.labs.lodestar.models.Engagement;
19+
import com.redhat.labs.lodestar.models.EngagementUser;
20+
import com.redhat.labs.lodestar.models.gitlab.File;
21+
import com.redhat.labs.lodestar.models.gitlab.Project;
22+
23+
import io.quarkus.runtime.StartupEvent;
24+
25+
@ApplicationScoped
26+
public class MigrationService {
27+
private static final Logger LOGGER = LoggerFactory.getLogger(MigrationService.class);
28+
29+
private static final String userJson = "users.json";
30+
31+
@Inject
32+
EngagementService engagementService;
33+
34+
@Inject
35+
FileService fileService;
36+
37+
@Inject
38+
ProjectService projectService;
39+
40+
@Inject
41+
JsonMarshaller json;
42+
43+
@ConfigProperty(name = "engagements.repository.id")
44+
int engagementRepositoryId;
45+
46+
@ConfigProperty(name = "migrate.users")
47+
boolean migrateUsers;
48+
49+
@ConfigProperty(name = "migrate.uuid")
50+
boolean migrateUuids;
51+
52+
private Map<Integer, Engagement> allEngagements = new HashMap<>();
53+
54+
/**
55+
* Currently the migration will only occur if config properties are true.
56+
* The migration is idempotent so no harm in rerunning. It will only update
57+
* engagements that haven't been migrated. As we get closer to migration time
58+
* we should evaluate whether this is the right approach.
59+
* Once the start up is complete this service can not be called.
60+
* @param ev
61+
*/
62+
void onStart(@Observes StartupEvent ev) {
63+
64+
if(migrateUsers) {
65+
LOGGER.info("Migrate users: {}", migrateUsers);
66+
migrateUsers();
67+
LOGGER.info("End Migrate users");
68+
}
69+
70+
if(migrateUuids) {
71+
LOGGER.info("Migrate uuids: {}", migrateUuids);
72+
migrateUuids();
73+
LOGGER.info("End Migrate uuids");
74+
}
75+
76+
}
77+
/**
78+
* Get all projects and split for individual update
79+
*/
80+
private void migrateUuids() {
81+
List<Project> allProjects = projectService.getProjectsByGroup(engagementRepositoryId, true);
82+
allProjects.parallelStream().forEach(this::updateProjectWithUuid);
83+
}
84+
85+
/**
86+
* Update a single project if the description is not already set and an engagement has been found
87+
* @param project
88+
*/
89+
private void updateProjectWithUuid(Project project) {
90+
91+
if(project.getDescription() == null && getAllEngagements().containsKey(project.getId())) {
92+
String uuid = allEngagements.get(project.getId()).getUuid();
93+
project.setDescription(String.format(ProjectStructureService.ENGAGEMENT_PROJECT_DESCRIPTION, uuid));
94+
projectService.updateProject(project);
95+
96+
LOGGER.info("Added uuid {} to project {} {}", uuid, project.getId(), project);
97+
}
98+
}
99+
100+
/**
101+
* Get all engagements (engagement.json) and split for individual update
102+
*/
103+
private void migrateUsers() {
104+
getAllEngagements().values().parallelStream().forEach(this::migrateUsersToGitlab);
105+
}
106+
107+
/**
108+
* Get All engagements. This is run by migrate users and uuids so only run once if both are active
109+
* @return
110+
*/
111+
private Map<Integer, Engagement> getAllEngagements() {
112+
if (allEngagements.isEmpty()) {
113+
List<Engagement> engagements = engagementService.getAllEngagements(Optional.of(false), Optional.of(false));
114+
engagements.parallelStream().forEach(this::addToMap);
115+
LOGGER.debug("fetched engagements for migration {} ", allEngagements.size());
116+
}
117+
118+
return allEngagements;
119+
}
120+
121+
/**
122+
* Do a quick project id to engagement mapping to easily find the engagement by project id
123+
* @param engagement
124+
*/
125+
private void addToMap(Engagement engagement) {
126+
LOGGER.debug("mcanoy {}",engagement);
127+
allEngagements.put(engagement.getProjectId(), engagement);
128+
}
129+
130+
/**
131+
* This will write the user json to gitlab. Should you wish to rollback or redo you could add this code
132+
* fileService.deleteFile(engagement.getProjectId(), userJson);
133+
* @param engagement
134+
*/
135+
private void migrateUsersToGitlab(Engagement engagement) {
136+
137+
List<EngagementUser> users = engagement.getEngagementUsers();
138+
139+
if(users == null) {
140+
users = Collections.emptyList();
141+
}
142+
143+
if(fileService.getFile(engagement.getProjectId(), userJson).isEmpty()) {
144+
String content = json.toJson(users);
145+
File file = File.builder().content(content).authorEmail("[email protected]").authorName("Jim Bot").branch("master").commitMessage("migrating users").build();
146+
147+
fileService.createFile(engagement.getProjectId(), userJson, file);
148+
LOGGER.info("Migrated {} users for engagement {}", users.size(), engagement.getUuid());
149+
}
150+
}
151+
}

0 commit comments

Comments
 (0)