Get All Engagement Paging and Status Performance (#124)

* initial paging for get all engagements

* remove unused call

* cleanup

* fixes for get status perf, rest exception handling, tests

* added pagination test for get all

* cleanup commented code

* fix for query param name and updated readme

* moved pagedresults to pagination package

* rm accidental commit of route template

Author: dwasinge

README.md

@@ -16,21 +16,84 @@ The JSON REST APIs consist of the following resources:
 
 ### Config Resource
 
-The config resource exposes an API that allows clients to retrieve a configured `Config` file from GitLab.
+The config resource exposes an API that allows clients to retrieve a configuration files from GitLab.
+
+#### LodeStar Runtime Configuration File
 
 ```
 GET  /api/v1/config
+GET  /api/v2/config
 ```
 
+Version 1 of the API returns the file in YAML and version 2 of the API returns the file in JSON.
+
+#### Webhooks Configuration File
+
+```
+GET  /api/v2/config/webhooks
+```
+
+Returns a JSON representation of the configured webhooks.
+
 ### Engagement Resource
 
 The engagement resource exposes an API that allows clients to create or update and engagement resource in GitLab.
 
 ```
-POST /api/v1/engagements
+POST ​  /api​/v1​/engagements
+```
+Used to create or update an engagement.  This endpoint will create the expected group/project structure in GitLab.  Then, will update the `engagement.json` file if it already exists or create it if it does not.
+```
+POST   ​/api​/v1​/engagements​/customer​/{customer}​/{engagement}​/hooks
 ```
+Used to create a webhook in GitLab for the project associated to the customer and engagement name combination.
+```
+GET ​   /api​/v1​/engagements
+```
+Optional query parameters:
+
+- `includeCommits` - adds commit data to engagement if set to `true`
+- `includeStatus`  - adds status data to engagement if set to `true`
+- `pagination` - returns a single page of engagements if true.  all if false
+- `page` - page number to retrieve from GitLab
+- `per_page` - number of engagements to return per page
+
+Pagination Headers Returned:
+- `Link` - contains links for rel `first`, `last`, and `next`.  `next` omitted if last page is requested
+- `x-first-page` number of first page
+- `x-next-page`  number of next page (omitted if last page)
+- `x-last-page` number of last page
 
-This endpoint will create the expected group/project structure in GitLab.  Then, will update the `engagement.json` file if it already exists or create it if it does not.
+```
+```
+GET  ​  /api​/v1​/engagements​/customer​/{customer}​/{engagement}
+```
+Returns the engagement associated with the customer and engagement name combination.
+```
+```
+GET    ​/api​/v1​/engagements​/customer​/{customer}​/{engagement}​/commits
+```
+Returns a list of commits associated with the customer and engagement name combination.
+```
+GET    ​/api​/v1​/engagements​/customer​/{customer}​/{engagement}​/hooks
+```
+Returns the webhooks for the engagement associated with the customer and engagement name combination.
+```
+GET    ​/api​/v1​/engagements​/customer​/{customer}​/{engagement}​/status
+```
+Returns the status data for the engagement associated with the customer and engagement name combination.
+```
+GET    ​/api​/v1​/engagements​/namespace​/{namespace}
+```
+Returns the engagement associated with the provided namespace.
+```
+DELETE ​/api​/v1​/engagements​/customer​/{customer}​/{engagement}
+```
+Deletes the engagement associated with the customer and engagement name combination.  Will only delete if engagement has not been lost.
+```
+DELETE ​/api​/v1​/engagements​/hooks
+```
+Deletes all webhooks in all configured engagement projects.
 
 ### Version Resource
 
@@ -75,13 +138,15 @@ Deployment template will read from the above secret and inject following env var
 | CONFIG_FILE | my-config.yml | True |
 | CONFIG_GITLAB_REF | master | False |
 | WEBHOOK_FILE | webhooks.yml | False |
+| CONFIG_RELOAD | true | False |
 
 ### Engagements Resource
 
 | Name | Example Value | Required |
 |------|---------------|----------|
 | ENGAGEMENTS_REPOSITORY_ID | 2 | True |
 | WEBHOOK_DEFAULT_TOKEN | tolkien | False | 
+| ENGAGEMENTS_PRESERVE | true | False |
 
 ### Version Resource
 
@@ -90,6 +155,13 @@ Deployment template will read from the above secret and inject following env var
 | GIT_API_GIT_COMMIT | a2adfk | False |
 | GIT_API_GIT_TAG | v1.2 | False |
 
+### Other
+
+| Name | Example Value | Required |
+|------|---------------|----------|
+| ENV_ID | TEST | False |
+
+
 ## Development
 
 See [the deployment README](deployment/README.md) for details on how to spin up a deployment for developing on OpenShift.

pom.xml

@@ -14,11 +14,11 @@
 		<maven.compiler.target>11</maven.compiler.target>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 		<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
-		<quarkus-plugin.version>1.6.1.Final</quarkus-plugin.version>
+		<quarkus-plugin.version>1.11.3.Final</quarkus-plugin.version>
 		<quarkus.platform.artifact-id>quarkus-bom</quarkus.platform.artifact-id>
 		<quarkus.platform.group-id>io.quarkus</quarkus.platform.group-id>
-		<quarkus.platform.version>1.6.1.Final</quarkus.platform.version>
-		<surefire-plugin.version>2.22.1</surefire-plugin.version>
+		<quarkus.platform.version>1.11.3.Final</quarkus.platform.version>
+		<surefire-plugin.version>3.0.0-M5</surefire-plugin.version>
 		<lombok.version>1.18.12</lombok.version>
 		<sonar.host.url>https://sonarcloud.io</sonar.host.url>
 		<sonar.project.key>rht-labs_lodestar-git-api</sonar.project.key>

src/main/java/com/redhat/labs/lodestar/exception/mapper/ApiExceptionMapper.java

@@ -9,23 +9,30 @@
 import org.eclipse.microprofile.rest.client.ext.ResponseExceptionMapper;
 
 @Priority(4000)
-public class GitLabServiceResponseMapper implements 
-             ResponseExceptionMapper<RuntimeException> {
-  @Override
-  public RuntimeException toThrowable(Response response) {
-    int status = response.getStatus();
+public class GitLabServiceResponseMapper implements ResponseExceptionMapper<RuntimeException> {
+    @Override
+    public RuntimeException toThrowable(Response response) {
+        int status = response.getStatus();
 
-    String msg = getBody(response);
+        String msg = getBody(response);
 
-    return new WebApplicationException(msg, status);
+        return new WebApplicationException(msg, status);
 
-  }
+    }
+
+    private String getBody(Response response) {
+
+        if (response.hasEntity()) {
+
+            ByteArrayInputStream is = (ByteArrayInputStream) response.getEntity();
+            byte[] bytes = new byte[is.available()];
+            is.read(bytes, 0, is.available());
+            return new String(bytes);
+
+        }
+
+        return null;
 
-  private String getBody(Response response) {
-      ByteArrayInputStream is = (ByteArrayInputStream) response.getEntity();
-      byte[] bytes = new byte[is.available()];
-      is.read(bytes,0,is.available());
-      return new String(bytes);
     }
 
 }

src/main/java/com/redhat/labs/lodestar/exception/mapper/GitLabServiceResponseMapper.java

@@ -0,0 +1,20 @@
+package com.redhat.labs.lodestar.models.gitlab;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class ProjectTreeNode {
+
+    private String id;
+    private String name;
+    private String type;
+    private String path;
+    private String mode;
+
+}

src/main/java/com/redhat/labs/lodestar/models/gitlab/ProjectTreeNode.java

@@ -0,0 +1,132 @@
+package com.redhat.labs.lodestar.models.pagination;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.ws.rs.core.Link;
+import javax.ws.rs.core.UriBuilder;
+
+import com.redhat.labs.lodestar.models.Engagement;
+import com.redhat.labs.lodestar.models.gitlab.Project;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class Page {
+
+    private static final String CURRENT_PAGE = "page";
+    private static final String PER_PAGE = "per_page";
+    private static final String PAGE_HEADER_FORMAT = "x-%s-page";
+    private static final String PER_PAGE_HEADER = "x-per-page";
+
+    private static final String FIRST = "first";
+    private static final String FIRST_PAGE_HEADER = String.format(PAGE_HEADER_FORMAT, FIRST);
+
+    private static final String NEXT = "next";
+    private static final String NEXT_PAGE_HEADER = String.format(PAGE_HEADER_FORMAT, NEXT);
+
+    private static final String LAST = "last";
+    private static final String LAST_PAGE_HEADER = String.format(PAGE_HEADER_FORMAT, LAST);
+
+    private static final String REGEX = ".*&page=(\\d+)&.*";
+
+    @Builder.Default
+    Integer perPage = 20;
+    String gitLabLinkHeader;
+
+    @Builder.Default
+    Map<String, Map<String, Integer>> linkHeaders = new HashMap<>();
+
+    @Builder.Default
+    Map<String, Object> headers = new HashMap<>();
+
+    @Builder.Default
+    List<Project> results = new ArrayList<>();
+
+    @Builder.Default
+    List<Engagement> engagements = new ArrayList<>();
+
+    public void setHeadersFromGitLabHeader() {
+
+        if (null == this.gitLabLinkHeader) {
+            return;
+        }
+
+        // clear before reloading
+        linkHeaders.clear();
+
+        setHeadersForRelations();
+
+    }
+
+    private void setHeadersForRelations() {
+
+        // split links
+        String[] links = gitLabLinkHeader.split(",");
+
+        headers.put(PER_PAGE_HEADER, perPage);
+
+        for (String link : links) {
+
+            String rel;
+            String pageHeader;
+            Integer pageValue = Integer.valueOf(link.replaceAll(REGEX, "$1"));
+
+            if (link.contains(FIRST)) {
+
+                rel = FIRST;
+                linkHeaders.put(FIRST, new HashMap<>());
+                pageHeader = FIRST_PAGE_HEADER;
+
+            } else if (link.contains(NEXT)) {
+
+                rel = NEXT;
+                linkHeaders.put(NEXT, new HashMap<>());
+                pageHeader = NEXT_PAGE_HEADER;
+
+            } else if (link.contains(LAST)) {
+
+                rel = LAST;
+                linkHeaders.put(LAST, new HashMap<>());
+                pageHeader = LAST_PAGE_HEADER;
+
+            } else {
+                continue;
+            }
+
+            headers.put(pageHeader, pageValue);
+            linkHeaders.get(rel).put(CURRENT_PAGE, pageValue);
+            linkHeaders.get(rel).put(PER_PAGE, perPage);
+
+        }
+
+    }
+
+    public Link[] getLinks(UriBuilder uriBuilder) {
+
+        List<Link> links = linkHeaders.entrySet().stream().map(e1 -> {
+
+            String rel = e1.getKey();
+            Map<String, Integer> headerMap = e1.getValue();
+
+            javax.ws.rs.core.Link.Builder builder = Link.fromUriBuilder(uriBuilder).rel(rel);
+            headerMap.entrySet().forEach(e2 -> builder.param(e2.getKey(), String.valueOf(e2.getValue())));  
+
+            return builder.build();
+
+        }).collect(Collectors.toList());
+
+        return links.toArray(new Link[links.size()]);
+
+    }
+
+}

src/main/java/com/redhat/labs/lodestar/models/pagination/Page.java

@@ -1,4 +1,4 @@
-package com.redhat.labs.lodestar.models;
+package com.redhat.labs.lodestar.models.pagination;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -51,7 +51,7 @@ public void update(Response response, GenericType<List<T>> type) {
 
             if(totalPageString == null) {
                 total = UNDEFINED_PAGE_COUNT; //Should not be able to get this high
-                LOGGER.debug("X-Total-Pages header is missing");
+                LOGGER.trace("X-Total-Pages header is missing");
             } else {
                 total = Integer.valueOf(totalPageString);
             }