Add fault tolerance. Catch 404 errors and return empty list. Jacoco update. dev mode mongo connection. remove paagedresults.

mcanoy · mcanoy · commit 4b8309f9f041 · 2021-07-09T13:53:31.000-07:00
diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml
@@ -27,7 +27,7 @@ jobs:
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
-      run: mvn -B verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
+      run: mvn -B -q verify org.sonarsource.scanner.maven:sonar-maven-plugin:sonar
     - name: Find and Replace Commit
       uses: jacobtomlinson/gha-find-replace@0.1.1
       with:
diff --git a/.gitignore b/.gitignore
@@ -39,3 +39,4 @@ nb-configuration.xml
 .env
 
 deployment/mongo-volume
+.adhoc
diff --git a/README.md b/README.md
@@ -21,9 +21,9 @@ The following environment variables are available:
 | MONGODB_DATABASE | artifacts | The database name |
 
 | GITLAB_API_URL | https://acmegit.com | The url to Gitlab |
+| ENGAGEMENT_API_URL | https://acmegit.com | The url to Gitlab |
 | GITLAB_TOKEN | t | The Access Token for Gitlab |
 
-| GROUP_PARENT_ID | 1234 | Gitlab group ID containing engagements |
 | DEFAULT_BRANCH | master | Default branch to use if default not found for project |
 | DEFAULT_COMMIT_MESSAGE | updated artifacts list | Default commit message used if diff fails |
 | DEFAULT_PAGE_SIZE | 20 | Default number of artifacts that will be returned if pageSize not specified |
@@ -49,9 +49,9 @@ You can run your application in dev mode that enables live coding using:
 
 ```
 export GITLAB_API_URL=https://gitlab.com/ 
+export ENGAGEMENT_API_URL=http://git-api:8080
 export GITLAB_TOKEN=token
-export GROUP_PARENT_ID=<your-group-id>
 mvn quarkus:dev
 ```
 
-> **_NOTE:_**  Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/.
+> **_NOTE:_**  Quarkus now ships with a Dev UI, which is available in dev mode only at http://localhost:8080/q/dev/.
diff --git a/deployment/values-dev.yaml b/deployment/values-dev.yaml
@@ -22,10 +22,10 @@ db:
 
 api:
   gitlab: http://gitlab.com
-  engagement: http://git-api:8080
+  engagement: http://lodestar-git-api:8080
 
 tokens:
   gitlab: nope
 
 config:
-  gitlabGroupId: 0
+  gitlabGroupId: 0
diff --git a/pom.xml b/pom.xml
@@ -62,6 +62,11 @@
 			<groupId>io.quarkus</groupId>
 			<artifactId>quarkus-smallrye-openapi</artifactId>
 		</dependency>
+		<!-- Fault tolerance -->
+		<dependency>
+			<groupId>io.quarkus</groupId>
+			<artifactId>quarkus-smallrye-fault-tolerance</artifactId>
+		</dependency>
 		<!-- REST Client -->
 		<dependency>
 			<groupId>io.quarkus</groupId>
@@ -73,6 +78,10 @@
 			<artifactId>quarkus-smallrye-health</artifactId>
 		</dependency>
 		<!-- Other -->
+		<dependency>
+			<groupId>io.quarkus</groupId>
+			<artifactId>quarkus-jacoco</artifactId>
+		</dependency>
 		<dependency>
 			<groupId>io.quarkus</groupId>
 			<artifactId>quarkus-arc</artifactId>
@@ -145,27 +154,6 @@
 					</systemPropertyVariables>
 				</configuration>
 			</plugin>
-			<plugin>
-				<groupId>org.jacoco</groupId>
-				<artifactId>jacoco-maven-plugin</artifactId>
-				<version>0.8.5</version>
-				<executions>
-					<execution>
-						<id>jacoco-initialize</id>
-						<goals>
-							<goal>prepare-agent</goal>
-						</goals>
-						<phase>test-compile</phase>
-					</execution>
-					<execution>
-						<id>jacoco-site</id>
-						<phase>verify</phase>
-						<goals>
-							<goal>report</goal>
-						</goals>
-					</execution>
-				</executions>
-			</plugin>
 		</plugins>
 	</build>
 	<profiles>
diff --git a/src/main/java/com/redhat/labs/lodestar/model/pagination/PagedResults.java b/src/main/java/com/redhat/labs/lodestar/model/pagination/PagedResults.java
diff --git a/src/main/java/com/redhat/labs/lodestar/resource/ArtifactResource.java b/src/main/java/com/redhat/labs/lodestar/resource/ArtifactResource.java
@@ -59,7 +59,7 @@ public Response getArtifacts(@BeanParam GetListOptions options) {
         ArtifactCount count = service.countArtifacts(options);
 
         return Response.ok(artifacts).header("x-page", options.getPage()).header("x-per-page", options.getPageSize())
-                .header("x-total-activity", count.getCount())
+                .header("x-total-artifacts", count.getCount())
                 .header("x-total-pages", (count.getCount() / options.getPageSize()) + 1).build();
 
     }
diff --git a/src/main/java/com/redhat/labs/lodestar/rest/client/GitlabRestClient.java b/src/main/java/com/redhat/labs/lodestar/rest/client/GitlabRestClient.java
@@ -7,13 +7,17 @@
 import javax.ws.rs.PathParam;
 import javax.ws.rs.Produces;
 import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
 
+import org.apache.http.NoHttpResponseException;
+import org.eclipse.microprofile.faulttolerance.Retry;
 import org.eclipse.microprofile.rest.client.annotation.RegisterClientHeaders;
 import org.eclipse.microprofile.rest.client.annotation.RegisterProvider;
 import org.eclipse.microprofile.rest.client.inject.RegisterRestClient;
 
 import com.redhat.labs.lodestar.model.gitlab.File;
 
+@Retry(maxRetries = 5, delay = 1200, retryOn = NoHttpResponseException.class, abortOn = WebApplicationException.class)
 @Path("/api/v4")
 @RegisterRestClient(configKey = "gitlab.api")
 @RegisterClientHeaders(GitlabTokenFactory.class)
diff --git a/src/main/java/com/redhat/labs/lodestar/service/ArtifactService.java b/src/main/java/com/redhat/labs/lodestar/service/ArtifactService.java
@@ -5,6 +5,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -14,6 +15,7 @@
 import javax.enterprise.context.ApplicationScoped;
 import javax.inject.Inject;
 import javax.json.bind.Jsonb;
+import javax.ws.rs.WebApplicationException;
 
 import org.eclipse.microprofile.config.inject.ConfigProperty;
 import org.eclipse.microprofile.rest.client.inject.RestClient;
@@ -22,6 +24,8 @@
 import org.javers.core.JaversBuilder;
 import org.javers.core.diff.Diff;
 import org.javers.core.diff.ListCompareAlgorithm;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import com.redhat.labs.lodestar.model.Artifact;
 import com.redhat.labs.lodestar.model.ArtifactCount;
@@ -34,6 +38,7 @@
 
 @ApplicationScoped
 public class ArtifactService {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ArtifactService.class);
 
     @ConfigProperty(name = "artifacts.file", defaultValue = "artifacts.json")
     private String artifactsFile;
@@ -78,7 +83,7 @@ public void purge() {
     public void refresh() {
 
         engagementRestClient.getAllEngagementProjects().stream().map(Engagement::getProjectId)
-                .map(this::getArtifactsFileByProjectId).map(this::parseArtifactsFile).flatMap(Collection::stream)
+                .map(this::getArtifactsFromGitlabByProjectId).flatMap(Collection::stream)
                 .forEach(a -> {
 
                     // set uuid if missing
@@ -93,34 +98,31 @@ public void refresh() {
 
     }
 
-    /**
-     * Returns a {@link File} containing the artifacts content. Otherwise, null is
-     * returned.
-     * 
-     * @param projectId
-     * @return
-     */
-    File getArtifactsFileByProjectId(long projectId) {
-        return gitlabRestClient.getFile(projectId, artifactsFile, defaultBranch);
-    }
-
     /**
      * Returns a {@link List} of {@link Artifact}s for the given {@link File}. An
      * empty {@link List} is returned if {@link File} or its contents are null or
      * blank.
      * 
-     * @param file
+     * @param projectId
      * @return
      */
-    List<Artifact> parseArtifactsFile(File file) {
-
-        if (null == file || null == file.getContent() || file.getContent().isBlank()) {
-            return new ArrayList<>();
+    List<Artifact> getArtifactsFromGitlabByProjectId(long projectId) {
+        try {
+            File file = gitlabRestClient.getFile(projectId, artifactsFile, defaultBranch);
+            if(null == file.getContent() || file.getContent().isBlank()) {
+                LOGGER.error("NO FILE DATA FROM GITLAB FOR PROJECT {}. THIS SHALL NOT STAND", projectId);
+                return Collections.emptyList();
+            }
+            
+            file.decodeFileAttributes();
+            return Arrays.asList(jsonb.fromJson(file.getContent(), Artifact[].class));
+            
+        } catch(WebApplicationException wae) {
+            if(wae.getResponse().getStatus() != 404) {
+                throw wae;
+            }
+            return Collections.emptyList();
         }
-
-        file.decodeFileAttributes();
-        return Arrays.asList(jsonb.fromJson(file.getContent(), Artifact[].class));
-
     }
 
     /**
@@ -353,7 +355,7 @@ void updateArtifactsFile(String engagementUuid, List<Artifact> artifacts, Option
         // create
         File file = createArtifactsFile(content, defaultBranch, authorName, authorEmail, commitMessage);
 
-        // create or udpate in git
+        // update in git
         gitlabRestClient.updateFile(project.getProjectId(), artifactsFile, file);
 
     }
diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties
@@ -19,6 +19,7 @@ mongo.service.name=${DATABASE_SERVICE_NAME:localhost:27017}
 quarkus.mongodb.database=${MONGODB_DATABASE:artifacts}
 
 quarkus.mongodb.connection-string=mongodb://${mongo.user}:${mongo.password}@${mongo.service.name}/${quarkus.mongodb.database}?uuidRepresentation=javaLegacy
+%dev.quarkus.mongodb.connection-string=mongodb://${mongo.user}:${mongo.password}@${mongo.service.name}/${quarkus.mongodb.database}?uuidRepresentation=javaLegacy&authSource=admin
 
 %test.quarkus.mongodb.connection-string=mongodb://${mongo.service.name}/${quarkus.mongodb.database}?uuidRepresentation=javaLegacy
 %test.quarkus.mongodb.write-concern.journal=false

Original file line number	Diff line number	Diff line change
`@@ -39,3 +39,4 @@ nb-configuration.xml`
`39`	`39`	`.env`
`40`	`40`
`41`	`41`	`deployment/mongo-volume`
	`42`	`+.adhoc`
Original file line number	Diff line number	Diff line change
`@@ -59,7 +59,7 @@ public Response getArtifacts(@BeanParam GetListOptions options) {`
`59`	`59`	`ArtifactCount count = service.countArtifacts(options);`
`60`	`60`
`61`	`61`	`return Response.ok(artifacts).header("x-page", options.getPage()).header("x-per-page", options.getPageSize())`
`62`		`- .header("x-total-activity", count.getCount())`
	`62`	`+ .header("x-total-artifacts", count.getCount())`
`63`	`63`	`.header("x-total-pages", (count.getCount() / options.getPageSize()) + 1).build();`
`64`	`64`
`65`	`65`	`}`